From 24aade4b6b8ebe61f31d7665af20bc7ba7315166 Mon Sep 17 00:00:00 2001 From: oOPa <5878650+oOPa@users.noreply.github.com> Date: Fri, 19 Dec 2025 02:51:12 +0000 Subject: [PATCH 1/9] Support passing thinkingConfig to gemini realtime models --- .../google/src/beta/realtime/realtime_api.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index 30a8404f0..a7718ed50 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -102,6 +102,7 @@ interface RealtimeOptions { contextWindowCompression?: ContextWindowCompressionConfig; apiVersion?: string; geminiTools?: LLMTools; + thinkingConfig?: types.ThinkingConfig; } /** @@ -273,6 +274,11 @@ export class RealtimeModel extends llm.RealtimeModel { * Gemini-specific tools to use for the session */ geminiTools?: LLMTools; + + /** + * The thinking configuration for response generation + */ + thinkingConfig?: types.ThinkingConfig; } = {}, ) { const inputAudioTranscription = @@ -330,6 +336,7 @@ export class RealtimeModel extends llm.RealtimeModel { contextWindowCompression: options.contextWindowCompression, apiVersion: options.apiVersion, geminiTools: options.geminiTools, + thinkingConfig: options.thinkingConfig, }; } @@ -1152,6 +1159,12 @@ export class RealtimeSession extends llm.RealtimeSession { config.contextWindowCompression = opts.contextWindowCompression; } + if (opts.thinkingConfig !== undefined) { + config.generationConfig = { + thinkingConfig: opts.thinkingConfig, + }; + } + return config; } @@ -1410,9 +1423,9 @@ export class RealtimeSession extends llm.RealtimeSession { this.sessionShouldClose.set(); } - async commitAudio() {} + async commitAudio() { } - async clearAudio() {} + async clearAudio() { } private *resampleAudio(frame: AudioFrame): Generator { if (this.inputResampler) { From e451de17b79deac6181606130c89fbde43a19750 Mon Sep 17 00:00:00 2001 From: oOPa <5878650+oOPa@users.noreply.github.com> Date: Sun, 21 Dec 2025 09:55:06 +0000 Subject: [PATCH 2/9] Build google plugin dist --- plugins/google/dist/beta/gemini_tts.cjs | 243 ++++ plugins/google/dist/beta/gemini_tts.cjs.map | 1 + plugins/google/dist/beta/gemini_tts.d.cts | 47 + plugins/google/dist/beta/gemini_tts.d.ts | 47 + plugins/google/dist/beta/gemini_tts.d.ts.map | 1 + plugins/google/dist/beta/gemini_tts.js | 225 ++++ plugins/google/dist/beta/gemini_tts.js.map | 1 + plugins/google/dist/beta/gemini_tts.test.cjs | 9 + .../google/dist/beta/gemini_tts.test.cjs.map | 1 + .../google/dist/beta/gemini_tts.test.d.cts | 2 + plugins/google/dist/beta/gemini_tts.test.d.ts | 2 + .../google/dist/beta/gemini_tts.test.d.ts.map | 1 + plugins/google/dist/beta/gemini_tts.test.js | 8 + .../google/dist/beta/gemini_tts.test.js.map | 1 + plugins/google/dist/beta/index.cjs | 42 + plugins/google/dist/beta/index.cjs.map | 1 + plugins/google/dist/beta/index.d.cts | 3 + plugins/google/dist/beta/index.d.ts | 3 + plugins/google/dist/beta/index.d.ts.map | 1 + plugins/google/dist/beta/index.js | 7 + plugins/google/dist/beta/index.js.map | 1 + .../google/dist/beta/realtime/api_proto.cjs | 17 + .../dist/beta/realtime/api_proto.cjs.map | 1 + .../google/dist/beta/realtime/api_proto.d.cts | 26 + .../google/dist/beta/realtime/api_proto.d.ts | 26 + .../dist/beta/realtime/api_proto.d.ts.map | 1 + .../google/dist/beta/realtime/api_proto.js | 1 + .../dist/beta/realtime/api_proto.js.map | 1 + plugins/google/dist/beta/realtime/index.cjs | 29 + .../google/dist/beta/realtime/index.cjs.map | 1 + plugins/google/dist/beta/realtime/index.d.cts | 3 + plugins/google/dist/beta/realtime/index.d.ts | 3 + .../google/dist/beta/realtime/index.d.ts.map | 1 + plugins/google/dist/beta/realtime/index.js | 5 + .../google/dist/beta/realtime/index.js.map | 1 + .../dist/beta/realtime/realtime_api.cjs | 1001 +++++++++++++++++ .../dist/beta/realtime/realtime_api.cjs.map | 1 + .../dist/beta/realtime/realtime_api.d.cts | 273 +++++ .../dist/beta/realtime/realtime_api.d.ts | 273 +++++ .../dist/beta/realtime/realtime_api.d.ts.map | 1 + .../google/dist/beta/realtime/realtime_api.js | 983 ++++++++++++++++ .../dist/beta/realtime/realtime_api.js.map | 1 + plugins/google/dist/index.cjs | 58 + plugins/google/dist/index.cjs.map | 1 + plugins/google/dist/index.d.cts | 4 + plugins/google/dist/index.d.ts | 4 + plugins/google/dist/index.d.ts.map | 1 + plugins/google/dist/index.js | 20 + plugins/google/dist/index.js.map | 1 + plugins/google/dist/llm.cjs | 381 +++++++ plugins/google/dist/llm.cjs.map | 1 + plugins/google/dist/llm.d.cts | 82 ++ plugins/google/dist/llm.d.ts | 82 ++ plugins/google/dist/llm.d.ts.map | 1 + plugins/google/dist/llm.js | 362 ++++++ plugins/google/dist/llm.js.map | 1 + plugins/google/dist/llm.test.cjs | 14 + plugins/google/dist/llm.test.cjs.map | 1 + plugins/google/dist/llm.test.d.cts | 2 + plugins/google/dist/llm.test.d.ts | 2 + plugins/google/dist/llm.test.d.ts.map | 1 + plugins/google/dist/llm.test.js | 13 + plugins/google/dist/llm.test.js.map | 1 + plugins/google/dist/models.cjs | 17 + plugins/google/dist/models.cjs.map | 1 + plugins/google/dist/models.d.cts | 5 + plugins/google/dist/models.d.ts | 5 + plugins/google/dist/models.d.ts.map | 1 + plugins/google/dist/models.js | 1 + plugins/google/dist/models.js.map | 1 + plugins/google/dist/tools.cjs | 17 + plugins/google/dist/tools.cjs.map | 1 + plugins/google/dist/tools.d.cts | 3 + plugins/google/dist/tools.d.ts | 3 + plugins/google/dist/tools.d.ts.map | 1 + plugins/google/dist/tools.js | 1 + plugins/google/dist/tools.js.map | 1 + plugins/google/dist/utils.cjs | 137 +++ plugins/google/dist/utils.cjs.map | 1 + plugins/google/dist/utils.d.cts | 14 + plugins/google/dist/utils.d.ts | 14 + plugins/google/dist/utils.d.ts.map | 1 + plugins/google/dist/utils.js | 112 ++ plugins/google/dist/utils.js.map | 1 + 84 files changed, 4667 insertions(+) create mode 100644 plugins/google/dist/beta/gemini_tts.cjs create mode 100644 plugins/google/dist/beta/gemini_tts.cjs.map create mode 100644 plugins/google/dist/beta/gemini_tts.d.cts create mode 100644 plugins/google/dist/beta/gemini_tts.d.ts create mode 100644 plugins/google/dist/beta/gemini_tts.d.ts.map create mode 100644 plugins/google/dist/beta/gemini_tts.js create mode 100644 plugins/google/dist/beta/gemini_tts.js.map create mode 100644 plugins/google/dist/beta/gemini_tts.test.cjs create mode 100644 plugins/google/dist/beta/gemini_tts.test.cjs.map create mode 100644 plugins/google/dist/beta/gemini_tts.test.d.cts create mode 100644 plugins/google/dist/beta/gemini_tts.test.d.ts create mode 100644 plugins/google/dist/beta/gemini_tts.test.d.ts.map create mode 100644 plugins/google/dist/beta/gemini_tts.test.js create mode 100644 plugins/google/dist/beta/gemini_tts.test.js.map create mode 100644 plugins/google/dist/beta/index.cjs create mode 100644 plugins/google/dist/beta/index.cjs.map create mode 100644 plugins/google/dist/beta/index.d.cts create mode 100644 plugins/google/dist/beta/index.d.ts create mode 100644 plugins/google/dist/beta/index.d.ts.map create mode 100644 plugins/google/dist/beta/index.js create mode 100644 plugins/google/dist/beta/index.js.map create mode 100644 plugins/google/dist/beta/realtime/api_proto.cjs create mode 100644 plugins/google/dist/beta/realtime/api_proto.cjs.map create mode 100644 plugins/google/dist/beta/realtime/api_proto.d.cts create mode 100644 plugins/google/dist/beta/realtime/api_proto.d.ts create mode 100644 plugins/google/dist/beta/realtime/api_proto.d.ts.map create mode 100644 plugins/google/dist/beta/realtime/api_proto.js create mode 100644 plugins/google/dist/beta/realtime/api_proto.js.map create mode 100644 plugins/google/dist/beta/realtime/index.cjs create mode 100644 plugins/google/dist/beta/realtime/index.cjs.map create mode 100644 plugins/google/dist/beta/realtime/index.d.cts create mode 100644 plugins/google/dist/beta/realtime/index.d.ts create mode 100644 plugins/google/dist/beta/realtime/index.d.ts.map create mode 100644 plugins/google/dist/beta/realtime/index.js create mode 100644 plugins/google/dist/beta/realtime/index.js.map create mode 100644 plugins/google/dist/beta/realtime/realtime_api.cjs create mode 100644 plugins/google/dist/beta/realtime/realtime_api.cjs.map create mode 100644 plugins/google/dist/beta/realtime/realtime_api.d.cts create mode 100644 plugins/google/dist/beta/realtime/realtime_api.d.ts create mode 100644 plugins/google/dist/beta/realtime/realtime_api.d.ts.map create mode 100644 plugins/google/dist/beta/realtime/realtime_api.js create mode 100644 plugins/google/dist/beta/realtime/realtime_api.js.map create mode 100644 plugins/google/dist/index.cjs create mode 100644 plugins/google/dist/index.cjs.map create mode 100644 plugins/google/dist/index.d.cts create mode 100644 plugins/google/dist/index.d.ts create mode 100644 plugins/google/dist/index.d.ts.map create mode 100644 plugins/google/dist/index.js create mode 100644 plugins/google/dist/index.js.map create mode 100644 plugins/google/dist/llm.cjs create mode 100644 plugins/google/dist/llm.cjs.map create mode 100644 plugins/google/dist/llm.d.cts create mode 100644 plugins/google/dist/llm.d.ts create mode 100644 plugins/google/dist/llm.d.ts.map create mode 100644 plugins/google/dist/llm.js create mode 100644 plugins/google/dist/llm.js.map create mode 100644 plugins/google/dist/llm.test.cjs create mode 100644 plugins/google/dist/llm.test.cjs.map create mode 100644 plugins/google/dist/llm.test.d.cts create mode 100644 plugins/google/dist/llm.test.d.ts create mode 100644 plugins/google/dist/llm.test.d.ts.map create mode 100644 plugins/google/dist/llm.test.js create mode 100644 plugins/google/dist/llm.test.js.map create mode 100644 plugins/google/dist/models.cjs create mode 100644 plugins/google/dist/models.cjs.map create mode 100644 plugins/google/dist/models.d.cts create mode 100644 plugins/google/dist/models.d.ts create mode 100644 plugins/google/dist/models.d.ts.map create mode 100644 plugins/google/dist/models.js create mode 100644 plugins/google/dist/models.js.map create mode 100644 plugins/google/dist/tools.cjs create mode 100644 plugins/google/dist/tools.cjs.map create mode 100644 plugins/google/dist/tools.d.cts create mode 100644 plugins/google/dist/tools.d.ts create mode 100644 plugins/google/dist/tools.d.ts.map create mode 100644 plugins/google/dist/tools.js create mode 100644 plugins/google/dist/tools.js.map create mode 100644 plugins/google/dist/utils.cjs create mode 100644 plugins/google/dist/utils.cjs.map create mode 100644 plugins/google/dist/utils.d.cts create mode 100644 plugins/google/dist/utils.d.ts create mode 100644 plugins/google/dist/utils.d.ts.map create mode 100644 plugins/google/dist/utils.js create mode 100644 plugins/google/dist/utils.js.map diff --git a/plugins/google/dist/beta/gemini_tts.cjs b/plugins/google/dist/beta/gemini_tts.cjs new file mode 100644 index 000000000..cc9117628 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.cjs @@ -0,0 +1,243 @@ +"use strict"; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var gemini_tts_exports = {}; +__export(gemini_tts_exports, { + ChunkedStream: () => ChunkedStream, + TTS: () => TTS +}); +module.exports = __toCommonJS(gemini_tts_exports); +var import_genai = require("@google/genai"); +var import_agents = require("@livekit/agents"); +const DEFAULT_MODEL = "gemini-2.5-flash-preview-tts"; +const DEFAULT_VOICE = "Kore"; +const DEFAULT_SAMPLE_RATE = 24e3; +const NUM_CHANNELS = 1; +const DEFAULT_INSTRUCTIONS = "Say the text with a proper tone, don't omit or add any words"; +class TTS extends import_agents.tts.TTS { + #opts; + #client; + label = "google.gemini.TTS"; + /** + * Create a new instance of Gemini TTS. + * + * Environment Requirements: + * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file. + * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. + * + * @param opts - Configuration options for Gemini TTS + */ + constructor({ + model = DEFAULT_MODEL, + voiceName = DEFAULT_VOICE, + apiKey, + vertexai, + project, + location, + instructions + } = {}) { + super(DEFAULT_SAMPLE_RATE, NUM_CHANNELS, { streaming: false }); + const gcpProject = project || process.env.GOOGLE_CLOUD_PROJECT; + const gcpLocation = location || process.env.GOOGLE_CLOUD_LOCATION || "us-central1"; + const useVertexai = vertexai ?? process.env.GOOGLE_GENAI_USE_VERTEXAI === "true"; + const geminiApiKey = apiKey || process.env.GOOGLE_API_KEY; + let finalProject = gcpProject; + let finalLocation = gcpLocation; + let finalApiKey = geminiApiKey; + if (useVertexai) { + if (!finalProject) { + throw new import_agents.APIConnectionError({ + message: "Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable" + }); + } + finalApiKey = void 0; + } else { + finalProject = void 0; + finalLocation = void 0; + if (!finalApiKey) { + throw new import_agents.APIConnectionError({ + message: "API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable" + }); + } + } + this.#opts = { + model, + voiceName, + vertexai: useVertexai, + project: finalProject, + location: finalLocation, + instructions: instructions ?? DEFAULT_INSTRUCTIONS + }; + const clientOptions = useVertexai ? { + vertexai: true, + project: finalProject, + location: finalLocation + } : { + apiKey: finalApiKey + }; + this.#client = new import_genai.GoogleGenAI(clientOptions); + } + synthesize(text, connOptions, abortSignal) { + return new ChunkedStream(text, this, connOptions, abortSignal); + } + /** + * Update the TTS options. + * + * @param opts - Options to update + */ + updateOptions(opts) { + if (opts.voiceName !== void 0) { + this.#opts.voiceName = opts.voiceName; + } + } + stream() { + throw new Error("Streaming is not supported on Gemini TTS"); + } + get opts() { + return this.#opts; + } + get client() { + return this.#client; + } +} +class ChunkedStream extends import_agents.tts.ChunkedStream { + #tts; + label = "google.gemini.ChunkedStream"; + constructor(inputText, tts2, connOptions, abortSignal) { + super(inputText, tts2, connOptions, abortSignal); + this.#tts = tts2; + } + async run() { + const requestId = (0, import_agents.shortuuid)(); + const bstream = new import_agents.AudioByteStream(this.#tts.sampleRate, this.#tts.numChannels); + const config = { + responseModalities: ["AUDIO"], + speechConfig: { + voiceConfig: { + prebuiltVoiceConfig: { + voiceName: this.#tts.opts.voiceName + } + } + }, + abortSignal: this.abortSignal + }; + let inputText = this.inputText; + if (this.#tts.opts.instructions) { + inputText = `${this.#tts.opts.instructions}: +"${inputText}"`; + } + const contents = [ + { + role: "user", + parts: [{ text: inputText }] + } + ]; + const responseStream = await this.#tts.client.models.generateContentStream({ + model: this.#tts.opts.model, + contents, + config + }); + try { + for await (const response of responseStream) { + await this.#processResponse(response, bstream, requestId); + } + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + return; + } + if ((0, import_agents.isAPIError)(error)) throw error; + const err = error; + if (err.code && err.code >= 400 && err.code < 500) { + if (err.code === 429) { + throw new import_agents.APIStatusError({ + message: `Gemini TTS: Rate limit error - ${err.message || "Unknown error"}`, + options: { + statusCode: 429, + retryable: true + } + }); + } else { + throw new import_agents.APIStatusError({ + message: `Gemini TTS: Client error (${err.code}) - ${err.message || "Unknown error"}`, + options: { + statusCode: err.code, + retryable: false + } + }); + } + } + if (err.code && err.code >= 500) { + throw new import_agents.APIStatusError({ + message: `Gemini TTS: Server error (${err.code}) - ${err.message || "Unknown error"}`, + options: { + statusCode: err.code, + retryable: true + } + }); + } + throw new import_agents.APIConnectionError({ + message: `Gemini TTS: Connection error - ${err.message || "Unknown error"}`, + options: { retryable: true } + }); + } finally { + this.queue.close(); + } + } + async #processResponse(response, bstream, requestId) { + var _a, _b, _c; + if (!response.candidates || response.candidates.length === 0) { + return; + } + const candidate = response.candidates[0]; + if (!candidate || !((_a = candidate.content) == null ? void 0 : _a.parts)) { + return; + } + let lastFrame; + const sendLastFrame = (final) => { + if (lastFrame) { + this.queue.put({ + requestId, + frame: lastFrame, + segmentId: requestId, + final + }); + lastFrame = void 0; + } + }; + for (const part of candidate.content.parts) { + if (((_b = part.inlineData) == null ? void 0 : _b.data) && ((_c = part.inlineData.mimeType) == null ? void 0 : _c.startsWith("audio/"))) { + const audioBuffer = Buffer.from(part.inlineData.data, "base64"); + for (const frame of bstream.write(audioBuffer)) { + sendLastFrame(false); + lastFrame = frame; + } + } + } + for (const frame of bstream.flush()) { + sendLastFrame(false); + lastFrame = frame; + } + sendLastFrame(true); + } +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + ChunkedStream, + TTS +}); +//# sourceMappingURL=gemini_tts.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.cjs.map b/plugins/google/dist/beta/gemini_tts.cjs.map new file mode 100644 index 000000000..050d3cfd4 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../src/beta/gemini_tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as types from '@google/genai';\nimport { GoogleGenAI } from '@google/genai';\nimport {\n type APIConnectOptions,\n APIConnectionError,\n APIStatusError,\n AudioByteStream,\n isAPIError,\n shortuuid,\n tts,\n} from '@livekit/agents';\nimport type { AudioFrame } from '@livekit/rtc-node';\n\nexport type GeminiTTSModels = 'gemini-2.5-flash-preview-tts' | 'gemini-2.5-pro-preview-tts';\n\nexport type GeminiVoices =\n | 'Zephyr'\n | 'Puck'\n | 'Charon'\n | 'Kore'\n | 'Fenrir'\n | 'Leda'\n | 'Orus'\n | 'Aoede'\n | 'Callirrhoe'\n | 'Autonoe'\n | 'Enceladus'\n | 'Iapetus'\n | 'Umbriel'\n | 'Algieba'\n | 'Despina'\n | 'Erinome'\n | 'Algenib'\n | 'Rasalgethi'\n | 'Laomedeia'\n | 'Achernar'\n | 'Alnilam'\n | 'Schedar'\n | 'Gacrux'\n | 'Pulcherrima'\n | 'Achird'\n | 'Zubenelgenubi'\n | 'Vindemiatrix'\n | 'Sadachbia'\n | 'Sadaltager'\n | 'Sulafat';\n\nconst DEFAULT_MODEL: GeminiTTSModels = 'gemini-2.5-flash-preview-tts';\nconst DEFAULT_VOICE: GeminiVoices = 'Kore';\nconst DEFAULT_SAMPLE_RATE = 24000; // not configurable\nconst NUM_CHANNELS = 1;\nconst DEFAULT_INSTRUCTIONS = \"Say the text with a proper tone, don't omit or add any words\";\n\nexport interface TTSOptions {\n model: GeminiTTSModels | string;\n voiceName: GeminiVoices | string;\n vertexai: boolean;\n project?: string;\n location?: string;\n instructions?: string;\n}\n\nexport class TTS extends tts.TTS {\n #opts: TTSOptions;\n #client: GoogleGenAI;\n label = 'google.gemini.TTS';\n\n /**\n * Create a new instance of Gemini TTS.\n *\n * Environment Requirements:\n * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file.\n * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable.\n *\n * @param opts - Configuration options for Gemini TTS\n */\n constructor({\n model = DEFAULT_MODEL,\n voiceName = DEFAULT_VOICE,\n apiKey,\n vertexai,\n project,\n location,\n instructions,\n }: Partial = {}) {\n super(DEFAULT_SAMPLE_RATE, NUM_CHANNELS, { streaming: false });\n\n const gcpProject: string | undefined = project || process.env.GOOGLE_CLOUD_PROJECT;\n const gcpLocation: string | undefined =\n location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const useVertexai = vertexai ?? process.env.GOOGLE_GENAI_USE_VERTEXAI === 'true';\n const geminiApiKey = apiKey || process.env.GOOGLE_API_KEY;\n\n let finalProject: string | undefined = gcpProject;\n let finalLocation: string | undefined = gcpLocation;\n let finalApiKey: string | undefined = geminiApiKey;\n\n if (useVertexai) {\n if (!finalProject) {\n throw new APIConnectionError({\n message:\n 'Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable',\n });\n }\n finalApiKey = undefined;\n } else {\n finalProject = undefined;\n finalLocation = undefined;\n if (!finalApiKey) {\n throw new APIConnectionError({\n message:\n 'API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable',\n });\n }\n }\n\n this.#opts = {\n model,\n voiceName,\n vertexai: useVertexai,\n project: finalProject,\n location: finalLocation,\n instructions: instructions ?? DEFAULT_INSTRUCTIONS,\n };\n\n const clientOptions: types.GoogleGenAIOptions = useVertexai\n ? {\n vertexai: true,\n project: finalProject,\n location: finalLocation,\n }\n : {\n apiKey: finalApiKey,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n }\n\n synthesize(\n text: string,\n connOptions?: APIConnectOptions,\n abortSignal?: AbortSignal,\n ): ChunkedStream {\n return new ChunkedStream(text, this, connOptions, abortSignal);\n }\n\n /**\n * Update the TTS options.\n *\n * @param opts - Options to update\n */\n updateOptions(opts: { voiceName?: GeminiVoices | string }) {\n if (opts.voiceName !== undefined) {\n this.#opts.voiceName = opts.voiceName;\n }\n }\n\n stream(): tts.SynthesizeStream {\n throw new Error('Streaming is not supported on Gemini TTS');\n }\n\n get opts(): TTSOptions {\n return this.#opts;\n }\n\n get client(): GoogleGenAI {\n return this.#client;\n }\n}\n\nexport class ChunkedStream extends tts.ChunkedStream {\n #tts: TTS;\n label = 'google.gemini.ChunkedStream';\n\n constructor(\n inputText: string,\n tts: TTS,\n connOptions?: APIConnectOptions,\n abortSignal?: AbortSignal,\n ) {\n super(inputText, tts, connOptions, abortSignal);\n this.#tts = tts;\n }\n\n protected async run() {\n const requestId = shortuuid();\n const bstream = new AudioByteStream(this.#tts.sampleRate, this.#tts.numChannels);\n\n const config: types.GenerateContentConfig = {\n responseModalities: ['AUDIO'],\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: this.#tts.opts.voiceName,\n },\n },\n },\n abortSignal: this.abortSignal,\n };\n\n let inputText = this.inputText;\n if (this.#tts.opts.instructions) {\n inputText = `${this.#tts.opts.instructions}:\\n\"${inputText}\"`;\n }\n\n const contents: types.Content[] = [\n {\n role: 'user',\n parts: [{ text: inputText }],\n },\n ];\n\n const responseStream = await this.#tts.client.models.generateContentStream({\n model: this.#tts.opts.model,\n contents,\n config,\n });\n\n try {\n for await (const response of responseStream) {\n await this.#processResponse(response, bstream, requestId);\n }\n } catch (error: unknown) {\n if (error instanceof Error && error.name === 'AbortError') {\n return;\n }\n if (isAPIError(error)) throw error;\n\n const err = error as {\n code?: number;\n message?: string;\n status?: string;\n type?: string;\n };\n\n if (err.code && err.code >= 400 && err.code < 500) {\n if (err.code === 429) {\n throw new APIStatusError({\n message: `Gemini TTS: Rate limit error - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: 429,\n retryable: true,\n },\n });\n } else {\n throw new APIStatusError({\n message: `Gemini TTS: Client error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: false,\n },\n });\n }\n }\n\n if (err.code && err.code >= 500) {\n throw new APIStatusError({\n message: `Gemini TTS: Server error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: true,\n },\n });\n }\n\n throw new APIConnectionError({\n message: `Gemini TTS: Connection error - ${err.message || 'Unknown error'}`,\n options: { retryable: true },\n });\n } finally {\n this.queue.close();\n }\n }\n\n async #processResponse(\n response: types.GenerateContentResponse,\n bstream: AudioByteStream,\n requestId: string,\n ) {\n if (!response.candidates || response.candidates.length === 0) {\n return;\n }\n\n const candidate = response.candidates[0];\n if (!candidate || !candidate.content?.parts) {\n return;\n }\n\n let lastFrame: AudioFrame | undefined;\n const sendLastFrame = (final: boolean) => {\n if (lastFrame) {\n this.queue.put({\n requestId,\n frame: lastFrame,\n segmentId: requestId,\n final,\n });\n lastFrame = undefined;\n }\n };\n\n for (const part of candidate.content.parts) {\n if (part.inlineData?.data && part.inlineData.mimeType?.startsWith('audio/')) {\n const audioBuffer = Buffer.from(part.inlineData.data, 'base64');\n\n for (const frame of bstream.write(audioBuffer)) {\n sendLastFrame(false);\n lastFrame = frame;\n }\n }\n }\n\n for (const frame of bstream.flush()) {\n sendLastFrame(false);\n lastFrame = frame;\n }\n\n sendLastFrame(true);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAA4B;AAC5B,oBAQO;AAqCP,MAAM,gBAAiC;AACvC,MAAM,gBAA8B;AACpC,MAAM,sBAAsB;AAC5B,MAAM,eAAe;AACrB,MAAM,uBAAuB;AAWtB,MAAM,YAAY,kBAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EACA,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWR,YAAY;AAAA,IACV,QAAQ;AAAA,IACR,YAAY;AAAA,IACZ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,IAA8C,CAAC,GAAG;AAChD,UAAM,qBAAqB,cAAc,EAAE,WAAW,MAAM,CAAC;AAE7D,UAAM,aAAiC,WAAW,QAAQ,IAAI;AAC9D,UAAM,cACJ,YAAY,QAAQ,IAAI,yBAAyB;AACnD,UAAM,cAAc,YAAY,QAAQ,IAAI,8BAA8B;AAC1E,UAAM,eAAe,UAAU,QAAQ,IAAI;AAE3C,QAAI,eAAmC;AACvC,QAAI,gBAAoC;AACxC,QAAI,cAAkC;AAEtC,QAAI,aAAa;AACf,UAAI,CAAC,cAAc;AACjB,cAAM,IAAI,iCAAmB;AAAA,UAC3B,SACE;AAAA,QACJ,CAAC;AAAA,MACH;AACA,oBAAc;AAAA,IAChB,OAAO;AACL,qBAAe;AACf,sBAAgB;AAChB,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,iCAAmB;AAAA,UAC3B,SACE;AAAA,QACJ,CAAC;AAAA,MACH;AAAA,IACF;AAEA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,MACV,cAAc,gBAAgB;AAAA,IAChC;AAEA,UAAM,gBAA0C,cAC5C;AAAA,MACE,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,IACZ,IACA;AAAA,MACE,QAAQ;AAAA,IACV;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAAA,EAC9C;AAAA,EAEA,WACE,MACA,aACA,aACe;AACf,WAAO,IAAI,cAAc,MAAM,MAAM,aAAa,WAAW;AAAA,EAC/D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,cAAc,MAA6C;AACzD,QAAI,KAAK,cAAc,QAAW;AAChC,WAAK,MAAM,YAAY,KAAK;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,SAA+B;AAC7B,UAAM,IAAI,MAAM,0CAA0C;AAAA,EAC5D;AAAA,EAEA,IAAI,OAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AACF;AAEO,MAAM,sBAAsB,kBAAI,cAAc;AAAA,EACnD;AAAA,EACA,QAAQ;AAAA,EAER,YACE,WACAA,MACA,aACA,aACA;AACA,UAAM,WAAWA,MAAK,aAAa,WAAW;AAC9C,SAAK,OAAOA;AAAA,EACd;AAAA,EAEA,MAAgB,MAAM;AACpB,UAAM,gBAAY,yBAAU;AAC5B,UAAM,UAAU,IAAI,8BAAgB,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW;AAE/E,UAAM,SAAsC;AAAA,MAC1C,oBAAoB,CAAC,OAAO;AAAA,MAC5B,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK,KAAK,KAAK;AAAA,UAC5B;AAAA,QACF;AAAA,MACF;AAAA,MACA,aAAa,KAAK;AAAA,IACpB;AAEA,QAAI,YAAY,KAAK;AACrB,QAAI,KAAK,KAAK,KAAK,cAAc;AAC/B,kBAAY,GAAG,KAAK,KAAK,KAAK,YAAY;AAAA,GAAO,SAAS;AAAA,IAC5D;AAEA,UAAM,WAA4B;AAAA,MAChC;AAAA,QACE,MAAM;AAAA,QACN,OAAO,CAAC,EAAE,MAAM,UAAU,CAAC;AAAA,MAC7B;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,KAAK,OAAO,OAAO,sBAAsB;AAAA,MACzE,OAAO,KAAK,KAAK,KAAK;AAAA,MACtB;AAAA,MACA;AAAA,IACF,CAAC;AAED,QAAI;AACF,uBAAiB,YAAY,gBAAgB;AAC3C,cAAM,KAAK,iBAAiB,UAAU,SAAS,SAAS;AAAA,MAC1D;AAAA,IACF,SAAS,OAAgB;AACvB,UAAI,iBAAiB,SAAS,MAAM,SAAS,cAAc;AACzD;AAAA,MACF;AACA,cAAI,0BAAW,KAAK,EAAG,OAAM;AAE7B,YAAM,MAAM;AAOZ,UAAI,IAAI,QAAQ,IAAI,QAAQ,OAAO,IAAI,OAAO,KAAK;AACjD,YAAI,IAAI,SAAS,KAAK;AACpB,gBAAM,IAAI,6BAAe;AAAA,YACvB,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,YACzE,SAAS;AAAA,cACP,YAAY;AAAA,cACZ,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH,OAAO;AACL,gBAAM,IAAI,6BAAe;AAAA,YACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,YACnF,SAAS;AAAA,cACP,YAAY,IAAI;AAAA,cAChB,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,IAAI,QAAQ,IAAI,QAAQ,KAAK;AAC/B,cAAM,IAAI,6BAAe;AAAA,UACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,UACnF,SAAS;AAAA,YACP,YAAY,IAAI;AAAA,YAChB,WAAW;AAAA,UACb;AAAA,QACF,CAAC;AAAA,MACH;AAEA,YAAM,IAAI,iCAAmB;AAAA,QAC3B,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,QACzE,SAAS,EAAE,WAAW,KAAK;AAAA,MAC7B,CAAC;AAAA,IACH,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAM,iBACJ,UACA,SACA,WACA;AAzRJ;AA0RI,QAAI,CAAC,SAAS,cAAc,SAAS,WAAW,WAAW,GAAG;AAC5D;AAAA,IACF;AAEA,UAAM,YAAY,SAAS,WAAW,CAAC;AACvC,QAAI,CAAC,aAAa,GAAC,eAAU,YAAV,mBAAmB,QAAO;AAC3C;AAAA,IACF;AAEA,QAAI;AACJ,UAAM,gBAAgB,CAAC,UAAmB;AACxC,UAAI,WAAW;AACb,aAAK,MAAM,IAAI;AAAA,UACb;AAAA,UACA,OAAO;AAAA,UACP,WAAW;AAAA,UACX;AAAA,QACF,CAAC;AACD,oBAAY;AAAA,MACd;AAAA,IACF;AAEA,eAAW,QAAQ,UAAU,QAAQ,OAAO;AAC1C,YAAI,UAAK,eAAL,mBAAiB,WAAQ,UAAK,WAAW,aAAhB,mBAA0B,WAAW,YAAW;AAC3E,cAAM,cAAc,OAAO,KAAK,KAAK,WAAW,MAAM,QAAQ;AAE9D,mBAAW,SAAS,QAAQ,MAAM,WAAW,GAAG;AAC9C,wBAAc,KAAK;AACnB,sBAAY;AAAA,QACd;AAAA,MACF;AAAA,IACF;AAEA,eAAW,SAAS,QAAQ,MAAM,GAAG;AACnC,oBAAc,KAAK;AACnB,kBAAY;AAAA,IACd;AAEA,kBAAc,IAAI;AAAA,EACpB;AACF;","names":["tts"]} \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.d.cts b/plugins/google/dist/beta/gemini_tts.d.cts new file mode 100644 index 000000000..3f228c356 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.d.cts @@ -0,0 +1,47 @@ +import { GoogleGenAI } from '@google/genai'; +import { type APIConnectOptions, tts } from '@livekit/agents'; +export type GeminiTTSModels = 'gemini-2.5-flash-preview-tts' | 'gemini-2.5-pro-preview-tts'; +export type GeminiVoices = 'Zephyr' | 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Leda' | 'Orus' | 'Aoede' | 'Callirrhoe' | 'Autonoe' | 'Enceladus' | 'Iapetus' | 'Umbriel' | 'Algieba' | 'Despina' | 'Erinome' | 'Algenib' | 'Rasalgethi' | 'Laomedeia' | 'Achernar' | 'Alnilam' | 'Schedar' | 'Gacrux' | 'Pulcherrima' | 'Achird' | 'Zubenelgenubi' | 'Vindemiatrix' | 'Sadachbia' | 'Sadaltager' | 'Sulafat'; +export interface TTSOptions { + model: GeminiTTSModels | string; + voiceName: GeminiVoices | string; + vertexai: boolean; + project?: string; + location?: string; + instructions?: string; +} +export declare class TTS extends tts.TTS { + #private; + label: string; + /** + * Create a new instance of Gemini TTS. + * + * Environment Requirements: + * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file. + * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. + * + * @param opts - Configuration options for Gemini TTS + */ + constructor({ model, voiceName, apiKey, vertexai, project, location, instructions, }?: Partial); + synthesize(text: string, connOptions?: APIConnectOptions, abortSignal?: AbortSignal): ChunkedStream; + /** + * Update the TTS options. + * + * @param opts - Options to update + */ + updateOptions(opts: { + voiceName?: GeminiVoices | string; + }): void; + stream(): tts.SynthesizeStream; + get opts(): TTSOptions; + get client(): GoogleGenAI; +} +export declare class ChunkedStream extends tts.ChunkedStream { + #private; + label: string; + constructor(inputText: string, tts: TTS, connOptions?: APIConnectOptions, abortSignal?: AbortSignal); + protected run(): Promise; +} +//# sourceMappingURL=gemini_tts.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.d.ts b/plugins/google/dist/beta/gemini_tts.d.ts new file mode 100644 index 000000000..3f228c356 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.d.ts @@ -0,0 +1,47 @@ +import { GoogleGenAI } from '@google/genai'; +import { type APIConnectOptions, tts } from '@livekit/agents'; +export type GeminiTTSModels = 'gemini-2.5-flash-preview-tts' | 'gemini-2.5-pro-preview-tts'; +export type GeminiVoices = 'Zephyr' | 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Leda' | 'Orus' | 'Aoede' | 'Callirrhoe' | 'Autonoe' | 'Enceladus' | 'Iapetus' | 'Umbriel' | 'Algieba' | 'Despina' | 'Erinome' | 'Algenib' | 'Rasalgethi' | 'Laomedeia' | 'Achernar' | 'Alnilam' | 'Schedar' | 'Gacrux' | 'Pulcherrima' | 'Achird' | 'Zubenelgenubi' | 'Vindemiatrix' | 'Sadachbia' | 'Sadaltager' | 'Sulafat'; +export interface TTSOptions { + model: GeminiTTSModels | string; + voiceName: GeminiVoices | string; + vertexai: boolean; + project?: string; + location?: string; + instructions?: string; +} +export declare class TTS extends tts.TTS { + #private; + label: string; + /** + * Create a new instance of Gemini TTS. + * + * Environment Requirements: + * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file. + * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. + * + * @param opts - Configuration options for Gemini TTS + */ + constructor({ model, voiceName, apiKey, vertexai, project, location, instructions, }?: Partial); + synthesize(text: string, connOptions?: APIConnectOptions, abortSignal?: AbortSignal): ChunkedStream; + /** + * Update the TTS options. + * + * @param opts - Options to update + */ + updateOptions(opts: { + voiceName?: GeminiVoices | string; + }): void; + stream(): tts.SynthesizeStream; + get opts(): TTSOptions; + get client(): GoogleGenAI; +} +export declare class ChunkedStream extends tts.ChunkedStream { + #private; + label: string; + constructor(inputText: string, tts: TTS, connOptions?: APIConnectOptions, abortSignal?: AbortSignal); + protected run(): Promise; +} +//# sourceMappingURL=gemini_tts.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.d.ts.map b/plugins/google/dist/beta/gemini_tts.d.ts.map new file mode 100644 index 000000000..a4f33ca24 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"gemini_tts.d.ts","sourceRoot":"","sources":["../../src/beta/gemini_tts.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EACL,KAAK,iBAAiB,EAMtB,GAAG,EACJ,MAAM,iBAAiB,CAAC;AAGzB,MAAM,MAAM,eAAe,GAAG,8BAA8B,GAAG,4BAA4B,CAAC;AAE5F,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,MAAM,GACN,QAAQ,GACR,MAAM,GACN,QAAQ,GACR,MAAM,GACN,MAAM,GACN,OAAO,GACP,YAAY,GACZ,SAAS,GACT,WAAW,GACX,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,YAAY,GACZ,WAAW,GACX,UAAU,GACV,SAAS,GACT,SAAS,GACT,QAAQ,GACR,aAAa,GACb,QAAQ,GACR,eAAe,GACf,cAAc,GACd,WAAW,GACX,YAAY,GACZ,SAAS,CAAC;AAQd,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,eAAe,GAAG,MAAM,CAAC;IAChC,SAAS,EAAE,YAAY,GAAG,MAAM,CAAC;IACjC,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;;IAG9B,KAAK,SAAuB;IAE5B;;;;;;;;OAQG;gBACS,EACV,KAAqB,EACrB,SAAyB,EACzB,MAAM,EACN,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,YAAY,GACb,GAAE,OAAO,CAAC,UAAU,GAAG;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAM;IAsDhD,UAAU,CACR,IAAI,EAAE,MAAM,EACZ,WAAW,CAAC,EAAE,iBAAiB,EAC/B,WAAW,CAAC,EAAE,WAAW,GACxB,aAAa;IAIhB;;;;OAIG;IACH,aAAa,CAAC,IAAI,EAAE;QAAE,SAAS,CAAC,EAAE,YAAY,GAAG,MAAM,CAAA;KAAE;IAMzD,MAAM,IAAI,GAAG,CAAC,gBAAgB;IAI9B,IAAI,IAAI,IAAI,UAAU,CAErB;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;CACF;AAED,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;;IAElD,KAAK,SAAiC;gBAGpC,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,GAAG,EACR,WAAW,CAAC,EAAE,iBAAiB,EAC/B,WAAW,CAAC,EAAE,WAAW;cAMX,GAAG;CAuIpB"} \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.js b/plugins/google/dist/beta/gemini_tts.js new file mode 100644 index 000000000..747d356e5 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.js @@ -0,0 +1,225 @@ +import { GoogleGenAI } from "@google/genai"; +import { + APIConnectionError, + APIStatusError, + AudioByteStream, + isAPIError, + shortuuid, + tts +} from "@livekit/agents"; +const DEFAULT_MODEL = "gemini-2.5-flash-preview-tts"; +const DEFAULT_VOICE = "Kore"; +const DEFAULT_SAMPLE_RATE = 24e3; +const NUM_CHANNELS = 1; +const DEFAULT_INSTRUCTIONS = "Say the text with a proper tone, don't omit or add any words"; +class TTS extends tts.TTS { + #opts; + #client; + label = "google.gemini.TTS"; + /** + * Create a new instance of Gemini TTS. + * + * Environment Requirements: + * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file. + * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. + * + * @param opts - Configuration options for Gemini TTS + */ + constructor({ + model = DEFAULT_MODEL, + voiceName = DEFAULT_VOICE, + apiKey, + vertexai, + project, + location, + instructions + } = {}) { + super(DEFAULT_SAMPLE_RATE, NUM_CHANNELS, { streaming: false }); + const gcpProject = project || process.env.GOOGLE_CLOUD_PROJECT; + const gcpLocation = location || process.env.GOOGLE_CLOUD_LOCATION || "us-central1"; + const useVertexai = vertexai ?? process.env.GOOGLE_GENAI_USE_VERTEXAI === "true"; + const geminiApiKey = apiKey || process.env.GOOGLE_API_KEY; + let finalProject = gcpProject; + let finalLocation = gcpLocation; + let finalApiKey = geminiApiKey; + if (useVertexai) { + if (!finalProject) { + throw new APIConnectionError({ + message: "Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable" + }); + } + finalApiKey = void 0; + } else { + finalProject = void 0; + finalLocation = void 0; + if (!finalApiKey) { + throw new APIConnectionError({ + message: "API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable" + }); + } + } + this.#opts = { + model, + voiceName, + vertexai: useVertexai, + project: finalProject, + location: finalLocation, + instructions: instructions ?? DEFAULT_INSTRUCTIONS + }; + const clientOptions = useVertexai ? { + vertexai: true, + project: finalProject, + location: finalLocation + } : { + apiKey: finalApiKey + }; + this.#client = new GoogleGenAI(clientOptions); + } + synthesize(text, connOptions, abortSignal) { + return new ChunkedStream(text, this, connOptions, abortSignal); + } + /** + * Update the TTS options. + * + * @param opts - Options to update + */ + updateOptions(opts) { + if (opts.voiceName !== void 0) { + this.#opts.voiceName = opts.voiceName; + } + } + stream() { + throw new Error("Streaming is not supported on Gemini TTS"); + } + get opts() { + return this.#opts; + } + get client() { + return this.#client; + } +} +class ChunkedStream extends tts.ChunkedStream { + #tts; + label = "google.gemini.ChunkedStream"; + constructor(inputText, tts2, connOptions, abortSignal) { + super(inputText, tts2, connOptions, abortSignal); + this.#tts = tts2; + } + async run() { + const requestId = shortuuid(); + const bstream = new AudioByteStream(this.#tts.sampleRate, this.#tts.numChannels); + const config = { + responseModalities: ["AUDIO"], + speechConfig: { + voiceConfig: { + prebuiltVoiceConfig: { + voiceName: this.#tts.opts.voiceName + } + } + }, + abortSignal: this.abortSignal + }; + let inputText = this.inputText; + if (this.#tts.opts.instructions) { + inputText = `${this.#tts.opts.instructions}: +"${inputText}"`; + } + const contents = [ + { + role: "user", + parts: [{ text: inputText }] + } + ]; + const responseStream = await this.#tts.client.models.generateContentStream({ + model: this.#tts.opts.model, + contents, + config + }); + try { + for await (const response of responseStream) { + await this.#processResponse(response, bstream, requestId); + } + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + return; + } + if (isAPIError(error)) throw error; + const err = error; + if (err.code && err.code >= 400 && err.code < 500) { + if (err.code === 429) { + throw new APIStatusError({ + message: `Gemini TTS: Rate limit error - ${err.message || "Unknown error"}`, + options: { + statusCode: 429, + retryable: true + } + }); + } else { + throw new APIStatusError({ + message: `Gemini TTS: Client error (${err.code}) - ${err.message || "Unknown error"}`, + options: { + statusCode: err.code, + retryable: false + } + }); + } + } + if (err.code && err.code >= 500) { + throw new APIStatusError({ + message: `Gemini TTS: Server error (${err.code}) - ${err.message || "Unknown error"}`, + options: { + statusCode: err.code, + retryable: true + } + }); + } + throw new APIConnectionError({ + message: `Gemini TTS: Connection error - ${err.message || "Unknown error"}`, + options: { retryable: true } + }); + } finally { + this.queue.close(); + } + } + async #processResponse(response, bstream, requestId) { + var _a, _b, _c; + if (!response.candidates || response.candidates.length === 0) { + return; + } + const candidate = response.candidates[0]; + if (!candidate || !((_a = candidate.content) == null ? void 0 : _a.parts)) { + return; + } + let lastFrame; + const sendLastFrame = (final) => { + if (lastFrame) { + this.queue.put({ + requestId, + frame: lastFrame, + segmentId: requestId, + final + }); + lastFrame = void 0; + } + }; + for (const part of candidate.content.parts) { + if (((_b = part.inlineData) == null ? void 0 : _b.data) && ((_c = part.inlineData.mimeType) == null ? void 0 : _c.startsWith("audio/"))) { + const audioBuffer = Buffer.from(part.inlineData.data, "base64"); + for (const frame of bstream.write(audioBuffer)) { + sendLastFrame(false); + lastFrame = frame; + } + } + } + for (const frame of bstream.flush()) { + sendLastFrame(false); + lastFrame = frame; + } + sendLastFrame(true); + } +} +export { + ChunkedStream, + TTS +}; +//# sourceMappingURL=gemini_tts.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.js.map b/plugins/google/dist/beta/gemini_tts.js.map new file mode 100644 index 000000000..a4e3e1088 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../src/beta/gemini_tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as types from '@google/genai';\nimport { GoogleGenAI } from '@google/genai';\nimport {\n type APIConnectOptions,\n APIConnectionError,\n APIStatusError,\n AudioByteStream,\n isAPIError,\n shortuuid,\n tts,\n} from '@livekit/agents';\nimport type { AudioFrame } from '@livekit/rtc-node';\n\nexport type GeminiTTSModels = 'gemini-2.5-flash-preview-tts' | 'gemini-2.5-pro-preview-tts';\n\nexport type GeminiVoices =\n | 'Zephyr'\n | 'Puck'\n | 'Charon'\n | 'Kore'\n | 'Fenrir'\n | 'Leda'\n | 'Orus'\n | 'Aoede'\n | 'Callirrhoe'\n | 'Autonoe'\n | 'Enceladus'\n | 'Iapetus'\n | 'Umbriel'\n | 'Algieba'\n | 'Despina'\n | 'Erinome'\n | 'Algenib'\n | 'Rasalgethi'\n | 'Laomedeia'\n | 'Achernar'\n | 'Alnilam'\n | 'Schedar'\n | 'Gacrux'\n | 'Pulcherrima'\n | 'Achird'\n | 'Zubenelgenubi'\n | 'Vindemiatrix'\n | 'Sadachbia'\n | 'Sadaltager'\n | 'Sulafat';\n\nconst DEFAULT_MODEL: GeminiTTSModels = 'gemini-2.5-flash-preview-tts';\nconst DEFAULT_VOICE: GeminiVoices = 'Kore';\nconst DEFAULT_SAMPLE_RATE = 24000; // not configurable\nconst NUM_CHANNELS = 1;\nconst DEFAULT_INSTRUCTIONS = \"Say the text with a proper tone, don't omit or add any words\";\n\nexport interface TTSOptions {\n model: GeminiTTSModels | string;\n voiceName: GeminiVoices | string;\n vertexai: boolean;\n project?: string;\n location?: string;\n instructions?: string;\n}\n\nexport class TTS extends tts.TTS {\n #opts: TTSOptions;\n #client: GoogleGenAI;\n label = 'google.gemini.TTS';\n\n /**\n * Create a new instance of Gemini TTS.\n *\n * Environment Requirements:\n * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file.\n * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable.\n *\n * @param opts - Configuration options for Gemini TTS\n */\n constructor({\n model = DEFAULT_MODEL,\n voiceName = DEFAULT_VOICE,\n apiKey,\n vertexai,\n project,\n location,\n instructions,\n }: Partial = {}) {\n super(DEFAULT_SAMPLE_RATE, NUM_CHANNELS, { streaming: false });\n\n const gcpProject: string | undefined = project || process.env.GOOGLE_CLOUD_PROJECT;\n const gcpLocation: string | undefined =\n location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const useVertexai = vertexai ?? process.env.GOOGLE_GENAI_USE_VERTEXAI === 'true';\n const geminiApiKey = apiKey || process.env.GOOGLE_API_KEY;\n\n let finalProject: string | undefined = gcpProject;\n let finalLocation: string | undefined = gcpLocation;\n let finalApiKey: string | undefined = geminiApiKey;\n\n if (useVertexai) {\n if (!finalProject) {\n throw new APIConnectionError({\n message:\n 'Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable',\n });\n }\n finalApiKey = undefined;\n } else {\n finalProject = undefined;\n finalLocation = undefined;\n if (!finalApiKey) {\n throw new APIConnectionError({\n message:\n 'API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable',\n });\n }\n }\n\n this.#opts = {\n model,\n voiceName,\n vertexai: useVertexai,\n project: finalProject,\n location: finalLocation,\n instructions: instructions ?? DEFAULT_INSTRUCTIONS,\n };\n\n const clientOptions: types.GoogleGenAIOptions = useVertexai\n ? {\n vertexai: true,\n project: finalProject,\n location: finalLocation,\n }\n : {\n apiKey: finalApiKey,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n }\n\n synthesize(\n text: string,\n connOptions?: APIConnectOptions,\n abortSignal?: AbortSignal,\n ): ChunkedStream {\n return new ChunkedStream(text, this, connOptions, abortSignal);\n }\n\n /**\n * Update the TTS options.\n *\n * @param opts - Options to update\n */\n updateOptions(opts: { voiceName?: GeminiVoices | string }) {\n if (opts.voiceName !== undefined) {\n this.#opts.voiceName = opts.voiceName;\n }\n }\n\n stream(): tts.SynthesizeStream {\n throw new Error('Streaming is not supported on Gemini TTS');\n }\n\n get opts(): TTSOptions {\n return this.#opts;\n }\n\n get client(): GoogleGenAI {\n return this.#client;\n }\n}\n\nexport class ChunkedStream extends tts.ChunkedStream {\n #tts: TTS;\n label = 'google.gemini.ChunkedStream';\n\n constructor(\n inputText: string,\n tts: TTS,\n connOptions?: APIConnectOptions,\n abortSignal?: AbortSignal,\n ) {\n super(inputText, tts, connOptions, abortSignal);\n this.#tts = tts;\n }\n\n protected async run() {\n const requestId = shortuuid();\n const bstream = new AudioByteStream(this.#tts.sampleRate, this.#tts.numChannels);\n\n const config: types.GenerateContentConfig = {\n responseModalities: ['AUDIO'],\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: this.#tts.opts.voiceName,\n },\n },\n },\n abortSignal: this.abortSignal,\n };\n\n let inputText = this.inputText;\n if (this.#tts.opts.instructions) {\n inputText = `${this.#tts.opts.instructions}:\\n\"${inputText}\"`;\n }\n\n const contents: types.Content[] = [\n {\n role: 'user',\n parts: [{ text: inputText }],\n },\n ];\n\n const responseStream = await this.#tts.client.models.generateContentStream({\n model: this.#tts.opts.model,\n contents,\n config,\n });\n\n try {\n for await (const response of responseStream) {\n await this.#processResponse(response, bstream, requestId);\n }\n } catch (error: unknown) {\n if (error instanceof Error && error.name === 'AbortError') {\n return;\n }\n if (isAPIError(error)) throw error;\n\n const err = error as {\n code?: number;\n message?: string;\n status?: string;\n type?: string;\n };\n\n if (err.code && err.code >= 400 && err.code < 500) {\n if (err.code === 429) {\n throw new APIStatusError({\n message: `Gemini TTS: Rate limit error - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: 429,\n retryable: true,\n },\n });\n } else {\n throw new APIStatusError({\n message: `Gemini TTS: Client error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: false,\n },\n });\n }\n }\n\n if (err.code && err.code >= 500) {\n throw new APIStatusError({\n message: `Gemini TTS: Server error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: true,\n },\n });\n }\n\n throw new APIConnectionError({\n message: `Gemini TTS: Connection error - ${err.message || 'Unknown error'}`,\n options: { retryable: true },\n });\n } finally {\n this.queue.close();\n }\n }\n\n async #processResponse(\n response: types.GenerateContentResponse,\n bstream: AudioByteStream,\n requestId: string,\n ) {\n if (!response.candidates || response.candidates.length === 0) {\n return;\n }\n\n const candidate = response.candidates[0];\n if (!candidate || !candidate.content?.parts) {\n return;\n }\n\n let lastFrame: AudioFrame | undefined;\n const sendLastFrame = (final: boolean) => {\n if (lastFrame) {\n this.queue.put({\n requestId,\n frame: lastFrame,\n segmentId: requestId,\n final,\n });\n lastFrame = undefined;\n }\n };\n\n for (const part of candidate.content.parts) {\n if (part.inlineData?.data && part.inlineData.mimeType?.startsWith('audio/')) {\n const audioBuffer = Buffer.from(part.inlineData.data, 'base64');\n\n for (const frame of bstream.write(audioBuffer)) {\n sendLastFrame(false);\n lastFrame = frame;\n }\n }\n }\n\n for (const frame of bstream.flush()) {\n sendLastFrame(false);\n lastFrame = frame;\n }\n\n sendLastFrame(true);\n }\n}\n"],"mappings":"AAIA,SAAS,mBAAmB;AAC5B;AAAA,EAEE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAqCP,MAAM,gBAAiC;AACvC,MAAM,gBAA8B;AACpC,MAAM,sBAAsB;AAC5B,MAAM,eAAe;AACrB,MAAM,uBAAuB;AAWtB,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EACA,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWR,YAAY;AAAA,IACV,QAAQ;AAAA,IACR,YAAY;AAAA,IACZ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,IAA8C,CAAC,GAAG;AAChD,UAAM,qBAAqB,cAAc,EAAE,WAAW,MAAM,CAAC;AAE7D,UAAM,aAAiC,WAAW,QAAQ,IAAI;AAC9D,UAAM,cACJ,YAAY,QAAQ,IAAI,yBAAyB;AACnD,UAAM,cAAc,YAAY,QAAQ,IAAI,8BAA8B;AAC1E,UAAM,eAAe,UAAU,QAAQ,IAAI;AAE3C,QAAI,eAAmC;AACvC,QAAI,gBAAoC;AACxC,QAAI,cAAkC;AAEtC,QAAI,aAAa;AACf,UAAI,CAAC,cAAc;AACjB,cAAM,IAAI,mBAAmB;AAAA,UAC3B,SACE;AAAA,QACJ,CAAC;AAAA,MACH;AACA,oBAAc;AAAA,IAChB,OAAO;AACL,qBAAe;AACf,sBAAgB;AAChB,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,mBAAmB;AAAA,UAC3B,SACE;AAAA,QACJ,CAAC;AAAA,MACH;AAAA,IACF;AAEA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,MACV,cAAc,gBAAgB;AAAA,IAChC;AAEA,UAAM,gBAA0C,cAC5C;AAAA,MACE,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,IACZ,IACA;AAAA,MACE,QAAQ;AAAA,IACV;AAEJ,SAAK,UAAU,IAAI,YAAY,aAAa;AAAA,EAC9C;AAAA,EAEA,WACE,MACA,aACA,aACe;AACf,WAAO,IAAI,cAAc,MAAM,MAAM,aAAa,WAAW;AAAA,EAC/D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,cAAc,MAA6C;AACzD,QAAI,KAAK,cAAc,QAAW;AAChC,WAAK,MAAM,YAAY,KAAK;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,SAA+B;AAC7B,UAAM,IAAI,MAAM,0CAA0C;AAAA,EAC5D;AAAA,EAEA,IAAI,OAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AACF;AAEO,MAAM,sBAAsB,IAAI,cAAc;AAAA,EACnD;AAAA,EACA,QAAQ;AAAA,EAER,YACE,WACAA,MACA,aACA,aACA;AACA,UAAM,WAAWA,MAAK,aAAa,WAAW;AAC9C,SAAK,OAAOA;AAAA,EACd;AAAA,EAEA,MAAgB,MAAM;AACpB,UAAM,YAAY,UAAU;AAC5B,UAAM,UAAU,IAAI,gBAAgB,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW;AAE/E,UAAM,SAAsC;AAAA,MAC1C,oBAAoB,CAAC,OAAO;AAAA,MAC5B,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK,KAAK,KAAK;AAAA,UAC5B;AAAA,QACF;AAAA,MACF;AAAA,MACA,aAAa,KAAK;AAAA,IACpB;AAEA,QAAI,YAAY,KAAK;AACrB,QAAI,KAAK,KAAK,KAAK,cAAc;AAC/B,kBAAY,GAAG,KAAK,KAAK,KAAK,YAAY;AAAA,GAAO,SAAS;AAAA,IAC5D;AAEA,UAAM,WAA4B;AAAA,MAChC;AAAA,QACE,MAAM;AAAA,QACN,OAAO,CAAC,EAAE,MAAM,UAAU,CAAC;AAAA,MAC7B;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,KAAK,OAAO,OAAO,sBAAsB;AAAA,MACzE,OAAO,KAAK,KAAK,KAAK;AAAA,MACtB;AAAA,MACA;AAAA,IACF,CAAC;AAED,QAAI;AACF,uBAAiB,YAAY,gBAAgB;AAC3C,cAAM,KAAK,iBAAiB,UAAU,SAAS,SAAS;AAAA,MAC1D;AAAA,IACF,SAAS,OAAgB;AACvB,UAAI,iBAAiB,SAAS,MAAM,SAAS,cAAc;AACzD;AAAA,MACF;AACA,UAAI,WAAW,KAAK,EAAG,OAAM;AAE7B,YAAM,MAAM;AAOZ,UAAI,IAAI,QAAQ,IAAI,QAAQ,OAAO,IAAI,OAAO,KAAK;AACjD,YAAI,IAAI,SAAS,KAAK;AACpB,gBAAM,IAAI,eAAe;AAAA,YACvB,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,YACzE,SAAS;AAAA,cACP,YAAY;AAAA,cACZ,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH,OAAO;AACL,gBAAM,IAAI,eAAe;AAAA,YACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,YACnF,SAAS;AAAA,cACP,YAAY,IAAI;AAAA,cAChB,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,IAAI,QAAQ,IAAI,QAAQ,KAAK;AAC/B,cAAM,IAAI,eAAe;AAAA,UACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,UACnF,SAAS;AAAA,YACP,YAAY,IAAI;AAAA,YAChB,WAAW;AAAA,UACb;AAAA,QACF,CAAC;AAAA,MACH;AAEA,YAAM,IAAI,mBAAmB;AAAA,QAC3B,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,QACzE,SAAS,EAAE,WAAW,KAAK;AAAA,MAC7B,CAAC;AAAA,IACH,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAM,iBACJ,UACA,SACA,WACA;AAzRJ;AA0RI,QAAI,CAAC,SAAS,cAAc,SAAS,WAAW,WAAW,GAAG;AAC5D;AAAA,IACF;AAEA,UAAM,YAAY,SAAS,WAAW,CAAC;AACvC,QAAI,CAAC,aAAa,GAAC,eAAU,YAAV,mBAAmB,QAAO;AAC3C;AAAA,IACF;AAEA,QAAI;AACJ,UAAM,gBAAgB,CAAC,UAAmB;AACxC,UAAI,WAAW;AACb,aAAK,MAAM,IAAI;AAAA,UACb;AAAA,UACA,OAAO;AAAA,UACP,WAAW;AAAA,UACX;AAAA,QACF,CAAC;AACD,oBAAY;AAAA,MACd;AAAA,IACF;AAEA,eAAW,QAAQ,UAAU,QAAQ,OAAO;AAC1C,YAAI,UAAK,eAAL,mBAAiB,WAAQ,UAAK,WAAW,aAAhB,mBAA0B,WAAW,YAAW;AAC3E,cAAM,cAAc,OAAO,KAAK,KAAK,WAAW,MAAM,QAAQ;AAE9D,mBAAW,SAAS,QAAQ,MAAM,WAAW,GAAG;AAC9C,wBAAc,KAAK;AACnB,sBAAY;AAAA,QACd;AAAA,MACF;AAAA,IACF;AAEA,eAAW,SAAS,QAAQ,MAAM,GAAG;AACnC,oBAAc,KAAK;AACnB,kBAAY;AAAA,IACd;AAEA,kBAAc,IAAI;AAAA,EACpB;AACF;","names":["tts"]} \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.cjs b/plugins/google/dist/beta/gemini_tts.test.cjs new file mode 100644 index 000000000..207bb62be --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.test.cjs @@ -0,0 +1,9 @@ +"use strict"; +var import_agents_plugin_openai = require("@livekit/agents-plugin-openai"); +var import_agents_plugins_test = require("@livekit/agents-plugins-test"); +var import_vitest = require("vitest"); +var import_gemini_tts = require("./gemini_tts.cjs"); +import_vitest.describe.skip("Google Gemini TTS", async () => { + await (0, import_agents_plugins_test.tts)(new import_gemini_tts.TTS(), new import_agents_plugin_openai.STT()); +}); +//# sourceMappingURL=gemini_tts.test.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.cjs.map b/plugins/google/dist/beta/gemini_tts.test.cjs.map new file mode 100644 index 000000000..f0d1500b9 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.test.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../src/beta/gemini_tts.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { STT } from '@livekit/agents-plugin-openai';\nimport { tts } from '@livekit/agents-plugins-test';\nimport { describe } from 'vitest';\nimport { TTS } from './gemini_tts.js';\n\ndescribe.skip('Google Gemini TTS', async () => {\n await tts(new TTS(), new STT());\n});\n"],"mappings":";AAGA,kCAAoB;AACpB,iCAAoB;AACpB,oBAAyB;AACzB,wBAAoB;AAEpB,uBAAS,KAAK,qBAAqB,YAAY;AAC7C,YAAM,gCAAI,IAAI,sBAAI,GAAG,IAAI,gCAAI,CAAC;AAChC,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.d.cts b/plugins/google/dist/beta/gemini_tts.test.d.cts new file mode 100644 index 000000000..3578eadc7 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.test.d.cts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=gemini_tts.test.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.d.ts b/plugins/google/dist/beta/gemini_tts.test.d.ts new file mode 100644 index 000000000..3578eadc7 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.test.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=gemini_tts.test.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.d.ts.map b/plugins/google/dist/beta/gemini_tts.test.d.ts.map new file mode 100644 index 000000000..e4b03e46f --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.test.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"gemini_tts.test.d.ts","sourceRoot":"","sources":["../../src/beta/gemini_tts.test.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.js b/plugins/google/dist/beta/gemini_tts.test.js new file mode 100644 index 000000000..c01b1f996 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.test.js @@ -0,0 +1,8 @@ +import { STT } from "@livekit/agents-plugin-openai"; +import { tts } from "@livekit/agents-plugins-test"; +import { describe } from "vitest"; +import { TTS } from "./gemini_tts.js"; +describe.skip("Google Gemini TTS", async () => { + await tts(new TTS(), new STT()); +}); +//# sourceMappingURL=gemini_tts.test.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.js.map b/plugins/google/dist/beta/gemini_tts.test.js.map new file mode 100644 index 000000000..7e419a023 --- /dev/null +++ b/plugins/google/dist/beta/gemini_tts.test.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../src/beta/gemini_tts.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { STT } from '@livekit/agents-plugin-openai';\nimport { tts } from '@livekit/agents-plugins-test';\nimport { describe } from 'vitest';\nimport { TTS } from './gemini_tts.js';\n\ndescribe.skip('Google Gemini TTS', async () => {\n await tts(new TTS(), new STT());\n});\n"],"mappings":"AAGA,SAAS,WAAW;AACpB,SAAS,WAAW;AACpB,SAAS,gBAAgB;AACzB,SAAS,WAAW;AAEpB,SAAS,KAAK,qBAAqB,YAAY;AAC7C,QAAM,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,CAAC;AAChC,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/index.cjs b/plugins/google/dist/beta/index.cjs new file mode 100644 index 000000000..c84230339 --- /dev/null +++ b/plugins/google/dist/beta/index.cjs @@ -0,0 +1,42 @@ +"use strict"; +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + // If the importer is in node compatibility mode or this is not an ESM + // file that has been converted to a CommonJS file using a Babel- + // compatible transform (i.e. "__esModule" has not been set), then set + // "default" to the CommonJS "module.exports" for node compatibility. + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var beta_exports = {}; +__export(beta_exports, { + TTS: () => import_gemini_tts.TTS, + realtime: () => realtime +}); +module.exports = __toCommonJS(beta_exports); +var import_gemini_tts = require("./gemini_tts.cjs"); +var realtime = __toESM(require("./realtime/index.cjs"), 1); +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + TTS, + realtime +}); +//# sourceMappingURL=index.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/index.cjs.map b/plugins/google/dist/beta/index.cjs.map new file mode 100644 index 000000000..0795a40b9 --- /dev/null +++ b/plugins/google/dist/beta/index.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../src/beta/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport { TTS, type GeminiTTSModels, type GeminiVoices, type TTSOptions } from './gemini_tts.js';\nexport * as realtime from './realtime/index.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,wBAA8E;AAC9E,eAA0B;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/index.d.cts b/plugins/google/dist/beta/index.d.cts new file mode 100644 index 000000000..2536cd29d --- /dev/null +++ b/plugins/google/dist/beta/index.d.cts @@ -0,0 +1,3 @@ +export { TTS, type GeminiTTSModels, type GeminiVoices, type TTSOptions } from './gemini_tts.js'; +export * as realtime from './realtime/index.js'; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/index.d.ts b/plugins/google/dist/beta/index.d.ts new file mode 100644 index 000000000..2536cd29d --- /dev/null +++ b/plugins/google/dist/beta/index.d.ts @@ -0,0 +1,3 @@ +export { TTS, type GeminiTTSModels, type GeminiVoices, type TTSOptions } from './gemini_tts.js'; +export * as realtime from './realtime/index.js'; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/index.d.ts.map b/plugins/google/dist/beta/index.d.ts.map new file mode 100644 index 000000000..d5701f348 --- /dev/null +++ b/plugins/google/dist/beta/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/beta/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,GAAG,EAAE,KAAK,eAAe,EAAE,KAAK,YAAY,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAChG,OAAO,KAAK,QAAQ,MAAM,qBAAqB,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/beta/index.js b/plugins/google/dist/beta/index.js new file mode 100644 index 000000000..31bcddc7e --- /dev/null +++ b/plugins/google/dist/beta/index.js @@ -0,0 +1,7 @@ +import { TTS } from "./gemini_tts.js"; +import * as realtime from "./realtime/index.js"; +export { + TTS, + realtime +}; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/index.js.map b/plugins/google/dist/beta/index.js.map new file mode 100644 index 000000000..d366ee8e1 --- /dev/null +++ b/plugins/google/dist/beta/index.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../src/beta/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport { TTS, type GeminiTTSModels, type GeminiVoices, type TTSOptions } from './gemini_tts.js';\nexport * as realtime from './realtime/index.js';\n"],"mappings":"AAIA,SAAS,WAAqE;AAC9E,YAAY,cAAc;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.cjs b/plugins/google/dist/beta/realtime/api_proto.cjs new file mode 100644 index 000000000..d13608ec1 --- /dev/null +++ b/plugins/google/dist/beta/realtime/api_proto.cjs @@ -0,0 +1,17 @@ +"use strict"; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var api_proto_exports = {}; +module.exports = __toCommonJS(api_proto_exports); +//# sourceMappingURL=api_proto.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.cjs.map b/plugins/google/dist/beta/realtime/api_proto.cjs.map new file mode 100644 index 000000000..14ddc4d1a --- /dev/null +++ b/plugins/google/dist/beta/realtime/api_proto.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../../src/beta/realtime/api_proto.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as types from '@google/genai';\n\n/**\n * Supported Google Live API models\n */\nexport type LiveAPIModels =\n | 'gemini-2.0-flash-exp'\n // models supported on Gemini API\n | 'gemini-2.0-flash-live-001'\n | 'gemini-2.5-flash-preview-native-audio-dialog'\n | 'gemini-2.5-flash-exp-native-audio-thinking-dialog';\n\n/**\n * Available voice options for Google Realtime API\n */\nexport type Voice = 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Aoede' | 'Leda' | 'Orus' | 'Zephyr';\n\n/**\n * Union type for all possible client events\n */\n\nexport type ClientEvents =\n | {\n type: 'content';\n value: types.LiveClientContent;\n }\n | {\n type: 'realtime_input';\n value: types.LiveClientRealtimeInput;\n }\n | {\n type: 'tool_response';\n value: types.LiveClientToolResponse;\n }\n | {\n type: 'function_response';\n value: types.FunctionResponse;\n };\n"],"mappings":";;;;;;;;;;;;;;AAAA;AAAA;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.d.cts b/plugins/google/dist/beta/realtime/api_proto.d.cts new file mode 100644 index 000000000..953af7083 --- /dev/null +++ b/plugins/google/dist/beta/realtime/api_proto.d.cts @@ -0,0 +1,26 @@ +import type * as types from '@google/genai'; +/** + * Supported Google Live API models + */ +export type LiveAPIModels = 'gemini-2.0-flash-exp' | 'gemini-2.0-flash-live-001' | 'gemini-2.5-flash-preview-native-audio-dialog' | 'gemini-2.5-flash-exp-native-audio-thinking-dialog'; +/** + * Available voice options for Google Realtime API + */ +export type Voice = 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Aoede' | 'Leda' | 'Orus' | 'Zephyr'; +/** + * Union type for all possible client events + */ +export type ClientEvents = { + type: 'content'; + value: types.LiveClientContent; +} | { + type: 'realtime_input'; + value: types.LiveClientRealtimeInput; +} | { + type: 'tool_response'; + value: types.LiveClientToolResponse; +} | { + type: 'function_response'; + value: types.FunctionResponse; +}; +//# sourceMappingURL=api_proto.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.d.ts b/plugins/google/dist/beta/realtime/api_proto.d.ts new file mode 100644 index 000000000..953af7083 --- /dev/null +++ b/plugins/google/dist/beta/realtime/api_proto.d.ts @@ -0,0 +1,26 @@ +import type * as types from '@google/genai'; +/** + * Supported Google Live API models + */ +export type LiveAPIModels = 'gemini-2.0-flash-exp' | 'gemini-2.0-flash-live-001' | 'gemini-2.5-flash-preview-native-audio-dialog' | 'gemini-2.5-flash-exp-native-audio-thinking-dialog'; +/** + * Available voice options for Google Realtime API + */ +export type Voice = 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Aoede' | 'Leda' | 'Orus' | 'Zephyr'; +/** + * Union type for all possible client events + */ +export type ClientEvents = { + type: 'content'; + value: types.LiveClientContent; +} | { + type: 'realtime_input'; + value: types.LiveClientRealtimeInput; +} | { + type: 'tool_response'; + value: types.LiveClientToolResponse; +} | { + type: 'function_response'; + value: types.FunctionResponse; +}; +//# sourceMappingURL=api_proto.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.d.ts.map b/plugins/google/dist/beta/realtime/api_proto.d.ts.map new file mode 100644 index 000000000..f49abf619 --- /dev/null +++ b/plugins/google/dist/beta/realtime/api_proto.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api_proto.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/api_proto.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,KAAK,KAAK,MAAM,eAAe,CAAC;AAE5C;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,sBAAsB,GAEtB,2BAA2B,GAC3B,8CAA8C,GAC9C,mDAAmD,CAAC;AAExD;;GAEG;AACH,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEjG;;GAEG;AAEH,MAAM,MAAM,YAAY,GACpB;IACE,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,KAAK,CAAC,iBAAiB,CAAC;CAChC,GACD;IACE,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,KAAK,CAAC,uBAAuB,CAAC;CACtC,GACD;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC;CACrC,GACD;IACE,IAAI,EAAE,mBAAmB,CAAC;IAC1B,KAAK,EAAE,KAAK,CAAC,gBAAgB,CAAC;CAC/B,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.js b/plugins/google/dist/beta/realtime/api_proto.js new file mode 100644 index 000000000..041de4df6 --- /dev/null +++ b/plugins/google/dist/beta/realtime/api_proto.js @@ -0,0 +1 @@ +//# sourceMappingURL=api_proto.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.js.map b/plugins/google/dist/beta/realtime/api_proto.js.map new file mode 100644 index 000000000..84c51b288 --- /dev/null +++ b/plugins/google/dist/beta/realtime/api_proto.js.map @@ -0,0 +1 @@ +{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.cjs b/plugins/google/dist/beta/realtime/index.cjs new file mode 100644 index 000000000..449158563 --- /dev/null +++ b/plugins/google/dist/beta/realtime/index.cjs @@ -0,0 +1,29 @@ +"use strict"; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var realtime_exports = {}; +__export(realtime_exports, { + RealtimeModel: () => import_realtime_api.RealtimeModel +}); +module.exports = __toCommonJS(realtime_exports); +var import_realtime_api = require("./realtime_api.cjs"); +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + RealtimeModel +}); +//# sourceMappingURL=index.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.cjs.map b/plugins/google/dist/beta/realtime/index.cjs.map new file mode 100644 index 000000000..cec90f754 --- /dev/null +++ b/plugins/google/dist/beta/realtime/index.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../../src/beta/realtime/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport type { ClientEvents, LiveAPIModels, Voice } from './api_proto.js';\nexport { RealtimeModel } from './realtime_api.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,0BAA8B;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.d.cts b/plugins/google/dist/beta/realtime/index.d.cts new file mode 100644 index 000000000..d1c48a43c --- /dev/null +++ b/plugins/google/dist/beta/realtime/index.d.cts @@ -0,0 +1,3 @@ +export type { ClientEvents, LiveAPIModels, Voice } from './api_proto.js'; +export { RealtimeModel } from './realtime_api.js'; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.d.ts b/plugins/google/dist/beta/realtime/index.d.ts new file mode 100644 index 000000000..d1c48a43c --- /dev/null +++ b/plugins/google/dist/beta/realtime/index.d.ts @@ -0,0 +1,3 @@ +export type { ClientEvents, LiveAPIModels, Voice } from './api_proto.js'; +export { RealtimeModel } from './realtime_api.js'; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.d.ts.map b/plugins/google/dist/beta/realtime/index.d.ts.map new file mode 100644 index 000000000..5b3374be5 --- /dev/null +++ b/plugins/google/dist/beta/realtime/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/index.ts"],"names":[],"mappings":"AAGA,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.js b/plugins/google/dist/beta/realtime/index.js new file mode 100644 index 000000000..2fec47d74 --- /dev/null +++ b/plugins/google/dist/beta/realtime/index.js @@ -0,0 +1,5 @@ +import { RealtimeModel } from "./realtime_api.js"; +export { + RealtimeModel +}; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.js.map b/plugins/google/dist/beta/realtime/index.js.map new file mode 100644 index 000000000..e46ec7404 --- /dev/null +++ b/plugins/google/dist/beta/realtime/index.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../../src/beta/realtime/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport type { ClientEvents, LiveAPIModels, Voice } from './api_proto.js';\nexport { RealtimeModel } from './realtime_api.js';\n"],"mappings":"AAIA,SAAS,qBAAqB;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.cjs b/plugins/google/dist/beta/realtime/realtime_api.cjs new file mode 100644 index 000000000..4cead1cd0 --- /dev/null +++ b/plugins/google/dist/beta/realtime/realtime_api.cjs @@ -0,0 +1,1001 @@ +"use strict"; +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + // If the importer is in node compatibility mode or this is not an ESM + // file that has been converted to a CommonJS file using a Babel- + // compatible transform (i.e. "__esModule" has not been set), then set + // "default" to the CommonJS "module.exports" for node compatibility. + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var realtime_api_exports = {}; +__export(realtime_api_exports, { + DEFAULT_IMAGE_ENCODE_OPTIONS: () => DEFAULT_IMAGE_ENCODE_OPTIONS, + RealtimeModel: () => RealtimeModel, + RealtimeSession: () => RealtimeSession +}); +module.exports = __toCommonJS(realtime_api_exports); +var types = __toESM(require("@google/genai"), 1); +var import_genai = require("@google/genai"); +var import_agents = require("@livekit/agents"); +var import_mutex = require("@livekit/mutex"); +var import_rtc_node = require("@livekit/rtc-node"); +var import_tools = require("../../tools.cjs"); +var import_utils = require("../../utils.cjs"); +const INPUT_AUDIO_SAMPLE_RATE = 16e3; +const INPUT_AUDIO_CHANNELS = 1; +const OUTPUT_AUDIO_SAMPLE_RATE = 24e3; +const OUTPUT_AUDIO_CHANNELS = 1; +const DEFAULT_IMAGE_ENCODE_OPTIONS = { + format: "JPEG", + quality: 75, + resizeOptions: { + width: 1024, + height: 1024, + strategy: "scale_aspect_fit" + } +}; +function setsEqual(a, b) { + return a.size === b.size && [...a].every((x) => b.has(x)); +} +class RealtimeModel extends import_agents.llm.RealtimeModel { + /** @internal */ + _options; + constructor(options = {}) { + var _a, _b, _c; + const inputAudioTranscription = options.inputAudioTranscription === void 0 ? {} : options.inputAudioTranscription; + const outputAudioTranscription = options.outputAudioTranscription === void 0 ? {} : options.outputAudioTranscription; + let serverTurnDetection = true; + if ((_b = (_a = options.realtimeInputConfig) == null ? void 0 : _a.automaticActivityDetection) == null ? void 0 : _b.disabled) { + serverTurnDetection = false; + } + super({ + messageTruncation: false, + turnDetection: serverTurnDetection, + userTranscription: inputAudioTranscription !== null, + autoToolReplyGeneration: true, + audioOutput: ((_c = options.modalities) == null ? void 0 : _c.includes(import_genai.Modality.AUDIO)) ?? true + }); + const apiKey = options.apiKey || process.env.GOOGLE_API_KEY; + const project = options.project || process.env.GOOGLE_CLOUD_PROJECT; + const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || "us-central1"; + const vertexai = options.vertexai ?? false; + const defaultModel = vertexai ? "gemini-2.0-flash-exp" : "gemini-2.0-flash-live-001"; + this._options = { + model: options.model || defaultModel, + apiKey, + voice: options.voice || "Puck", + language: options.language, + responseModalities: options.modalities || [import_genai.Modality.AUDIO], + vertexai, + project, + location, + candidateCount: options.candidateCount || 1, + temperature: options.temperature, + maxOutputTokens: options.maxOutputTokens, + topP: options.topP, + topK: options.topK, + presencePenalty: options.presencePenalty, + frequencyPenalty: options.frequencyPenalty, + instructions: options.instructions, + inputAudioTranscription: inputAudioTranscription || void 0, + outputAudioTranscription: outputAudioTranscription || void 0, + imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS, + connOptions: options.connOptions || import_agents.DEFAULT_API_CONNECT_OPTIONS, + httpOptions: options.httpOptions, + enableAffectiveDialog: options.enableAffectiveDialog, + proactivity: options.proactivity, + realtimeInputConfig: options.realtimeInputConfig, + contextWindowCompression: options.contextWindowCompression, + apiVersion: options.apiVersion, + geminiTools: options.geminiTools, + thinkingConfig: options.thinkingConfig + }; + } + /** + * Create a new realtime session + */ + session() { + return new RealtimeSession(this); + } + /** + * Update model options + */ + updateOptions(options) { + if (options.voice !== void 0) { + this._options.voice = options.voice; + } + if (options.temperature !== void 0) { + this._options.temperature = options.temperature; + } + } + /** + * Close the model and cleanup resources + */ + async close() { + } +} +class RealtimeSession extends import_agents.llm.RealtimeSession { + _tools = {}; + _chatCtx = import_agents.llm.ChatContext.empty(); + options; + geminiDeclarations = []; + messageChannel = new import_agents.Queue(); + inputResampler; + inputResamplerInputRate; + instructions; + currentGeneration; + bstream; + // Google-specific properties + activeSession; + sessionShouldClose = new import_agents.Event(); + responseCreatedFutures = {}; + pendingGenerationFut; + sessionResumptionHandle; + inUserActivity = false; + sessionLock = new import_mutex.Mutex(); + numRetries = 0; + hasReceivedAudioInput = false; + #client; + #task; + #logger = (0, import_agents.log)(); + #closed = false; + constructor(realtimeModel) { + super(realtimeModel); + this.options = realtimeModel._options; + this.bstream = new import_agents.AudioByteStream( + INPUT_AUDIO_SAMPLE_RATE, + INPUT_AUDIO_CHANNELS, + INPUT_AUDIO_SAMPLE_RATE / 20 + ); + const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } = this.options; + const apiVersion = !this.options.apiVersion && (enableAffectiveDialog || proactivity) ? "v1alpha" : this.options.apiVersion; + const httpOptions = { + ...this.options.httpOptions, + apiVersion, + timeout: this.options.connOptions.timeoutMs + }; + const clientOptions = vertexai ? { + vertexai: true, + project, + location, + httpOptions + } : { + apiKey, + httpOptions + }; + this.#client = new import_genai.GoogleGenAI(clientOptions); + this.#task = this.#mainTask(); + } + async closeActiveSession() { + const unlock = await this.sessionLock.lock(); + if (this.activeSession) { + try { + await this.activeSession.close(); + } catch (error) { + this.#logger.warn({ error }, "Error closing Gemini session"); + } finally { + this.activeSession = void 0; + } + } + unlock(); + } + markRestartNeeded() { + if (!this.sessionShouldClose.isSet) { + this.sessionShouldClose.set(); + this.messageChannel = new import_agents.Queue(); + } + } + getToolResultsForRealtime(ctx, vertexai) { + const toolResponses = []; + for (const item of ctx.items) { + if (item.type === "function_call_output") { + const response = { + id: item.callId, + name: item.name, + response: { output: item.output } + }; + if (!vertexai) { + response.id = item.callId; + } + toolResponses.push(response); + } + } + return toolResponses.length > 0 ? { functionResponses: toolResponses } : void 0; + } + updateOptions(options) { + let shouldRestart = false; + if (options.voice !== void 0 && this.options.voice !== options.voice) { + this.options.voice = options.voice; + shouldRestart = true; + } + if (options.temperature !== void 0 && this.options.temperature !== options.temperature) { + this.options.temperature = options.temperature; + shouldRestart = true; + } + if (shouldRestart) { + this.markRestartNeeded(); + } + } + async updateInstructions(instructions) { + if (this.options.instructions === void 0 || this.options.instructions !== instructions) { + this.options.instructions = instructions; + this.markRestartNeeded(); + } + } + async updateChatCtx(chatCtx) { + const unlock = await this.sessionLock.lock(); + try { + if (!this.activeSession) { + this._chatCtx = chatCtx.copy(); + return; + } + } finally { + unlock(); + } + const diffOps = import_agents.llm.computeChatCtxDiff(this._chatCtx, chatCtx); + if (diffOps.toRemove.length > 0) { + this.#logger.warn("Gemini Live does not support removing messages"); + } + const appendCtx = import_agents.llm.ChatContext.empty(); + for (const [, itemId] of diffOps.toCreate) { + const item = chatCtx.getById(itemId); + if (item) { + appendCtx.items.push(item); + } + } + if (appendCtx.items.length > 0) { + const [turns] = await appendCtx.copy({ + excludeFunctionCall: true + }).toProviderFormat("google", false); + const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai); + if (turns.length > 0) { + this.sendClientEvent({ + type: "content", + value: { + turns, + turnComplete: false + } + }); + } + if (toolResults) { + this.sendClientEvent({ + type: "tool_response", + value: toolResults + }); + } + } + this._chatCtx = chatCtx.copy(); + } + async updateTools(tools) { + const newDeclarations = (0, import_utils.toFunctionDeclarations)(tools); + const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name)); + const newToolNames = new Set(newDeclarations.map((f) => f.name)); + if (!setsEqual(currentToolNames, newToolNames)) { + this.geminiDeclarations = newDeclarations; + this._tools = tools; + this.markRestartNeeded(); + } + } + get chatCtx() { + return this._chatCtx.copy(); + } + get tools() { + return { ...this._tools }; + } + get manualActivityDetection() { + var _a, _b; + return ((_b = (_a = this.options.realtimeInputConfig) == null ? void 0 : _a.automaticActivityDetection) == null ? void 0 : _b.disabled) ?? false; + } + pushAudio(frame) { + this.hasReceivedAudioInput = true; + for (const f of this.resampleAudio(frame)) { + for (const nf of this.bstream.write(f.data.buffer)) { + const realtimeInput = { + mediaChunks: [ + { + mimeType: "audio/pcm", + data: Buffer.from(nf.data.buffer).toString("base64") + } + ] + }; + this.sendClientEvent({ + type: "realtime_input", + value: realtimeInput + }); + } + } + } + pushVideo(_) { + } + sendClientEvent(event) { + this.messageChannel.put(event); + } + async generateReply(instructions) { + if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { + this.#logger.warn( + "generateReply called while another generation is pending, cancelling previous." + ); + this.pendingGenerationFut.reject(new Error("Superseded by new generate_reply call")); + } + const fut = new import_agents.Future(); + this.pendingGenerationFut = fut; + if (this.inUserActivity) { + this.sendClientEvent({ + type: "realtime_input", + value: { + activityEnd: {} + } + }); + this.inUserActivity = false; + } + const turns = []; + if (instructions !== void 0) { + turns.push({ + parts: [{ text: instructions }], + role: "model" + }); + } + turns.push({ + parts: [{ text: "." }], + role: "user" + }); + this.sendClientEvent({ + type: "content", + value: { + turns, + turnComplete: true + } + }); + const timeoutHandle = setTimeout(() => { + if (!fut.done) { + fut.reject(new Error("generateReply timed out waiting for generation_created event.")); + if (this.pendingGenerationFut === fut) { + this.pendingGenerationFut = void 0; + } + } + }, 5e3); + fut.await.finally(() => clearTimeout(timeoutHandle)); + return fut.await; + } + startUserActivity() { + if (!this.manualActivityDetection) { + return; + } + if (!this.inUserActivity) { + this.inUserActivity = true; + this.sendClientEvent({ + type: "realtime_input", + value: { + activityStart: {} + } + }); + } + } + async interrupt() { + var _a; + if (((_a = this.options.realtimeInputConfig) == null ? void 0 : _a.activityHandling) === import_genai.ActivityHandling.NO_INTERRUPTION) { + return; + } + this.startUserActivity(); + } + async truncate(_options) { + this.#logger.warn("truncate is not supported by the Google Realtime API."); + } + async close() { + super.close(); + this.#closed = true; + this.sessionShouldClose.set(); + await this.closeActiveSession(); + if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { + this.pendingGenerationFut.reject(new Error("Session closed")); + } + for (const fut of Object.values(this.responseCreatedFutures)) { + if (!fut.done) { + fut.reject(new Error("Session closed before response created")); + } + } + this.responseCreatedFutures = {}; + if (this.currentGeneration) { + this.markCurrentGenerationDone(); + } + } + async #mainTask() { + const maxRetries = this.options.connOptions.maxRetry; + while (!this.#closed) { + await this.closeActiveSession(); + this.sessionShouldClose.clear(); + const config = this.buildConnectConfig(); + try { + this.#logger.debug("Connecting to Gemini Realtime API..."); + const sessionOpened = new import_agents.Event(); + const session = await this.#client.live.connect({ + model: this.options.model, + callbacks: { + onopen: () => sessionOpened.set(), + onmessage: (message) => { + this.onReceiveMessage(session, message); + }, + onerror: (error) => { + this.#logger.error("Gemini Live session error:", error); + if (!this.sessionShouldClose.isSet) { + this.markRestartNeeded(); + } + }, + onclose: (event) => { + this.#logger.debug("Gemini Live session closed:", event.code, event.reason); + this.markCurrentGenerationDone(); + } + }, + config + }); + await sessionOpened.wait(); + const unlock = await this.sessionLock.lock(); + try { + this.activeSession = session; + const [turns] = await this._chatCtx.copy({ + excludeFunctionCall: true + }).toProviderFormat("google", false); + if (turns.length > 0) { + await session.sendClientContent({ + turns, + turnComplete: false + }); + } + } finally { + unlock(); + } + const sendTask = import_agents.Task.from((controller) => this.sendTask(session, controller)); + const restartWaitTask = import_agents.Task.from(({ signal }) => { + const abortEvent = new import_agents.Event(); + signal.addEventListener("abort", () => abortEvent.set()); + return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]); + }); + await Promise.race([sendTask.result, restartWaitTask.result]); + if (!restartWaitTask.done && this.#closed) { + break; + } + await (0, import_agents.cancelAndWait)([sendTask, restartWaitTask], 2e3); + } catch (error) { + this.#logger.error(`Gemini Realtime API error: ${error}`); + if (this.#closed) break; + if (maxRetries === 0) { + this.emitError(error, false); + throw new import_agents.APIConnectionError({ + message: "Failed to connect to Gemini Live" + }); + } + if (this.numRetries >= maxRetries) { + this.emitError(error, false); + throw new import_agents.APIConnectionError({ + message: `Failed to connect to Gemini Live after ${maxRetries} attempts` + }); + } + const retryInterval = this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs; + this.#logger.warn( + { + attempt: this.numRetries, + maxRetries + }, + `Gemini Realtime API connection failed, retrying in ${retryInterval}ms` + ); + await (0, import_agents.delay)(retryInterval); + this.numRetries++; + } finally { + await this.closeActiveSession(); + } + } + } + async sendTask(session, controller) { + try { + while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) { + const msg = await this.messageChannel.get(); + if (controller.signal.aborted) break; + const unlock = await this.sessionLock.lock(); + try { + if (this.sessionShouldClose.isSet || this.activeSession !== session) { + break; + } + } finally { + unlock(); + } + switch (msg.type) { + case "content": + const { turns, turnComplete } = msg.value; + this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`); + await session.sendClientContent({ + turns, + turnComplete: turnComplete ?? true + }); + break; + case "tool_response": + const { functionResponses } = msg.value; + if (functionResponses) { + this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`); + await session.sendToolResponse({ + functionResponses + }); + } + break; + case "realtime_input": + const { mediaChunks, activityStart, activityEnd } = msg.value; + if (mediaChunks) { + for (const mediaChunk of mediaChunks) { + await session.sendRealtimeInput({ media: mediaChunk }); + } + } + if (activityStart) await session.sendRealtimeInput({ activityStart }); + if (activityEnd) await session.sendRealtimeInput({ activityEnd }); + break; + default: + this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`); + break; + } + } + } catch (e) { + if (!this.sessionShouldClose.isSet) { + this.#logger.error(`Error in send task: ${e}`); + this.markRestartNeeded(); + } + } finally { + this.#logger.debug( + { + closed: this.#closed, + sessionShouldClose: this.sessionShouldClose.isSet, + aborted: controller.signal.aborted + }, + "send task finished." + ); + } + } + async onReceiveMessage(session, response) { + var _a, _b, _c; + const hasAudioData = (_c = (_b = (_a = response.serverContent) == null ? void 0 : _a.modelTurn) == null ? void 0 : _b.parts) == null ? void 0 : _c.some( + (part) => { + var _a2; + return (_a2 = part.inlineData) == null ? void 0 : _a2.data; + } + ); + if (!hasAudioData) { + this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`); + } + const unlock = await this.sessionLock.lock(); + try { + if (this.sessionShouldClose.isSet || this.activeSession !== session) { + this.#logger.debug("onReceiveMessage: Session changed or closed, stopping receive."); + return; + } + } finally { + unlock(); + } + if ((!this.currentGeneration || this.currentGeneration._done) && (response.serverContent || response.toolCall)) { + this.startNewGeneration(); + } + if (response.sessionResumptionUpdate) { + if (response.sessionResumptionUpdate.resumable && response.sessionResumptionUpdate.newHandle) { + this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle; + } + } + try { + if (response.serverContent) { + this.handleServerContent(response.serverContent); + } + if (response.toolCall) { + this.handleToolCall(response.toolCall); + } + if (response.toolCallCancellation) { + this.handleToolCallCancellation(response.toolCallCancellation); + } + if (response.usageMetadata) { + this.handleUsageMetadata(response.usageMetadata); + } + if (response.goAway) { + this.handleGoAway(response.goAway); + } + if (this.numRetries > 0) { + this.numRetries = 0; + } + } catch (e) { + if (!this.sessionShouldClose.isSet) { + this.#logger.error(`Error in onReceiveMessage: ${e}`); + this.markRestartNeeded(); + } + } + } + /// Truncate large base64/audio payloads for logging to avoid flooding logs + truncateString(data, maxLength = 30) { + return data.length > maxLength ? `${data.slice(0, maxLength)}\u2026` : data; + } + loggableClientEvent(event, maxLength = 30) { + var _a; + const obj = { ...event }; + if (obj.type === "realtime_input" && ((_a = obj.value) == null ? void 0 : _a.mediaChunks)) { + obj.value = { + ...obj.value, + mediaChunks: obj.value.mediaChunks.map( + (mc) => ({ + ...mc, + data: typeof mc.data === "string" ? this.truncateString(mc.data, maxLength) : mc.data + }) + ) + }; + } + return obj; + } + loggableServerMessage(message, maxLength = 30) { + const obj = { ...message }; + if (obj.serverContent && obj.serverContent.modelTurn && Array.isArray(obj.serverContent.modelTurn.parts)) { + obj.serverContent = { ...obj.serverContent }; + obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn }; + obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part) => { + var _a; + if (((_a = part == null ? void 0 : part.inlineData) == null ? void 0 : _a.data) && typeof part.inlineData.data === "string") { + return { + ...part, + inlineData: { + ...part.inlineData, + data: this.truncateString(part.inlineData.data, maxLength) + } + }; + } + return part; + }); + } + return obj; + } + markCurrentGenerationDone() { + if (!this.currentGeneration || this.currentGeneration._done) { + return; + } + this.handleInputSpeechStopped(); + const gen = this.currentGeneration; + if (gen.inputTranscription) { + this.emit("input_audio_transcription_completed", { + itemId: gen.inputId, + transcript: gen.inputTranscription, + isFinal: true + }); + this._chatCtx.addMessage({ + role: "user", + content: gen.inputTranscription, + id: gen.inputId + }); + } + if (gen.outputText) { + this._chatCtx.addMessage({ + role: "assistant", + content: gen.outputText, + id: gen.responseId + }); + } + if (this.options.outputAudioTranscription === void 0) { + gen.textChannel.write(""); + } + gen.textChannel.close(); + gen.audioChannel.close(); + gen.functionChannel.close(); + gen.messageChannel.close(); + gen._done = true; + } + emitError(error, recoverable) { + this.emit("error", { + timestamp: Date.now(), + // TODO(brian): add label to realtime model + label: "google_realtime", + error, + recoverable + }); + } + buildConnectConfig() { + const opts = this.options; + const config = { + responseModalities: opts.responseModalities, + systemInstruction: opts.instructions ? { + parts: [{ text: opts.instructions }] + } : void 0, + speechConfig: { + voiceConfig: { + prebuiltVoiceConfig: { + voiceName: opts.voice + } + }, + languageCode: opts.language + }, + tools: [ + { + functionDeclarations: this.geminiDeclarations, + ...this.options.geminiTools + } + ], + inputAudioTranscription: opts.inputAudioTranscription, + outputAudioTranscription: opts.outputAudioTranscription, + sessionResumption: { + handle: this.sessionResumptionHandle + } + }; + if (opts.temperature !== void 0) { + config.temperature = opts.temperature; + } + if (opts.maxOutputTokens !== void 0) { + config.maxOutputTokens = opts.maxOutputTokens; + } + if (opts.topP !== void 0) { + config.topP = opts.topP; + } + if (opts.topK !== void 0) { + config.topK = opts.topK; + } + if (opts.proactivity !== void 0) { + config.proactivity = { proactiveAudio: opts.proactivity }; + } + if (opts.enableAffectiveDialog !== void 0) { + config.enableAffectiveDialog = opts.enableAffectiveDialog; + } + if (opts.realtimeInputConfig !== void 0) { + config.realtimeInputConfig = opts.realtimeInputConfig; + } + if (opts.contextWindowCompression !== void 0) { + config.contextWindowCompression = opts.contextWindowCompression; + } + if (opts.thinkingConfig !== void 0) { + config.generationConfig = { + thinkingConfig: opts.thinkingConfig + }; + } + return config; + } + startNewGeneration() { + if (this.currentGeneration && !this.currentGeneration._done) { + this.#logger.warn("Starting new generation while another is active. Finalizing previous."); + this.markCurrentGenerationDone(); + } + const responseId = (0, import_agents.shortuuid)("GR_"); + this.currentGeneration = { + messageChannel: import_agents.stream.createStreamChannel(), + functionChannel: import_agents.stream.createStreamChannel(), + responseId, + inputId: (0, import_agents.shortuuid)("GI_"), + textChannel: import_agents.stream.createStreamChannel(), + audioChannel: import_agents.stream.createStreamChannel(), + inputTranscription: "", + outputText: "", + _createdTimestamp: Date.now(), + _done: false + }; + if (!this._realtimeModel.capabilities.audioOutput) { + this.currentGeneration.audioChannel.close(); + } + const modalities = this._realtimeModel.capabilities.audioOutput ? ["audio", "text"] : ["text"]; + this.currentGeneration.messageChannel.write({ + messageId: responseId, + textStream: this.currentGeneration.textChannel.stream(), + audioStream: this.currentGeneration.audioChannel.stream(), + modalities: Promise.resolve(modalities) + }); + const generationEvent = { + messageStream: this.currentGeneration.messageChannel.stream(), + functionStream: this.currentGeneration.functionChannel.stream(), + userInitiated: false + }; + if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { + generationEvent.userInitiated = true; + this.pendingGenerationFut.resolve(generationEvent); + this.pendingGenerationFut = void 0; + } else { + this.handleInputSpeechStarted(); + } + this.emit("generation_created", generationEvent); + } + handleInputSpeechStarted() { + this.emit("input_speech_started", {}); + } + handleInputSpeechStopped() { + this.emit("input_speech_stopped", { + userTranscriptionEnabled: false + }); + } + handleServerContent(serverContent) { + if (!this.currentGeneration) { + this.#logger.warn("received server content but no active generation."); + return; + } + const gen = this.currentGeneration; + if (serverContent.modelTurn) { + const turn = serverContent.modelTurn; + for (const part of turn.parts || []) { + if (part.text) { + gen.outputText += part.text; + gen.textChannel.write(part.text); + } + if (part.inlineData) { + if (!gen._firstTokenTimestamp) { + gen._firstTokenTimestamp = Date.now(); + } + try { + if (!part.inlineData.data) { + throw new Error("frameData is not bytes"); + } + const binaryString = atob(part.inlineData.data); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + const int16Array = new Int16Array(bytes.buffer); + const audioFrame = new import_rtc_node.AudioFrame( + int16Array, + OUTPUT_AUDIO_SAMPLE_RATE, + OUTPUT_AUDIO_CHANNELS, + int16Array.length / OUTPUT_AUDIO_CHANNELS + ); + gen.audioChannel.write(audioFrame); + } catch (error) { + this.#logger.error("Error processing audio data:", error); + } + } + } + } + if (serverContent.inputTranscription && serverContent.inputTranscription.text) { + let text = serverContent.inputTranscription.text; + if (gen.inputTranscription === "") { + text = text.trimStart(); + } + gen.inputTranscription += text; + this.emit("input_audio_transcription_completed", { + itemId: gen.inputId, + transcript: gen.inputTranscription, + isFinal: false + }); + } + if (serverContent.outputTranscription && serverContent.outputTranscription.text) { + const text = serverContent.outputTranscription.text; + gen.outputText += text; + gen.textChannel.write(text); + } + if (serverContent.generationComplete || serverContent.turnComplete) { + gen._completedTimestamp = Date.now(); + } + if (serverContent.interrupted) { + this.handleInputSpeechStarted(); + } + if (serverContent.turnComplete) { + this.markCurrentGenerationDone(); + } + } + handleToolCall(toolCall) { + if (!this.currentGeneration) { + this.#logger.warn("received tool call but no active generation."); + return; + } + const gen = this.currentGeneration; + for (const fc of toolCall.functionCalls || []) { + gen.functionChannel.write({ + callId: fc.id || (0, import_agents.shortuuid)("fnc-call-"), + name: fc.name, + args: fc.args ? JSON.stringify(fc.args) : "" + }); + } + this.markCurrentGenerationDone(); + } + handleToolCallCancellation(cancellation) { + this.#logger.warn( + { + functionCallIds: cancellation.ids + }, + "server cancelled tool calls" + ); + } + handleUsageMetadata(usage) { + if (!this.currentGeneration) { + this.#logger.debug("Received usage metadata but no active generation"); + return; + } + const gen = this.currentGeneration; + const createdTimestamp = gen._createdTimestamp; + const firstTokenTimestamp = gen._firstTokenTimestamp; + const completedTimestamp = gen._completedTimestamp || Date.now(); + const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1; + const durationMs = completedTimestamp - createdTimestamp; + const inputTokens = usage.promptTokenCount || 0; + const outputTokens = usage.responseTokenCount || 0; + const totalTokens = usage.totalTokenCount || 0; + const realtimeMetrics = { + type: "realtime_model_metrics", + timestamp: createdTimestamp, + requestId: gen.responseId, + ttftMs, + durationMs, + cancelled: gen._done && !gen._completedTimestamp, + label: "google_realtime", + inputTokens, + outputTokens, + totalTokens, + tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1e3) : 0, + inputTokenDetails: { + ...this.tokenDetailsMap(usage.promptTokensDetails), + cachedTokens: (usage.cacheTokensDetails || []).reduce( + (sum, detail) => sum + (detail.tokenCount || 0), + 0 + ), + cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails) + }, + outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails) + }; + this.emit("metrics_collected", realtimeMetrics); + } + tokenDetailsMap(tokenDetails) { + const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 }; + if (!tokenDetails) { + return tokenDetailsMap; + } + for (const tokenDetail of tokenDetails) { + if (!tokenDetail.tokenCount) { + continue; + } + if (tokenDetail.modality === types.MediaModality.AUDIO) { + tokenDetailsMap.audioTokens += tokenDetail.tokenCount; + } else if (tokenDetail.modality === types.MediaModality.TEXT) { + tokenDetailsMap.textTokens += tokenDetail.tokenCount; + } else if (tokenDetail.modality === types.MediaModality.IMAGE) { + tokenDetailsMap.imageTokens += tokenDetail.tokenCount; + } + } + return tokenDetailsMap; + } + handleGoAway(goAway) { + this.#logger.warn({ timeLeft: goAway.timeLeft }, "Gemini server indicates disconnection soon."); + this.sessionShouldClose.set(); + } + async commitAudio() { + } + async clearAudio() { + } + *resampleAudio(frame) { + if (this.inputResampler) { + if (frame.sampleRate !== this.inputResamplerInputRate) { + this.inputResampler = void 0; + this.inputResamplerInputRate = void 0; + } + } + if (this.inputResampler === void 0 && (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)) { + this.inputResampler = new import_rtc_node.AudioResampler( + frame.sampleRate, + INPUT_AUDIO_SAMPLE_RATE, + INPUT_AUDIO_CHANNELS + ); + this.inputResamplerInputRate = frame.sampleRate; + } + if (this.inputResampler) { + for (const resampledFrame of this.inputResampler.push(frame)) { + yield resampledFrame; + } + } else { + yield frame; + } + } +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + DEFAULT_IMAGE_ENCODE_OPTIONS, + RealtimeModel, + RealtimeSession +}); +//# sourceMappingURL=realtime_api.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.cjs.map b/plugins/google/dist/beta/realtime/realtime_api.cjs.map new file mode 100644 index 000000000..f46212026 --- /dev/null +++ b/plugins/google/dist/beta/realtime/realtime_api.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n if (this.activeSession) {\n try {\n await this.activeSession.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n } finally {\n this.activeSession = undefined;\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n this.messageChannel = new Queue();\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error('Gemini Live session error:', error);\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n this.#logger.debug('Gemini Live session closed:', event.code, event.reason);\n this.markCurrentGenerationDone();\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n if (opts.thinkingConfig !== undefined) {\n config.generationConfig = {\n thinkingConfig: opts.thinkingConfig,\n };\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() { }\n\n async clearAudio() { }\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,YAAuB;AACvB,mBAQO;AAEP,oBAcO;AACP,mBAAsB;AACtB,sBAA4D;AAC5D,mBAA8B;AAC9B,mBAAuC;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,kBAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,sBAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,sBAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,kBAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,kBAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,oBAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,oBAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,mBAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI,KAAK,eAAe;AACtB,UAAI;AACF,cAAM,KAAK,cAAc,MAAM;AAAA,MACjC,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D,UAAE;AACA,aAAK,gBAAgB;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAC5B,WAAK,iBAAiB,IAAI,oBAAM;AAAA,IAClC;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,kBAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,kBAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,sBAAkB,qCAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AAzlBzC;AA0lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,qBAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AApsBpB;AAssBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,8BAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,oBAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,8BAA8B,KAAK;AACtD,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,+BAA+B,MAAM,MAAM,MAAM,MAAM;AAC1E,mBAAK,0BAA0B;AAAA,YACjC;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,mBAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,mBAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,oBAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,kBAAM,6BAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,kBAAM,qBAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA55BnB;AA85BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AA/5Bb,YAAAA;AA+5BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAv+B7B;AAw+BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAngC/F;AAogCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,IACF;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,QAAI,KAAK,mBAAmB,QAAW;AACrC,aAAO,mBAAmB;AAAA,QACxB,gBAAgB,KAAK;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,iBAAa,yBAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,qBAAO,oBAA2C;AAAA,MAClE,iBAAiB,qBAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,aAAS,yBAAU,KAAK;AAAA,MACxB,aAAa,qBAAO,oBAA4B;AAAA,MAChD,cAAc,qBAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,UAAM,yBAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAE;AAAA,EAEtB,MAAM,aAAa;AAAA,EAAE;AAAA,EAErB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.d.cts b/plugins/google/dist/beta/realtime/realtime_api.d.cts new file mode 100644 index 000000000..0b57b6a91 --- /dev/null +++ b/plugins/google/dist/beta/realtime/realtime_api.d.cts @@ -0,0 +1,273 @@ +import * as types from '@google/genai'; +import { type AudioTranscriptionConfig, type ContextWindowCompressionConfig, type HttpOptions, Modality, type RealtimeInputConfig } from '@google/genai'; +import type { APIConnectOptions } from '@livekit/agents'; +import { llm } from '@livekit/agents'; +import { AudioFrame, type VideoFrame } from '@livekit/rtc-node'; +import { type LLMTools } from '../../tools.js'; +import type { LiveAPIModels, Voice } from './api_proto.js'; +/** + * Default image encoding options for Google Realtime API + */ +export declare const DEFAULT_IMAGE_ENCODE_OPTIONS: { + format: "JPEG"; + quality: number; + resizeOptions: { + width: number; + height: number; + strategy: "scale_aspect_fit"; + }; +}; +/** + * Input transcription result + */ +export interface InputTranscription { + itemId: string; + transcript: string; +} +/** + * Internal realtime options for Google Realtime API + */ +interface RealtimeOptions { + model: LiveAPIModels | string; + apiKey?: string; + voice: Voice | string; + language?: string; + responseModalities: Modality[]; + vertexai: boolean; + project?: string; + location?: string; + candidateCount: number; + temperature?: number; + maxOutputTokens?: number; + topP?: number; + topK?: number; + presencePenalty?: number; + frequencyPenalty?: number; + instructions?: string; + inputAudioTranscription?: AudioTranscriptionConfig; + outputAudioTranscription?: AudioTranscriptionConfig; + imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS; + connOptions: APIConnectOptions; + httpOptions?: HttpOptions; + enableAffectiveDialog?: boolean; + proactivity?: boolean; + realtimeInputConfig?: RealtimeInputConfig; + contextWindowCompression?: ContextWindowCompressionConfig; + apiVersion?: string; + geminiTools?: LLMTools; + thinkingConfig?: types.ThinkingConfig; +} +/** + * Google Realtime Model for real-time voice conversations with Gemini models + */ +export declare class RealtimeModel extends llm.RealtimeModel { + /** @internal */ + _options: RealtimeOptions; + constructor(options?: { + /** + * Initial system instructions for the model + */ + instructions?: string; + /** + * The name of the model to use + */ + model?: LiveAPIModels | string; + /** + * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable + */ + apiKey?: string; + /** + * Voice setting for audio outputs + */ + voice?: Voice | string; + /** + * The language (BCP-47 Code) to use for the API + * See https://ai.google.dev/gemini-api/docs/live#supported-languages + */ + language?: string; + /** + * Modalities to use, such as [Modality.TEXT, Modality.AUDIO] + */ + modalities?: Modality[]; + /** + * Whether to use VertexAI for the API + */ + vertexai?: boolean; + /** + * The project ID to use for the API (for VertexAI) + */ + project?: string; + /** + * The location to use for the API (for VertexAI) + */ + location?: string; + /** + * The number of candidate responses to generate + */ + candidateCount?: number; + /** + * Sampling temperature for response generation + */ + temperature?: number; + /** + * Maximum number of tokens in the response + */ + maxOutputTokens?: number; + /** + * The top-p value for response generation + */ + topP?: number; + /** + * The top-k value for response generation + */ + topK?: number; + /** + * The presence penalty for response generation + */ + presencePenalty?: number; + /** + * The frequency penalty for response generation + */ + frequencyPenalty?: number; + /** + * The configuration for input audio transcription + */ + inputAudioTranscription?: AudioTranscriptionConfig | null; + /** + * The configuration for output audio transcription + */ + outputAudioTranscription?: AudioTranscriptionConfig | null; + /** + * The configuration for image encoding + */ + imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS; + /** + * Whether to enable affective dialog + */ + enableAffectiveDialog?: boolean; + /** + * Whether to enable proactive audio + */ + proactivity?: boolean; + /** + * The configuration for realtime input + */ + realtimeInputConfig?: RealtimeInputConfig; + /** + * The configuration for context window compression + */ + contextWindowCompression?: ContextWindowCompressionConfig; + /** + * API version to use + */ + apiVersion?: string; + /** + * The configuration for the API connection + */ + connOptions?: APIConnectOptions; + /** + * HTTP options for API requests + */ + httpOptions?: HttpOptions; + /** + * Gemini-specific tools to use for the session + */ + geminiTools?: LLMTools; + /** + * The thinking configuration for response generation + */ + thinkingConfig?: types.ThinkingConfig; + }); + /** + * Create a new realtime session + */ + session(): RealtimeSession; + /** + * Update model options + */ + updateOptions(options: { + voice?: Voice | string; + temperature?: number; + }): void; + /** + * Close the model and cleanup resources + */ + close(): Promise; +} +/** + * Google Realtime Session for real-time voice conversations + * + * This session provides real-time streaming capabilities with Google's Gemini models, + * supporting both text and audio modalities with function calling capabilities. + */ +export declare class RealtimeSession extends llm.RealtimeSession { + #private; + private _tools; + private _chatCtx; + private options; + private geminiDeclarations; + private messageChannel; + private inputResampler?; + private inputResamplerInputRate?; + private instructions?; + private currentGeneration?; + private bstream; + private activeSession?; + private sessionShouldClose; + private responseCreatedFutures; + private pendingGenerationFut?; + private sessionResumptionHandle?; + private inUserActivity; + private sessionLock; + private numRetries; + private hasReceivedAudioInput; + constructor(realtimeModel: RealtimeModel); + private closeActiveSession; + private markRestartNeeded; + private getToolResultsForRealtime; + updateOptions(options: { + voice?: Voice | string; + temperature?: number; + toolChoice?: llm.ToolChoice; + }): void; + updateInstructions(instructions: string): Promise; + updateChatCtx(chatCtx: llm.ChatContext): Promise; + updateTools(tools: llm.ToolContext): Promise; + get chatCtx(): llm.ChatContext; + get tools(): llm.ToolContext; + get manualActivityDetection(): boolean; + pushAudio(frame: AudioFrame): void; + pushVideo(_: VideoFrame): void; + private sendClientEvent; + generateReply(instructions?: string): Promise; + startUserActivity(): void; + interrupt(): Promise; + truncate(_options: { + messageId: string; + audioEndMs: number; + audioTranscript?: string; + }): Promise; + close(): Promise; + private sendTask; + private onReceiveMessage; + private truncateString; + private loggableClientEvent; + private loggableServerMessage; + private markCurrentGenerationDone; + private emitError; + private buildConnectConfig; + private startNewGeneration; + private handleInputSpeechStarted; + private handleInputSpeechStopped; + private handleServerContent; + private handleToolCall; + private handleToolCallCancellation; + private handleUsageMetadata; + private tokenDetailsMap; + private handleGoAway; + commitAudio(): Promise; + clearAudio(): Promise; + private resampleAudio; +} +export {}; +//# sourceMappingURL=realtime_api.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.d.ts b/plugins/google/dist/beta/realtime/realtime_api.d.ts new file mode 100644 index 000000000..0b57b6a91 --- /dev/null +++ b/plugins/google/dist/beta/realtime/realtime_api.d.ts @@ -0,0 +1,273 @@ +import * as types from '@google/genai'; +import { type AudioTranscriptionConfig, type ContextWindowCompressionConfig, type HttpOptions, Modality, type RealtimeInputConfig } from '@google/genai'; +import type { APIConnectOptions } from '@livekit/agents'; +import { llm } from '@livekit/agents'; +import { AudioFrame, type VideoFrame } from '@livekit/rtc-node'; +import { type LLMTools } from '../../tools.js'; +import type { LiveAPIModels, Voice } from './api_proto.js'; +/** + * Default image encoding options for Google Realtime API + */ +export declare const DEFAULT_IMAGE_ENCODE_OPTIONS: { + format: "JPEG"; + quality: number; + resizeOptions: { + width: number; + height: number; + strategy: "scale_aspect_fit"; + }; +}; +/** + * Input transcription result + */ +export interface InputTranscription { + itemId: string; + transcript: string; +} +/** + * Internal realtime options for Google Realtime API + */ +interface RealtimeOptions { + model: LiveAPIModels | string; + apiKey?: string; + voice: Voice | string; + language?: string; + responseModalities: Modality[]; + vertexai: boolean; + project?: string; + location?: string; + candidateCount: number; + temperature?: number; + maxOutputTokens?: number; + topP?: number; + topK?: number; + presencePenalty?: number; + frequencyPenalty?: number; + instructions?: string; + inputAudioTranscription?: AudioTranscriptionConfig; + outputAudioTranscription?: AudioTranscriptionConfig; + imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS; + connOptions: APIConnectOptions; + httpOptions?: HttpOptions; + enableAffectiveDialog?: boolean; + proactivity?: boolean; + realtimeInputConfig?: RealtimeInputConfig; + contextWindowCompression?: ContextWindowCompressionConfig; + apiVersion?: string; + geminiTools?: LLMTools; + thinkingConfig?: types.ThinkingConfig; +} +/** + * Google Realtime Model for real-time voice conversations with Gemini models + */ +export declare class RealtimeModel extends llm.RealtimeModel { + /** @internal */ + _options: RealtimeOptions; + constructor(options?: { + /** + * Initial system instructions for the model + */ + instructions?: string; + /** + * The name of the model to use + */ + model?: LiveAPIModels | string; + /** + * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable + */ + apiKey?: string; + /** + * Voice setting for audio outputs + */ + voice?: Voice | string; + /** + * The language (BCP-47 Code) to use for the API + * See https://ai.google.dev/gemini-api/docs/live#supported-languages + */ + language?: string; + /** + * Modalities to use, such as [Modality.TEXT, Modality.AUDIO] + */ + modalities?: Modality[]; + /** + * Whether to use VertexAI for the API + */ + vertexai?: boolean; + /** + * The project ID to use for the API (for VertexAI) + */ + project?: string; + /** + * The location to use for the API (for VertexAI) + */ + location?: string; + /** + * The number of candidate responses to generate + */ + candidateCount?: number; + /** + * Sampling temperature for response generation + */ + temperature?: number; + /** + * Maximum number of tokens in the response + */ + maxOutputTokens?: number; + /** + * The top-p value for response generation + */ + topP?: number; + /** + * The top-k value for response generation + */ + topK?: number; + /** + * The presence penalty for response generation + */ + presencePenalty?: number; + /** + * The frequency penalty for response generation + */ + frequencyPenalty?: number; + /** + * The configuration for input audio transcription + */ + inputAudioTranscription?: AudioTranscriptionConfig | null; + /** + * The configuration for output audio transcription + */ + outputAudioTranscription?: AudioTranscriptionConfig | null; + /** + * The configuration for image encoding + */ + imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS; + /** + * Whether to enable affective dialog + */ + enableAffectiveDialog?: boolean; + /** + * Whether to enable proactive audio + */ + proactivity?: boolean; + /** + * The configuration for realtime input + */ + realtimeInputConfig?: RealtimeInputConfig; + /** + * The configuration for context window compression + */ + contextWindowCompression?: ContextWindowCompressionConfig; + /** + * API version to use + */ + apiVersion?: string; + /** + * The configuration for the API connection + */ + connOptions?: APIConnectOptions; + /** + * HTTP options for API requests + */ + httpOptions?: HttpOptions; + /** + * Gemini-specific tools to use for the session + */ + geminiTools?: LLMTools; + /** + * The thinking configuration for response generation + */ + thinkingConfig?: types.ThinkingConfig; + }); + /** + * Create a new realtime session + */ + session(): RealtimeSession; + /** + * Update model options + */ + updateOptions(options: { + voice?: Voice | string; + temperature?: number; + }): void; + /** + * Close the model and cleanup resources + */ + close(): Promise; +} +/** + * Google Realtime Session for real-time voice conversations + * + * This session provides real-time streaming capabilities with Google's Gemini models, + * supporting both text and audio modalities with function calling capabilities. + */ +export declare class RealtimeSession extends llm.RealtimeSession { + #private; + private _tools; + private _chatCtx; + private options; + private geminiDeclarations; + private messageChannel; + private inputResampler?; + private inputResamplerInputRate?; + private instructions?; + private currentGeneration?; + private bstream; + private activeSession?; + private sessionShouldClose; + private responseCreatedFutures; + private pendingGenerationFut?; + private sessionResumptionHandle?; + private inUserActivity; + private sessionLock; + private numRetries; + private hasReceivedAudioInput; + constructor(realtimeModel: RealtimeModel); + private closeActiveSession; + private markRestartNeeded; + private getToolResultsForRealtime; + updateOptions(options: { + voice?: Voice | string; + temperature?: number; + toolChoice?: llm.ToolChoice; + }): void; + updateInstructions(instructions: string): Promise; + updateChatCtx(chatCtx: llm.ChatContext): Promise; + updateTools(tools: llm.ToolContext): Promise; + get chatCtx(): llm.ChatContext; + get tools(): llm.ToolContext; + get manualActivityDetection(): boolean; + pushAudio(frame: AudioFrame): void; + pushVideo(_: VideoFrame): void; + private sendClientEvent; + generateReply(instructions?: string): Promise; + startUserActivity(): void; + interrupt(): Promise; + truncate(_options: { + messageId: string; + audioEndMs: number; + audioTranscript?: string; + }): Promise; + close(): Promise; + private sendTask; + private onReceiveMessage; + private truncateString; + private loggableClientEvent; + private loggableServerMessage; + private markCurrentGenerationDone; + private emitError; + private buildConnectConfig; + private startNewGeneration; + private handleInputSpeechStarted; + private handleInputSpeechStopped; + private handleServerContent; + private handleToolCall; + private handleToolCallCancellation; + private handleUsageMetadata; + private tokenDetailsMap; + private handleGoAway; + commitAudio(): Promise; + clearAudio(): Promise; + private resampleAudio; +} +export {}; +//# sourceMappingURL=realtime_api.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.d.ts.map b/plugins/google/dist/beta/realtime/realtime_api.d.ts.map new file mode 100644 index 000000000..aad58218a --- /dev/null +++ b/plugins/google/dist/beta/realtime/realtime_api.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"realtime_api.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/realtime_api.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,eAAe,CAAC;AACvC,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EAEnC,KAAK,WAAW,EAChB,QAAQ,EACR,KAAK,mBAAmB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAUL,GAAG,EAIJ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,UAAU,EAAkB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAU3D;;GAEG;AACH,eAAO,MAAM,4BAA4B;;;;;;;;CAQxC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,UAAU,eAAe;IACvB,KAAK,EAAE,aAAa,GAAG,MAAM,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,QAAQ,EAAE,CAAC;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,wBAAwB,CAAC;IACnD,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;IACpD,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;IACzD,WAAW,EAAE,iBAAiB,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;CACvC;AA2BD;;GAEG;AACH,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,gBAAgB;IAChB,QAAQ,EAAE,eAAe,CAAC;gBAGxB,OAAO,GAAE;QACP;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;QAE/B;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAEvB;;;WAGG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;QAExB;;WAEG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAC;QAEnB;;WAEG;QACH,OAAO,CAAC,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,uBAAuB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE1D;;WAEG;QACH,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE3D;;WAEG;QACH,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;QAEzD;;WAEG;QACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,CAAC;QAEtB;;WAEG;QACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;QAE1C;;WAEG;QACH,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;QAE1D;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,WAAW,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,QAAQ,CAAC;QAEvB;;WAEG;QACH,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;KAClC;IA6DR;;OAEG;IACH,OAAO;IAIP;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAW9E;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,GAAG,CAAC,eAAe;;IACtD,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAA2B;IAE3C,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,cAAc,CAAuC;IAC7D,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAC,CAAqB;IAC/C,OAAO,CAAC,OAAO,CAAkB;IAGjC,OAAO,CAAC,aAAa,CAAC,CAAU;IAChC,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,sBAAsB,CAA4D;IAC1F,OAAO,CAAC,oBAAoB,CAAC,CAAqC;IAElE,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,qBAAqB,CAAS;gBAO1B,aAAa,EAAE,aAAa;YAwC1B,kBAAkB;IAgBhC,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,yBAAyB;IAyBjC,aAAa,CAAC,OAAO,EAAE;QACrB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;KAC7B;IAkBK,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOvD,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDtD,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAYxD,IAAI,OAAO,IAAI,GAAG,CAAC,WAAW,CAE7B;IAED,IAAI,KAAK,IAAI,GAAG,CAAC,WAAW,CAE3B;IAED,IAAI,uBAAuB,IAAI,OAAO,CAErC;IAED,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAsBlC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI;IAI9B,OAAO,CAAC,eAAe;IAIjB,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAyD/E,iBAAiB,IAAI,IAAI;IAgBnB,SAAS;IAQT,QAAQ,CAAC,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE;IAItF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAwId,QAAQ;YAiER,gBAAgB;IAuE9B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,mBAAmB;IAmB3B,OAAO,CAAC,qBAAqB;IA4B7B,OAAO,CAAC,yBAAyB;IA+CjC,OAAO,CAAC,SAAS;IAUjB,OAAO,CAAC,kBAAkB;IAsE1B,OAAO,CAAC,kBAAkB;IAwD1B,OAAO,CAAC,wBAAwB;IAIhC,OAAO,CAAC,wBAAwB;IAMhC,OAAO,CAAC,mBAAmB;IAoF3B,OAAO,CAAC,cAAc;IAmBtB,OAAO,CAAC,0BAA0B;IASlC,OAAO,CAAC,mBAAmB;IA6C3B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,YAAY;IAMd,WAAW;IAEX,UAAU;IAEhB,OAAO,CAAE,aAAa;CA8BvB"} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.js b/plugins/google/dist/beta/realtime/realtime_api.js new file mode 100644 index 000000000..d66061998 --- /dev/null +++ b/plugins/google/dist/beta/realtime/realtime_api.js @@ -0,0 +1,983 @@ +import * as types from "@google/genai"; +import { + ActivityHandling, + GoogleGenAI, + Modality +} from "@google/genai"; +import { + APIConnectionError, + AudioByteStream, + DEFAULT_API_CONNECT_OPTIONS, + Event, + Future, + Queue, + Task, + cancelAndWait, + delay, + llm, + log, + shortuuid, + stream +} from "@livekit/agents"; +import { Mutex } from "@livekit/mutex"; +import { AudioFrame, AudioResampler } from "@livekit/rtc-node"; +import {} from "../../tools.js"; +import { toFunctionDeclarations } from "../../utils.js"; +const INPUT_AUDIO_SAMPLE_RATE = 16e3; +const INPUT_AUDIO_CHANNELS = 1; +const OUTPUT_AUDIO_SAMPLE_RATE = 24e3; +const OUTPUT_AUDIO_CHANNELS = 1; +const DEFAULT_IMAGE_ENCODE_OPTIONS = { + format: "JPEG", + quality: 75, + resizeOptions: { + width: 1024, + height: 1024, + strategy: "scale_aspect_fit" + } +}; +function setsEqual(a, b) { + return a.size === b.size && [...a].every((x) => b.has(x)); +} +class RealtimeModel extends llm.RealtimeModel { + /** @internal */ + _options; + constructor(options = {}) { + var _a, _b, _c; + const inputAudioTranscription = options.inputAudioTranscription === void 0 ? {} : options.inputAudioTranscription; + const outputAudioTranscription = options.outputAudioTranscription === void 0 ? {} : options.outputAudioTranscription; + let serverTurnDetection = true; + if ((_b = (_a = options.realtimeInputConfig) == null ? void 0 : _a.automaticActivityDetection) == null ? void 0 : _b.disabled) { + serverTurnDetection = false; + } + super({ + messageTruncation: false, + turnDetection: serverTurnDetection, + userTranscription: inputAudioTranscription !== null, + autoToolReplyGeneration: true, + audioOutput: ((_c = options.modalities) == null ? void 0 : _c.includes(Modality.AUDIO)) ?? true + }); + const apiKey = options.apiKey || process.env.GOOGLE_API_KEY; + const project = options.project || process.env.GOOGLE_CLOUD_PROJECT; + const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || "us-central1"; + const vertexai = options.vertexai ?? false; + const defaultModel = vertexai ? "gemini-2.0-flash-exp" : "gemini-2.0-flash-live-001"; + this._options = { + model: options.model || defaultModel, + apiKey, + voice: options.voice || "Puck", + language: options.language, + responseModalities: options.modalities || [Modality.AUDIO], + vertexai, + project, + location, + candidateCount: options.candidateCount || 1, + temperature: options.temperature, + maxOutputTokens: options.maxOutputTokens, + topP: options.topP, + topK: options.topK, + presencePenalty: options.presencePenalty, + frequencyPenalty: options.frequencyPenalty, + instructions: options.instructions, + inputAudioTranscription: inputAudioTranscription || void 0, + outputAudioTranscription: outputAudioTranscription || void 0, + imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS, + connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS, + httpOptions: options.httpOptions, + enableAffectiveDialog: options.enableAffectiveDialog, + proactivity: options.proactivity, + realtimeInputConfig: options.realtimeInputConfig, + contextWindowCompression: options.contextWindowCompression, + apiVersion: options.apiVersion, + geminiTools: options.geminiTools, + thinkingConfig: options.thinkingConfig + }; + } + /** + * Create a new realtime session + */ + session() { + return new RealtimeSession(this); + } + /** + * Update model options + */ + updateOptions(options) { + if (options.voice !== void 0) { + this._options.voice = options.voice; + } + if (options.temperature !== void 0) { + this._options.temperature = options.temperature; + } + } + /** + * Close the model and cleanup resources + */ + async close() { + } +} +class RealtimeSession extends llm.RealtimeSession { + _tools = {}; + _chatCtx = llm.ChatContext.empty(); + options; + geminiDeclarations = []; + messageChannel = new Queue(); + inputResampler; + inputResamplerInputRate; + instructions; + currentGeneration; + bstream; + // Google-specific properties + activeSession; + sessionShouldClose = new Event(); + responseCreatedFutures = {}; + pendingGenerationFut; + sessionResumptionHandle; + inUserActivity = false; + sessionLock = new Mutex(); + numRetries = 0; + hasReceivedAudioInput = false; + #client; + #task; + #logger = log(); + #closed = false; + constructor(realtimeModel) { + super(realtimeModel); + this.options = realtimeModel._options; + this.bstream = new AudioByteStream( + INPUT_AUDIO_SAMPLE_RATE, + INPUT_AUDIO_CHANNELS, + INPUT_AUDIO_SAMPLE_RATE / 20 + ); + const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } = this.options; + const apiVersion = !this.options.apiVersion && (enableAffectiveDialog || proactivity) ? "v1alpha" : this.options.apiVersion; + const httpOptions = { + ...this.options.httpOptions, + apiVersion, + timeout: this.options.connOptions.timeoutMs + }; + const clientOptions = vertexai ? { + vertexai: true, + project, + location, + httpOptions + } : { + apiKey, + httpOptions + }; + this.#client = new GoogleGenAI(clientOptions); + this.#task = this.#mainTask(); + } + async closeActiveSession() { + const unlock = await this.sessionLock.lock(); + if (this.activeSession) { + try { + await this.activeSession.close(); + } catch (error) { + this.#logger.warn({ error }, "Error closing Gemini session"); + } finally { + this.activeSession = void 0; + } + } + unlock(); + } + markRestartNeeded() { + if (!this.sessionShouldClose.isSet) { + this.sessionShouldClose.set(); + this.messageChannel = new Queue(); + } + } + getToolResultsForRealtime(ctx, vertexai) { + const toolResponses = []; + for (const item of ctx.items) { + if (item.type === "function_call_output") { + const response = { + id: item.callId, + name: item.name, + response: { output: item.output } + }; + if (!vertexai) { + response.id = item.callId; + } + toolResponses.push(response); + } + } + return toolResponses.length > 0 ? { functionResponses: toolResponses } : void 0; + } + updateOptions(options) { + let shouldRestart = false; + if (options.voice !== void 0 && this.options.voice !== options.voice) { + this.options.voice = options.voice; + shouldRestart = true; + } + if (options.temperature !== void 0 && this.options.temperature !== options.temperature) { + this.options.temperature = options.temperature; + shouldRestart = true; + } + if (shouldRestart) { + this.markRestartNeeded(); + } + } + async updateInstructions(instructions) { + if (this.options.instructions === void 0 || this.options.instructions !== instructions) { + this.options.instructions = instructions; + this.markRestartNeeded(); + } + } + async updateChatCtx(chatCtx) { + const unlock = await this.sessionLock.lock(); + try { + if (!this.activeSession) { + this._chatCtx = chatCtx.copy(); + return; + } + } finally { + unlock(); + } + const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx); + if (diffOps.toRemove.length > 0) { + this.#logger.warn("Gemini Live does not support removing messages"); + } + const appendCtx = llm.ChatContext.empty(); + for (const [, itemId] of diffOps.toCreate) { + const item = chatCtx.getById(itemId); + if (item) { + appendCtx.items.push(item); + } + } + if (appendCtx.items.length > 0) { + const [turns] = await appendCtx.copy({ + excludeFunctionCall: true + }).toProviderFormat("google", false); + const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai); + if (turns.length > 0) { + this.sendClientEvent({ + type: "content", + value: { + turns, + turnComplete: false + } + }); + } + if (toolResults) { + this.sendClientEvent({ + type: "tool_response", + value: toolResults + }); + } + } + this._chatCtx = chatCtx.copy(); + } + async updateTools(tools) { + const newDeclarations = toFunctionDeclarations(tools); + const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name)); + const newToolNames = new Set(newDeclarations.map((f) => f.name)); + if (!setsEqual(currentToolNames, newToolNames)) { + this.geminiDeclarations = newDeclarations; + this._tools = tools; + this.markRestartNeeded(); + } + } + get chatCtx() { + return this._chatCtx.copy(); + } + get tools() { + return { ...this._tools }; + } + get manualActivityDetection() { + var _a, _b; + return ((_b = (_a = this.options.realtimeInputConfig) == null ? void 0 : _a.automaticActivityDetection) == null ? void 0 : _b.disabled) ?? false; + } + pushAudio(frame) { + this.hasReceivedAudioInput = true; + for (const f of this.resampleAudio(frame)) { + for (const nf of this.bstream.write(f.data.buffer)) { + const realtimeInput = { + mediaChunks: [ + { + mimeType: "audio/pcm", + data: Buffer.from(nf.data.buffer).toString("base64") + } + ] + }; + this.sendClientEvent({ + type: "realtime_input", + value: realtimeInput + }); + } + } + } + pushVideo(_) { + } + sendClientEvent(event) { + this.messageChannel.put(event); + } + async generateReply(instructions) { + if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { + this.#logger.warn( + "generateReply called while another generation is pending, cancelling previous." + ); + this.pendingGenerationFut.reject(new Error("Superseded by new generate_reply call")); + } + const fut = new Future(); + this.pendingGenerationFut = fut; + if (this.inUserActivity) { + this.sendClientEvent({ + type: "realtime_input", + value: { + activityEnd: {} + } + }); + this.inUserActivity = false; + } + const turns = []; + if (instructions !== void 0) { + turns.push({ + parts: [{ text: instructions }], + role: "model" + }); + } + turns.push({ + parts: [{ text: "." }], + role: "user" + }); + this.sendClientEvent({ + type: "content", + value: { + turns, + turnComplete: true + } + }); + const timeoutHandle = setTimeout(() => { + if (!fut.done) { + fut.reject(new Error("generateReply timed out waiting for generation_created event.")); + if (this.pendingGenerationFut === fut) { + this.pendingGenerationFut = void 0; + } + } + }, 5e3); + fut.await.finally(() => clearTimeout(timeoutHandle)); + return fut.await; + } + startUserActivity() { + if (!this.manualActivityDetection) { + return; + } + if (!this.inUserActivity) { + this.inUserActivity = true; + this.sendClientEvent({ + type: "realtime_input", + value: { + activityStart: {} + } + }); + } + } + async interrupt() { + var _a; + if (((_a = this.options.realtimeInputConfig) == null ? void 0 : _a.activityHandling) === ActivityHandling.NO_INTERRUPTION) { + return; + } + this.startUserActivity(); + } + async truncate(_options) { + this.#logger.warn("truncate is not supported by the Google Realtime API."); + } + async close() { + super.close(); + this.#closed = true; + this.sessionShouldClose.set(); + await this.closeActiveSession(); + if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { + this.pendingGenerationFut.reject(new Error("Session closed")); + } + for (const fut of Object.values(this.responseCreatedFutures)) { + if (!fut.done) { + fut.reject(new Error("Session closed before response created")); + } + } + this.responseCreatedFutures = {}; + if (this.currentGeneration) { + this.markCurrentGenerationDone(); + } + } + async #mainTask() { + const maxRetries = this.options.connOptions.maxRetry; + while (!this.#closed) { + await this.closeActiveSession(); + this.sessionShouldClose.clear(); + const config = this.buildConnectConfig(); + try { + this.#logger.debug("Connecting to Gemini Realtime API..."); + const sessionOpened = new Event(); + const session = await this.#client.live.connect({ + model: this.options.model, + callbacks: { + onopen: () => sessionOpened.set(), + onmessage: (message) => { + this.onReceiveMessage(session, message); + }, + onerror: (error) => { + this.#logger.error("Gemini Live session error:", error); + if (!this.sessionShouldClose.isSet) { + this.markRestartNeeded(); + } + }, + onclose: (event) => { + this.#logger.debug("Gemini Live session closed:", event.code, event.reason); + this.markCurrentGenerationDone(); + } + }, + config + }); + await sessionOpened.wait(); + const unlock = await this.sessionLock.lock(); + try { + this.activeSession = session; + const [turns] = await this._chatCtx.copy({ + excludeFunctionCall: true + }).toProviderFormat("google", false); + if (turns.length > 0) { + await session.sendClientContent({ + turns, + turnComplete: false + }); + } + } finally { + unlock(); + } + const sendTask = Task.from((controller) => this.sendTask(session, controller)); + const restartWaitTask = Task.from(({ signal }) => { + const abortEvent = new Event(); + signal.addEventListener("abort", () => abortEvent.set()); + return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]); + }); + await Promise.race([sendTask.result, restartWaitTask.result]); + if (!restartWaitTask.done && this.#closed) { + break; + } + await cancelAndWait([sendTask, restartWaitTask], 2e3); + } catch (error) { + this.#logger.error(`Gemini Realtime API error: ${error}`); + if (this.#closed) break; + if (maxRetries === 0) { + this.emitError(error, false); + throw new APIConnectionError({ + message: "Failed to connect to Gemini Live" + }); + } + if (this.numRetries >= maxRetries) { + this.emitError(error, false); + throw new APIConnectionError({ + message: `Failed to connect to Gemini Live after ${maxRetries} attempts` + }); + } + const retryInterval = this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs; + this.#logger.warn( + { + attempt: this.numRetries, + maxRetries + }, + `Gemini Realtime API connection failed, retrying in ${retryInterval}ms` + ); + await delay(retryInterval); + this.numRetries++; + } finally { + await this.closeActiveSession(); + } + } + } + async sendTask(session, controller) { + try { + while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) { + const msg = await this.messageChannel.get(); + if (controller.signal.aborted) break; + const unlock = await this.sessionLock.lock(); + try { + if (this.sessionShouldClose.isSet || this.activeSession !== session) { + break; + } + } finally { + unlock(); + } + switch (msg.type) { + case "content": + const { turns, turnComplete } = msg.value; + this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`); + await session.sendClientContent({ + turns, + turnComplete: turnComplete ?? true + }); + break; + case "tool_response": + const { functionResponses } = msg.value; + if (functionResponses) { + this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`); + await session.sendToolResponse({ + functionResponses + }); + } + break; + case "realtime_input": + const { mediaChunks, activityStart, activityEnd } = msg.value; + if (mediaChunks) { + for (const mediaChunk of mediaChunks) { + await session.sendRealtimeInput({ media: mediaChunk }); + } + } + if (activityStart) await session.sendRealtimeInput({ activityStart }); + if (activityEnd) await session.sendRealtimeInput({ activityEnd }); + break; + default: + this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`); + break; + } + } + } catch (e) { + if (!this.sessionShouldClose.isSet) { + this.#logger.error(`Error in send task: ${e}`); + this.markRestartNeeded(); + } + } finally { + this.#logger.debug( + { + closed: this.#closed, + sessionShouldClose: this.sessionShouldClose.isSet, + aborted: controller.signal.aborted + }, + "send task finished." + ); + } + } + async onReceiveMessage(session, response) { + var _a, _b, _c; + const hasAudioData = (_c = (_b = (_a = response.serverContent) == null ? void 0 : _a.modelTurn) == null ? void 0 : _b.parts) == null ? void 0 : _c.some( + (part) => { + var _a2; + return (_a2 = part.inlineData) == null ? void 0 : _a2.data; + } + ); + if (!hasAudioData) { + this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`); + } + const unlock = await this.sessionLock.lock(); + try { + if (this.sessionShouldClose.isSet || this.activeSession !== session) { + this.#logger.debug("onReceiveMessage: Session changed or closed, stopping receive."); + return; + } + } finally { + unlock(); + } + if ((!this.currentGeneration || this.currentGeneration._done) && (response.serverContent || response.toolCall)) { + this.startNewGeneration(); + } + if (response.sessionResumptionUpdate) { + if (response.sessionResumptionUpdate.resumable && response.sessionResumptionUpdate.newHandle) { + this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle; + } + } + try { + if (response.serverContent) { + this.handleServerContent(response.serverContent); + } + if (response.toolCall) { + this.handleToolCall(response.toolCall); + } + if (response.toolCallCancellation) { + this.handleToolCallCancellation(response.toolCallCancellation); + } + if (response.usageMetadata) { + this.handleUsageMetadata(response.usageMetadata); + } + if (response.goAway) { + this.handleGoAway(response.goAway); + } + if (this.numRetries > 0) { + this.numRetries = 0; + } + } catch (e) { + if (!this.sessionShouldClose.isSet) { + this.#logger.error(`Error in onReceiveMessage: ${e}`); + this.markRestartNeeded(); + } + } + } + /// Truncate large base64/audio payloads for logging to avoid flooding logs + truncateString(data, maxLength = 30) { + return data.length > maxLength ? `${data.slice(0, maxLength)}\u2026` : data; + } + loggableClientEvent(event, maxLength = 30) { + var _a; + const obj = { ...event }; + if (obj.type === "realtime_input" && ((_a = obj.value) == null ? void 0 : _a.mediaChunks)) { + obj.value = { + ...obj.value, + mediaChunks: obj.value.mediaChunks.map( + (mc) => ({ + ...mc, + data: typeof mc.data === "string" ? this.truncateString(mc.data, maxLength) : mc.data + }) + ) + }; + } + return obj; + } + loggableServerMessage(message, maxLength = 30) { + const obj = { ...message }; + if (obj.serverContent && obj.serverContent.modelTurn && Array.isArray(obj.serverContent.modelTurn.parts)) { + obj.serverContent = { ...obj.serverContent }; + obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn }; + obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part) => { + var _a; + if (((_a = part == null ? void 0 : part.inlineData) == null ? void 0 : _a.data) && typeof part.inlineData.data === "string") { + return { + ...part, + inlineData: { + ...part.inlineData, + data: this.truncateString(part.inlineData.data, maxLength) + } + }; + } + return part; + }); + } + return obj; + } + markCurrentGenerationDone() { + if (!this.currentGeneration || this.currentGeneration._done) { + return; + } + this.handleInputSpeechStopped(); + const gen = this.currentGeneration; + if (gen.inputTranscription) { + this.emit("input_audio_transcription_completed", { + itemId: gen.inputId, + transcript: gen.inputTranscription, + isFinal: true + }); + this._chatCtx.addMessage({ + role: "user", + content: gen.inputTranscription, + id: gen.inputId + }); + } + if (gen.outputText) { + this._chatCtx.addMessage({ + role: "assistant", + content: gen.outputText, + id: gen.responseId + }); + } + if (this.options.outputAudioTranscription === void 0) { + gen.textChannel.write(""); + } + gen.textChannel.close(); + gen.audioChannel.close(); + gen.functionChannel.close(); + gen.messageChannel.close(); + gen._done = true; + } + emitError(error, recoverable) { + this.emit("error", { + timestamp: Date.now(), + // TODO(brian): add label to realtime model + label: "google_realtime", + error, + recoverable + }); + } + buildConnectConfig() { + const opts = this.options; + const config = { + responseModalities: opts.responseModalities, + systemInstruction: opts.instructions ? { + parts: [{ text: opts.instructions }] + } : void 0, + speechConfig: { + voiceConfig: { + prebuiltVoiceConfig: { + voiceName: opts.voice + } + }, + languageCode: opts.language + }, + tools: [ + { + functionDeclarations: this.geminiDeclarations, + ...this.options.geminiTools + } + ], + inputAudioTranscription: opts.inputAudioTranscription, + outputAudioTranscription: opts.outputAudioTranscription, + sessionResumption: { + handle: this.sessionResumptionHandle + } + }; + if (opts.temperature !== void 0) { + config.temperature = opts.temperature; + } + if (opts.maxOutputTokens !== void 0) { + config.maxOutputTokens = opts.maxOutputTokens; + } + if (opts.topP !== void 0) { + config.topP = opts.topP; + } + if (opts.topK !== void 0) { + config.topK = opts.topK; + } + if (opts.proactivity !== void 0) { + config.proactivity = { proactiveAudio: opts.proactivity }; + } + if (opts.enableAffectiveDialog !== void 0) { + config.enableAffectiveDialog = opts.enableAffectiveDialog; + } + if (opts.realtimeInputConfig !== void 0) { + config.realtimeInputConfig = opts.realtimeInputConfig; + } + if (opts.contextWindowCompression !== void 0) { + config.contextWindowCompression = opts.contextWindowCompression; + } + if (opts.thinkingConfig !== void 0) { + config.generationConfig = { + thinkingConfig: opts.thinkingConfig + }; + } + return config; + } + startNewGeneration() { + if (this.currentGeneration && !this.currentGeneration._done) { + this.#logger.warn("Starting new generation while another is active. Finalizing previous."); + this.markCurrentGenerationDone(); + } + const responseId = shortuuid("GR_"); + this.currentGeneration = { + messageChannel: stream.createStreamChannel(), + functionChannel: stream.createStreamChannel(), + responseId, + inputId: shortuuid("GI_"), + textChannel: stream.createStreamChannel(), + audioChannel: stream.createStreamChannel(), + inputTranscription: "", + outputText: "", + _createdTimestamp: Date.now(), + _done: false + }; + if (!this._realtimeModel.capabilities.audioOutput) { + this.currentGeneration.audioChannel.close(); + } + const modalities = this._realtimeModel.capabilities.audioOutput ? ["audio", "text"] : ["text"]; + this.currentGeneration.messageChannel.write({ + messageId: responseId, + textStream: this.currentGeneration.textChannel.stream(), + audioStream: this.currentGeneration.audioChannel.stream(), + modalities: Promise.resolve(modalities) + }); + const generationEvent = { + messageStream: this.currentGeneration.messageChannel.stream(), + functionStream: this.currentGeneration.functionChannel.stream(), + userInitiated: false + }; + if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { + generationEvent.userInitiated = true; + this.pendingGenerationFut.resolve(generationEvent); + this.pendingGenerationFut = void 0; + } else { + this.handleInputSpeechStarted(); + } + this.emit("generation_created", generationEvent); + } + handleInputSpeechStarted() { + this.emit("input_speech_started", {}); + } + handleInputSpeechStopped() { + this.emit("input_speech_stopped", { + userTranscriptionEnabled: false + }); + } + handleServerContent(serverContent) { + if (!this.currentGeneration) { + this.#logger.warn("received server content but no active generation."); + return; + } + const gen = this.currentGeneration; + if (serverContent.modelTurn) { + const turn = serverContent.modelTurn; + for (const part of turn.parts || []) { + if (part.text) { + gen.outputText += part.text; + gen.textChannel.write(part.text); + } + if (part.inlineData) { + if (!gen._firstTokenTimestamp) { + gen._firstTokenTimestamp = Date.now(); + } + try { + if (!part.inlineData.data) { + throw new Error("frameData is not bytes"); + } + const binaryString = atob(part.inlineData.data); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + const int16Array = new Int16Array(bytes.buffer); + const audioFrame = new AudioFrame( + int16Array, + OUTPUT_AUDIO_SAMPLE_RATE, + OUTPUT_AUDIO_CHANNELS, + int16Array.length / OUTPUT_AUDIO_CHANNELS + ); + gen.audioChannel.write(audioFrame); + } catch (error) { + this.#logger.error("Error processing audio data:", error); + } + } + } + } + if (serverContent.inputTranscription && serverContent.inputTranscription.text) { + let text = serverContent.inputTranscription.text; + if (gen.inputTranscription === "") { + text = text.trimStart(); + } + gen.inputTranscription += text; + this.emit("input_audio_transcription_completed", { + itemId: gen.inputId, + transcript: gen.inputTranscription, + isFinal: false + }); + } + if (serverContent.outputTranscription && serverContent.outputTranscription.text) { + const text = serverContent.outputTranscription.text; + gen.outputText += text; + gen.textChannel.write(text); + } + if (serverContent.generationComplete || serverContent.turnComplete) { + gen._completedTimestamp = Date.now(); + } + if (serverContent.interrupted) { + this.handleInputSpeechStarted(); + } + if (serverContent.turnComplete) { + this.markCurrentGenerationDone(); + } + } + handleToolCall(toolCall) { + if (!this.currentGeneration) { + this.#logger.warn("received tool call but no active generation."); + return; + } + const gen = this.currentGeneration; + for (const fc of toolCall.functionCalls || []) { + gen.functionChannel.write({ + callId: fc.id || shortuuid("fnc-call-"), + name: fc.name, + args: fc.args ? JSON.stringify(fc.args) : "" + }); + } + this.markCurrentGenerationDone(); + } + handleToolCallCancellation(cancellation) { + this.#logger.warn( + { + functionCallIds: cancellation.ids + }, + "server cancelled tool calls" + ); + } + handleUsageMetadata(usage) { + if (!this.currentGeneration) { + this.#logger.debug("Received usage metadata but no active generation"); + return; + } + const gen = this.currentGeneration; + const createdTimestamp = gen._createdTimestamp; + const firstTokenTimestamp = gen._firstTokenTimestamp; + const completedTimestamp = gen._completedTimestamp || Date.now(); + const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1; + const durationMs = completedTimestamp - createdTimestamp; + const inputTokens = usage.promptTokenCount || 0; + const outputTokens = usage.responseTokenCount || 0; + const totalTokens = usage.totalTokenCount || 0; + const realtimeMetrics = { + type: "realtime_model_metrics", + timestamp: createdTimestamp, + requestId: gen.responseId, + ttftMs, + durationMs, + cancelled: gen._done && !gen._completedTimestamp, + label: "google_realtime", + inputTokens, + outputTokens, + totalTokens, + tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1e3) : 0, + inputTokenDetails: { + ...this.tokenDetailsMap(usage.promptTokensDetails), + cachedTokens: (usage.cacheTokensDetails || []).reduce( + (sum, detail) => sum + (detail.tokenCount || 0), + 0 + ), + cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails) + }, + outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails) + }; + this.emit("metrics_collected", realtimeMetrics); + } + tokenDetailsMap(tokenDetails) { + const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 }; + if (!tokenDetails) { + return tokenDetailsMap; + } + for (const tokenDetail of tokenDetails) { + if (!tokenDetail.tokenCount) { + continue; + } + if (tokenDetail.modality === types.MediaModality.AUDIO) { + tokenDetailsMap.audioTokens += tokenDetail.tokenCount; + } else if (tokenDetail.modality === types.MediaModality.TEXT) { + tokenDetailsMap.textTokens += tokenDetail.tokenCount; + } else if (tokenDetail.modality === types.MediaModality.IMAGE) { + tokenDetailsMap.imageTokens += tokenDetail.tokenCount; + } + } + return tokenDetailsMap; + } + handleGoAway(goAway) { + this.#logger.warn({ timeLeft: goAway.timeLeft }, "Gemini server indicates disconnection soon."); + this.sessionShouldClose.set(); + } + async commitAudio() { + } + async clearAudio() { + } + *resampleAudio(frame) { + if (this.inputResampler) { + if (frame.sampleRate !== this.inputResamplerInputRate) { + this.inputResampler = void 0; + this.inputResamplerInputRate = void 0; + } + } + if (this.inputResampler === void 0 && (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)) { + this.inputResampler = new AudioResampler( + frame.sampleRate, + INPUT_AUDIO_SAMPLE_RATE, + INPUT_AUDIO_CHANNELS + ); + this.inputResamplerInputRate = frame.sampleRate; + } + if (this.inputResampler) { + for (const resampledFrame of this.inputResampler.push(frame)) { + yield resampledFrame; + } + } else { + yield frame; + } + } +} +export { + DEFAULT_IMAGE_ENCODE_OPTIONS, + RealtimeModel, + RealtimeSession +}; +//# sourceMappingURL=realtime_api.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.js.map b/plugins/google/dist/beta/realtime/realtime_api.js.map new file mode 100644 index 000000000..2d84ebf4c --- /dev/null +++ b/plugins/google/dist/beta/realtime/realtime_api.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n if (this.activeSession) {\n try {\n await this.activeSession.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n } finally {\n this.activeSession = undefined;\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n this.messageChannel = new Queue();\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error('Gemini Live session error:', error);\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n this.#logger.debug('Gemini Live session closed:', event.code, event.reason);\n this.markCurrentGenerationDone();\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n if (opts.thinkingConfig !== undefined) {\n config.generationConfig = {\n thinkingConfig: opts.thinkingConfig,\n };\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() { }\n\n async clearAudio() { }\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":"AAIA,YAAY,WAAW;AACvB;AAAA,EACE;AAAA,EAGA;AAAA,EAEA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,aAAa;AACtB,SAAS,YAAY,sBAAuC;AAC5D,eAA8B;AAC9B,SAAS,8BAA8B;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,IAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,SAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,SAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,IAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,IAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,MAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,MAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,MAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,YAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI,KAAK,eAAe;AACtB,UAAI;AACF,cAAM,KAAK,cAAc,MAAM;AAAA,MACjC,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D,UAAE;AACA,aAAK,gBAAgB;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAC5B,WAAK,iBAAiB,IAAI,MAAM;AAAA,IAClC;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,IAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,IAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,kBAAkB,uBAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AAzlBzC;AA0lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,OAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AApsBpB;AAssBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,iBAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,MAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,8BAA8B,KAAK;AACtD,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,+BAA+B,MAAM,MAAM,MAAM,MAAM;AAC1E,mBAAK,0BAA0B;AAAA,YACjC;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,KAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,MAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,cAAM,cAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,cAAM,MAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA55BnB;AA85BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AA/5Bb,YAAAA;AA+5BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAv+B7B;AAw+BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAngC/F;AAogCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,IACF;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,QAAI,KAAK,mBAAmB,QAAW;AACrC,aAAO,mBAAmB;AAAA,QACxB,gBAAgB,KAAK;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,aAAa,UAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,OAAO,oBAA2C;AAAA,MAClE,iBAAiB,OAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,SAAS,UAAU,KAAK;AAAA,MACxB,aAAa,OAAO,oBAA4B;AAAA,MAChD,cAAc,OAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,MAAM,UAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAE;AAAA,EAEtB,MAAM,aAAa;AAAA,EAAE;AAAA,EAErB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file diff --git a/plugins/google/dist/index.cjs b/plugins/google/dist/index.cjs new file mode 100644 index 000000000..6f9040d48 --- /dev/null +++ b/plugins/google/dist/index.cjs @@ -0,0 +1,58 @@ +"use strict"; +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default")); +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + // If the importer is in node compatibility mode or this is not an ESM + // file that has been converted to a CommonJS file using a Babel- + // compatible transform (i.e. "__esModule" has not been set), then set + // "default" to the CommonJS "module.exports" for node compatibility. + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var index_exports = {}; +__export(index_exports, { + LLM: () => import_llm.LLM, + LLMStream: () => import_llm.LLMStream, + beta: () => beta +}); +module.exports = __toCommonJS(index_exports); +var import_agents = require("@livekit/agents"); +var beta = __toESM(require("./beta/index.cjs"), 1); +var import_llm = require("./llm.cjs"); +__reExport(index_exports, require("./models.cjs"), module.exports); +class GooglePlugin extends import_agents.Plugin { + constructor() { + super({ + title: "google", + version: "0.1.0", + package: "@livekit/agents-plugin-google" + }); + } +} +import_agents.Plugin.registerPlugin(new GooglePlugin()); +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + LLM, + LLMStream, + beta, + ...require("./models.cjs") +}); +//# sourceMappingURL=index.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/index.cjs.map b/plugins/google/dist/index.cjs.map new file mode 100644 index 000000000..7d7d35f2a --- /dev/null +++ b/plugins/google/dist/index.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Plugin } from '@livekit/agents';\n\nexport * as beta from './beta/index.js';\nexport { LLM, LLMStream, type LLMOptions } from './llm.js';\nexport * from './models.js';\n\nclass GooglePlugin extends Plugin {\n constructor() {\n super({\n title: 'google',\n version: '0.1.0',\n package: '@livekit/agents-plugin-google',\n });\n }\n}\n\nPlugin.registerPlugin(new GooglePlugin());\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAuB;AAEvB,WAAsB;AACtB,iBAAgD;AAChD,0BAAc,wBAPd;AASA,MAAM,qBAAqB,qBAAO;AAAA,EAChC,cAAc;AACZ,UAAM;AAAA,MACJ,OAAO;AAAA,MACP,SAAS;AAAA,MACT,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAEA,qBAAO,eAAe,IAAI,aAAa,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/index.d.cts b/plugins/google/dist/index.d.cts new file mode 100644 index 000000000..fc4113b2e --- /dev/null +++ b/plugins/google/dist/index.d.cts @@ -0,0 +1,4 @@ +export * as beta from './beta/index.js'; +export { LLM, LLMStream, type LLMOptions } from './llm.js'; +export * from './models.js'; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/index.d.ts b/plugins/google/dist/index.d.ts new file mode 100644 index 000000000..fc4113b2e --- /dev/null +++ b/plugins/google/dist/index.d.ts @@ -0,0 +1,4 @@ +export * as beta from './beta/index.js'; +export { LLM, LLMStream, type LLMOptions } from './llm.js'; +export * from './models.js'; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/index.d.ts.map b/plugins/google/dist/index.d.ts.map new file mode 100644 index 000000000..44ab2e75e --- /dev/null +++ b/plugins/google/dist/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,IAAI,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3D,cAAc,aAAa,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/index.js b/plugins/google/dist/index.js new file mode 100644 index 000000000..a5ab61750 --- /dev/null +++ b/plugins/google/dist/index.js @@ -0,0 +1,20 @@ +import { Plugin } from "@livekit/agents"; +import * as beta from "./beta/index.js"; +import { LLM, LLMStream } from "./llm.js"; +export * from "./models.js"; +class GooglePlugin extends Plugin { + constructor() { + super({ + title: "google", + version: "0.1.0", + package: "@livekit/agents-plugin-google" + }); + } +} +Plugin.registerPlugin(new GooglePlugin()); +export { + LLM, + LLMStream, + beta +}; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/plugins/google/dist/index.js.map b/plugins/google/dist/index.js.map new file mode 100644 index 000000000..8f2b9d3ab --- /dev/null +++ b/plugins/google/dist/index.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Plugin } from '@livekit/agents';\n\nexport * as beta from './beta/index.js';\nexport { LLM, LLMStream, type LLMOptions } from './llm.js';\nexport * from './models.js';\n\nclass GooglePlugin extends Plugin {\n constructor() {\n super({\n title: 'google',\n version: '0.1.0',\n package: '@livekit/agents-plugin-google',\n });\n }\n}\n\nPlugin.registerPlugin(new GooglePlugin());\n"],"mappings":"AAGA,SAAS,cAAc;AAEvB,YAAY,UAAU;AACtB,SAAS,KAAK,iBAAkC;AAChD,cAAc;AAEd,MAAM,qBAAqB,OAAO;AAAA,EAChC,cAAc;AACZ,UAAM;AAAA,MACJ,OAAO;AAAA,MACP,SAAS;AAAA,MACT,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAEA,OAAO,eAAe,IAAI,aAAa,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/llm.cjs b/plugins/google/dist/llm.cjs new file mode 100644 index 000000000..2856ec4c1 --- /dev/null +++ b/plugins/google/dist/llm.cjs @@ -0,0 +1,381 @@ +"use strict"; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var llm_exports = {}; +__export(llm_exports, { + LLM: () => LLM, + LLMStream: () => LLMStream +}); +module.exports = __toCommonJS(llm_exports); +var import_genai = require("@google/genai"); +var import_agents = require("@livekit/agents"); +var import_utils = require("./utils.cjs"); +class LLM extends import_agents.llm.LLM { + #opts; + #client; + label() { + return "google.LLM"; + } + get model() { + return this.#opts.model; + } + /** + * Create a new instance of Google GenAI LLM. + * + * Environment Requirements: + * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods. + * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables + * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file, + * and the location defaults to "us-central1". + * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. + * + * @param model - The model name to use. Defaults to "gemini-2.0-flash-001". + * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable. + * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false. + * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined. + * @param location - The location to use for VertexAI API requests. Default value is "us-central1". + * @param temperature - Sampling temperature for response generation. Defaults to undefined. + * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined. + * @param topP - The nucleus sampling probability for response generation. Defaults to undefined. + * @param topK - The top-k sampling value for response generation. Defaults to undefined. + * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined. + * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined. + * @param toolChoice - Specifies whether to use tools during response generation. Defaults to "auto". + * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined. + * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined. + * @param geminiTools - The Gemini-specific tools to use for the session. + * @param httpOptions - The HTTP options to use for the session. + * @param seed - Random seed for reproducible results. Defaults to undefined. + */ + constructor({ + model, + apiKey, + vertexai, + project, + location, + temperature, + maxOutputTokens, + topP, + topK, + presencePenalty, + frequencyPenalty, + toolChoice, + thinkingConfig, + automaticFunctionCallingConfig, + geminiTools, + httpOptions, + seed + } = { + model: "gemini-2.0-flash-001" + }) { + super(); + const useVertexAI = vertexai ?? (process.env.GOOGLE_GENAI_USE_VERTEXAI === "true" || process.env.GOOGLE_GENAI_USE_VERTEXAI === "1"); + let gcpProject = project ?? process.env.GOOGLE_CLOUD_PROJECT; + let gcpLocation = location ?? process.env.GOOGLE_CLOUD_LOCATION; + let geminiApiKey = apiKey ?? process.env.GOOGLE_API_KEY; + if (useVertexAI) { + if (!gcpProject) { + throw new Error( + "Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable" + ); + } + geminiApiKey = void 0; + } else { + gcpProject = void 0; + gcpLocation = void 0; + if (!geminiApiKey) { + throw new Error( + "API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable" + ); + } + } + if ((thinkingConfig == null ? void 0 : thinkingConfig.thinkingBudget) !== void 0) { + const budget = thinkingConfig.thinkingBudget; + if (budget < 0 || budget > 24576) { + throw new Error("thinkingBudget inside thinkingConfig must be between 0 and 24576"); + } + } + const clientOptions = useVertexAI ? { + vertexai: true, + project: gcpProject, + location: gcpLocation + } : { + apiKey: geminiApiKey + }; + this.#client = new import_genai.GoogleGenAI(clientOptions); + this.#opts = { + model, + vertexai: useVertexAI, + project: gcpProject, + location: gcpLocation, + temperature, + maxOutputTokens, + topP, + topK, + presencePenalty, + frequencyPenalty, + toolChoice, + thinkingConfig, + automaticFunctionCallingConfig, + geminiTools, + httpOptions, + seed, + apiKey + }; + } + chat({ + chatCtx, + toolCtx, + connOptions = import_agents.DEFAULT_API_CONNECT_OPTIONS, + toolChoice, + extraKwargs, + geminiTools + }) { + const extras = { ...extraKwargs }; + toolChoice = toolChoice !== void 0 ? toolChoice : this.#opts.toolChoice; + if (toolChoice) { + let geminiToolConfig; + if (typeof toolChoice === "object" && toolChoice.type === "function") { + geminiToolConfig = { + functionCallingConfig: { + mode: import_genai.FunctionCallingConfigMode.ANY, + allowedFunctionNames: [toolChoice.function.name] + } + }; + } else if (toolChoice === "required") { + const toolNames = Object.entries(toolCtx || {}).map(([name]) => name); + geminiToolConfig = { + functionCallingConfig: { + mode: import_genai.FunctionCallingConfigMode.ANY, + allowedFunctionNames: toolNames.length > 0 ? toolNames : void 0 + } + }; + } else if (toolChoice === "auto") { + geminiToolConfig = { + functionCallingConfig: { + mode: import_genai.FunctionCallingConfigMode.AUTO + } + }; + } else if (toolChoice === "none") { + geminiToolConfig = { + functionCallingConfig: { + mode: import_genai.FunctionCallingConfigMode.NONE + } + }; + } else { + throw new Error(`Invalid tool choice: ${toolChoice}`); + } + extras.toolConfig = geminiToolConfig; + } + if (this.#opts.temperature !== void 0) { + extras.temperature = this.#opts.temperature; + } + if (this.#opts.maxOutputTokens !== void 0) { + extras.maxOutputTokens = this.#opts.maxOutputTokens; + } + if (this.#opts.topP !== void 0) { + extras.topP = this.#opts.topP; + } + if (this.#opts.topK !== void 0) { + extras.topK = this.#opts.topK; + } + if (this.#opts.presencePenalty !== void 0) { + extras.presencePenalty = this.#opts.presencePenalty; + } + if (this.#opts.frequencyPenalty !== void 0) { + extras.frequencyPenalty = this.#opts.frequencyPenalty; + } + if (this.#opts.seed !== void 0) { + extras.seed = this.#opts.seed; + } + if (this.#opts.thinkingConfig !== void 0) { + extras.thinkingConfig = this.#opts.thinkingConfig; + } + if (this.#opts.automaticFunctionCallingConfig !== void 0) { + extras.automaticFunctionCalling = this.#opts.automaticFunctionCallingConfig; + } + geminiTools = geminiTools !== void 0 ? geminiTools : this.#opts.geminiTools; + return new LLMStream(this, { + client: this.#client, + model: this.#opts.model, + chatCtx, + toolCtx, + connOptions, + geminiTools, + extraKwargs: extras + }); + } +} +class LLMStream extends import_agents.llm.LLMStream { + #client; + #model; + #geminiTools; + #extraKwargs; + constructor(llm2, { + client, + model, + chatCtx, + toolCtx, + connOptions, + geminiTools, + extraKwargs + }) { + super(llm2, { chatCtx, toolCtx, connOptions }); + this.#client = client; + this.#model = model; + this.#geminiTools = geminiTools; + this.#extraKwargs = extraKwargs; + } + async run() { + var _a, _b; + let retryable = true; + const requestId = `google_${Date.now()}`; + try { + const [turns, extraData] = await this.chatCtx.toProviderFormat("google"); + const contents = turns.map((turn) => ({ + role: turn.role, + parts: turn.parts + })); + const functionDeclarations = this.toolCtx ? (0, import_utils.toFunctionDeclarations)(this.toolCtx) : void 0; + const tools = functionDeclarations && functionDeclarations.length > 0 ? [{ functionDeclarations }] : void 0; + let systemInstruction = void 0; + if (extraData.systemMessages && extraData.systemMessages.length > 0) { + systemInstruction = { + parts: extraData.systemMessages.map((content) => ({ text: content })) + }; + } + const response = await this.#client.models.generateContentStream({ + model: this.#model, + contents, + config: { + ...this.#extraKwargs, + systemInstruction, + httpOptions: this.#extraKwargs.httpOptions ?? { + timeout: Math.floor(this.connOptions.timeoutMs) + }, + tools + } + }); + for await (const chunk of response) { + if (chunk.promptFeedback) { + throw new import_agents.APIStatusError({ + message: `Prompt feedback error: ${JSON.stringify(chunk.promptFeedback)}`, + options: { + retryable: false, + requestId + } + }); + } + if (!chunk.candidates || !((_b = (_a = chunk.candidates[0]) == null ? void 0 : _a.content) == null ? void 0 : _b.parts)) { + this.logger.warn(`No candidates in the response: ${JSON.stringify(chunk)}`); + continue; + } + if (chunk.candidates.length > 1) { + this.logger.warn( + "Google LLM: there are multiple candidates in the response, returning response from the first one." + ); + } + for (const part of chunk.candidates[0].content.parts) { + const chatChunk = this.#parsePart(requestId, part); + if (chatChunk) { + retryable = false; + this.queue.put(chatChunk); + } + } + if (chunk.usageMetadata) { + const usage = chunk.usageMetadata; + this.queue.put({ + id: requestId, + usage: { + completionTokens: usage.candidatesTokenCount || 0, + promptTokens: usage.promptTokenCount || 0, + promptCachedTokens: usage.cachedContentTokenCount || 0, + totalTokens: usage.totalTokenCount || 0 + } + }); + } + } + } catch (error) { + const err = error; + if (err.code && err.code >= 400 && err.code < 500) { + if (err.code === 429) { + throw new import_agents.APIStatusError({ + message: `Google LLM: Rate limit error - ${err.message || "Unknown error"}`, + options: { + statusCode: 429, + retryable: true + } + }); + } else { + throw new import_agents.APIStatusError({ + message: `Google LLM: Client error (${err.code}) - ${err.message || "Unknown error"}`, + options: { + statusCode: err.code, + retryable: false + } + }); + } + } + if (err.code && err.code >= 500) { + throw new import_agents.APIStatusError({ + message: `Google LLM: Server error (${err.code}) - ${err.message || "Unknown error"}`, + options: { + statusCode: err.code, + retryable + } + }); + } + throw new import_agents.APIConnectionError({ + message: `Google LLM: API error - ${err.message || "Unknown error"}`, + options: { + retryable + } + }); + } + } + #parsePart(id, part) { + if (part.functionCall) { + return { + id, + delta: { + role: "assistant", + toolCalls: [ + import_agents.llm.FunctionCall.create({ + callId: part.functionCall.id || (0, import_agents.shortuuid)("function_call_"), + name: part.functionCall.name, + args: JSON.stringify(part.functionCall.args) + }) + ] + } + }; + } + return { + id, + delta: { + content: part.text, + role: "assistant" + } + }; + } +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + LLM, + LLMStream +}); +//# sourceMappingURL=llm.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/llm.cjs.map b/plugins/google/dist/llm.cjs.map new file mode 100644 index 000000000..501f0cdc2 --- /dev/null +++ b/plugins/google/dist/llm.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as types from '@google/genai';\nimport { FunctionCallingConfigMode, type GenerateContentConfig, GoogleGenAI } from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n APIStatusError,\n DEFAULT_API_CONNECT_OPTIONS,\n llm,\n shortuuid,\n} from '@livekit/agents';\nimport type { ChatModels } from './models.js';\nimport type { LLMTools } from './tools.js';\nimport { toFunctionDeclarations } from './utils.js';\n\ninterface GoogleFormatData {\n systemMessages: string[] | null;\n}\n\nexport interface LLMOptions {\n model: string | ChatModels;\n apiKey?: string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n vertexai?: boolean;\n project?: string;\n location?: string;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n thinkingConfig?: types.ThinkingConfig;\n automaticFunctionCallingConfig?: types.AutomaticFunctionCallingConfig;\n geminiTools?: LLMTools;\n httpOptions?: types.HttpOptions;\n seed?: number;\n}\n\nexport class LLM extends llm.LLM {\n #opts: LLMOptions;\n #client: GoogleGenAI;\n\n label(): string {\n return 'google.LLM';\n }\n\n get model(): string {\n return this.#opts.model;\n }\n\n /**\n * Create a new instance of Google GenAI LLM.\n *\n * Environment Requirements:\n * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods.\n * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables\n * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file,\n * and the location defaults to \"us-central1\".\n * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable.\n *\n * @param model - The model name to use. Defaults to \"gemini-2.0-flash-001\".\n * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable.\n * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false.\n * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined.\n * @param location - The location to use for VertexAI API requests. Default value is \"us-central1\".\n * @param temperature - Sampling temperature for response generation. Defaults to undefined.\n * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined.\n * @param topP - The nucleus sampling probability for response generation. Defaults to undefined.\n * @param topK - The top-k sampling value for response generation. Defaults to undefined.\n * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined.\n * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined.\n * @param toolChoice - Specifies whether to use tools during response generation. Defaults to \"auto\".\n * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined.\n * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined.\n * @param geminiTools - The Gemini-specific tools to use for the session.\n * @param httpOptions - The HTTP options to use for the session.\n * @param seed - Random seed for reproducible results. Defaults to undefined.\n */\n constructor(\n {\n model,\n apiKey,\n vertexai,\n project,\n location,\n temperature,\n maxOutputTokens,\n topP,\n topK,\n presencePenalty,\n frequencyPenalty,\n toolChoice,\n thinkingConfig,\n automaticFunctionCallingConfig,\n geminiTools,\n httpOptions,\n seed,\n }: LLMOptions = {\n model: 'gemini-2.0-flash-001',\n },\n ) {\n super();\n\n const useVertexAI =\n vertexai ??\n (process.env.GOOGLE_GENAI_USE_VERTEXAI === 'true' ||\n process.env.GOOGLE_GENAI_USE_VERTEXAI === '1');\n\n let gcpProject: string | undefined = project ?? process.env.GOOGLE_CLOUD_PROJECT;\n let gcpLocation: string | undefined = location ?? process.env.GOOGLE_CLOUD_LOCATION;\n let geminiApiKey: string | undefined = apiKey ?? process.env.GOOGLE_API_KEY;\n\n if (useVertexAI) {\n if (!gcpProject) {\n // TODO(brian): use default_async to get the project ID\n throw new Error(\n 'Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable',\n );\n }\n geminiApiKey = undefined;\n } else {\n gcpProject = undefined;\n gcpLocation = undefined;\n if (!geminiApiKey) {\n throw new Error(\n 'API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable',\n );\n }\n }\n\n // Validate thinkingConfig\n if (thinkingConfig?.thinkingBudget !== undefined) {\n const budget = thinkingConfig.thinkingBudget;\n if (budget < 0 || budget > 24576) {\n throw new Error('thinkingBudget inside thinkingConfig must be between 0 and 24576');\n }\n }\n\n const clientOptions: types.GoogleGenAIOptions = useVertexAI\n ? {\n vertexai: true,\n project: gcpProject,\n location: gcpLocation,\n }\n : {\n apiKey: geminiApiKey,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n\n this.#opts = {\n model,\n vertexai: useVertexAI,\n project: gcpProject,\n location: gcpLocation,\n temperature,\n maxOutputTokens,\n topP,\n topK,\n presencePenalty,\n frequencyPenalty,\n toolChoice,\n thinkingConfig,\n automaticFunctionCallingConfig,\n geminiTools,\n httpOptions,\n seed,\n apiKey,\n };\n }\n\n chat({\n chatCtx,\n toolCtx,\n connOptions = DEFAULT_API_CONNECT_OPTIONS,\n toolChoice,\n extraKwargs,\n geminiTools,\n }: {\n chatCtx: llm.ChatContext;\n toolCtx?: llm.ToolContext;\n connOptions?: APIConnectOptions;\n parallelToolCalls?: boolean;\n toolChoice?: llm.ToolChoice;\n extraKwargs?: Record;\n geminiTools?: LLMTools;\n }): LLMStream {\n const extras: GenerateContentConfig = { ...extraKwargs } as GenerateContentConfig;\n\n toolChoice = toolChoice !== undefined ? toolChoice : this.#opts.toolChoice;\n\n if (toolChoice) {\n let geminiToolConfig: types.ToolConfig;\n\n if (typeof toolChoice === 'object' && toolChoice.type === 'function') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.ANY,\n allowedFunctionNames: [toolChoice.function.name],\n },\n };\n } else if (toolChoice === 'required') {\n const toolNames = Object.entries(toolCtx || {}).map(([name]) => name);\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.ANY,\n allowedFunctionNames: toolNames.length > 0 ? toolNames : undefined,\n },\n };\n } else if (toolChoice === 'auto') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.AUTO,\n },\n };\n } else if (toolChoice === 'none') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.NONE,\n },\n };\n } else {\n throw new Error(`Invalid tool choice: ${toolChoice}`);\n }\n\n extras.toolConfig = geminiToolConfig;\n }\n\n if (this.#opts.temperature !== undefined) {\n extras.temperature = this.#opts.temperature;\n }\n if (this.#opts.maxOutputTokens !== undefined) {\n extras.maxOutputTokens = this.#opts.maxOutputTokens;\n }\n if (this.#opts.topP !== undefined) {\n extras.topP = this.#opts.topP;\n }\n if (this.#opts.topK !== undefined) {\n extras.topK = this.#opts.topK;\n }\n if (this.#opts.presencePenalty !== undefined) {\n extras.presencePenalty = this.#opts.presencePenalty;\n }\n if (this.#opts.frequencyPenalty !== undefined) {\n extras.frequencyPenalty = this.#opts.frequencyPenalty;\n }\n if (this.#opts.seed !== undefined) {\n extras.seed = this.#opts.seed;\n }\n\n if (this.#opts.thinkingConfig !== undefined) {\n extras.thinkingConfig = this.#opts.thinkingConfig;\n }\n\n if (this.#opts.automaticFunctionCallingConfig !== undefined) {\n extras.automaticFunctionCalling = this.#opts.automaticFunctionCallingConfig;\n }\n\n geminiTools = geminiTools !== undefined ? geminiTools : this.#opts.geminiTools;\n\n return new LLMStream(this, {\n client: this.#client,\n model: this.#opts.model,\n chatCtx,\n toolCtx,\n connOptions,\n geminiTools,\n extraKwargs: extras,\n });\n }\n}\n\nexport class LLMStream extends llm.LLMStream {\n #client: GoogleGenAI;\n #model: string;\n #geminiTools?: LLMTools;\n #extraKwargs: GenerateContentConfig;\n\n constructor(\n llm: LLM,\n {\n client,\n model,\n chatCtx,\n toolCtx,\n connOptions,\n geminiTools,\n extraKwargs,\n }: {\n client: GoogleGenAI;\n model: string;\n chatCtx: llm.ChatContext;\n toolCtx?: llm.ToolContext;\n connOptions: APIConnectOptions;\n geminiTools?: LLMTools;\n extraKwargs: GenerateContentConfig;\n },\n ) {\n // Call base constructor with dev 1.0 object parameter pattern\n super(llm, { chatCtx, toolCtx, connOptions });\n this.#client = client;\n this.#model = model;\n this.#geminiTools = geminiTools;\n this.#extraKwargs = extraKwargs;\n }\n\n protected async run(): Promise {\n let retryable = true;\n const requestId = `google_${Date.now()}`;\n\n try {\n const [turns, extraData] = (await this.chatCtx.toProviderFormat('google')) as [\n Record[],\n GoogleFormatData,\n ];\n\n const contents: types.Content[] = turns.map((turn: Record) => ({\n role: turn.role as types.Content['role'],\n parts: turn.parts as types.Part[],\n }));\n\n const functionDeclarations = this.toolCtx ? toFunctionDeclarations(this.toolCtx) : undefined;\n const tools =\n functionDeclarations && functionDeclarations.length > 0\n ? [{ functionDeclarations }]\n : undefined;\n\n let systemInstruction: types.Content | undefined = undefined;\n if (extraData.systemMessages && extraData.systemMessages.length > 0) {\n systemInstruction = {\n parts: extraData.systemMessages.map((content: string) => ({ text: content })),\n };\n }\n\n const response = await this.#client.models.generateContentStream({\n model: this.#model,\n contents,\n config: {\n ...this.#extraKwargs,\n systemInstruction,\n httpOptions: this.#extraKwargs.httpOptions ?? {\n timeout: Math.floor(this.connOptions.timeoutMs),\n },\n tools,\n },\n });\n\n for await (const chunk of response) {\n if (chunk.promptFeedback) {\n throw new APIStatusError({\n message: `Prompt feedback error: ${JSON.stringify(chunk.promptFeedback)}`,\n options: {\n retryable: false,\n requestId,\n },\n });\n }\n\n if (!chunk.candidates || !chunk.candidates[0]?.content?.parts) {\n this.logger.warn(`No candidates in the response: ${JSON.stringify(chunk)}`);\n continue;\n }\n\n if (chunk.candidates.length > 1) {\n this.logger.warn(\n 'Google LLM: there are multiple candidates in the response, returning response from the first one.',\n );\n }\n\n for (const part of chunk.candidates[0].content.parts) {\n const chatChunk = this.#parsePart(requestId, part);\n if (chatChunk) {\n retryable = false;\n this.queue.put(chatChunk);\n }\n }\n\n if (chunk.usageMetadata) {\n const usage = chunk.usageMetadata;\n this.queue.put({\n id: requestId,\n usage: {\n completionTokens: usage.candidatesTokenCount || 0,\n promptTokens: usage.promptTokenCount || 0,\n promptCachedTokens: usage.cachedContentTokenCount || 0,\n totalTokens: usage.totalTokenCount || 0,\n },\n });\n }\n }\n } catch (error: unknown) {\n const err = error as {\n code?: number;\n message?: string;\n status?: string;\n type?: string;\n };\n\n if (err.code && err.code >= 400 && err.code < 500) {\n if (err.code === 429) {\n throw new APIStatusError({\n message: `Google LLM: Rate limit error - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: 429,\n retryable: true,\n },\n });\n } else {\n throw new APIStatusError({\n message: `Google LLM: Client error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: false,\n },\n });\n }\n }\n\n if (err.code && err.code >= 500) {\n throw new APIStatusError({\n message: `Google LLM: Server error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable,\n },\n });\n }\n\n throw new APIConnectionError({\n message: `Google LLM: API error - ${err.message || 'Unknown error'}`,\n options: {\n retryable,\n },\n });\n }\n }\n\n #parsePart(id: string, part: types.Part): llm.ChatChunk | null {\n if (part.functionCall) {\n return {\n id,\n delta: {\n role: 'assistant',\n toolCalls: [\n llm.FunctionCall.create({\n callId: part.functionCall.id || shortuuid('function_call_'),\n name: part.functionCall.name!,\n args: JSON.stringify(part.functionCall.args!),\n }),\n ],\n },\n };\n }\n\n return {\n id,\n delta: {\n content: part.text,\n role: 'assistant',\n },\n };\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAmF;AAEnF,oBAMO;AAGP,mBAAuC;AA0BhC,MAAM,YAAY,kBAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EAEA,QAAgB;AACd,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA8BA,YACE;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,IAAgB;AAAA,IACd,OAAO;AAAA,EACT,GACA;AACA,UAAM;AAEN,UAAM,cACJ,aACC,QAAQ,IAAI,8BAA8B,UACzC,QAAQ,IAAI,8BAA8B;AAE9C,QAAI,aAAiC,WAAW,QAAQ,IAAI;AAC5D,QAAI,cAAkC,YAAY,QAAQ,IAAI;AAC9D,QAAI,eAAmC,UAAU,QAAQ,IAAI;AAE7D,QAAI,aAAa;AACf,UAAI,CAAC,YAAY;AAEf,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,qBAAe;AAAA,IACjB,OAAO;AACL,mBAAa;AACb,oBAAc;AACd,UAAI,CAAC,cAAc;AACjB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,SAAI,iDAAgB,oBAAmB,QAAW;AAChD,YAAM,SAAS,eAAe;AAC9B,UAAI,SAAS,KAAK,SAAS,OAAO;AAChC,cAAM,IAAI,MAAM,kEAAkE;AAAA,MACpF;AAAA,IACF;AAEA,UAAM,gBAA0C,cAC5C;AAAA,MACE,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,IACZ,IACA;AAAA,MACE,QAAQ;AAAA,IACV;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAE5C,SAAK,QAAQ;AAAA,MACX;AAAA,MACA,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAQc;AACZ,UAAM,SAAgC,EAAE,GAAG,YAAY;AAEvD,iBAAa,eAAe,SAAY,aAAa,KAAK,MAAM;AAEhE,QAAI,YAAY;AACd,UAAI;AAEJ,UAAI,OAAO,eAAe,YAAY,WAAW,SAAS,YAAY;AACpE,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,uCAA0B;AAAA,YAChC,sBAAsB,CAAC,WAAW,SAAS,IAAI;AAAA,UACjD;AAAA,QACF;AAAA,MACF,WAAW,eAAe,YAAY;AACpC,cAAM,YAAY,OAAO,QAAQ,WAAW,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,MAAM,IAAI;AACpE,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,uCAA0B;AAAA,YAChC,sBAAsB,UAAU,SAAS,IAAI,YAAY;AAAA,UAC3D;AAAA,QACF;AAAA,MACF,WAAW,eAAe,QAAQ;AAChC,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,uCAA0B;AAAA,UAClC;AAAA,QACF;AAAA,MACF,WAAW,eAAe,QAAQ;AAChC,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,uCAA0B;AAAA,UAClC;AAAA,QACF;AAAA,MACF,OAAO;AACL,cAAM,IAAI,MAAM,wBAAwB,UAAU,EAAE;AAAA,MACtD;AAEA,aAAO,aAAa;AAAA,IACtB;AAEA,QAAI,KAAK,MAAM,gBAAgB,QAAW;AACxC,aAAO,cAAc,KAAK,MAAM;AAAA,IAClC;AACA,QAAI,KAAK,MAAM,oBAAoB,QAAW;AAC5C,aAAO,kBAAkB,KAAK,MAAM;AAAA,IACtC;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AACA,QAAI,KAAK,MAAM,oBAAoB,QAAW;AAC5C,aAAO,kBAAkB,KAAK,MAAM;AAAA,IACtC;AACA,QAAI,KAAK,MAAM,qBAAqB,QAAW;AAC7C,aAAO,mBAAmB,KAAK,MAAM;AAAA,IACvC;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AAEA,QAAI,KAAK,MAAM,mBAAmB,QAAW;AAC3C,aAAO,iBAAiB,KAAK,MAAM;AAAA,IACrC;AAEA,QAAI,KAAK,MAAM,mCAAmC,QAAW;AAC3D,aAAO,2BAA2B,KAAK,MAAM;AAAA,IAC/C;AAEA,kBAAc,gBAAgB,SAAY,cAAc,KAAK,MAAM;AAEnE,WAAO,IAAI,UAAU,MAAM;AAAA,MACzB,QAAQ,KAAK;AAAA,MACb,OAAO,KAAK,MAAM;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa;AAAA,IACf,CAAC;AAAA,EACH;AACF;AAEO,MAAM,kBAAkB,kBAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACEA,MACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GASA;AAEA,UAAMA,MAAK,EAAE,SAAS,SAAS,YAAY,CAAC;AAC5C,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,eAAe;AACpB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,MAAqB;AArTvC;AAsTI,QAAI,YAAY;AAChB,UAAM,YAAY,UAAU,KAAK,IAAI,CAAC;AAEtC,QAAI;AACF,YAAM,CAAC,OAAO,SAAS,IAAK,MAAM,KAAK,QAAQ,iBAAiB,QAAQ;AAKxE,YAAM,WAA4B,MAAM,IAAI,CAAC,UAAmC;AAAA,QAC9E,MAAM,KAAK;AAAA,QACX,OAAO,KAAK;AAAA,MACd,EAAE;AAEF,YAAM,uBAAuB,KAAK,cAAU,qCAAuB,KAAK,OAAO,IAAI;AACnF,YAAM,QACJ,wBAAwB,qBAAqB,SAAS,IAClD,CAAC,EAAE,qBAAqB,CAAC,IACzB;AAEN,UAAI,oBAA+C;AACnD,UAAI,UAAU,kBAAkB,UAAU,eAAe,SAAS,GAAG;AACnE,4BAAoB;AAAA,UAClB,OAAO,UAAU,eAAe,IAAI,CAAC,aAAqB,EAAE,MAAM,QAAQ,EAAE;AAAA,QAC9E;AAAA,MACF;AAEA,YAAM,WAAW,MAAM,KAAK,QAAQ,OAAO,sBAAsB;AAAA,QAC/D,OAAO,KAAK;AAAA,QACZ;AAAA,QACA,QAAQ;AAAA,UACN,GAAG,KAAK;AAAA,UACR;AAAA,UACA,aAAa,KAAK,aAAa,eAAe;AAAA,YAC5C,SAAS,KAAK,MAAM,KAAK,YAAY,SAAS;AAAA,UAChD;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAED,uBAAiB,SAAS,UAAU;AAClC,YAAI,MAAM,gBAAgB;AACxB,gBAAM,IAAI,6BAAe;AAAA,YACvB,SAAS,0BAA0B,KAAK,UAAU,MAAM,cAAc,CAAC;AAAA,YACvE,SAAS;AAAA,cACP,WAAW;AAAA,cACX;AAAA,YACF;AAAA,UACF,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,MAAM,cAAc,GAAC,iBAAM,WAAW,CAAC,MAAlB,mBAAqB,YAArB,mBAA8B,QAAO;AAC7D,eAAK,OAAO,KAAK,kCAAkC,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1E;AAAA,QACF;AAEA,YAAI,MAAM,WAAW,SAAS,GAAG;AAC/B,eAAK,OAAO;AAAA,YACV;AAAA,UACF;AAAA,QACF;AAEA,mBAAW,QAAQ,MAAM,WAAW,CAAC,EAAE,QAAQ,OAAO;AACpD,gBAAM,YAAY,KAAK,WAAW,WAAW,IAAI;AACjD,cAAI,WAAW;AACb,wBAAY;AACZ,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAAA,QACF;AAEA,YAAI,MAAM,eAAe;AACvB,gBAAM,QAAQ,MAAM;AACpB,eAAK,MAAM,IAAI;AAAA,YACb,IAAI;AAAA,YACJ,OAAO;AAAA,cACL,kBAAkB,MAAM,wBAAwB;AAAA,cAChD,cAAc,MAAM,oBAAoB;AAAA,cACxC,oBAAoB,MAAM,2BAA2B;AAAA,cACrD,aAAa,MAAM,mBAAmB;AAAA,YACxC;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,SAAS,OAAgB;AACvB,YAAM,MAAM;AAOZ,UAAI,IAAI,QAAQ,IAAI,QAAQ,OAAO,IAAI,OAAO,KAAK;AACjD,YAAI,IAAI,SAAS,KAAK;AACpB,gBAAM,IAAI,6BAAe;AAAA,YACvB,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,YACzE,SAAS;AAAA,cACP,YAAY;AAAA,cACZ,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH,OAAO;AACL,gBAAM,IAAI,6BAAe;AAAA,YACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,YACnF,SAAS;AAAA,cACP,YAAY,IAAI;AAAA,cAChB,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,IAAI,QAAQ,IAAI,QAAQ,KAAK;AAC/B,cAAM,IAAI,6BAAe;AAAA,UACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,UACnF,SAAS;AAAA,YACP,YAAY,IAAI;AAAA,YAChB;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH;AAEA,YAAM,IAAI,iCAAmB;AAAA,QAC3B,SAAS,2BAA2B,IAAI,WAAW,eAAe;AAAA,QAClE,SAAS;AAAA,UACP;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,WAAW,IAAY,MAAwC;AAC7D,QAAI,KAAK,cAAc;AACrB,aAAO;AAAA,QACL;AAAA,QACA,OAAO;AAAA,UACL,MAAM;AAAA,UACN,WAAW;AAAA,YACT,kBAAI,aAAa,OAAO;AAAA,cACtB,QAAQ,KAAK,aAAa,UAAM,yBAAU,gBAAgB;AAAA,cAC1D,MAAM,KAAK,aAAa;AAAA,cACxB,MAAM,KAAK,UAAU,KAAK,aAAa,IAAK;AAAA,YAC9C,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL;AAAA,MACA,OAAO;AAAA,QACL,SAAS,KAAK;AAAA,QACd,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;","names":["llm"]} \ No newline at end of file diff --git a/plugins/google/dist/llm.d.cts b/plugins/google/dist/llm.d.cts new file mode 100644 index 000000000..8961efcae --- /dev/null +++ b/plugins/google/dist/llm.d.cts @@ -0,0 +1,82 @@ +import type * as types from '@google/genai'; +import { type GenerateContentConfig, GoogleGenAI } from '@google/genai'; +import type { APIConnectOptions } from '@livekit/agents'; +import { llm } from '@livekit/agents'; +import type { ChatModels } from './models.js'; +import type { LLMTools } from './tools.js'; +export interface LLMOptions { + model: string | ChatModels; + apiKey?: string; + temperature?: number; + toolChoice?: llm.ToolChoice; + vertexai?: boolean; + project?: string; + location?: string; + maxOutputTokens?: number; + topP?: number; + topK?: number; + presencePenalty?: number; + frequencyPenalty?: number; + thinkingConfig?: types.ThinkingConfig; + automaticFunctionCallingConfig?: types.AutomaticFunctionCallingConfig; + geminiTools?: LLMTools; + httpOptions?: types.HttpOptions; + seed?: number; +} +export declare class LLM extends llm.LLM { + #private; + label(): string; + get model(): string; + /** + * Create a new instance of Google GenAI LLM. + * + * Environment Requirements: + * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods. + * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables + * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file, + * and the location defaults to "us-central1". + * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. + * + * @param model - The model name to use. Defaults to "gemini-2.0-flash-001". + * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable. + * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false. + * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined. + * @param location - The location to use for VertexAI API requests. Default value is "us-central1". + * @param temperature - Sampling temperature for response generation. Defaults to undefined. + * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined. + * @param topP - The nucleus sampling probability for response generation. Defaults to undefined. + * @param topK - The top-k sampling value for response generation. Defaults to undefined. + * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined. + * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined. + * @param toolChoice - Specifies whether to use tools during response generation. Defaults to "auto". + * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined. + * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined. + * @param geminiTools - The Gemini-specific tools to use for the session. + * @param httpOptions - The HTTP options to use for the session. + * @param seed - Random seed for reproducible results. Defaults to undefined. + */ + constructor({ model, apiKey, vertexai, project, location, temperature, maxOutputTokens, topP, topK, presencePenalty, frequencyPenalty, toolChoice, thinkingConfig, automaticFunctionCallingConfig, geminiTools, httpOptions, seed, }?: LLMOptions); + chat({ chatCtx, toolCtx, connOptions, toolChoice, extraKwargs, geminiTools, }: { + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContext; + connOptions?: APIConnectOptions; + parallelToolCalls?: boolean; + toolChoice?: llm.ToolChoice; + extraKwargs?: Record; + geminiTools?: LLMTools; + }): LLMStream; +} +export declare class LLMStream extends llm.LLMStream { + #private; + constructor(llm: LLM, { client, model, chatCtx, toolCtx, connOptions, geminiTools, extraKwargs, }: { + client: GoogleGenAI; + model: string; + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContext; + connOptions: APIConnectOptions; + geminiTools?: LLMTools; + extraKwargs: GenerateContentConfig; + }); + protected run(): Promise; +} +//# sourceMappingURL=llm.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/llm.d.ts b/plugins/google/dist/llm.d.ts new file mode 100644 index 000000000..8961efcae --- /dev/null +++ b/plugins/google/dist/llm.d.ts @@ -0,0 +1,82 @@ +import type * as types from '@google/genai'; +import { type GenerateContentConfig, GoogleGenAI } from '@google/genai'; +import type { APIConnectOptions } from '@livekit/agents'; +import { llm } from '@livekit/agents'; +import type { ChatModels } from './models.js'; +import type { LLMTools } from './tools.js'; +export interface LLMOptions { + model: string | ChatModels; + apiKey?: string; + temperature?: number; + toolChoice?: llm.ToolChoice; + vertexai?: boolean; + project?: string; + location?: string; + maxOutputTokens?: number; + topP?: number; + topK?: number; + presencePenalty?: number; + frequencyPenalty?: number; + thinkingConfig?: types.ThinkingConfig; + automaticFunctionCallingConfig?: types.AutomaticFunctionCallingConfig; + geminiTools?: LLMTools; + httpOptions?: types.HttpOptions; + seed?: number; +} +export declare class LLM extends llm.LLM { + #private; + label(): string; + get model(): string; + /** + * Create a new instance of Google GenAI LLM. + * + * Environment Requirements: + * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods. + * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables + * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file, + * and the location defaults to "us-central1". + * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. + * + * @param model - The model name to use. Defaults to "gemini-2.0-flash-001". + * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable. + * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false. + * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined. + * @param location - The location to use for VertexAI API requests. Default value is "us-central1". + * @param temperature - Sampling temperature for response generation. Defaults to undefined. + * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined. + * @param topP - The nucleus sampling probability for response generation. Defaults to undefined. + * @param topK - The top-k sampling value for response generation. Defaults to undefined. + * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined. + * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined. + * @param toolChoice - Specifies whether to use tools during response generation. Defaults to "auto". + * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined. + * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined. + * @param geminiTools - The Gemini-specific tools to use for the session. + * @param httpOptions - The HTTP options to use for the session. + * @param seed - Random seed for reproducible results. Defaults to undefined. + */ + constructor({ model, apiKey, vertexai, project, location, temperature, maxOutputTokens, topP, topK, presencePenalty, frequencyPenalty, toolChoice, thinkingConfig, automaticFunctionCallingConfig, geminiTools, httpOptions, seed, }?: LLMOptions); + chat({ chatCtx, toolCtx, connOptions, toolChoice, extraKwargs, geminiTools, }: { + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContext; + connOptions?: APIConnectOptions; + parallelToolCalls?: boolean; + toolChoice?: llm.ToolChoice; + extraKwargs?: Record; + geminiTools?: LLMTools; + }): LLMStream; +} +export declare class LLMStream extends llm.LLMStream { + #private; + constructor(llm: LLM, { client, model, chatCtx, toolCtx, connOptions, geminiTools, extraKwargs, }: { + client: GoogleGenAI; + model: string; + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContext; + connOptions: APIConnectOptions; + geminiTools?: LLMTools; + extraKwargs: GenerateContentConfig; + }); + protected run(): Promise; +} +//# sourceMappingURL=llm.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/llm.d.ts.map b/plugins/google/dist/llm.d.ts.map new file mode 100644 index 000000000..7f9b262b9 --- /dev/null +++ b/plugins/google/dist/llm.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,KAAK,KAAK,MAAM,eAAe,CAAC;AAC5C,OAAO,EAA6B,KAAK,qBAAqB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACnG,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAIL,GAAG,EAEJ,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAO3C,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,GAAG,UAAU,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;IACtC,8BAA8B,CAAC,EAAE,KAAK,CAAC,8BAA8B,CAAC;IACtE,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,WAAW,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;;IAI9B,KAAK,IAAI,MAAM;IAIf,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;gBAED,EACE,KAAK,EACL,MAAM,EACN,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,WAAW,EACX,eAAe,EACf,IAAI,EACJ,IAAI,EACJ,eAAe,EACf,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,8BAA8B,EAC9B,WAAW,EACX,WAAW,EACX,IAAI,GACL,GAAE,UAEF;IAwEH,IAAI,CAAC,EACH,OAAO,EACP,OAAO,EACP,WAAyC,EACzC,UAAU,EACV,WAAW,EACX,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;QAC5B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,WAAW,CAAC,EAAE,QAAQ,CAAC;KACxB,GAAG,SAAS;CAoFd;AAED,qBAAa,SAAU,SAAQ,GAAG,CAAC,SAAS;;gBAOxC,GAAG,EAAE,GAAG,EACR,EACE,MAAM,EACN,KAAK,EACL,OAAO,EACP,OAAO,EACP,WAAW,EACX,WAAW,EACX,WAAW,GACZ,EAAE;QACD,MAAM,EAAE,WAAW,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,WAAW,EAAE,iBAAiB,CAAC;QAC/B,WAAW,CAAC,EAAE,QAAQ,CAAC;QACvB,WAAW,EAAE,qBAAqB,CAAC;KACpC;cAUa,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CA4JrC"} \ No newline at end of file diff --git a/plugins/google/dist/llm.js b/plugins/google/dist/llm.js new file mode 100644 index 000000000..ed4702692 --- /dev/null +++ b/plugins/google/dist/llm.js @@ -0,0 +1,362 @@ +import { FunctionCallingConfigMode, GoogleGenAI } from "@google/genai"; +import { + APIConnectionError, + APIStatusError, + DEFAULT_API_CONNECT_OPTIONS, + llm, + shortuuid +} from "@livekit/agents"; +import { toFunctionDeclarations } from "./utils.js"; +class LLM extends llm.LLM { + #opts; + #client; + label() { + return "google.LLM"; + } + get model() { + return this.#opts.model; + } + /** + * Create a new instance of Google GenAI LLM. + * + * Environment Requirements: + * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods. + * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables + * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file, + * and the location defaults to "us-central1". + * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. + * + * @param model - The model name to use. Defaults to "gemini-2.0-flash-001". + * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable. + * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false. + * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined. + * @param location - The location to use for VertexAI API requests. Default value is "us-central1". + * @param temperature - Sampling temperature for response generation. Defaults to undefined. + * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined. + * @param topP - The nucleus sampling probability for response generation. Defaults to undefined. + * @param topK - The top-k sampling value for response generation. Defaults to undefined. + * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined. + * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined. + * @param toolChoice - Specifies whether to use tools during response generation. Defaults to "auto". + * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined. + * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined. + * @param geminiTools - The Gemini-specific tools to use for the session. + * @param httpOptions - The HTTP options to use for the session. + * @param seed - Random seed for reproducible results. Defaults to undefined. + */ + constructor({ + model, + apiKey, + vertexai, + project, + location, + temperature, + maxOutputTokens, + topP, + topK, + presencePenalty, + frequencyPenalty, + toolChoice, + thinkingConfig, + automaticFunctionCallingConfig, + geminiTools, + httpOptions, + seed + } = { + model: "gemini-2.0-flash-001" + }) { + super(); + const useVertexAI = vertexai ?? (process.env.GOOGLE_GENAI_USE_VERTEXAI === "true" || process.env.GOOGLE_GENAI_USE_VERTEXAI === "1"); + let gcpProject = project ?? process.env.GOOGLE_CLOUD_PROJECT; + let gcpLocation = location ?? process.env.GOOGLE_CLOUD_LOCATION; + let geminiApiKey = apiKey ?? process.env.GOOGLE_API_KEY; + if (useVertexAI) { + if (!gcpProject) { + throw new Error( + "Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable" + ); + } + geminiApiKey = void 0; + } else { + gcpProject = void 0; + gcpLocation = void 0; + if (!geminiApiKey) { + throw new Error( + "API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable" + ); + } + } + if ((thinkingConfig == null ? void 0 : thinkingConfig.thinkingBudget) !== void 0) { + const budget = thinkingConfig.thinkingBudget; + if (budget < 0 || budget > 24576) { + throw new Error("thinkingBudget inside thinkingConfig must be between 0 and 24576"); + } + } + const clientOptions = useVertexAI ? { + vertexai: true, + project: gcpProject, + location: gcpLocation + } : { + apiKey: geminiApiKey + }; + this.#client = new GoogleGenAI(clientOptions); + this.#opts = { + model, + vertexai: useVertexAI, + project: gcpProject, + location: gcpLocation, + temperature, + maxOutputTokens, + topP, + topK, + presencePenalty, + frequencyPenalty, + toolChoice, + thinkingConfig, + automaticFunctionCallingConfig, + geminiTools, + httpOptions, + seed, + apiKey + }; + } + chat({ + chatCtx, + toolCtx, + connOptions = DEFAULT_API_CONNECT_OPTIONS, + toolChoice, + extraKwargs, + geminiTools + }) { + const extras = { ...extraKwargs }; + toolChoice = toolChoice !== void 0 ? toolChoice : this.#opts.toolChoice; + if (toolChoice) { + let geminiToolConfig; + if (typeof toolChoice === "object" && toolChoice.type === "function") { + geminiToolConfig = { + functionCallingConfig: { + mode: FunctionCallingConfigMode.ANY, + allowedFunctionNames: [toolChoice.function.name] + } + }; + } else if (toolChoice === "required") { + const toolNames = Object.entries(toolCtx || {}).map(([name]) => name); + geminiToolConfig = { + functionCallingConfig: { + mode: FunctionCallingConfigMode.ANY, + allowedFunctionNames: toolNames.length > 0 ? toolNames : void 0 + } + }; + } else if (toolChoice === "auto") { + geminiToolConfig = { + functionCallingConfig: { + mode: FunctionCallingConfigMode.AUTO + } + }; + } else if (toolChoice === "none") { + geminiToolConfig = { + functionCallingConfig: { + mode: FunctionCallingConfigMode.NONE + } + }; + } else { + throw new Error(`Invalid tool choice: ${toolChoice}`); + } + extras.toolConfig = geminiToolConfig; + } + if (this.#opts.temperature !== void 0) { + extras.temperature = this.#opts.temperature; + } + if (this.#opts.maxOutputTokens !== void 0) { + extras.maxOutputTokens = this.#opts.maxOutputTokens; + } + if (this.#opts.topP !== void 0) { + extras.topP = this.#opts.topP; + } + if (this.#opts.topK !== void 0) { + extras.topK = this.#opts.topK; + } + if (this.#opts.presencePenalty !== void 0) { + extras.presencePenalty = this.#opts.presencePenalty; + } + if (this.#opts.frequencyPenalty !== void 0) { + extras.frequencyPenalty = this.#opts.frequencyPenalty; + } + if (this.#opts.seed !== void 0) { + extras.seed = this.#opts.seed; + } + if (this.#opts.thinkingConfig !== void 0) { + extras.thinkingConfig = this.#opts.thinkingConfig; + } + if (this.#opts.automaticFunctionCallingConfig !== void 0) { + extras.automaticFunctionCalling = this.#opts.automaticFunctionCallingConfig; + } + geminiTools = geminiTools !== void 0 ? geminiTools : this.#opts.geminiTools; + return new LLMStream(this, { + client: this.#client, + model: this.#opts.model, + chatCtx, + toolCtx, + connOptions, + geminiTools, + extraKwargs: extras + }); + } +} +class LLMStream extends llm.LLMStream { + #client; + #model; + #geminiTools; + #extraKwargs; + constructor(llm2, { + client, + model, + chatCtx, + toolCtx, + connOptions, + geminiTools, + extraKwargs + }) { + super(llm2, { chatCtx, toolCtx, connOptions }); + this.#client = client; + this.#model = model; + this.#geminiTools = geminiTools; + this.#extraKwargs = extraKwargs; + } + async run() { + var _a, _b; + let retryable = true; + const requestId = `google_${Date.now()}`; + try { + const [turns, extraData] = await this.chatCtx.toProviderFormat("google"); + const contents = turns.map((turn) => ({ + role: turn.role, + parts: turn.parts + })); + const functionDeclarations = this.toolCtx ? toFunctionDeclarations(this.toolCtx) : void 0; + const tools = functionDeclarations && functionDeclarations.length > 0 ? [{ functionDeclarations }] : void 0; + let systemInstruction = void 0; + if (extraData.systemMessages && extraData.systemMessages.length > 0) { + systemInstruction = { + parts: extraData.systemMessages.map((content) => ({ text: content })) + }; + } + const response = await this.#client.models.generateContentStream({ + model: this.#model, + contents, + config: { + ...this.#extraKwargs, + systemInstruction, + httpOptions: this.#extraKwargs.httpOptions ?? { + timeout: Math.floor(this.connOptions.timeoutMs) + }, + tools + } + }); + for await (const chunk of response) { + if (chunk.promptFeedback) { + throw new APIStatusError({ + message: `Prompt feedback error: ${JSON.stringify(chunk.promptFeedback)}`, + options: { + retryable: false, + requestId + } + }); + } + if (!chunk.candidates || !((_b = (_a = chunk.candidates[0]) == null ? void 0 : _a.content) == null ? void 0 : _b.parts)) { + this.logger.warn(`No candidates in the response: ${JSON.stringify(chunk)}`); + continue; + } + if (chunk.candidates.length > 1) { + this.logger.warn( + "Google LLM: there are multiple candidates in the response, returning response from the first one." + ); + } + for (const part of chunk.candidates[0].content.parts) { + const chatChunk = this.#parsePart(requestId, part); + if (chatChunk) { + retryable = false; + this.queue.put(chatChunk); + } + } + if (chunk.usageMetadata) { + const usage = chunk.usageMetadata; + this.queue.put({ + id: requestId, + usage: { + completionTokens: usage.candidatesTokenCount || 0, + promptTokens: usage.promptTokenCount || 0, + promptCachedTokens: usage.cachedContentTokenCount || 0, + totalTokens: usage.totalTokenCount || 0 + } + }); + } + } + } catch (error) { + const err = error; + if (err.code && err.code >= 400 && err.code < 500) { + if (err.code === 429) { + throw new APIStatusError({ + message: `Google LLM: Rate limit error - ${err.message || "Unknown error"}`, + options: { + statusCode: 429, + retryable: true + } + }); + } else { + throw new APIStatusError({ + message: `Google LLM: Client error (${err.code}) - ${err.message || "Unknown error"}`, + options: { + statusCode: err.code, + retryable: false + } + }); + } + } + if (err.code && err.code >= 500) { + throw new APIStatusError({ + message: `Google LLM: Server error (${err.code}) - ${err.message || "Unknown error"}`, + options: { + statusCode: err.code, + retryable + } + }); + } + throw new APIConnectionError({ + message: `Google LLM: API error - ${err.message || "Unknown error"}`, + options: { + retryable + } + }); + } + } + #parsePart(id, part) { + if (part.functionCall) { + return { + id, + delta: { + role: "assistant", + toolCalls: [ + llm.FunctionCall.create({ + callId: part.functionCall.id || shortuuid("function_call_"), + name: part.functionCall.name, + args: JSON.stringify(part.functionCall.args) + }) + ] + } + }; + } + return { + id, + delta: { + content: part.text, + role: "assistant" + } + }; + } +} +export { + LLM, + LLMStream +}; +//# sourceMappingURL=llm.js.map \ No newline at end of file diff --git a/plugins/google/dist/llm.js.map b/plugins/google/dist/llm.js.map new file mode 100644 index 000000000..eea21e4b3 --- /dev/null +++ b/plugins/google/dist/llm.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as types from '@google/genai';\nimport { FunctionCallingConfigMode, type GenerateContentConfig, GoogleGenAI } from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n APIStatusError,\n DEFAULT_API_CONNECT_OPTIONS,\n llm,\n shortuuid,\n} from '@livekit/agents';\nimport type { ChatModels } from './models.js';\nimport type { LLMTools } from './tools.js';\nimport { toFunctionDeclarations } from './utils.js';\n\ninterface GoogleFormatData {\n systemMessages: string[] | null;\n}\n\nexport interface LLMOptions {\n model: string | ChatModels;\n apiKey?: string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n vertexai?: boolean;\n project?: string;\n location?: string;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n thinkingConfig?: types.ThinkingConfig;\n automaticFunctionCallingConfig?: types.AutomaticFunctionCallingConfig;\n geminiTools?: LLMTools;\n httpOptions?: types.HttpOptions;\n seed?: number;\n}\n\nexport class LLM extends llm.LLM {\n #opts: LLMOptions;\n #client: GoogleGenAI;\n\n label(): string {\n return 'google.LLM';\n }\n\n get model(): string {\n return this.#opts.model;\n }\n\n /**\n * Create a new instance of Google GenAI LLM.\n *\n * Environment Requirements:\n * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods.\n * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables\n * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file,\n * and the location defaults to \"us-central1\".\n * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable.\n *\n * @param model - The model name to use. Defaults to \"gemini-2.0-flash-001\".\n * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable.\n * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false.\n * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined.\n * @param location - The location to use for VertexAI API requests. Default value is \"us-central1\".\n * @param temperature - Sampling temperature for response generation. Defaults to undefined.\n * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined.\n * @param topP - The nucleus sampling probability for response generation. Defaults to undefined.\n * @param topK - The top-k sampling value for response generation. Defaults to undefined.\n * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined.\n * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined.\n * @param toolChoice - Specifies whether to use tools during response generation. Defaults to \"auto\".\n * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined.\n * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined.\n * @param geminiTools - The Gemini-specific tools to use for the session.\n * @param httpOptions - The HTTP options to use for the session.\n * @param seed - Random seed for reproducible results. Defaults to undefined.\n */\n constructor(\n {\n model,\n apiKey,\n vertexai,\n project,\n location,\n temperature,\n maxOutputTokens,\n topP,\n topK,\n presencePenalty,\n frequencyPenalty,\n toolChoice,\n thinkingConfig,\n automaticFunctionCallingConfig,\n geminiTools,\n httpOptions,\n seed,\n }: LLMOptions = {\n model: 'gemini-2.0-flash-001',\n },\n ) {\n super();\n\n const useVertexAI =\n vertexai ??\n (process.env.GOOGLE_GENAI_USE_VERTEXAI === 'true' ||\n process.env.GOOGLE_GENAI_USE_VERTEXAI === '1');\n\n let gcpProject: string | undefined = project ?? process.env.GOOGLE_CLOUD_PROJECT;\n let gcpLocation: string | undefined = location ?? process.env.GOOGLE_CLOUD_LOCATION;\n let geminiApiKey: string | undefined = apiKey ?? process.env.GOOGLE_API_KEY;\n\n if (useVertexAI) {\n if (!gcpProject) {\n // TODO(brian): use default_async to get the project ID\n throw new Error(\n 'Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable',\n );\n }\n geminiApiKey = undefined;\n } else {\n gcpProject = undefined;\n gcpLocation = undefined;\n if (!geminiApiKey) {\n throw new Error(\n 'API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable',\n );\n }\n }\n\n // Validate thinkingConfig\n if (thinkingConfig?.thinkingBudget !== undefined) {\n const budget = thinkingConfig.thinkingBudget;\n if (budget < 0 || budget > 24576) {\n throw new Error('thinkingBudget inside thinkingConfig must be between 0 and 24576');\n }\n }\n\n const clientOptions: types.GoogleGenAIOptions = useVertexAI\n ? {\n vertexai: true,\n project: gcpProject,\n location: gcpLocation,\n }\n : {\n apiKey: geminiApiKey,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n\n this.#opts = {\n model,\n vertexai: useVertexAI,\n project: gcpProject,\n location: gcpLocation,\n temperature,\n maxOutputTokens,\n topP,\n topK,\n presencePenalty,\n frequencyPenalty,\n toolChoice,\n thinkingConfig,\n automaticFunctionCallingConfig,\n geminiTools,\n httpOptions,\n seed,\n apiKey,\n };\n }\n\n chat({\n chatCtx,\n toolCtx,\n connOptions = DEFAULT_API_CONNECT_OPTIONS,\n toolChoice,\n extraKwargs,\n geminiTools,\n }: {\n chatCtx: llm.ChatContext;\n toolCtx?: llm.ToolContext;\n connOptions?: APIConnectOptions;\n parallelToolCalls?: boolean;\n toolChoice?: llm.ToolChoice;\n extraKwargs?: Record;\n geminiTools?: LLMTools;\n }): LLMStream {\n const extras: GenerateContentConfig = { ...extraKwargs } as GenerateContentConfig;\n\n toolChoice = toolChoice !== undefined ? toolChoice : this.#opts.toolChoice;\n\n if (toolChoice) {\n let geminiToolConfig: types.ToolConfig;\n\n if (typeof toolChoice === 'object' && toolChoice.type === 'function') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.ANY,\n allowedFunctionNames: [toolChoice.function.name],\n },\n };\n } else if (toolChoice === 'required') {\n const toolNames = Object.entries(toolCtx || {}).map(([name]) => name);\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.ANY,\n allowedFunctionNames: toolNames.length > 0 ? toolNames : undefined,\n },\n };\n } else if (toolChoice === 'auto') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.AUTO,\n },\n };\n } else if (toolChoice === 'none') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.NONE,\n },\n };\n } else {\n throw new Error(`Invalid tool choice: ${toolChoice}`);\n }\n\n extras.toolConfig = geminiToolConfig;\n }\n\n if (this.#opts.temperature !== undefined) {\n extras.temperature = this.#opts.temperature;\n }\n if (this.#opts.maxOutputTokens !== undefined) {\n extras.maxOutputTokens = this.#opts.maxOutputTokens;\n }\n if (this.#opts.topP !== undefined) {\n extras.topP = this.#opts.topP;\n }\n if (this.#opts.topK !== undefined) {\n extras.topK = this.#opts.topK;\n }\n if (this.#opts.presencePenalty !== undefined) {\n extras.presencePenalty = this.#opts.presencePenalty;\n }\n if (this.#opts.frequencyPenalty !== undefined) {\n extras.frequencyPenalty = this.#opts.frequencyPenalty;\n }\n if (this.#opts.seed !== undefined) {\n extras.seed = this.#opts.seed;\n }\n\n if (this.#opts.thinkingConfig !== undefined) {\n extras.thinkingConfig = this.#opts.thinkingConfig;\n }\n\n if (this.#opts.automaticFunctionCallingConfig !== undefined) {\n extras.automaticFunctionCalling = this.#opts.automaticFunctionCallingConfig;\n }\n\n geminiTools = geminiTools !== undefined ? geminiTools : this.#opts.geminiTools;\n\n return new LLMStream(this, {\n client: this.#client,\n model: this.#opts.model,\n chatCtx,\n toolCtx,\n connOptions,\n geminiTools,\n extraKwargs: extras,\n });\n }\n}\n\nexport class LLMStream extends llm.LLMStream {\n #client: GoogleGenAI;\n #model: string;\n #geminiTools?: LLMTools;\n #extraKwargs: GenerateContentConfig;\n\n constructor(\n llm: LLM,\n {\n client,\n model,\n chatCtx,\n toolCtx,\n connOptions,\n geminiTools,\n extraKwargs,\n }: {\n client: GoogleGenAI;\n model: string;\n chatCtx: llm.ChatContext;\n toolCtx?: llm.ToolContext;\n connOptions: APIConnectOptions;\n geminiTools?: LLMTools;\n extraKwargs: GenerateContentConfig;\n },\n ) {\n // Call base constructor with dev 1.0 object parameter pattern\n super(llm, { chatCtx, toolCtx, connOptions });\n this.#client = client;\n this.#model = model;\n this.#geminiTools = geminiTools;\n this.#extraKwargs = extraKwargs;\n }\n\n protected async run(): Promise {\n let retryable = true;\n const requestId = `google_${Date.now()}`;\n\n try {\n const [turns, extraData] = (await this.chatCtx.toProviderFormat('google')) as [\n Record[],\n GoogleFormatData,\n ];\n\n const contents: types.Content[] = turns.map((turn: Record) => ({\n role: turn.role as types.Content['role'],\n parts: turn.parts as types.Part[],\n }));\n\n const functionDeclarations = this.toolCtx ? toFunctionDeclarations(this.toolCtx) : undefined;\n const tools =\n functionDeclarations && functionDeclarations.length > 0\n ? [{ functionDeclarations }]\n : undefined;\n\n let systemInstruction: types.Content | undefined = undefined;\n if (extraData.systemMessages && extraData.systemMessages.length > 0) {\n systemInstruction = {\n parts: extraData.systemMessages.map((content: string) => ({ text: content })),\n };\n }\n\n const response = await this.#client.models.generateContentStream({\n model: this.#model,\n contents,\n config: {\n ...this.#extraKwargs,\n systemInstruction,\n httpOptions: this.#extraKwargs.httpOptions ?? {\n timeout: Math.floor(this.connOptions.timeoutMs),\n },\n tools,\n },\n });\n\n for await (const chunk of response) {\n if (chunk.promptFeedback) {\n throw new APIStatusError({\n message: `Prompt feedback error: ${JSON.stringify(chunk.promptFeedback)}`,\n options: {\n retryable: false,\n requestId,\n },\n });\n }\n\n if (!chunk.candidates || !chunk.candidates[0]?.content?.parts) {\n this.logger.warn(`No candidates in the response: ${JSON.stringify(chunk)}`);\n continue;\n }\n\n if (chunk.candidates.length > 1) {\n this.logger.warn(\n 'Google LLM: there are multiple candidates in the response, returning response from the first one.',\n );\n }\n\n for (const part of chunk.candidates[0].content.parts) {\n const chatChunk = this.#parsePart(requestId, part);\n if (chatChunk) {\n retryable = false;\n this.queue.put(chatChunk);\n }\n }\n\n if (chunk.usageMetadata) {\n const usage = chunk.usageMetadata;\n this.queue.put({\n id: requestId,\n usage: {\n completionTokens: usage.candidatesTokenCount || 0,\n promptTokens: usage.promptTokenCount || 0,\n promptCachedTokens: usage.cachedContentTokenCount || 0,\n totalTokens: usage.totalTokenCount || 0,\n },\n });\n }\n }\n } catch (error: unknown) {\n const err = error as {\n code?: number;\n message?: string;\n status?: string;\n type?: string;\n };\n\n if (err.code && err.code >= 400 && err.code < 500) {\n if (err.code === 429) {\n throw new APIStatusError({\n message: `Google LLM: Rate limit error - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: 429,\n retryable: true,\n },\n });\n } else {\n throw new APIStatusError({\n message: `Google LLM: Client error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: false,\n },\n });\n }\n }\n\n if (err.code && err.code >= 500) {\n throw new APIStatusError({\n message: `Google LLM: Server error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable,\n },\n });\n }\n\n throw new APIConnectionError({\n message: `Google LLM: API error - ${err.message || 'Unknown error'}`,\n options: {\n retryable,\n },\n });\n }\n }\n\n #parsePart(id: string, part: types.Part): llm.ChatChunk | null {\n if (part.functionCall) {\n return {\n id,\n delta: {\n role: 'assistant',\n toolCalls: [\n llm.FunctionCall.create({\n callId: part.functionCall.id || shortuuid('function_call_'),\n name: part.functionCall.name!,\n args: JSON.stringify(part.functionCall.args!),\n }),\n ],\n },\n };\n }\n\n return {\n id,\n delta: {\n content: part.text,\n role: 'assistant',\n },\n };\n }\n}\n"],"mappings":"AAIA,SAAS,2BAAuD,mBAAmB;AAEnF;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAGP,SAAS,8BAA8B;AA0BhC,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EAEA,QAAgB;AACd,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA8BA,YACE;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,IAAgB;AAAA,IACd,OAAO;AAAA,EACT,GACA;AACA,UAAM;AAEN,UAAM,cACJ,aACC,QAAQ,IAAI,8BAA8B,UACzC,QAAQ,IAAI,8BAA8B;AAE9C,QAAI,aAAiC,WAAW,QAAQ,IAAI;AAC5D,QAAI,cAAkC,YAAY,QAAQ,IAAI;AAC9D,QAAI,eAAmC,UAAU,QAAQ,IAAI;AAE7D,QAAI,aAAa;AACf,UAAI,CAAC,YAAY;AAEf,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,qBAAe;AAAA,IACjB,OAAO;AACL,mBAAa;AACb,oBAAc;AACd,UAAI,CAAC,cAAc;AACjB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,SAAI,iDAAgB,oBAAmB,QAAW;AAChD,YAAM,SAAS,eAAe;AAC9B,UAAI,SAAS,KAAK,SAAS,OAAO;AAChC,cAAM,IAAI,MAAM,kEAAkE;AAAA,MACpF;AAAA,IACF;AAEA,UAAM,gBAA0C,cAC5C;AAAA,MACE,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,IACZ,IACA;AAAA,MACE,QAAQ;AAAA,IACV;AAEJ,SAAK,UAAU,IAAI,YAAY,aAAa;AAE5C,SAAK,QAAQ;AAAA,MACX;AAAA,MACA,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAQc;AACZ,UAAM,SAAgC,EAAE,GAAG,YAAY;AAEvD,iBAAa,eAAe,SAAY,aAAa,KAAK,MAAM;AAEhE,QAAI,YAAY;AACd,UAAI;AAEJ,UAAI,OAAO,eAAe,YAAY,WAAW,SAAS,YAAY;AACpE,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,0BAA0B;AAAA,YAChC,sBAAsB,CAAC,WAAW,SAAS,IAAI;AAAA,UACjD;AAAA,QACF;AAAA,MACF,WAAW,eAAe,YAAY;AACpC,cAAM,YAAY,OAAO,QAAQ,WAAW,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,MAAM,IAAI;AACpE,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,0BAA0B;AAAA,YAChC,sBAAsB,UAAU,SAAS,IAAI,YAAY;AAAA,UAC3D;AAAA,QACF;AAAA,MACF,WAAW,eAAe,QAAQ;AAChC,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,0BAA0B;AAAA,UAClC;AAAA,QACF;AAAA,MACF,WAAW,eAAe,QAAQ;AAChC,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,0BAA0B;AAAA,UAClC;AAAA,QACF;AAAA,MACF,OAAO;AACL,cAAM,IAAI,MAAM,wBAAwB,UAAU,EAAE;AAAA,MACtD;AAEA,aAAO,aAAa;AAAA,IACtB;AAEA,QAAI,KAAK,MAAM,gBAAgB,QAAW;AACxC,aAAO,cAAc,KAAK,MAAM;AAAA,IAClC;AACA,QAAI,KAAK,MAAM,oBAAoB,QAAW;AAC5C,aAAO,kBAAkB,KAAK,MAAM;AAAA,IACtC;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AACA,QAAI,KAAK,MAAM,oBAAoB,QAAW;AAC5C,aAAO,kBAAkB,KAAK,MAAM;AAAA,IACtC;AACA,QAAI,KAAK,MAAM,qBAAqB,QAAW;AAC7C,aAAO,mBAAmB,KAAK,MAAM;AAAA,IACvC;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AAEA,QAAI,KAAK,MAAM,mBAAmB,QAAW;AAC3C,aAAO,iBAAiB,KAAK,MAAM;AAAA,IACrC;AAEA,QAAI,KAAK,MAAM,mCAAmC,QAAW;AAC3D,aAAO,2BAA2B,KAAK,MAAM;AAAA,IAC/C;AAEA,kBAAc,gBAAgB,SAAY,cAAc,KAAK,MAAM;AAEnE,WAAO,IAAI,UAAU,MAAM;AAAA,MACzB,QAAQ,KAAK;AAAA,MACb,OAAO,KAAK,MAAM;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa;AAAA,IACf,CAAC;AAAA,EACH;AACF;AAEO,MAAM,kBAAkB,IAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACEA,MACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GASA;AAEA,UAAMA,MAAK,EAAE,SAAS,SAAS,YAAY,CAAC;AAC5C,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,eAAe;AACpB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,MAAqB;AArTvC;AAsTI,QAAI,YAAY;AAChB,UAAM,YAAY,UAAU,KAAK,IAAI,CAAC;AAEtC,QAAI;AACF,YAAM,CAAC,OAAO,SAAS,IAAK,MAAM,KAAK,QAAQ,iBAAiB,QAAQ;AAKxE,YAAM,WAA4B,MAAM,IAAI,CAAC,UAAmC;AAAA,QAC9E,MAAM,KAAK;AAAA,QACX,OAAO,KAAK;AAAA,MACd,EAAE;AAEF,YAAM,uBAAuB,KAAK,UAAU,uBAAuB,KAAK,OAAO,IAAI;AACnF,YAAM,QACJ,wBAAwB,qBAAqB,SAAS,IAClD,CAAC,EAAE,qBAAqB,CAAC,IACzB;AAEN,UAAI,oBAA+C;AACnD,UAAI,UAAU,kBAAkB,UAAU,eAAe,SAAS,GAAG;AACnE,4BAAoB;AAAA,UAClB,OAAO,UAAU,eAAe,IAAI,CAAC,aAAqB,EAAE,MAAM,QAAQ,EAAE;AAAA,QAC9E;AAAA,MACF;AAEA,YAAM,WAAW,MAAM,KAAK,QAAQ,OAAO,sBAAsB;AAAA,QAC/D,OAAO,KAAK;AAAA,QACZ;AAAA,QACA,QAAQ;AAAA,UACN,GAAG,KAAK;AAAA,UACR;AAAA,UACA,aAAa,KAAK,aAAa,eAAe;AAAA,YAC5C,SAAS,KAAK,MAAM,KAAK,YAAY,SAAS;AAAA,UAChD;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAED,uBAAiB,SAAS,UAAU;AAClC,YAAI,MAAM,gBAAgB;AACxB,gBAAM,IAAI,eAAe;AAAA,YACvB,SAAS,0BAA0B,KAAK,UAAU,MAAM,cAAc,CAAC;AAAA,YACvE,SAAS;AAAA,cACP,WAAW;AAAA,cACX;AAAA,YACF;AAAA,UACF,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,MAAM,cAAc,GAAC,iBAAM,WAAW,CAAC,MAAlB,mBAAqB,YAArB,mBAA8B,QAAO;AAC7D,eAAK,OAAO,KAAK,kCAAkC,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1E;AAAA,QACF;AAEA,YAAI,MAAM,WAAW,SAAS,GAAG;AAC/B,eAAK,OAAO;AAAA,YACV;AAAA,UACF;AAAA,QACF;AAEA,mBAAW,QAAQ,MAAM,WAAW,CAAC,EAAE,QAAQ,OAAO;AACpD,gBAAM,YAAY,KAAK,WAAW,WAAW,IAAI;AACjD,cAAI,WAAW;AACb,wBAAY;AACZ,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAAA,QACF;AAEA,YAAI,MAAM,eAAe;AACvB,gBAAM,QAAQ,MAAM;AACpB,eAAK,MAAM,IAAI;AAAA,YACb,IAAI;AAAA,YACJ,OAAO;AAAA,cACL,kBAAkB,MAAM,wBAAwB;AAAA,cAChD,cAAc,MAAM,oBAAoB;AAAA,cACxC,oBAAoB,MAAM,2BAA2B;AAAA,cACrD,aAAa,MAAM,mBAAmB;AAAA,YACxC;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,SAAS,OAAgB;AACvB,YAAM,MAAM;AAOZ,UAAI,IAAI,QAAQ,IAAI,QAAQ,OAAO,IAAI,OAAO,KAAK;AACjD,YAAI,IAAI,SAAS,KAAK;AACpB,gBAAM,IAAI,eAAe;AAAA,YACvB,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,YACzE,SAAS;AAAA,cACP,YAAY;AAAA,cACZ,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH,OAAO;AACL,gBAAM,IAAI,eAAe;AAAA,YACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,YACnF,SAAS;AAAA,cACP,YAAY,IAAI;AAAA,cAChB,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,IAAI,QAAQ,IAAI,QAAQ,KAAK;AAC/B,cAAM,IAAI,eAAe;AAAA,UACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,UACnF,SAAS;AAAA,YACP,YAAY,IAAI;AAAA,YAChB;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH;AAEA,YAAM,IAAI,mBAAmB;AAAA,QAC3B,SAAS,2BAA2B,IAAI,WAAW,eAAe;AAAA,QAClE,SAAS;AAAA,UACP;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,WAAW,IAAY,MAAwC;AAC7D,QAAI,KAAK,cAAc;AACrB,aAAO;AAAA,QACL;AAAA,QACA,OAAO;AAAA,UACL,MAAM;AAAA,UACN,WAAW;AAAA,YACT,IAAI,aAAa,OAAO;AAAA,cACtB,QAAQ,KAAK,aAAa,MAAM,UAAU,gBAAgB;AAAA,cAC1D,MAAM,KAAK,aAAa;AAAA,cACxB,MAAM,KAAK,UAAU,KAAK,aAAa,IAAK;AAAA,YAC9C,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL;AAAA,MACA,OAAO;AAAA,QACL,SAAS,KAAK;AAAA,QACd,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;","names":["llm"]} \ No newline at end of file diff --git a/plugins/google/dist/llm.test.cjs b/plugins/google/dist/llm.test.cjs new file mode 100644 index 000000000..79eb0be52 --- /dev/null +++ b/plugins/google/dist/llm.test.cjs @@ -0,0 +1,14 @@ +"use strict"; +var import_agents_plugins_test = require("@livekit/agents-plugins-test"); +var import_vitest = require("vitest"); +var import_llm = require("./llm.cjs"); +(0, import_vitest.describe)("Google", async () => { + await (0, import_agents_plugins_test.llm)( + new import_llm.LLM({ + model: "gemini-2.5-flash", + temperature: 0 + }), + true + ); +}); +//# sourceMappingURL=llm.test.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/llm.test.cjs.map b/plugins/google/dist/llm.test.cjs.map new file mode 100644 index 000000000..40d74b59f --- /dev/null +++ b/plugins/google/dist/llm.test.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/llm.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm } from '@livekit/agents-plugins-test';\nimport { describe } from 'vitest';\nimport { LLM } from './llm.js';\n\ndescribe('Google', async () => {\n await llm(\n new LLM({\n model: 'gemini-2.5-flash',\n temperature: 0,\n }),\n true,\n );\n});\n"],"mappings":";AAGA,iCAAoB;AACpB,oBAAyB;AACzB,iBAAoB;AAAA,IAEpB,wBAAS,UAAU,YAAY;AAC7B,YAAM;AAAA,IACJ,IAAI,eAAI;AAAA,MACN,OAAO;AAAA,MACP,aAAa;AAAA,IACf,CAAC;AAAA,IACD;AAAA,EACF;AACF,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/llm.test.d.cts b/plugins/google/dist/llm.test.d.cts new file mode 100644 index 000000000..e16fd8dca --- /dev/null +++ b/plugins/google/dist/llm.test.d.cts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=llm.test.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/llm.test.d.ts b/plugins/google/dist/llm.test.d.ts new file mode 100644 index 000000000..e16fd8dca --- /dev/null +++ b/plugins/google/dist/llm.test.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=llm.test.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/llm.test.d.ts.map b/plugins/google/dist/llm.test.d.ts.map new file mode 100644 index 000000000..7666c651f --- /dev/null +++ b/plugins/google/dist/llm.test.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"llm.test.d.ts","sourceRoot":"","sources":["../src/llm.test.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/plugins/google/dist/llm.test.js b/plugins/google/dist/llm.test.js new file mode 100644 index 000000000..06c96c3b1 --- /dev/null +++ b/plugins/google/dist/llm.test.js @@ -0,0 +1,13 @@ +import { llm } from "@livekit/agents-plugins-test"; +import { describe } from "vitest"; +import { LLM } from "./llm.js"; +describe("Google", async () => { + await llm( + new LLM({ + model: "gemini-2.5-flash", + temperature: 0 + }), + true + ); +}); +//# sourceMappingURL=llm.test.js.map \ No newline at end of file diff --git a/plugins/google/dist/llm.test.js.map b/plugins/google/dist/llm.test.js.map new file mode 100644 index 000000000..1b9abe082 --- /dev/null +++ b/plugins/google/dist/llm.test.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/llm.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm } from '@livekit/agents-plugins-test';\nimport { describe } from 'vitest';\nimport { LLM } from './llm.js';\n\ndescribe('Google', async () => {\n await llm(\n new LLM({\n model: 'gemini-2.5-flash',\n temperature: 0,\n }),\n true,\n );\n});\n"],"mappings":"AAGA,SAAS,WAAW;AACpB,SAAS,gBAAgB;AACzB,SAAS,WAAW;AAEpB,SAAS,UAAU,YAAY;AAC7B,QAAM;AAAA,IACJ,IAAI,IAAI;AAAA,MACN,OAAO;AAAA,MACP,aAAa;AAAA,IACf,CAAC;AAAA,IACD;AAAA,EACF;AACF,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/models.cjs b/plugins/google/dist/models.cjs new file mode 100644 index 000000000..d97268935 --- /dev/null +++ b/plugins/google/dist/models.cjs @@ -0,0 +1,17 @@ +"use strict"; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var models_exports = {}; +module.exports = __toCommonJS(models_exports); +//# sourceMappingURL=models.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/models.cjs.map b/plugins/google/dist/models.cjs.map new file mode 100644 index 000000000..5d92bc5c1 --- /dev/null +++ b/plugins/google/dist/models.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/models.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n// Speech to Text v2\nexport type SpeechModels =\n | 'long'\n | 'short'\n | 'telephony'\n | 'medical_dictation'\n | 'medical_conversation'\n | 'chirp'\n | 'chirp_2'\n | 'latest_long'\n | 'latest_short';\n\nexport type SpeechLanguages =\n | 'en-US'\n | 'ja-JP'\n | 'en-IN'\n | 'en-GB'\n | 'hi-IN'\n | 'af-ZA'\n | 'sq-AL'\n | 'am-ET'\n | 'ar-EG'\n | 'hy-AM'\n | 'ast-ES'\n | 'az-AZ'\n | 'eu-ES'\n | 'be-BY'\n | 'bs-BA'\n | 'bg-BG'\n | 'my-MM'\n | 'ca-ES'\n | 'ceb-PH'\n | 'ckb-IQ'\n | 'zh-Hans-CN'\n | 'yue-Hant-HK'\n | 'zh-TW'\n | 'hr-HR'\n | 'cs-CZ'\n | 'da-DK'\n | 'nl-NL'\n | 'en-AU'\n | 'et-EE'\n | 'fil-PH'\n | 'fi-FI'\n | 'fr-CA'\n | 'fr-FR'\n | 'gl-ES'\n | 'ka-GE'\n | 'de-DE'\n | 'el-GR'\n | 'gu-IN'\n | 'ha-NG'\n | 'iw-IL'\n | 'hu-HU'\n | 'is-IS'\n | 'id-ID'\n | 'it-IT'\n | 'jv-ID'\n | 'kea-CV'\n | 'kam-KE'\n | 'kn-IN'\n | 'kk-KZ'\n | 'km-KH'\n | 'ko-KR'\n | 'ky-KG'\n | 'lo-LA'\n | 'lv-LV'\n | 'ln-CD'\n | 'lt-LT'\n | 'luo-KE'\n | 'lb-LU'\n | 'mk-MK'\n | 'no-NO'\n | 'pl-PL'\n | 'pt-BR'\n | 'pt-PT'\n | 'ro-RO'\n | 'ru-RU'\n | 'es-CO'\n | 'es-MX'\n | 'es-US'\n | 'th-TH'\n | 'tr-TR'\n | 'uk-UA'\n | 'vi-VN';\n\nexport type Gender = 'male' | 'female' | 'neutral';\n\nexport type ChatModels =\n | 'gemini-2.5-pro-preview-05-06'\n | 'gemini-2.5-flash-preview-04-17'\n | 'gemini-2.5-flash-preview-05-20'\n | 'gemini-2.0-flash-001'\n | 'gemini-2.0-flash-lite-preview-02-05'\n | 'gemini-2.0-pro-exp-02-05'\n | 'gemini-1.5-pro';\n"],"mappings":";;;;;;;;;;;;;;AAAA;AAAA;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/models.d.cts b/plugins/google/dist/models.d.cts new file mode 100644 index 000000000..1534100fd --- /dev/null +++ b/plugins/google/dist/models.d.cts @@ -0,0 +1,5 @@ +export type SpeechModels = 'long' | 'short' | 'telephony' | 'medical_dictation' | 'medical_conversation' | 'chirp' | 'chirp_2' | 'latest_long' | 'latest_short'; +export type SpeechLanguages = 'en-US' | 'ja-JP' | 'en-IN' | 'en-GB' | 'hi-IN' | 'af-ZA' | 'sq-AL' | 'am-ET' | 'ar-EG' | 'hy-AM' | 'ast-ES' | 'az-AZ' | 'eu-ES' | 'be-BY' | 'bs-BA' | 'bg-BG' | 'my-MM' | 'ca-ES' | 'ceb-PH' | 'ckb-IQ' | 'zh-Hans-CN' | 'yue-Hant-HK' | 'zh-TW' | 'hr-HR' | 'cs-CZ' | 'da-DK' | 'nl-NL' | 'en-AU' | 'et-EE' | 'fil-PH' | 'fi-FI' | 'fr-CA' | 'fr-FR' | 'gl-ES' | 'ka-GE' | 'de-DE' | 'el-GR' | 'gu-IN' | 'ha-NG' | 'iw-IL' | 'hu-HU' | 'is-IS' | 'id-ID' | 'it-IT' | 'jv-ID' | 'kea-CV' | 'kam-KE' | 'kn-IN' | 'kk-KZ' | 'km-KH' | 'ko-KR' | 'ky-KG' | 'lo-LA' | 'lv-LV' | 'ln-CD' | 'lt-LT' | 'luo-KE' | 'lb-LU' | 'mk-MK' | 'no-NO' | 'pl-PL' | 'pt-BR' | 'pt-PT' | 'ro-RO' | 'ru-RU' | 'es-CO' | 'es-MX' | 'es-US' | 'th-TH' | 'tr-TR' | 'uk-UA' | 'vi-VN'; +export type Gender = 'male' | 'female' | 'neutral'; +export type ChatModels = 'gemini-2.5-pro-preview-05-06' | 'gemini-2.5-flash-preview-04-17' | 'gemini-2.5-flash-preview-05-20' | 'gemini-2.0-flash-001' | 'gemini-2.0-flash-lite-preview-02-05' | 'gemini-2.0-pro-exp-02-05' | 'gemini-1.5-pro'; +//# sourceMappingURL=models.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/models.d.ts b/plugins/google/dist/models.d.ts new file mode 100644 index 000000000..1534100fd --- /dev/null +++ b/plugins/google/dist/models.d.ts @@ -0,0 +1,5 @@ +export type SpeechModels = 'long' | 'short' | 'telephony' | 'medical_dictation' | 'medical_conversation' | 'chirp' | 'chirp_2' | 'latest_long' | 'latest_short'; +export type SpeechLanguages = 'en-US' | 'ja-JP' | 'en-IN' | 'en-GB' | 'hi-IN' | 'af-ZA' | 'sq-AL' | 'am-ET' | 'ar-EG' | 'hy-AM' | 'ast-ES' | 'az-AZ' | 'eu-ES' | 'be-BY' | 'bs-BA' | 'bg-BG' | 'my-MM' | 'ca-ES' | 'ceb-PH' | 'ckb-IQ' | 'zh-Hans-CN' | 'yue-Hant-HK' | 'zh-TW' | 'hr-HR' | 'cs-CZ' | 'da-DK' | 'nl-NL' | 'en-AU' | 'et-EE' | 'fil-PH' | 'fi-FI' | 'fr-CA' | 'fr-FR' | 'gl-ES' | 'ka-GE' | 'de-DE' | 'el-GR' | 'gu-IN' | 'ha-NG' | 'iw-IL' | 'hu-HU' | 'is-IS' | 'id-ID' | 'it-IT' | 'jv-ID' | 'kea-CV' | 'kam-KE' | 'kn-IN' | 'kk-KZ' | 'km-KH' | 'ko-KR' | 'ky-KG' | 'lo-LA' | 'lv-LV' | 'ln-CD' | 'lt-LT' | 'luo-KE' | 'lb-LU' | 'mk-MK' | 'no-NO' | 'pl-PL' | 'pt-BR' | 'pt-PT' | 'ro-RO' | 'ru-RU' | 'es-CO' | 'es-MX' | 'es-US' | 'th-TH' | 'tr-TR' | 'uk-UA' | 'vi-VN'; +export type Gender = 'male' | 'female' | 'neutral'; +export type ChatModels = 'gemini-2.5-pro-preview-05-06' | 'gemini-2.5-flash-preview-04-17' | 'gemini-2.5-flash-preview-05-20' | 'gemini-2.0-flash-001' | 'gemini-2.0-flash-lite-preview-02-05' | 'gemini-2.0-pro-exp-02-05' | 'gemini-1.5-pro'; +//# sourceMappingURL=models.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/models.d.ts.map b/plugins/google/dist/models.d.ts.map new file mode 100644 index 000000000..8204511ae --- /dev/null +++ b/plugins/google/dist/models.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAKA,MAAM,MAAM,YAAY,GACpB,MAAM,GACN,OAAO,GACP,WAAW,GACX,mBAAmB,GACnB,sBAAsB,GACtB,OAAO,GACP,SAAS,GACT,aAAa,GACb,cAAc,CAAC;AAEnB,MAAM,MAAM,eAAe,GACvB,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,YAAY,GACZ,aAAa,GACb,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,CAAC;AAEZ,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEnD,MAAM,MAAM,UAAU,GAClB,8BAA8B,GAC9B,gCAAgC,GAChC,gCAAgC,GAChC,sBAAsB,GACtB,qCAAqC,GACrC,0BAA0B,GAC1B,gBAAgB,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/models.js b/plugins/google/dist/models.js new file mode 100644 index 000000000..5e38071cf --- /dev/null +++ b/plugins/google/dist/models.js @@ -0,0 +1 @@ +//# sourceMappingURL=models.js.map \ No newline at end of file diff --git a/plugins/google/dist/models.js.map b/plugins/google/dist/models.js.map new file mode 100644 index 000000000..84c51b288 --- /dev/null +++ b/plugins/google/dist/models.js.map @@ -0,0 +1 @@ +{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/tools.cjs b/plugins/google/dist/tools.cjs new file mode 100644 index 000000000..f14d3006c --- /dev/null +++ b/plugins/google/dist/tools.cjs @@ -0,0 +1,17 @@ +"use strict"; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var tools_exports = {}; +module.exports = __toCommonJS(tools_exports); +//# sourceMappingURL=tools.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/tools.cjs.map b/plugins/google/dist/tools.cjs.map new file mode 100644 index 000000000..174549578 --- /dev/null +++ b/plugins/google/dist/tools.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/tools.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Tool } from '@google/genai';\n\nexport type LLMTools = Omit;\n"],"mappings":";;;;;;;;;;;;;;AAAA;AAAA;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/tools.d.cts b/plugins/google/dist/tools.d.cts new file mode 100644 index 000000000..583504a9f --- /dev/null +++ b/plugins/google/dist/tools.d.cts @@ -0,0 +1,3 @@ +import type { Tool } from '@google/genai'; +export type LLMTools = Omit; +//# sourceMappingURL=tools.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/tools.d.ts b/plugins/google/dist/tools.d.ts new file mode 100644 index 000000000..583504a9f --- /dev/null +++ b/plugins/google/dist/tools.d.ts @@ -0,0 +1,3 @@ +import type { Tool } from '@google/genai'; +export type LLMTools = Omit; +//# sourceMappingURL=tools.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/tools.d.ts.map b/plugins/google/dist/tools.d.ts.map new file mode 100644 index 000000000..c82959cd7 --- /dev/null +++ b/plugins/google/dist/tools.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAE1C,MAAM,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,sBAAsB,CAAC,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/tools.js b/plugins/google/dist/tools.js new file mode 100644 index 000000000..4a36c860e --- /dev/null +++ b/plugins/google/dist/tools.js @@ -0,0 +1 @@ +//# sourceMappingURL=tools.js.map \ No newline at end of file diff --git a/plugins/google/dist/tools.js.map b/plugins/google/dist/tools.js.map new file mode 100644 index 000000000..84c51b288 --- /dev/null +++ b/plugins/google/dist/tools.js.map @@ -0,0 +1 @@ +{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/utils.cjs b/plugins/google/dist/utils.cjs new file mode 100644 index 000000000..e64730020 --- /dev/null +++ b/plugins/google/dist/utils.cjs @@ -0,0 +1,137 @@ +"use strict"; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var utils_exports = {}; +__export(utils_exports, { + convertJSONSchemaToOpenAPISchema: () => convertJSONSchemaToOpenAPISchema, + toFunctionDeclarations: () => toFunctionDeclarations +}); +module.exports = __toCommonJS(utils_exports); +var import_agents = require("@livekit/agents"); +function convertJSONSchemaToOpenAPISchema(jsonSchema) { + if (jsonSchema == null || isEmptyObjectSchema(jsonSchema)) { + return void 0; + } + if (typeof jsonSchema === "boolean") { + return { type: "boolean", properties: {} }; + } + const { + type, + description, + required, + properties, + items, + allOf, + anyOf, + oneOf, + format, + const: constValue, + minLength, + enum: enumValues + } = jsonSchema; + const result = {}; + if (description) result.description = description; + if (required) result.required = required; + if (format) result.format = format; + if (constValue !== void 0) { + result.enum = [constValue]; + } + if (type) { + if (Array.isArray(type)) { + if (type.includes("null")) { + result.type = type.filter((t) => t !== "null")[0]; + result.nullable = true; + } else { + result.type = type; + } + } else if (type === "null") { + result.type = "null"; + } else { + result.type = type; + } + } + if (enumValues !== void 0) { + result.enum = enumValues; + } + if (properties != null) { + result.properties = Object.entries(properties).reduce( + (acc, [key, value]) => { + acc[key] = convertJSONSchemaToOpenAPISchema(value); + return acc; + }, + {} + ); + } + if (items) { + result.items = Array.isArray(items) ? items.map(convertJSONSchemaToOpenAPISchema) : convertJSONSchemaToOpenAPISchema(items); + } + if (allOf) { + result.allOf = allOf.map(convertJSONSchemaToOpenAPISchema); + } + if (anyOf) { + if (anyOf.some((schema) => typeof schema === "object" && (schema == null ? void 0 : schema.type) === "null")) { + const nonNullSchemas = anyOf.filter( + (schema) => !(typeof schema === "object" && (schema == null ? void 0 : schema.type) === "null") + ); + if (nonNullSchemas.length === 1) { + const converted = convertJSONSchemaToOpenAPISchema( + nonNullSchemas[0] + ); + if (typeof converted === "object") { + result.nullable = true; + Object.assign(result, converted); + } + } else { + result.anyOf = nonNullSchemas.map(convertJSONSchemaToOpenAPISchema); + result.nullable = true; + } + } else { + result.anyOf = anyOf.map(convertJSONSchemaToOpenAPISchema); + } + } + if (oneOf) { + result.oneOf = oneOf.map(convertJSONSchemaToOpenAPISchema); + } + if (minLength !== void 0) { + result.minLength = minLength; + } + return result; +} +function isEmptyObjectSchema(jsonSchema) { + return jsonSchema != null && typeof jsonSchema === "object" && jsonSchema.type === "object" && (jsonSchema.properties == null || Object.keys(jsonSchema.properties).length === 0) && !jsonSchema.additionalProperties; +} +function toFunctionDeclarations(toolCtx) { + const functionDeclarations = []; + for (const [name, tool] of Object.entries(toolCtx)) { + const { description, parameters } = tool; + const jsonSchema = import_agents.llm.toJsonSchema(parameters, false); + const schemaCopy = JSON.parse(JSON.stringify(jsonSchema)); + functionDeclarations.push({ + name, + description, + parameters: convertJSONSchemaToOpenAPISchema(schemaCopy) + }); + } + return functionDeclarations; +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + convertJSONSchemaToOpenAPISchema, + toFunctionDeclarations +}); +//# sourceMappingURL=utils.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/utils.cjs.map b/plugins/google/dist/utils.cjs.map new file mode 100644 index 000000000..80fb8001b --- /dev/null +++ b/plugins/google/dist/utils.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { FunctionDeclaration, Schema } from '@google/genai';\nimport { llm } from '@livekit/agents';\nimport type { JSONSchema7 } from 'json-schema';\n\n/**\n * JSON Schema v7\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01\n */\nexport type JSONSchema7Definition = JSONSchema7 | boolean;\n\n/**\n * Converts JSON Schema 7 to OpenAPI Schema 3.0\n */\n// Adapted from https://github.com/vercel/ai/blob/main/packages/google/src/convert-json-schema-to-openapi-schema.ts\nexport function convertJSONSchemaToOpenAPISchema(jsonSchema: JSONSchema7Definition): unknown {\n // parameters need to be undefined if they are empty objects:\n if (jsonSchema == null || isEmptyObjectSchema(jsonSchema)) {\n return undefined;\n }\n\n if (typeof jsonSchema === 'boolean') {\n return { type: 'boolean', properties: {} };\n }\n\n const {\n type,\n description,\n required,\n properties,\n items,\n allOf,\n anyOf,\n oneOf,\n format,\n const: constValue,\n minLength,\n enum: enumValues,\n } = jsonSchema;\n\n const result: Record = {};\n\n if (description) result.description = description;\n if (required) result.required = required;\n if (format) result.format = format;\n\n if (constValue !== undefined) {\n result.enum = [constValue];\n }\n\n // Handle type\n if (type) {\n if (Array.isArray(type)) {\n if (type.includes('null')) {\n result.type = type.filter((t) => t !== 'null')[0];\n result.nullable = true;\n } else {\n result.type = type;\n }\n } else if (type === 'null') {\n result.type = 'null';\n } else {\n result.type = type;\n }\n }\n\n // Handle enum\n if (enumValues !== undefined) {\n result.enum = enumValues;\n }\n\n if (properties != null) {\n result.properties = Object.entries(properties).reduce(\n (acc, [key, value]) => {\n acc[key] = convertJSONSchemaToOpenAPISchema(value);\n return acc;\n },\n {} as Record,\n );\n }\n\n if (items) {\n result.items = Array.isArray(items)\n ? items.map(convertJSONSchemaToOpenAPISchema)\n : convertJSONSchemaToOpenAPISchema(items);\n }\n\n if (allOf) {\n result.allOf = allOf.map(convertJSONSchemaToOpenAPISchema);\n }\n if (anyOf) {\n // Handle cases where anyOf includes a null type\n if (anyOf.some((schema) => typeof schema === 'object' && schema?.type === 'null')) {\n const nonNullSchemas = anyOf.filter(\n (schema) => !(typeof schema === 'object' && schema?.type === 'null'),\n );\n\n if (nonNullSchemas.length === 1) {\n // If there's only one non-null schema, convert it and make it nullable\n const converted = convertJSONSchemaToOpenAPISchema(\n nonNullSchemas[0] as JSONSchema7Definition,\n );\n if (typeof converted === 'object') {\n result.nullable = true;\n Object.assign(result, converted);\n }\n } else {\n // If there are multiple non-null schemas, keep them in anyOf\n result.anyOf = nonNullSchemas.map(convertJSONSchemaToOpenAPISchema);\n result.nullable = true;\n }\n } else {\n result.anyOf = anyOf.map(convertJSONSchemaToOpenAPISchema);\n }\n }\n if (oneOf) {\n result.oneOf = oneOf.map(convertJSONSchemaToOpenAPISchema);\n }\n\n if (minLength !== undefined) {\n result.minLength = minLength;\n }\n\n return result;\n}\n\nfunction isEmptyObjectSchema(jsonSchema: JSONSchema7Definition): boolean {\n return (\n jsonSchema != null &&\n typeof jsonSchema === 'object' &&\n jsonSchema.type === 'object' &&\n (jsonSchema.properties == null || Object.keys(jsonSchema.properties).length === 0) &&\n !jsonSchema.additionalProperties\n );\n}\n\nexport function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[] {\n const functionDeclarations: FunctionDeclaration[] = [];\n\n for (const [name, tool] of Object.entries(toolCtx)) {\n const { description, parameters } = tool;\n const jsonSchema = llm.toJsonSchema(parameters, false);\n\n // Create a deep copy to prevent the Google GenAI library from mutating the schema\n const schemaCopy = JSON.parse(JSON.stringify(jsonSchema));\n\n functionDeclarations.push({\n name,\n description,\n parameters: convertJSONSchemaToOpenAPISchema(schemaCopy) as Schema,\n });\n }\n\n return functionDeclarations;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,oBAAoB;AAab,SAAS,iCAAiC,YAA4C;AAE3F,MAAI,cAAc,QAAQ,oBAAoB,UAAU,GAAG;AACzD,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,eAAe,WAAW;AACnC,WAAO,EAAE,MAAM,WAAW,YAAY,CAAC,EAAE;AAAA,EAC3C;AAEA,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP;AAAA,IACA,MAAM;AAAA,EACR,IAAI;AAEJ,QAAM,SAAkC,CAAC;AAEzC,MAAI,YAAa,QAAO,cAAc;AACtC,MAAI,SAAU,QAAO,WAAW;AAChC,MAAI,OAAQ,QAAO,SAAS;AAE5B,MAAI,eAAe,QAAW;AAC5B,WAAO,OAAO,CAAC,UAAU;AAAA,EAC3B;AAGA,MAAI,MAAM;AACR,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,UAAI,KAAK,SAAS,MAAM,GAAG;AACzB,eAAO,OAAO,KAAK,OAAO,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC;AAChD,eAAO,WAAW;AAAA,MACpB,OAAO;AACL,eAAO,OAAO;AAAA,MAChB;AAAA,IACF,WAAW,SAAS,QAAQ;AAC1B,aAAO,OAAO;AAAA,IAChB,OAAO;AACL,aAAO,OAAO;AAAA,IAChB;AAAA,EACF;AAGA,MAAI,eAAe,QAAW;AAC5B,WAAO,OAAO;AAAA,EAChB;AAEA,MAAI,cAAc,MAAM;AACtB,WAAO,aAAa,OAAO,QAAQ,UAAU,EAAE;AAAA,MAC7C,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM;AACrB,YAAI,GAAG,IAAI,iCAAiC,KAAK;AACjD,eAAO;AAAA,MACT;AAAA,MACA,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,QAAQ,KAAK,IAC9B,MAAM,IAAI,gCAAgC,IAC1C,iCAAiC,KAAK;AAAA,EAC5C;AAEA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,EAC3D;AACA,MAAI,OAAO;AAET,QAAI,MAAM,KAAK,CAAC,WAAW,OAAO,WAAW,aAAY,iCAAQ,UAAS,MAAM,GAAG;AACjF,YAAM,iBAAiB,MAAM;AAAA,QAC3B,CAAC,WAAW,EAAE,OAAO,WAAW,aAAY,iCAAQ,UAAS;AAAA,MAC/D;AAEA,UAAI,eAAe,WAAW,GAAG;AAE/B,cAAM,YAAY;AAAA,UAChB,eAAe,CAAC;AAAA,QAClB;AACA,YAAI,OAAO,cAAc,UAAU;AACjC,iBAAO,WAAW;AAClB,iBAAO,OAAO,QAAQ,SAAS;AAAA,QACjC;AAAA,MACF,OAAO;AAEL,eAAO,QAAQ,eAAe,IAAI,gCAAgC;AAClE,eAAO,WAAW;AAAA,MACpB;AAAA,IACF,OAAO;AACL,aAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,IAC3D;AAAA,EACF;AACA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,EAC3D;AAEA,MAAI,cAAc,QAAW;AAC3B,WAAO,YAAY;AAAA,EACrB;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,YAA4C;AACvE,SACE,cAAc,QACd,OAAO,eAAe,YACtB,WAAW,SAAS,aACnB,WAAW,cAAc,QAAQ,OAAO,KAAK,WAAW,UAAU,EAAE,WAAW,MAChF,CAAC,WAAW;AAEhB;AAEO,SAAS,uBAAuB,SAAiD;AACtF,QAAM,uBAA8C,CAAC;AAErD,aAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AAClD,UAAM,EAAE,aAAa,WAAW,IAAI;AACpC,UAAM,aAAa,kBAAI,aAAa,YAAY,KAAK;AAGrD,UAAM,aAAa,KAAK,MAAM,KAAK,UAAU,UAAU,CAAC;AAExD,yBAAqB,KAAK;AAAA,MACxB;AAAA,MACA;AAAA,MACA,YAAY,iCAAiC,UAAU;AAAA,IACzD,CAAC;AAAA,EACH;AAEA,SAAO;AACT;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/utils.d.cts b/plugins/google/dist/utils.d.cts new file mode 100644 index 000000000..a4826d8a4 --- /dev/null +++ b/plugins/google/dist/utils.d.cts @@ -0,0 +1,14 @@ +import type { FunctionDeclaration } from '@google/genai'; +import { llm } from '@livekit/agents'; +import type { JSONSchema7 } from 'json-schema'; +/** + * JSON Schema v7 + * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01 + */ +export type JSONSchema7Definition = JSONSchema7 | boolean; +/** + * Converts JSON Schema 7 to OpenAPI Schema 3.0 + */ +export declare function convertJSONSchemaToOpenAPISchema(jsonSchema: JSONSchema7Definition): unknown; +export declare function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[]; +//# sourceMappingURL=utils.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/utils.d.ts b/plugins/google/dist/utils.d.ts new file mode 100644 index 000000000..a4826d8a4 --- /dev/null +++ b/plugins/google/dist/utils.d.ts @@ -0,0 +1,14 @@ +import type { FunctionDeclaration } from '@google/genai'; +import { llm } from '@livekit/agents'; +import type { JSONSchema7 } from 'json-schema'; +/** + * JSON Schema v7 + * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01 + */ +export type JSONSchema7Definition = JSONSchema7 | boolean; +/** + * Converts JSON Schema 7 to OpenAPI Schema 3.0 + */ +export declare function convertJSONSchemaToOpenAPISchema(jsonSchema: JSONSchema7Definition): unknown; +export declare function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[]; +//# sourceMappingURL=utils.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/utils.d.ts.map b/plugins/google/dist/utils.d.ts.map new file mode 100644 index 000000000..9b3bef578 --- /dev/null +++ b/plugins/google/dist/utils.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,mBAAmB,EAAU,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,WAAW,GAAG,OAAO,CAAC;AAE1D;;GAEG;AAEH,wBAAgB,gCAAgC,CAAC,UAAU,EAAE,qBAAqB,GAAG,OAAO,CA6G3F;AAYD,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,mBAAmB,EAAE,CAkBtF"} \ No newline at end of file diff --git a/plugins/google/dist/utils.js b/plugins/google/dist/utils.js new file mode 100644 index 000000000..f551ada4d --- /dev/null +++ b/plugins/google/dist/utils.js @@ -0,0 +1,112 @@ +import { llm } from "@livekit/agents"; +function convertJSONSchemaToOpenAPISchema(jsonSchema) { + if (jsonSchema == null || isEmptyObjectSchema(jsonSchema)) { + return void 0; + } + if (typeof jsonSchema === "boolean") { + return { type: "boolean", properties: {} }; + } + const { + type, + description, + required, + properties, + items, + allOf, + anyOf, + oneOf, + format, + const: constValue, + minLength, + enum: enumValues + } = jsonSchema; + const result = {}; + if (description) result.description = description; + if (required) result.required = required; + if (format) result.format = format; + if (constValue !== void 0) { + result.enum = [constValue]; + } + if (type) { + if (Array.isArray(type)) { + if (type.includes("null")) { + result.type = type.filter((t) => t !== "null")[0]; + result.nullable = true; + } else { + result.type = type; + } + } else if (type === "null") { + result.type = "null"; + } else { + result.type = type; + } + } + if (enumValues !== void 0) { + result.enum = enumValues; + } + if (properties != null) { + result.properties = Object.entries(properties).reduce( + (acc, [key, value]) => { + acc[key] = convertJSONSchemaToOpenAPISchema(value); + return acc; + }, + {} + ); + } + if (items) { + result.items = Array.isArray(items) ? items.map(convertJSONSchemaToOpenAPISchema) : convertJSONSchemaToOpenAPISchema(items); + } + if (allOf) { + result.allOf = allOf.map(convertJSONSchemaToOpenAPISchema); + } + if (anyOf) { + if (anyOf.some((schema) => typeof schema === "object" && (schema == null ? void 0 : schema.type) === "null")) { + const nonNullSchemas = anyOf.filter( + (schema) => !(typeof schema === "object" && (schema == null ? void 0 : schema.type) === "null") + ); + if (nonNullSchemas.length === 1) { + const converted = convertJSONSchemaToOpenAPISchema( + nonNullSchemas[0] + ); + if (typeof converted === "object") { + result.nullable = true; + Object.assign(result, converted); + } + } else { + result.anyOf = nonNullSchemas.map(convertJSONSchemaToOpenAPISchema); + result.nullable = true; + } + } else { + result.anyOf = anyOf.map(convertJSONSchemaToOpenAPISchema); + } + } + if (oneOf) { + result.oneOf = oneOf.map(convertJSONSchemaToOpenAPISchema); + } + if (minLength !== void 0) { + result.minLength = minLength; + } + return result; +} +function isEmptyObjectSchema(jsonSchema) { + return jsonSchema != null && typeof jsonSchema === "object" && jsonSchema.type === "object" && (jsonSchema.properties == null || Object.keys(jsonSchema.properties).length === 0) && !jsonSchema.additionalProperties; +} +function toFunctionDeclarations(toolCtx) { + const functionDeclarations = []; + for (const [name, tool] of Object.entries(toolCtx)) { + const { description, parameters } = tool; + const jsonSchema = llm.toJsonSchema(parameters, false); + const schemaCopy = JSON.parse(JSON.stringify(jsonSchema)); + functionDeclarations.push({ + name, + description, + parameters: convertJSONSchemaToOpenAPISchema(schemaCopy) + }); + } + return functionDeclarations; +} +export { + convertJSONSchemaToOpenAPISchema, + toFunctionDeclarations +}; +//# sourceMappingURL=utils.js.map \ No newline at end of file diff --git a/plugins/google/dist/utils.js.map b/plugins/google/dist/utils.js.map new file mode 100644 index 000000000..a5ec8f6c1 --- /dev/null +++ b/plugins/google/dist/utils.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { FunctionDeclaration, Schema } from '@google/genai';\nimport { llm } from '@livekit/agents';\nimport type { JSONSchema7 } from 'json-schema';\n\n/**\n * JSON Schema v7\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01\n */\nexport type JSONSchema7Definition = JSONSchema7 | boolean;\n\n/**\n * Converts JSON Schema 7 to OpenAPI Schema 3.0\n */\n// Adapted from https://github.com/vercel/ai/blob/main/packages/google/src/convert-json-schema-to-openapi-schema.ts\nexport function convertJSONSchemaToOpenAPISchema(jsonSchema: JSONSchema7Definition): unknown {\n // parameters need to be undefined if they are empty objects:\n if (jsonSchema == null || isEmptyObjectSchema(jsonSchema)) {\n return undefined;\n }\n\n if (typeof jsonSchema === 'boolean') {\n return { type: 'boolean', properties: {} };\n }\n\n const {\n type,\n description,\n required,\n properties,\n items,\n allOf,\n anyOf,\n oneOf,\n format,\n const: constValue,\n minLength,\n enum: enumValues,\n } = jsonSchema;\n\n const result: Record = {};\n\n if (description) result.description = description;\n if (required) result.required = required;\n if (format) result.format = format;\n\n if (constValue !== undefined) {\n result.enum = [constValue];\n }\n\n // Handle type\n if (type) {\n if (Array.isArray(type)) {\n if (type.includes('null')) {\n result.type = type.filter((t) => t !== 'null')[0];\n result.nullable = true;\n } else {\n result.type = type;\n }\n } else if (type === 'null') {\n result.type = 'null';\n } else {\n result.type = type;\n }\n }\n\n // Handle enum\n if (enumValues !== undefined) {\n result.enum = enumValues;\n }\n\n if (properties != null) {\n result.properties = Object.entries(properties).reduce(\n (acc, [key, value]) => {\n acc[key] = convertJSONSchemaToOpenAPISchema(value);\n return acc;\n },\n {} as Record,\n );\n }\n\n if (items) {\n result.items = Array.isArray(items)\n ? items.map(convertJSONSchemaToOpenAPISchema)\n : convertJSONSchemaToOpenAPISchema(items);\n }\n\n if (allOf) {\n result.allOf = allOf.map(convertJSONSchemaToOpenAPISchema);\n }\n if (anyOf) {\n // Handle cases where anyOf includes a null type\n if (anyOf.some((schema) => typeof schema === 'object' && schema?.type === 'null')) {\n const nonNullSchemas = anyOf.filter(\n (schema) => !(typeof schema === 'object' && schema?.type === 'null'),\n );\n\n if (nonNullSchemas.length === 1) {\n // If there's only one non-null schema, convert it and make it nullable\n const converted = convertJSONSchemaToOpenAPISchema(\n nonNullSchemas[0] as JSONSchema7Definition,\n );\n if (typeof converted === 'object') {\n result.nullable = true;\n Object.assign(result, converted);\n }\n } else {\n // If there are multiple non-null schemas, keep them in anyOf\n result.anyOf = nonNullSchemas.map(convertJSONSchemaToOpenAPISchema);\n result.nullable = true;\n }\n } else {\n result.anyOf = anyOf.map(convertJSONSchemaToOpenAPISchema);\n }\n }\n if (oneOf) {\n result.oneOf = oneOf.map(convertJSONSchemaToOpenAPISchema);\n }\n\n if (minLength !== undefined) {\n result.minLength = minLength;\n }\n\n return result;\n}\n\nfunction isEmptyObjectSchema(jsonSchema: JSONSchema7Definition): boolean {\n return (\n jsonSchema != null &&\n typeof jsonSchema === 'object' &&\n jsonSchema.type === 'object' &&\n (jsonSchema.properties == null || Object.keys(jsonSchema.properties).length === 0) &&\n !jsonSchema.additionalProperties\n );\n}\n\nexport function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[] {\n const functionDeclarations: FunctionDeclaration[] = [];\n\n for (const [name, tool] of Object.entries(toolCtx)) {\n const { description, parameters } = tool;\n const jsonSchema = llm.toJsonSchema(parameters, false);\n\n // Create a deep copy to prevent the Google GenAI library from mutating the schema\n const schemaCopy = JSON.parse(JSON.stringify(jsonSchema));\n\n functionDeclarations.push({\n name,\n description,\n parameters: convertJSONSchemaToOpenAPISchema(schemaCopy) as Schema,\n });\n }\n\n return functionDeclarations;\n}\n"],"mappings":"AAIA,SAAS,WAAW;AAab,SAAS,iCAAiC,YAA4C;AAE3F,MAAI,cAAc,QAAQ,oBAAoB,UAAU,GAAG;AACzD,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,eAAe,WAAW;AACnC,WAAO,EAAE,MAAM,WAAW,YAAY,CAAC,EAAE;AAAA,EAC3C;AAEA,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP;AAAA,IACA,MAAM;AAAA,EACR,IAAI;AAEJ,QAAM,SAAkC,CAAC;AAEzC,MAAI,YAAa,QAAO,cAAc;AACtC,MAAI,SAAU,QAAO,WAAW;AAChC,MAAI,OAAQ,QAAO,SAAS;AAE5B,MAAI,eAAe,QAAW;AAC5B,WAAO,OAAO,CAAC,UAAU;AAAA,EAC3B;AAGA,MAAI,MAAM;AACR,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,UAAI,KAAK,SAAS,MAAM,GAAG;AACzB,eAAO,OAAO,KAAK,OAAO,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC;AAChD,eAAO,WAAW;AAAA,MACpB,OAAO;AACL,eAAO,OAAO;AAAA,MAChB;AAAA,IACF,WAAW,SAAS,QAAQ;AAC1B,aAAO,OAAO;AAAA,IAChB,OAAO;AACL,aAAO,OAAO;AAAA,IAChB;AAAA,EACF;AAGA,MAAI,eAAe,QAAW;AAC5B,WAAO,OAAO;AAAA,EAChB;AAEA,MAAI,cAAc,MAAM;AACtB,WAAO,aAAa,OAAO,QAAQ,UAAU,EAAE;AAAA,MAC7C,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM;AACrB,YAAI,GAAG,IAAI,iCAAiC,KAAK;AACjD,eAAO;AAAA,MACT;AAAA,MACA,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,QAAQ,KAAK,IAC9B,MAAM,IAAI,gCAAgC,IAC1C,iCAAiC,KAAK;AAAA,EAC5C;AAEA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,EAC3D;AACA,MAAI,OAAO;AAET,QAAI,MAAM,KAAK,CAAC,WAAW,OAAO,WAAW,aAAY,iCAAQ,UAAS,MAAM,GAAG;AACjF,YAAM,iBAAiB,MAAM;AAAA,QAC3B,CAAC,WAAW,EAAE,OAAO,WAAW,aAAY,iCAAQ,UAAS;AAAA,MAC/D;AAEA,UAAI,eAAe,WAAW,GAAG;AAE/B,cAAM,YAAY;AAAA,UAChB,eAAe,CAAC;AAAA,QAClB;AACA,YAAI,OAAO,cAAc,UAAU;AACjC,iBAAO,WAAW;AAClB,iBAAO,OAAO,QAAQ,SAAS;AAAA,QACjC;AAAA,MACF,OAAO;AAEL,eAAO,QAAQ,eAAe,IAAI,gCAAgC;AAClE,eAAO,WAAW;AAAA,MACpB;AAAA,IACF,OAAO;AACL,aAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,IAC3D;AAAA,EACF;AACA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,EAC3D;AAEA,MAAI,cAAc,QAAW;AAC3B,WAAO,YAAY;AAAA,EACrB;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,YAA4C;AACvE,SACE,cAAc,QACd,OAAO,eAAe,YACtB,WAAW,SAAS,aACnB,WAAW,cAAc,QAAQ,OAAO,KAAK,WAAW,UAAU,EAAE,WAAW,MAChF,CAAC,WAAW;AAEhB;AAEO,SAAS,uBAAuB,SAAiD;AACtF,QAAM,uBAA8C,CAAC;AAErD,aAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AAClD,UAAM,EAAE,aAAa,WAAW,IAAI;AACpC,UAAM,aAAa,IAAI,aAAa,YAAY,KAAK;AAGrD,UAAM,aAAa,KAAK,MAAM,KAAK,UAAU,UAAU,CAAC;AAExD,yBAAqB,KAAK;AAAA,MACxB;AAAA,MACA;AAAA,MACA,YAAY,iCAAiC,UAAU;AAAA,IACzD,CAAC;AAAA,EACH;AAEA,SAAO;AACT;","names":[]} \ No newline at end of file From 73092589ab3e51138155662421e3576d914ab9cd Mon Sep 17 00:00:00 2001 From: oOPa <5878650+oOPa@users.noreply.github.com> Date: Sun, 21 Dec 2025 13:48:17 +0000 Subject: [PATCH 3/9] fix: update `@google/genai` dependency and pass `thinkingConfig` at the top-level of the session configuration. --- .../dist/beta/realtime/realtime_api.cjs | 8 +- .../dist/beta/realtime/realtime_api.cjs.map | 2 +- .../dist/beta/realtime/realtime_api.d.ts.map | 2 +- .../google/dist/beta/realtime/realtime_api.js | 8 +- .../dist/beta/realtime/realtime_api.js.map | 2 +- plugins/google/package.json | 2 +- .../google/src/beta/realtime/realtime_api.ts | 11 +- pnpm-lock.yaml | 158 +++++++++++------- 8 files changed, 111 insertions(+), 82 deletions(-) diff --git a/plugins/google/dist/beta/realtime/realtime_api.cjs b/plugins/google/dist/beta/realtime/realtime_api.cjs index 4cead1cd0..7c21ebd6c 100644 --- a/plugins/google/dist/beta/realtime/realtime_api.cjs +++ b/plugins/google/dist/beta/realtime/realtime_api.cjs @@ -729,7 +729,8 @@ class RealtimeSession extends import_agents.llm.RealtimeSession { outputAudioTranscription: opts.outputAudioTranscription, sessionResumption: { handle: this.sessionResumptionHandle - } + }, + thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : void 0 }; if (opts.temperature !== void 0) { config.temperature = opts.temperature; @@ -755,11 +756,6 @@ class RealtimeSession extends import_agents.llm.RealtimeSession { if (opts.contextWindowCompression !== void 0) { config.contextWindowCompression = opts.contextWindowCompression; } - if (opts.thinkingConfig !== void 0) { - config.generationConfig = { - thinkingConfig: opts.thinkingConfig - }; - } return config; } startNewGeneration() { diff --git a/plugins/google/dist/beta/realtime/realtime_api.cjs.map b/plugins/google/dist/beta/realtime/realtime_api.cjs.map index f46212026..7afd6c23b 100644 --- a/plugins/google/dist/beta/realtime/realtime_api.cjs.map +++ b/plugins/google/dist/beta/realtime/realtime_api.cjs.map @@ -1 +1 @@ -{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n if (this.activeSession) {\n try {\n await this.activeSession.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n } finally {\n this.activeSession = undefined;\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n this.messageChannel = new Queue();\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error('Gemini Live session error:', error);\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n this.#logger.debug('Gemini Live session closed:', event.code, event.reason);\n this.markCurrentGenerationDone();\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n if (opts.thinkingConfig !== undefined) {\n config.generationConfig = {\n thinkingConfig: opts.thinkingConfig,\n };\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() { }\n\n async clearAudio() { }\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,YAAuB;AACvB,mBAQO;AAEP,oBAcO;AACP,mBAAsB;AACtB,sBAA4D;AAC5D,mBAA8B;AAC9B,mBAAuC;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,kBAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,sBAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,sBAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,kBAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,kBAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,oBAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,oBAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,mBAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI,KAAK,eAAe;AACtB,UAAI;AACF,cAAM,KAAK,cAAc,MAAM;AAAA,MACjC,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D,UAAE;AACA,aAAK,gBAAgB;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAC5B,WAAK,iBAAiB,IAAI,oBAAM;AAAA,IAClC;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,kBAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,kBAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,sBAAkB,qCAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AAzlBzC;AA0lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,qBAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AApsBpB;AAssBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,8BAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,oBAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,8BAA8B,KAAK;AACtD,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,+BAA+B,MAAM,MAAM,MAAM,MAAM;AAC1E,mBAAK,0BAA0B;AAAA,YACjC;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,mBAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,mBAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,oBAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,kBAAM,6BAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,kBAAM,qBAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA55BnB;AA85BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AA/5Bb,YAAAA;AA+5BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAv+B7B;AAw+BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAngC/F;AAogCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,IACF;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,QAAI,KAAK,mBAAmB,QAAW;AACrC,aAAO,mBAAmB;AAAA,QACxB,gBAAgB,KAAK;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,iBAAa,yBAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,qBAAO,oBAA2C;AAAA,MAClE,iBAAiB,qBAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,aAAS,yBAAU,KAAK;AAAA,MACxB,aAAa,qBAAO,oBAA4B;AAAA,MAChD,cAAc,qBAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,UAAM,yBAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAE;AAAA,EAEtB,MAAM,aAAa;AAAA,EAAE;AAAA,EAErB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file +{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n if (this.activeSession) {\n try {\n await this.activeSession.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n } finally {\n this.activeSession = undefined;\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n this.messageChannel = new Queue();\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error('Gemini Live session error:', error);\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n this.#logger.debug('Gemini Live session closed:', event.code, event.reason);\n this.markCurrentGenerationDone();\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined,\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() { }\n\n async clearAudio() { }\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,YAAuB;AACvB,mBAQO;AAEP,oBAcO;AACP,mBAAsB;AACtB,sBAA4D;AAC5D,mBAA8B;AAC9B,mBAAuC;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,kBAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,sBAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,sBAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,kBAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,kBAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,oBAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,oBAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,mBAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI,KAAK,eAAe;AACtB,UAAI;AACF,cAAM,KAAK,cAAc,MAAM;AAAA,MACjC,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D,UAAE;AACA,aAAK,gBAAgB;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAC5B,WAAK,iBAAiB,IAAI,oBAAM;AAAA,IAClC;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,kBAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,kBAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,sBAAkB,qCAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AAzlBzC;AA0lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,qBAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AApsBpB;AAssBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,8BAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,oBAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,8BAA8B,KAAK;AACtD,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,+BAA+B,MAAM,MAAM,MAAM,MAAM;AAC1E,mBAAK,0BAA0B;AAAA,YACjC;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,mBAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,mBAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,oBAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,kBAAM,6BAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,kBAAM,qBAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA55BnB;AA85BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AA/5Bb,YAAAA;AA+5BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAv+B7B;AAw+BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAngC/F;AAogCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,MACA,gBAAgB,KAAK,iBAAiB,KAAK,iBAAiB;AAAA,IAC9D;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,iBAAa,yBAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,qBAAO,oBAA2C;AAAA,MAClE,iBAAiB,qBAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,aAAS,yBAAU,KAAK;AAAA,MACxB,aAAa,qBAAO,oBAA4B;AAAA,MAChD,cAAc,qBAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,UAAM,yBAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAE;AAAA,EAEtB,MAAM,aAAa;AAAA,EAAE;AAAA,EAErB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.d.ts.map b/plugins/google/dist/beta/realtime/realtime_api.d.ts.map index aad58218a..a0494b73d 100644 --- a/plugins/google/dist/beta/realtime/realtime_api.d.ts.map +++ b/plugins/google/dist/beta/realtime/realtime_api.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"realtime_api.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/realtime_api.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,eAAe,CAAC;AACvC,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EAEnC,KAAK,WAAW,EAChB,QAAQ,EACR,KAAK,mBAAmB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAUL,GAAG,EAIJ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,UAAU,EAAkB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAU3D;;GAEG;AACH,eAAO,MAAM,4BAA4B;;;;;;;;CAQxC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,UAAU,eAAe;IACvB,KAAK,EAAE,aAAa,GAAG,MAAM,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,QAAQ,EAAE,CAAC;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,wBAAwB,CAAC;IACnD,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;IACpD,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;IACzD,WAAW,EAAE,iBAAiB,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;CACvC;AA2BD;;GAEG;AACH,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,gBAAgB;IAChB,QAAQ,EAAE,eAAe,CAAC;gBAGxB,OAAO,GAAE;QACP;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;QAE/B;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAEvB;;;WAGG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;QAExB;;WAEG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAC;QAEnB;;WAEG;QACH,OAAO,CAAC,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,uBAAuB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE1D;;WAEG;QACH,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE3D;;WAEG;QACH,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;QAEzD;;WAEG;QACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,CAAC;QAEtB;;WAEG;QACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;QAE1C;;WAEG;QACH,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;QAE1D;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,WAAW,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,QAAQ,CAAC;QAEvB;;WAEG;QACH,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;KAClC;IA6DR;;OAEG;IACH,OAAO;IAIP;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAW9E;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,GAAG,CAAC,eAAe;;IACtD,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAA2B;IAE3C,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,cAAc,CAAuC;IAC7D,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAC,CAAqB;IAC/C,OAAO,CAAC,OAAO,CAAkB;IAGjC,OAAO,CAAC,aAAa,CAAC,CAAU;IAChC,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,sBAAsB,CAA4D;IAC1F,OAAO,CAAC,oBAAoB,CAAC,CAAqC;IAElE,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,qBAAqB,CAAS;gBAO1B,aAAa,EAAE,aAAa;YAwC1B,kBAAkB;IAgBhC,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,yBAAyB;IAyBjC,aAAa,CAAC,OAAO,EAAE;QACrB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;KAC7B;IAkBK,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOvD,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDtD,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAYxD,IAAI,OAAO,IAAI,GAAG,CAAC,WAAW,CAE7B;IAED,IAAI,KAAK,IAAI,GAAG,CAAC,WAAW,CAE3B;IAED,IAAI,uBAAuB,IAAI,OAAO,CAErC;IAED,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAsBlC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI;IAI9B,OAAO,CAAC,eAAe;IAIjB,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAyD/E,iBAAiB,IAAI,IAAI;IAgBnB,SAAS;IAQT,QAAQ,CAAC,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE;IAItF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAwId,QAAQ;YAiER,gBAAgB;IAuE9B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,mBAAmB;IAmB3B,OAAO,CAAC,qBAAqB;IA4B7B,OAAO,CAAC,yBAAyB;IA+CjC,OAAO,CAAC,SAAS;IAUjB,OAAO,CAAC,kBAAkB;IAsE1B,OAAO,CAAC,kBAAkB;IAwD1B,OAAO,CAAC,wBAAwB;IAIhC,OAAO,CAAC,wBAAwB;IAMhC,OAAO,CAAC,mBAAmB;IAoF3B,OAAO,CAAC,cAAc;IAmBtB,OAAO,CAAC,0BAA0B;IASlC,OAAO,CAAC,mBAAmB;IA6C3B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,YAAY;IAMd,WAAW;IAEX,UAAU;IAEhB,OAAO,CAAE,aAAa;CA8BvB"} \ No newline at end of file +{"version":3,"file":"realtime_api.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/realtime_api.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,eAAe,CAAC;AACvC,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EAEnC,KAAK,WAAW,EAChB,QAAQ,EACR,KAAK,mBAAmB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAUL,GAAG,EAIJ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,UAAU,EAAkB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAU3D;;GAEG;AACH,eAAO,MAAM,4BAA4B;;;;;;;;CAQxC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,UAAU,eAAe;IACvB,KAAK,EAAE,aAAa,GAAG,MAAM,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,QAAQ,EAAE,CAAC;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,wBAAwB,CAAC;IACnD,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;IACpD,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;IACzD,WAAW,EAAE,iBAAiB,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;CACvC;AA2BD;;GAEG;AACH,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,gBAAgB;IAChB,QAAQ,EAAE,eAAe,CAAC;gBAGxB,OAAO,GAAE;QACP;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;QAE/B;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAEvB;;;WAGG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;QAExB;;WAEG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAC;QAEnB;;WAEG;QACH,OAAO,CAAC,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,uBAAuB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE1D;;WAEG;QACH,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE3D;;WAEG;QACH,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;QAEzD;;WAEG;QACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,CAAC;QAEtB;;WAEG;QACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;QAE1C;;WAEG;QACH,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;QAE1D;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,WAAW,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,QAAQ,CAAC;QAEvB;;WAEG;QACH,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;KAClC;IA6DR;;OAEG;IACH,OAAO;IAIP;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAW9E;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,GAAG,CAAC,eAAe;;IACtD,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAA2B;IAE3C,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,cAAc,CAAuC;IAC7D,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAC,CAAqB;IAC/C,OAAO,CAAC,OAAO,CAAkB;IAGjC,OAAO,CAAC,aAAa,CAAC,CAAU;IAChC,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,sBAAsB,CAA4D;IAC1F,OAAO,CAAC,oBAAoB,CAAC,CAAqC;IAElE,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,qBAAqB,CAAS;gBAO1B,aAAa,EAAE,aAAa;YAwC1B,kBAAkB;IAgBhC,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,yBAAyB;IAyBjC,aAAa,CAAC,OAAO,EAAE;QACrB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;KAC7B;IAkBK,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOvD,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDtD,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAYxD,IAAI,OAAO,IAAI,GAAG,CAAC,WAAW,CAE7B;IAED,IAAI,KAAK,IAAI,GAAG,CAAC,WAAW,CAE3B;IAED,IAAI,uBAAuB,IAAI,OAAO,CAErC;IAED,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAsBlC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI;IAI9B,OAAO,CAAC,eAAe;IAIjB,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAyD/E,iBAAiB,IAAI,IAAI;IAgBnB,SAAS;IAQT,QAAQ,CAAC,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE;IAItF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAwId,QAAQ;YAiER,gBAAgB;IAuE9B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,mBAAmB;IAmB3B,OAAO,CAAC,qBAAqB;IA4B7B,OAAO,CAAC,yBAAyB;IA+CjC,OAAO,CAAC,SAAS;IAUjB,OAAO,CAAC,kBAAkB;IAiE1B,OAAO,CAAC,kBAAkB;IAwD1B,OAAO,CAAC,wBAAwB;IAIhC,OAAO,CAAC,wBAAwB;IAMhC,OAAO,CAAC,mBAAmB;IAoF3B,OAAO,CAAC,cAAc;IAmBtB,OAAO,CAAC,0BAA0B;IASlC,OAAO,CAAC,mBAAmB;IA6C3B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,YAAY;IAMd,WAAW;IAEX,UAAU;IAEhB,OAAO,CAAE,aAAa;CA8BvB"} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.js b/plugins/google/dist/beta/realtime/realtime_api.js index d66061998..f8b573fd3 100644 --- a/plugins/google/dist/beta/realtime/realtime_api.js +++ b/plugins/google/dist/beta/realtime/realtime_api.js @@ -712,7 +712,8 @@ class RealtimeSession extends llm.RealtimeSession { outputAudioTranscription: opts.outputAudioTranscription, sessionResumption: { handle: this.sessionResumptionHandle - } + }, + thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : void 0 }; if (opts.temperature !== void 0) { config.temperature = opts.temperature; @@ -738,11 +739,6 @@ class RealtimeSession extends llm.RealtimeSession { if (opts.contextWindowCompression !== void 0) { config.contextWindowCompression = opts.contextWindowCompression; } - if (opts.thinkingConfig !== void 0) { - config.generationConfig = { - thinkingConfig: opts.thinkingConfig - }; - } return config; } startNewGeneration() { diff --git a/plugins/google/dist/beta/realtime/realtime_api.js.map b/plugins/google/dist/beta/realtime/realtime_api.js.map index 2d84ebf4c..0cbc30e0d 100644 --- a/plugins/google/dist/beta/realtime/realtime_api.js.map +++ b/plugins/google/dist/beta/realtime/realtime_api.js.map @@ -1 +1 @@ -{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n if (this.activeSession) {\n try {\n await this.activeSession.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n } finally {\n this.activeSession = undefined;\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n this.messageChannel = new Queue();\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error('Gemini Live session error:', error);\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n this.#logger.debug('Gemini Live session closed:', event.code, event.reason);\n this.markCurrentGenerationDone();\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n if (opts.thinkingConfig !== undefined) {\n config.generationConfig = {\n thinkingConfig: opts.thinkingConfig,\n };\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() { }\n\n async clearAudio() { }\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":"AAIA,YAAY,WAAW;AACvB;AAAA,EACE;AAAA,EAGA;AAAA,EAEA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,aAAa;AACtB,SAAS,YAAY,sBAAuC;AAC5D,eAA8B;AAC9B,SAAS,8BAA8B;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,IAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,SAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,SAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,IAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,IAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,MAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,MAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,MAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,YAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI,KAAK,eAAe;AACtB,UAAI;AACF,cAAM,KAAK,cAAc,MAAM;AAAA,MACjC,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D,UAAE;AACA,aAAK,gBAAgB;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAC5B,WAAK,iBAAiB,IAAI,MAAM;AAAA,IAClC;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,IAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,IAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,kBAAkB,uBAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AAzlBzC;AA0lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,OAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AApsBpB;AAssBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,iBAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,MAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,8BAA8B,KAAK;AACtD,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,+BAA+B,MAAM,MAAM,MAAM,MAAM;AAC1E,mBAAK,0BAA0B;AAAA,YACjC;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,KAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,MAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,cAAM,cAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,cAAM,MAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA55BnB;AA85BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AA/5Bb,YAAAA;AA+5BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAv+B7B;AAw+BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAngC/F;AAogCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,IACF;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,QAAI,KAAK,mBAAmB,QAAW;AACrC,aAAO,mBAAmB;AAAA,QACxB,gBAAgB,KAAK;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,aAAa,UAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,OAAO,oBAA2C;AAAA,MAClE,iBAAiB,OAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,SAAS,UAAU,KAAK;AAAA,MACxB,aAAa,OAAO,oBAA4B;AAAA,MAChD,cAAc,OAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,MAAM,UAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAE;AAAA,EAEtB,MAAM,aAAa;AAAA,EAAE;AAAA,EAErB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file +{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n if (this.activeSession) {\n try {\n await this.activeSession.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n } finally {\n this.activeSession = undefined;\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n this.messageChannel = new Queue();\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error('Gemini Live session error:', error);\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n this.#logger.debug('Gemini Live session closed:', event.code, event.reason);\n this.markCurrentGenerationDone();\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined,\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() { }\n\n async clearAudio() { }\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":"AAIA,YAAY,WAAW;AACvB;AAAA,EACE;AAAA,EAGA;AAAA,EAEA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,aAAa;AACtB,SAAS,YAAY,sBAAuC;AAC5D,eAA8B;AAC9B,SAAS,8BAA8B;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,IAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,SAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,SAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,IAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,IAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,MAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,MAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,MAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,YAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI,KAAK,eAAe;AACtB,UAAI;AACF,cAAM,KAAK,cAAc,MAAM;AAAA,MACjC,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D,UAAE;AACA,aAAK,gBAAgB;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAC5B,WAAK,iBAAiB,IAAI,MAAM;AAAA,IAClC;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,IAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,IAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,kBAAkB,uBAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AAzlBzC;AA0lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,OAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AApsBpB;AAssBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,iBAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,MAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,8BAA8B,KAAK;AACtD,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,+BAA+B,MAAM,MAAM,MAAM,MAAM;AAC1E,mBAAK,0BAA0B;AAAA,YACjC;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,KAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,MAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,cAAM,cAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,cAAM,MAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA55BnB;AA85BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AA/5Bb,YAAAA;AA+5BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAv+B7B;AAw+BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAngC/F;AAogCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,MACA,gBAAgB,KAAK,iBAAiB,KAAK,iBAAiB;AAAA,IAC9D;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,aAAa,UAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,OAAO,oBAA2C;AAAA,MAClE,iBAAiB,OAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,SAAS,UAAU,KAAK;AAAA,MACxB,aAAa,OAAO,oBAA4B;AAAA,MAChD,cAAc,OAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,MAAM,UAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAE;AAAA,EAEtB,MAAM,aAAa;AAAA,EAAE;AAAA,EAErB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file diff --git a/plugins/google/package.json b/plugins/google/package.json index 373b63082..6559e1d19 100644 --- a/plugins/google/package.json +++ b/plugins/google/package.json @@ -43,7 +43,7 @@ "typescript": "^5.0.0" }, "dependencies": { - "@google/genai": "^1.13.0", + "@google/genai": "^1.34.0", "@livekit/mutex": "^1.1.1", "@types/json-schema": "^7.0.15", "json-schema": "^0.4.0" diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index a7718ed50..ab5aaeabe 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -1127,6 +1127,7 @@ export class RealtimeSession extends llm.RealtimeSession { sessionResumption: { handle: this.sessionResumptionHandle, }, + thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined, }; // Add generation fields at TOP LEVEL (NO generationConfig!) @@ -1159,12 +1160,6 @@ export class RealtimeSession extends llm.RealtimeSession { config.contextWindowCompression = opts.contextWindowCompression; } - if (opts.thinkingConfig !== undefined) { - config.generationConfig = { - thinkingConfig: opts.thinkingConfig, - }; - } - return config; } @@ -1423,9 +1418,9 @@ export class RealtimeSession extends llm.RealtimeSession { this.sessionShouldClose.set(); } - async commitAudio() { } + async commitAudio() {} - async clearAudio() { } + async clearAudio() {} private *resampleAudio(frame: AudioFrame): Generator { if (this.inputResampler) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ec83bd27b..14a47d194 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -511,8 +511,8 @@ importers: plugins/google: dependencies: '@google/genai': - specifier: ^1.13.0 - version: 1.13.0 + specifier: ^1.34.0 + version: 1.34.0 '@livekit/mutex': specifier: ^1.1.1 version: 1.1.1 @@ -1445,11 +1445,11 @@ packages: cpu: [x64] os: [win32] - '@google/genai@1.13.0': - resolution: {integrity: sha512-BxilXzE8cJ0zt5/lXk6KwuBcIT9P2Lbi2WXhwWMbxf1RNeC68/8DmYQqMrzQP333CieRMdbDXs0eNCphLoScWg==} + '@google/genai@1.34.0': + resolution: {integrity: sha512-vu53UMPvjmb7PGzlYu6Tzxso8Dfhn+a7eQFaS2uNemVtDZKwzSpJ5+ikqBbXplF7RGB1STcVDqCkPvquiwb2sw==} engines: {node: '>=20.0.0'} peerDependencies: - '@modelcontextprotocol/sdk': ^1.11.0 + '@modelcontextprotocol/sdk': ^1.24.0 peerDependenciesMeta: '@modelcontextprotocol/sdk': optional: true @@ -2703,6 +2703,10 @@ packages: damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + data-view-buffer@1.0.1: resolution: {integrity: sha512-0lht7OugA5x3iJLOWFhWK/5ehONdprk0ISXqVFn/NFrDu+cuc8iADFrGQz5BnRK7LLU3JmkbXSxaqX+/mXYtUA==} engines: {node: '>= 0.4'} @@ -2756,6 +2760,15 @@ packages: supports-color: optional: true + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + deep-eql@4.1.3: resolution: {integrity: sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==} engines: {node: '>=6'} @@ -3170,6 +3183,10 @@ packages: picomatch: optional: true + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -3216,6 +3233,10 @@ packages: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + fs-extra@7.0.1: resolution: {integrity: sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==} engines: {node: '>=6 <7 || >=8'} @@ -3242,13 +3263,13 @@ packages: functions-have-names@1.2.3: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} - gaxios@6.7.1: - resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} - engines: {node: '>=14'} + gaxios@7.1.3: + resolution: {integrity: sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==} + engines: {node: '>=18'} - gcp-metadata@6.1.1: - resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==} - engines: {node: '>=14'} + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} get-func-name@2.0.2: resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} @@ -3324,12 +3345,12 @@ packages: resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} engines: {node: '>=10'} - google-auth-library@9.15.1: - resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} - engines: {node: '>=14'} + google-auth-library@10.5.0: + resolution: {integrity: sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==} + engines: {node: '>=18'} - google-logging-utils@0.0.2: - resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} engines: {node: '>=14'} gopd@1.0.1: @@ -3345,9 +3366,9 @@ packages: graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} - gtoken@7.1.0: - resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} - engines: {node: '>=14.0.0'} + gtoken@8.0.0: + resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==} + engines: {node: '>=18'} guid-typescript@1.0.9: resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} @@ -3537,10 +3558,6 @@ packages: resolution: {integrity: sha512-nA2hv5XIhLR3uVzDDfCIknerhx8XUKnstuOERPNNIinXG7v9u+ohXF67vxm4TPTEPU6lm61ZkwP3c9PCB97rhg==} engines: {node: '>= 0.4'} - is-stream@2.0.1: - resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} - engines: {node: '>=8'} - is-stream@3.0.0: resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -3682,8 +3699,8 @@ packages: jwa@2.0.1: resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} - jws@4.0.0: - resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==} + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -3892,6 +3909,11 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + node-fetch@2.7.0: resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} engines: {node: 4.x || >=6.0.0} @@ -3901,6 +3923,10 @@ packages: encoding: optional: true + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + npm-run-path@5.3.0: resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -4768,10 +4794,6 @@ packages: resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} hasBin: true - uuid@9.0.1: - resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} - hasBin: true - validator@13.12.0: resolution: {integrity: sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg==} engines: {node: '>= 0.10'} @@ -4877,6 +4899,10 @@ packages: resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==} engines: {node: '>=10.13.0'} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -5535,13 +5561,12 @@ snapshots: '@ffmpeg-installer/win32-x64@4.1.0': optional: true - '@google/genai@1.13.0': + '@google/genai@1.34.0': dependencies: - google-auth-library: 9.15.1 + google-auth-library: 10.5.0 ws: 8.18.3 transitivePeerDependencies: - bufferutil - - encoding - supports-color - utf-8-validate @@ -6881,6 +6906,8 @@ snapshots: damerau-levenshtein@1.0.8: {} + data-uri-to-buffer@4.0.1: {} + data-view-buffer@1.0.1: dependencies: call-bind: 1.0.7 @@ -6919,6 +6946,10 @@ snapshots: dependencies: ms: 2.1.3 + debug@4.4.3: + dependencies: + ms: 2.1.3 + deep-eql@4.1.3: dependencies: type-detect: 4.0.8 @@ -7516,6 +7547,11 @@ snapshots: optionalDependencies: picomatch: 4.0.2 + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + file-entry-cache@6.0.1: dependencies: flat-cache: 3.2.0 @@ -7570,6 +7606,10 @@ snapshots: hasown: 2.0.2 mime-types: 2.1.35 + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + fs-extra@7.0.1: dependencies: graceful-fs: 4.2.11 @@ -7598,24 +7638,21 @@ snapshots: functions-have-names@1.2.3: {} - gaxios@6.7.1: + gaxios@7.1.3: dependencies: extend: 3.0.2 https-proxy-agent: 7.0.6 - is-stream: 2.0.1 - node-fetch: 2.7.0 - uuid: 9.0.1 + node-fetch: 3.3.2 + rimraf: 5.0.10 transitivePeerDependencies: - - encoding - supports-color - gcp-metadata@6.1.1: + gcp-metadata@8.1.2: dependencies: - gaxios: 6.7.1 - google-logging-utils: 0.0.2 + gaxios: 7.1.3 + google-logging-utils: 1.1.3 json-bigint: 1.0.0 transitivePeerDependencies: - - encoding - supports-color get-func-name@2.0.2: {} @@ -7725,19 +7762,19 @@ snapshots: merge2: 1.4.1 slash: 3.0.0 - google-auth-library@9.15.1: + google-auth-library@10.5.0: dependencies: base64-js: 1.5.1 ecdsa-sig-formatter: 1.0.11 - gaxios: 6.7.1 - gcp-metadata: 6.1.1 - gtoken: 7.1.0 - jws: 4.0.0 + gaxios: 7.1.3 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 + gtoken: 8.0.0 + jws: 4.0.1 transitivePeerDependencies: - - encoding - supports-color - google-logging-utils@0.0.2: {} + google-logging-utils@1.1.3: {} gopd@1.0.1: dependencies: @@ -7749,12 +7786,11 @@ snapshots: graphemer@1.4.0: {} - gtoken@7.1.0: + gtoken@8.0.0: dependencies: - gaxios: 6.7.1 - jws: 4.0.0 + gaxios: 7.1.3 + jws: 4.0.1 transitivePeerDependencies: - - encoding - supports-color guid-typescript@1.0.9: {} @@ -7792,7 +7828,7 @@ snapshots: https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.4 - debug: 4.4.1 + debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -7916,8 +7952,6 @@ snapshots: dependencies: call-bind: 1.0.7 - is-stream@2.0.1: {} - is-stream@3.0.0: {} is-string@1.0.7: @@ -8058,7 +8092,7 @@ snapshots: ecdsa-sig-formatter: 1.0.11 safe-buffer: 5.2.1 - jws@4.0.0: + jws@4.0.1: dependencies: jwa: 2.0.1 safe-buffer: 5.2.1 @@ -8247,10 +8281,18 @@ snapshots: natural-compare@1.4.0: {} + node-domexception@1.0.0: {} + node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + npm-run-path@5.3.0: dependencies: path-key: 4.0.0 @@ -9277,8 +9319,6 @@ snapshots: uuid@11.1.0: {} - uuid@9.0.1: {} - validator@13.12.0: {} vite-node@1.6.0(@types/node@22.15.30): @@ -9403,6 +9443,8 @@ snapshots: glob-to-regexp: 0.4.1 graceful-fs: 4.2.11 + web-streams-polyfill@3.3.3: {} + webidl-conversions@3.0.1: {} webidl-conversions@4.0.2: {} From 1df9feafcafe1db26172ae347d26f33351fd50e2 Mon Sep 17 00:00:00 2001 From: oOPa <5878650+oOPa@users.noreply.github.com> Date: Sun, 21 Dec 2025 14:27:24 +0000 Subject: [PATCH 4/9] fix: Improve RealtimeModel session closing, error handling, and restart logic by clearing active sessions earlier, waking up old message queues, and ensuring proper session state transitions on error or unexpected closure. --- .../dist/beta/realtime/realtime_api.cjs | 25 ++++++++++++---- .../dist/beta/realtime/realtime_api.cjs.map | 2 +- .../dist/beta/realtime/realtime_api.d.ts.map | 2 +- .../google/dist/beta/realtime/realtime_api.js | 25 ++++++++++++---- .../dist/beta/realtime/realtime_api.js.map | 2 +- .../google/src/beta/realtime/realtime_api.ts | 29 +++++++++++++++---- 6 files changed, 64 insertions(+), 21 deletions(-) diff --git a/plugins/google/dist/beta/realtime/realtime_api.cjs b/plugins/google/dist/beta/realtime/realtime_api.cjs index 7c21ebd6c..a37b33cba 100644 --- a/plugins/google/dist/beta/realtime/realtime_api.cjs +++ b/plugins/google/dist/beta/realtime/realtime_api.cjs @@ -187,13 +187,13 @@ class RealtimeSession extends import_agents.llm.RealtimeSession { } async closeActiveSession() { const unlock = await this.sessionLock.lock(); - if (this.activeSession) { + const session = this.activeSession; + this.activeSession = void 0; + if (session) { try { - await this.activeSession.close(); + await session.close(); } catch (error) { this.#logger.warn({ error }, "Error closing Gemini session"); - } finally { - this.activeSession = void 0; } } unlock(); @@ -201,7 +201,9 @@ class RealtimeSession extends import_agents.llm.RealtimeSession { markRestartNeeded() { if (!this.sessionShouldClose.isSet) { this.sessionShouldClose.set(); + const oldQueue = this.messageChannel; this.messageChannel = new import_agents.Queue(); + oldQueue.put({ type: "__internal_wakeup__" }); } } getToolResultsForRealtime(ctx, vertexai) { @@ -435,14 +437,25 @@ class RealtimeSession extends import_agents.llm.RealtimeSession { this.onReceiveMessage(session, message); }, onerror: (error) => { - this.#logger.error("Gemini Live session error:", error); + this.#logger.error({ error }, "Gemini Live session error"); + sessionOpened.set(); if (!this.sessionShouldClose.isSet) { this.markRestartNeeded(); } }, onclose: (event) => { - this.#logger.debug("Gemini Live session closed:", event.code, event.reason); + if (this.activeSession !== session) { + return; + } + this.#logger.debug( + { code: event.code, reason: event.reason }, + "Gemini Live session closed" + ); + sessionOpened.set(); this.markCurrentGenerationDone(); + if (!this.sessionShouldClose.isSet && !this.#closed) { + this.markRestartNeeded(); + } } }, config diff --git a/plugins/google/dist/beta/realtime/realtime_api.cjs.map b/plugins/google/dist/beta/realtime/realtime_api.cjs.map index 7afd6c23b..241e29755 100644 --- a/plugins/google/dist/beta/realtime/realtime_api.cjs.map +++ b/plugins/google/dist/beta/realtime/realtime_api.cjs.map @@ -1 +1 @@ -{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n if (this.activeSession) {\n try {\n await this.activeSession.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n } finally {\n this.activeSession = undefined;\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n this.messageChannel = new Queue();\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error('Gemini Live session error:', error);\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n this.#logger.debug('Gemini Live session closed:', event.code, event.reason);\n this.markCurrentGenerationDone();\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined,\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() { }\n\n async clearAudio() { }\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,YAAuB;AACvB,mBAQO;AAEP,oBAcO;AACP,mBAAsB;AACtB,sBAA4D;AAC5D,mBAA8B;AAC9B,mBAAuC;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,kBAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,sBAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,sBAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,kBAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,kBAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,oBAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,oBAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,mBAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI,KAAK,eAAe;AACtB,UAAI;AACF,cAAM,KAAK,cAAc,MAAM;AAAA,MACjC,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D,UAAE;AACA,aAAK,gBAAgB;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAC5B,WAAK,iBAAiB,IAAI,oBAAM;AAAA,IAClC;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,kBAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,kBAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,sBAAkB,qCAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AAzlBzC;AA0lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,qBAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AApsBpB;AAssBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,8BAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,oBAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,8BAA8B,KAAK;AACtD,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,+BAA+B,MAAM,MAAM,MAAM,MAAM;AAC1E,mBAAK,0BAA0B;AAAA,YACjC;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,mBAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,mBAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,oBAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,kBAAM,6BAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,kBAAM,qBAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA55BnB;AA85BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AA/5Bb,YAAAA;AA+5BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAv+B7B;AAw+BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAngC/F;AAogCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,MACA,gBAAgB,KAAK,iBAAiB,KAAK,iBAAiB;AAAA,IAC9D;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,iBAAa,yBAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,qBAAO,oBAA2C;AAAA,MAClE,iBAAiB,qBAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,aAAS,yBAAU,KAAK;AAAA,MACxB,aAAa,qBAAO,oBAA4B;AAAA,MAChD,cAAc,qBAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,UAAM,yBAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAE;AAAA,EAEtB,MAAM,aAAa;AAAA,EAAE;AAAA,EAErB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file +{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n const session = this.activeSession;\n this.activeSession = undefined;\n\n if (session) {\n try {\n await session.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n // Replace the queue so new messages don't go to the old (closing) session.\n const oldQueue = this.messageChannel;\n this.messageChannel = new Queue();\n // Wake up the current sendTask if it's waiting for a message.\n oldQueue.put({ type: '__internal_wakeup__' } as any);\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error({ error }, 'Gemini Live session error');\n sessionOpened.set(); // Break the wait if we were waiting for open\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n if (this.activeSession !== session) {\n return;\n }\n\n this.#logger.debug(\n { code: event.code, reason: event.reason },\n 'Gemini Live session closed',\n );\n sessionOpened.set(); // Break the wait if we were waiting for open\n this.markCurrentGenerationDone();\n if (!this.sessionShouldClose.isSet && !this.#closed) {\n this.markRestartNeeded();\n }\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined,\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() {}\n\n async clearAudio() {}\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,YAAuB;AACvB,mBAQO;AAEP,oBAcO;AACP,mBAAsB;AACtB,sBAA4D;AAC5D,mBAA8B;AAC9B,mBAAuC;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,kBAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,sBAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,sBAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,kBAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,kBAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,oBAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,oBAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,mBAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,UAAM,UAAU,KAAK;AACrB,SAAK,gBAAgB;AAErB,QAAI,SAAS;AACX,UAAI;AACF,cAAM,QAAQ,MAAM;AAAA,MACtB,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAE5B,YAAM,WAAW,KAAK;AACtB,WAAK,iBAAiB,IAAI,oBAAM;AAEhC,eAAS,IAAI,EAAE,MAAM,sBAAsB,CAAQ;AAAA,IACrD;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,kBAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,kBAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,sBAAkB,qCAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AA9lBzC;AA+lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,qBAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AAzsBpB;AA2sBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,8BAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,oBAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,EAAE,MAAM,GAAG,2BAA2B;AACzD,4BAAc,IAAI;AAClB,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,kBAAI,KAAK,kBAAkB,SAAS;AAClC;AAAA,cACF;AAEA,mBAAK,QAAQ;AAAA,gBACX,EAAE,MAAM,MAAM,MAAM,QAAQ,MAAM,OAAO;AAAA,gBACzC;AAAA,cACF;AACA,4BAAc,IAAI;AAClB,mBAAK,0BAA0B;AAC/B,kBAAI,CAAC,KAAK,mBAAmB,SAAS,CAAC,KAAK,SAAS;AACnD,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,mBAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,mBAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,oBAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,kBAAM,6BAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,kBAAM,qBAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA76BnB;AA+6BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AAh7Bb,YAAAA;AAg7BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAx/B7B;AAy/BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAphC/F;AAqhCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,MACA,gBAAgB,KAAK,iBAAiB,KAAK,iBAAiB;AAAA,IAC9D;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,iBAAa,yBAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,qBAAO,oBAA2C;AAAA,MAClE,iBAAiB,qBAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,aAAS,yBAAU,KAAK;AAAA,MACxB,aAAa,qBAAO,oBAA4B;AAAA,MAChD,cAAc,qBAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,UAAM,yBAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAC;AAAA,EAErB,MAAM,aAAa;AAAA,EAAC;AAAA,EAEpB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.d.ts.map b/plugins/google/dist/beta/realtime/realtime_api.d.ts.map index a0494b73d..94128e672 100644 --- a/plugins/google/dist/beta/realtime/realtime_api.d.ts.map +++ b/plugins/google/dist/beta/realtime/realtime_api.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"realtime_api.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/realtime_api.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,eAAe,CAAC;AACvC,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EAEnC,KAAK,WAAW,EAChB,QAAQ,EACR,KAAK,mBAAmB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAUL,GAAG,EAIJ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,UAAU,EAAkB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAU3D;;GAEG;AACH,eAAO,MAAM,4BAA4B;;;;;;;;CAQxC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,UAAU,eAAe;IACvB,KAAK,EAAE,aAAa,GAAG,MAAM,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,QAAQ,EAAE,CAAC;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,wBAAwB,CAAC;IACnD,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;IACpD,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;IACzD,WAAW,EAAE,iBAAiB,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;CACvC;AA2BD;;GAEG;AACH,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,gBAAgB;IAChB,QAAQ,EAAE,eAAe,CAAC;gBAGxB,OAAO,GAAE;QACP;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;QAE/B;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAEvB;;;WAGG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;QAExB;;WAEG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAC;QAEnB;;WAEG;QACH,OAAO,CAAC,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,uBAAuB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE1D;;WAEG;QACH,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE3D;;WAEG;QACH,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;QAEzD;;WAEG;QACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,CAAC;QAEtB;;WAEG;QACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;QAE1C;;WAEG;QACH,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;QAE1D;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,WAAW,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,QAAQ,CAAC;QAEvB;;WAEG;QACH,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;KAClC;IA6DR;;OAEG;IACH,OAAO;IAIP;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAW9E;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,GAAG,CAAC,eAAe;;IACtD,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAA2B;IAE3C,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,cAAc,CAAuC;IAC7D,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAC,CAAqB;IAC/C,OAAO,CAAC,OAAO,CAAkB;IAGjC,OAAO,CAAC,aAAa,CAAC,CAAU;IAChC,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,sBAAsB,CAA4D;IAC1F,OAAO,CAAC,oBAAoB,CAAC,CAAqC;IAElE,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,qBAAqB,CAAS;gBAO1B,aAAa,EAAE,aAAa;YAwC1B,kBAAkB;IAgBhC,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,yBAAyB;IAyBjC,aAAa,CAAC,OAAO,EAAE;QACrB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;KAC7B;IAkBK,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOvD,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDtD,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAYxD,IAAI,OAAO,IAAI,GAAG,CAAC,WAAW,CAE7B;IAED,IAAI,KAAK,IAAI,GAAG,CAAC,WAAW,CAE3B;IAED,IAAI,uBAAuB,IAAI,OAAO,CAErC;IAED,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAsBlC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI;IAI9B,OAAO,CAAC,eAAe;IAIjB,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAyD/E,iBAAiB,IAAI,IAAI;IAgBnB,SAAS;IAQT,QAAQ,CAAC,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE;IAItF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAwId,QAAQ;YAiER,gBAAgB;IAuE9B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,mBAAmB;IAmB3B,OAAO,CAAC,qBAAqB;IA4B7B,OAAO,CAAC,yBAAyB;IA+CjC,OAAO,CAAC,SAAS;IAUjB,OAAO,CAAC,kBAAkB;IAiE1B,OAAO,CAAC,kBAAkB;IAwD1B,OAAO,CAAC,wBAAwB;IAIhC,OAAO,CAAC,wBAAwB;IAMhC,OAAO,CAAC,mBAAmB;IAoF3B,OAAO,CAAC,cAAc;IAmBtB,OAAO,CAAC,0BAA0B;IASlC,OAAO,CAAC,mBAAmB;IA6C3B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,YAAY;IAMd,WAAW;IAEX,UAAU;IAEhB,OAAO,CAAE,aAAa;CA8BvB"} \ No newline at end of file +{"version":3,"file":"realtime_api.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/realtime_api.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,eAAe,CAAC;AACvC,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EAEnC,KAAK,WAAW,EAChB,QAAQ,EACR,KAAK,mBAAmB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAUL,GAAG,EAIJ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,UAAU,EAAkB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAU3D;;GAEG;AACH,eAAO,MAAM,4BAA4B;;;;;;;;CAQxC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,UAAU,eAAe;IACvB,KAAK,EAAE,aAAa,GAAG,MAAM,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,QAAQ,EAAE,CAAC;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,wBAAwB,CAAC;IACnD,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;IACpD,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;IACzD,WAAW,EAAE,iBAAiB,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;CACvC;AA2BD;;GAEG;AACH,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,gBAAgB;IAChB,QAAQ,EAAE,eAAe,CAAC;gBAGxB,OAAO,GAAE;QACP;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;QAE/B;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAEvB;;;WAGG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;QAExB;;WAEG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAC;QAEnB;;WAEG;QACH,OAAO,CAAC,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,uBAAuB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE1D;;WAEG;QACH,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE3D;;WAEG;QACH,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;QAEzD;;WAEG;QACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,CAAC;QAEtB;;WAEG;QACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;QAE1C;;WAEG;QACH,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;QAE1D;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,WAAW,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,QAAQ,CAAC;QAEvB;;WAEG;QACH,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;KAClC;IA6DR;;OAEG;IACH,OAAO;IAIP;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAW9E;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,GAAG,CAAC,eAAe;;IACtD,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAA2B;IAE3C,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,cAAc,CAAuC;IAC7D,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAC,CAAqB;IAC/C,OAAO,CAAC,OAAO,CAAkB;IAGjC,OAAO,CAAC,aAAa,CAAC,CAAU;IAChC,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,sBAAsB,CAA4D;IAC1F,OAAO,CAAC,oBAAoB,CAAC,CAAqC;IAElE,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,qBAAqB,CAAS;gBAO1B,aAAa,EAAE,aAAa;YAwC1B,kBAAkB;IAiBhC,OAAO,CAAC,iBAAiB;IAWzB,OAAO,CAAC,yBAAyB;IAyBjC,aAAa,CAAC,OAAO,EAAE;QACrB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;KAC7B;IAkBK,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOvD,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDtD,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAYxD,IAAI,OAAO,IAAI,GAAG,CAAC,WAAW,CAE7B;IAED,IAAI,KAAK,IAAI,GAAG,CAAC,WAAW,CAE3B;IAED,IAAI,uBAAuB,IAAI,OAAO,CAErC;IAED,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAsBlC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI;IAI9B,OAAO,CAAC,eAAe;IAIjB,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAyD/E,iBAAiB,IAAI,IAAI;IAgBnB,SAAS;IAQT,QAAQ,CAAC,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE;IAItF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAoJd,QAAQ;YAiER,gBAAgB;IAuE9B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,mBAAmB;IAmB3B,OAAO,CAAC,qBAAqB;IA4B7B,OAAO,CAAC,yBAAyB;IA+CjC,OAAO,CAAC,SAAS;IAUjB,OAAO,CAAC,kBAAkB;IAiE1B,OAAO,CAAC,kBAAkB;IAwD1B,OAAO,CAAC,wBAAwB;IAIhC,OAAO,CAAC,wBAAwB;IAMhC,OAAO,CAAC,mBAAmB;IAoF3B,OAAO,CAAC,cAAc;IAmBtB,OAAO,CAAC,0BAA0B;IASlC,OAAO,CAAC,mBAAmB;IA6C3B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,YAAY;IAMd,WAAW;IAEX,UAAU;IAEhB,OAAO,CAAE,aAAa;CA8BvB"} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.js b/plugins/google/dist/beta/realtime/realtime_api.js index f8b573fd3..42842fa2f 100644 --- a/plugins/google/dist/beta/realtime/realtime_api.js +++ b/plugins/google/dist/beta/realtime/realtime_api.js @@ -170,13 +170,13 @@ class RealtimeSession extends llm.RealtimeSession { } async closeActiveSession() { const unlock = await this.sessionLock.lock(); - if (this.activeSession) { + const session = this.activeSession; + this.activeSession = void 0; + if (session) { try { - await this.activeSession.close(); + await session.close(); } catch (error) { this.#logger.warn({ error }, "Error closing Gemini session"); - } finally { - this.activeSession = void 0; } } unlock(); @@ -184,7 +184,9 @@ class RealtimeSession extends llm.RealtimeSession { markRestartNeeded() { if (!this.sessionShouldClose.isSet) { this.sessionShouldClose.set(); + const oldQueue = this.messageChannel; this.messageChannel = new Queue(); + oldQueue.put({ type: "__internal_wakeup__" }); } } getToolResultsForRealtime(ctx, vertexai) { @@ -418,14 +420,25 @@ class RealtimeSession extends llm.RealtimeSession { this.onReceiveMessage(session, message); }, onerror: (error) => { - this.#logger.error("Gemini Live session error:", error); + this.#logger.error({ error }, "Gemini Live session error"); + sessionOpened.set(); if (!this.sessionShouldClose.isSet) { this.markRestartNeeded(); } }, onclose: (event) => { - this.#logger.debug("Gemini Live session closed:", event.code, event.reason); + if (this.activeSession !== session) { + return; + } + this.#logger.debug( + { code: event.code, reason: event.reason }, + "Gemini Live session closed" + ); + sessionOpened.set(); this.markCurrentGenerationDone(); + if (!this.sessionShouldClose.isSet && !this.#closed) { + this.markRestartNeeded(); + } } }, config diff --git a/plugins/google/dist/beta/realtime/realtime_api.js.map b/plugins/google/dist/beta/realtime/realtime_api.js.map index 0cbc30e0d..e56f1bd20 100644 --- a/plugins/google/dist/beta/realtime/realtime_api.js.map +++ b/plugins/google/dist/beta/realtime/realtime_api.js.map @@ -1 +1 @@ -{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n if (this.activeSession) {\n try {\n await this.activeSession.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n } finally {\n this.activeSession = undefined;\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n this.messageChannel = new Queue();\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error('Gemini Live session error:', error);\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n this.#logger.debug('Gemini Live session closed:', event.code, event.reason);\n this.markCurrentGenerationDone();\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined,\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() { }\n\n async clearAudio() { }\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":"AAIA,YAAY,WAAW;AACvB;AAAA,EACE;AAAA,EAGA;AAAA,EAEA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,aAAa;AACtB,SAAS,YAAY,sBAAuC;AAC5D,eAA8B;AAC9B,SAAS,8BAA8B;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,IAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,SAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,SAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,IAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,IAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,MAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,MAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,MAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,YAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI,KAAK,eAAe;AACtB,UAAI;AACF,cAAM,KAAK,cAAc,MAAM;AAAA,MACjC,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D,UAAE;AACA,aAAK,gBAAgB;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAC5B,WAAK,iBAAiB,IAAI,MAAM;AAAA,IAClC;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,IAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,IAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,kBAAkB,uBAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AAzlBzC;AA0lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,OAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AApsBpB;AAssBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,iBAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,MAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,8BAA8B,KAAK;AACtD,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,+BAA+B,MAAM,MAAM,MAAM,MAAM;AAC1E,mBAAK,0BAA0B;AAAA,YACjC;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,KAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,MAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,cAAM,cAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,cAAM,MAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA55BnB;AA85BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AA/5Bb,YAAAA;AA+5BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAv+B7B;AAw+BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAngC/F;AAogCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,MACA,gBAAgB,KAAK,iBAAiB,KAAK,iBAAiB;AAAA,IAC9D;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,aAAa,UAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,OAAO,oBAA2C;AAAA,MAClE,iBAAiB,OAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,SAAS,UAAU,KAAK;AAAA,MACxB,aAAa,OAAO,oBAA4B;AAAA,MAChD,cAAc,OAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,MAAM,UAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAE;AAAA,EAEtB,MAAM,aAAa;AAAA,EAAE;AAAA,EAErB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file +{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n const session = this.activeSession;\n this.activeSession = undefined;\n\n if (session) {\n try {\n await session.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n // Replace the queue so new messages don't go to the old (closing) session.\n const oldQueue = this.messageChannel;\n this.messageChannel = new Queue();\n // Wake up the current sendTask if it's waiting for a message.\n oldQueue.put({ type: '__internal_wakeup__' } as any);\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error({ error }, 'Gemini Live session error');\n sessionOpened.set(); // Break the wait if we were waiting for open\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n if (this.activeSession !== session) {\n return;\n }\n\n this.#logger.debug(\n { code: event.code, reason: event.reason },\n 'Gemini Live session closed',\n );\n sessionOpened.set(); // Break the wait if we were waiting for open\n this.markCurrentGenerationDone();\n if (!this.sessionShouldClose.isSet && !this.#closed) {\n this.markRestartNeeded();\n }\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined,\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() {}\n\n async clearAudio() {}\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":"AAIA,YAAY,WAAW;AACvB;AAAA,EACE;AAAA,EAGA;AAAA,EAEA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,aAAa;AACtB,SAAS,YAAY,sBAAuC;AAC5D,eAA8B;AAC9B,SAAS,8BAA8B;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,IAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,SAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,SAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,IAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,IAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,MAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,MAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,MAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,YAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,UAAM,UAAU,KAAK;AACrB,SAAK,gBAAgB;AAErB,QAAI,SAAS;AACX,UAAI;AACF,cAAM,QAAQ,MAAM;AAAA,MACtB,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAE5B,YAAM,WAAW,KAAK;AACtB,WAAK,iBAAiB,IAAI,MAAM;AAEhC,eAAS,IAAI,EAAE,MAAM,sBAAsB,CAAQ;AAAA,IACrD;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,IAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,IAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,kBAAkB,uBAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AA9lBzC;AA+lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,OAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AAzsBpB;AA2sBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,iBAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,MAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,EAAE,MAAM,GAAG,2BAA2B;AACzD,4BAAc,IAAI;AAClB,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,kBAAI,KAAK,kBAAkB,SAAS;AAClC;AAAA,cACF;AAEA,mBAAK,QAAQ;AAAA,gBACX,EAAE,MAAM,MAAM,MAAM,QAAQ,MAAM,OAAO;AAAA,gBACzC;AAAA,cACF;AACA,4BAAc,IAAI;AAClB,mBAAK,0BAA0B;AAC/B,kBAAI,CAAC,KAAK,mBAAmB,SAAS,CAAC,KAAK,SAAS;AACnD,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,KAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,MAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,cAAM,cAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,cAAM,MAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA76BnB;AA+6BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AAh7Bb,YAAAA;AAg7BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAx/B7B;AAy/BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAphC/F;AAqhCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,MACA,gBAAgB,KAAK,iBAAiB,KAAK,iBAAiB;AAAA,IAC9D;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,aAAa,UAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,OAAO,oBAA2C;AAAA,MAClE,iBAAiB,OAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,SAAS,UAAU,KAAK;AAAA,MACxB,aAAa,OAAO,oBAA4B;AAAA,MAChD,cAAc,OAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,MAAM,UAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAC;AAAA,EAErB,MAAM,aAAa;AAAA,EAAC;AAAA,EAEpB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index ab5aaeabe..5d2d3dbe0 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -448,13 +448,14 @@ export class RealtimeSession extends llm.RealtimeSession { private async closeActiveSession(): Promise { const unlock = await this.sessionLock.lock(); - if (this.activeSession) { + const session = this.activeSession; + this.activeSession = undefined; + + if (session) { try { - await this.activeSession.close(); + await session.close(); } catch (error) { this.#logger.warn({ error }, 'Error closing Gemini session'); - } finally { - this.activeSession = undefined; } } @@ -464,7 +465,11 @@ export class RealtimeSession extends llm.RealtimeSession { private markRestartNeeded(): void { if (!this.sessionShouldClose.isSet) { this.sessionShouldClose.set(); + // Replace the queue so new messages don't go to the old (closing) session. + const oldQueue = this.messageChannel; this.messageChannel = new Queue(); + // Wake up the current sendTask if it's waiting for a message. + oldQueue.put({ type: '__internal_wakeup__' } as any); } } @@ -764,14 +769,26 @@ export class RealtimeSession extends llm.RealtimeSession { this.onReceiveMessage(session, message); }, onerror: (error: ErrorEvent) => { - this.#logger.error('Gemini Live session error:', error); + this.#logger.error({ error }, 'Gemini Live session error'); + sessionOpened.set(); // Break the wait if we were waiting for open if (!this.sessionShouldClose.isSet) { this.markRestartNeeded(); } }, onclose: (event: CloseEvent) => { - this.#logger.debug('Gemini Live session closed:', event.code, event.reason); + if (this.activeSession !== session) { + return; + } + + this.#logger.debug( + { code: event.code, reason: event.reason }, + 'Gemini Live session closed', + ); + sessionOpened.set(); // Break the wait if we were waiting for open this.markCurrentGenerationDone(); + if (!this.sessionShouldClose.isSet && !this.#closed) { + this.markRestartNeeded(); + } }, }, config, From 4cf104b966946f08b8b61bb16c7ce7f9c8e00640 Mon Sep 17 00:00:00 2001 From: oOPa <5878650+oOPa@users.noreply.github.com> Date: Sun, 21 Dec 2025 19:17:19 +0000 Subject: [PATCH 5/9] remove build files --- plugins/google/dist/beta/gemini_tts.cjs | 243 ---- plugins/google/dist/beta/gemini_tts.cjs.map | 1 - plugins/google/dist/beta/gemini_tts.d.cts | 47 - plugins/google/dist/beta/gemini_tts.d.ts | 47 - plugins/google/dist/beta/gemini_tts.d.ts.map | 1 - plugins/google/dist/beta/gemini_tts.js | 225 ---- plugins/google/dist/beta/gemini_tts.js.map | 1 - plugins/google/dist/beta/gemini_tts.test.cjs | 9 - .../google/dist/beta/gemini_tts.test.cjs.map | 1 - .../google/dist/beta/gemini_tts.test.d.cts | 2 - plugins/google/dist/beta/gemini_tts.test.d.ts | 2 - .../google/dist/beta/gemini_tts.test.d.ts.map | 1 - plugins/google/dist/beta/gemini_tts.test.js | 8 - .../google/dist/beta/gemini_tts.test.js.map | 1 - plugins/google/dist/beta/index.cjs | 42 - plugins/google/dist/beta/index.cjs.map | 1 - plugins/google/dist/beta/index.d.cts | 3 - plugins/google/dist/beta/index.d.ts | 3 - plugins/google/dist/beta/index.d.ts.map | 1 - plugins/google/dist/beta/index.js | 7 - plugins/google/dist/beta/index.js.map | 1 - .../google/dist/beta/realtime/api_proto.cjs | 17 - .../dist/beta/realtime/api_proto.cjs.map | 1 - .../google/dist/beta/realtime/api_proto.d.cts | 26 - .../google/dist/beta/realtime/api_proto.d.ts | 26 - .../dist/beta/realtime/api_proto.d.ts.map | 1 - .../google/dist/beta/realtime/api_proto.js | 1 - .../dist/beta/realtime/api_proto.js.map | 1 - plugins/google/dist/beta/realtime/index.cjs | 29 - .../google/dist/beta/realtime/index.cjs.map | 1 - plugins/google/dist/beta/realtime/index.d.cts | 3 - plugins/google/dist/beta/realtime/index.d.ts | 3 - .../google/dist/beta/realtime/index.d.ts.map | 1 - plugins/google/dist/beta/realtime/index.js | 5 - .../google/dist/beta/realtime/index.js.map | 1 - .../dist/beta/realtime/realtime_api.cjs | 1010 ----------------- .../dist/beta/realtime/realtime_api.cjs.map | 1 - .../dist/beta/realtime/realtime_api.d.cts | 273 ----- .../dist/beta/realtime/realtime_api.d.ts | 273 ----- .../dist/beta/realtime/realtime_api.d.ts.map | 1 - .../google/dist/beta/realtime/realtime_api.js | 992 ---------------- .../dist/beta/realtime/realtime_api.js.map | 1 - plugins/google/dist/index.cjs | 58 - plugins/google/dist/index.cjs.map | 1 - plugins/google/dist/index.d.cts | 4 - plugins/google/dist/index.d.ts | 4 - plugins/google/dist/index.d.ts.map | 1 - plugins/google/dist/index.js | 20 - plugins/google/dist/index.js.map | 1 - plugins/google/dist/llm.cjs | 381 ------- plugins/google/dist/llm.cjs.map | 1 - plugins/google/dist/llm.d.cts | 82 -- plugins/google/dist/llm.d.ts | 82 -- plugins/google/dist/llm.d.ts.map | 1 - plugins/google/dist/llm.js | 362 ------ plugins/google/dist/llm.js.map | 1 - plugins/google/dist/llm.test.cjs | 14 - plugins/google/dist/llm.test.cjs.map | 1 - plugins/google/dist/llm.test.d.cts | 2 - plugins/google/dist/llm.test.d.ts | 2 - plugins/google/dist/llm.test.d.ts.map | 1 - plugins/google/dist/llm.test.js | 13 - plugins/google/dist/llm.test.js.map | 1 - plugins/google/dist/models.cjs | 17 - plugins/google/dist/models.cjs.map | 1 - plugins/google/dist/models.d.cts | 5 - plugins/google/dist/models.d.ts | 5 - plugins/google/dist/models.d.ts.map | 1 - plugins/google/dist/models.js | 1 - plugins/google/dist/models.js.map | 1 - plugins/google/dist/tools.cjs | 17 - plugins/google/dist/tools.cjs.map | 1 - plugins/google/dist/tools.d.cts | 3 - plugins/google/dist/tools.d.ts | 3 - plugins/google/dist/tools.d.ts.map | 1 - plugins/google/dist/tools.js | 1 - plugins/google/dist/tools.js.map | 1 - plugins/google/dist/utils.cjs | 137 --- plugins/google/dist/utils.cjs.map | 1 - plugins/google/dist/utils.d.cts | 14 - plugins/google/dist/utils.d.ts | 14 - plugins/google/dist/utils.d.ts.map | 1 - plugins/google/dist/utils.js | 112 -- plugins/google/dist/utils.js.map | 1 - 84 files changed, 4685 deletions(-) delete mode 100644 plugins/google/dist/beta/gemini_tts.cjs delete mode 100644 plugins/google/dist/beta/gemini_tts.cjs.map delete mode 100644 plugins/google/dist/beta/gemini_tts.d.cts delete mode 100644 plugins/google/dist/beta/gemini_tts.d.ts delete mode 100644 plugins/google/dist/beta/gemini_tts.d.ts.map delete mode 100644 plugins/google/dist/beta/gemini_tts.js delete mode 100644 plugins/google/dist/beta/gemini_tts.js.map delete mode 100644 plugins/google/dist/beta/gemini_tts.test.cjs delete mode 100644 plugins/google/dist/beta/gemini_tts.test.cjs.map delete mode 100644 plugins/google/dist/beta/gemini_tts.test.d.cts delete mode 100644 plugins/google/dist/beta/gemini_tts.test.d.ts delete mode 100644 plugins/google/dist/beta/gemini_tts.test.d.ts.map delete mode 100644 plugins/google/dist/beta/gemini_tts.test.js delete mode 100644 plugins/google/dist/beta/gemini_tts.test.js.map delete mode 100644 plugins/google/dist/beta/index.cjs delete mode 100644 plugins/google/dist/beta/index.cjs.map delete mode 100644 plugins/google/dist/beta/index.d.cts delete mode 100644 plugins/google/dist/beta/index.d.ts delete mode 100644 plugins/google/dist/beta/index.d.ts.map delete mode 100644 plugins/google/dist/beta/index.js delete mode 100644 plugins/google/dist/beta/index.js.map delete mode 100644 plugins/google/dist/beta/realtime/api_proto.cjs delete mode 100644 plugins/google/dist/beta/realtime/api_proto.cjs.map delete mode 100644 plugins/google/dist/beta/realtime/api_proto.d.cts delete mode 100644 plugins/google/dist/beta/realtime/api_proto.d.ts delete mode 100644 plugins/google/dist/beta/realtime/api_proto.d.ts.map delete mode 100644 plugins/google/dist/beta/realtime/api_proto.js delete mode 100644 plugins/google/dist/beta/realtime/api_proto.js.map delete mode 100644 plugins/google/dist/beta/realtime/index.cjs delete mode 100644 plugins/google/dist/beta/realtime/index.cjs.map delete mode 100644 plugins/google/dist/beta/realtime/index.d.cts delete mode 100644 plugins/google/dist/beta/realtime/index.d.ts delete mode 100644 plugins/google/dist/beta/realtime/index.d.ts.map delete mode 100644 plugins/google/dist/beta/realtime/index.js delete mode 100644 plugins/google/dist/beta/realtime/index.js.map delete mode 100644 plugins/google/dist/beta/realtime/realtime_api.cjs delete mode 100644 plugins/google/dist/beta/realtime/realtime_api.cjs.map delete mode 100644 plugins/google/dist/beta/realtime/realtime_api.d.cts delete mode 100644 plugins/google/dist/beta/realtime/realtime_api.d.ts delete mode 100644 plugins/google/dist/beta/realtime/realtime_api.d.ts.map delete mode 100644 plugins/google/dist/beta/realtime/realtime_api.js delete mode 100644 plugins/google/dist/beta/realtime/realtime_api.js.map delete mode 100644 plugins/google/dist/index.cjs delete mode 100644 plugins/google/dist/index.cjs.map delete mode 100644 plugins/google/dist/index.d.cts delete mode 100644 plugins/google/dist/index.d.ts delete mode 100644 plugins/google/dist/index.d.ts.map delete mode 100644 plugins/google/dist/index.js delete mode 100644 plugins/google/dist/index.js.map delete mode 100644 plugins/google/dist/llm.cjs delete mode 100644 plugins/google/dist/llm.cjs.map delete mode 100644 plugins/google/dist/llm.d.cts delete mode 100644 plugins/google/dist/llm.d.ts delete mode 100644 plugins/google/dist/llm.d.ts.map delete mode 100644 plugins/google/dist/llm.js delete mode 100644 plugins/google/dist/llm.js.map delete mode 100644 plugins/google/dist/llm.test.cjs delete mode 100644 plugins/google/dist/llm.test.cjs.map delete mode 100644 plugins/google/dist/llm.test.d.cts delete mode 100644 plugins/google/dist/llm.test.d.ts delete mode 100644 plugins/google/dist/llm.test.d.ts.map delete mode 100644 plugins/google/dist/llm.test.js delete mode 100644 plugins/google/dist/llm.test.js.map delete mode 100644 plugins/google/dist/models.cjs delete mode 100644 plugins/google/dist/models.cjs.map delete mode 100644 plugins/google/dist/models.d.cts delete mode 100644 plugins/google/dist/models.d.ts delete mode 100644 plugins/google/dist/models.d.ts.map delete mode 100644 plugins/google/dist/models.js delete mode 100644 plugins/google/dist/models.js.map delete mode 100644 plugins/google/dist/tools.cjs delete mode 100644 plugins/google/dist/tools.cjs.map delete mode 100644 plugins/google/dist/tools.d.cts delete mode 100644 plugins/google/dist/tools.d.ts delete mode 100644 plugins/google/dist/tools.d.ts.map delete mode 100644 plugins/google/dist/tools.js delete mode 100644 plugins/google/dist/tools.js.map delete mode 100644 plugins/google/dist/utils.cjs delete mode 100644 plugins/google/dist/utils.cjs.map delete mode 100644 plugins/google/dist/utils.d.cts delete mode 100644 plugins/google/dist/utils.d.ts delete mode 100644 plugins/google/dist/utils.d.ts.map delete mode 100644 plugins/google/dist/utils.js delete mode 100644 plugins/google/dist/utils.js.map diff --git a/plugins/google/dist/beta/gemini_tts.cjs b/plugins/google/dist/beta/gemini_tts.cjs deleted file mode 100644 index cc9117628..000000000 --- a/plugins/google/dist/beta/gemini_tts.cjs +++ /dev/null @@ -1,243 +0,0 @@ -"use strict"; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __export = (target, all) => { - for (var name in all) - __defProp(target, name, { get: all[name], enumerable: true }); -}; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); -var gemini_tts_exports = {}; -__export(gemini_tts_exports, { - ChunkedStream: () => ChunkedStream, - TTS: () => TTS -}); -module.exports = __toCommonJS(gemini_tts_exports); -var import_genai = require("@google/genai"); -var import_agents = require("@livekit/agents"); -const DEFAULT_MODEL = "gemini-2.5-flash-preview-tts"; -const DEFAULT_VOICE = "Kore"; -const DEFAULT_SAMPLE_RATE = 24e3; -const NUM_CHANNELS = 1; -const DEFAULT_INSTRUCTIONS = "Say the text with a proper tone, don't omit or add any words"; -class TTS extends import_agents.tts.TTS { - #opts; - #client; - label = "google.gemini.TTS"; - /** - * Create a new instance of Gemini TTS. - * - * Environment Requirements: - * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file. - * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. - * - * @param opts - Configuration options for Gemini TTS - */ - constructor({ - model = DEFAULT_MODEL, - voiceName = DEFAULT_VOICE, - apiKey, - vertexai, - project, - location, - instructions - } = {}) { - super(DEFAULT_SAMPLE_RATE, NUM_CHANNELS, { streaming: false }); - const gcpProject = project || process.env.GOOGLE_CLOUD_PROJECT; - const gcpLocation = location || process.env.GOOGLE_CLOUD_LOCATION || "us-central1"; - const useVertexai = vertexai ?? process.env.GOOGLE_GENAI_USE_VERTEXAI === "true"; - const geminiApiKey = apiKey || process.env.GOOGLE_API_KEY; - let finalProject = gcpProject; - let finalLocation = gcpLocation; - let finalApiKey = geminiApiKey; - if (useVertexai) { - if (!finalProject) { - throw new import_agents.APIConnectionError({ - message: "Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable" - }); - } - finalApiKey = void 0; - } else { - finalProject = void 0; - finalLocation = void 0; - if (!finalApiKey) { - throw new import_agents.APIConnectionError({ - message: "API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable" - }); - } - } - this.#opts = { - model, - voiceName, - vertexai: useVertexai, - project: finalProject, - location: finalLocation, - instructions: instructions ?? DEFAULT_INSTRUCTIONS - }; - const clientOptions = useVertexai ? { - vertexai: true, - project: finalProject, - location: finalLocation - } : { - apiKey: finalApiKey - }; - this.#client = new import_genai.GoogleGenAI(clientOptions); - } - synthesize(text, connOptions, abortSignal) { - return new ChunkedStream(text, this, connOptions, abortSignal); - } - /** - * Update the TTS options. - * - * @param opts - Options to update - */ - updateOptions(opts) { - if (opts.voiceName !== void 0) { - this.#opts.voiceName = opts.voiceName; - } - } - stream() { - throw new Error("Streaming is not supported on Gemini TTS"); - } - get opts() { - return this.#opts; - } - get client() { - return this.#client; - } -} -class ChunkedStream extends import_agents.tts.ChunkedStream { - #tts; - label = "google.gemini.ChunkedStream"; - constructor(inputText, tts2, connOptions, abortSignal) { - super(inputText, tts2, connOptions, abortSignal); - this.#tts = tts2; - } - async run() { - const requestId = (0, import_agents.shortuuid)(); - const bstream = new import_agents.AudioByteStream(this.#tts.sampleRate, this.#tts.numChannels); - const config = { - responseModalities: ["AUDIO"], - speechConfig: { - voiceConfig: { - prebuiltVoiceConfig: { - voiceName: this.#tts.opts.voiceName - } - } - }, - abortSignal: this.abortSignal - }; - let inputText = this.inputText; - if (this.#tts.opts.instructions) { - inputText = `${this.#tts.opts.instructions}: -"${inputText}"`; - } - const contents = [ - { - role: "user", - parts: [{ text: inputText }] - } - ]; - const responseStream = await this.#tts.client.models.generateContentStream({ - model: this.#tts.opts.model, - contents, - config - }); - try { - for await (const response of responseStream) { - await this.#processResponse(response, bstream, requestId); - } - } catch (error) { - if (error instanceof Error && error.name === "AbortError") { - return; - } - if ((0, import_agents.isAPIError)(error)) throw error; - const err = error; - if (err.code && err.code >= 400 && err.code < 500) { - if (err.code === 429) { - throw new import_agents.APIStatusError({ - message: `Gemini TTS: Rate limit error - ${err.message || "Unknown error"}`, - options: { - statusCode: 429, - retryable: true - } - }); - } else { - throw new import_agents.APIStatusError({ - message: `Gemini TTS: Client error (${err.code}) - ${err.message || "Unknown error"}`, - options: { - statusCode: err.code, - retryable: false - } - }); - } - } - if (err.code && err.code >= 500) { - throw new import_agents.APIStatusError({ - message: `Gemini TTS: Server error (${err.code}) - ${err.message || "Unknown error"}`, - options: { - statusCode: err.code, - retryable: true - } - }); - } - throw new import_agents.APIConnectionError({ - message: `Gemini TTS: Connection error - ${err.message || "Unknown error"}`, - options: { retryable: true } - }); - } finally { - this.queue.close(); - } - } - async #processResponse(response, bstream, requestId) { - var _a, _b, _c; - if (!response.candidates || response.candidates.length === 0) { - return; - } - const candidate = response.candidates[0]; - if (!candidate || !((_a = candidate.content) == null ? void 0 : _a.parts)) { - return; - } - let lastFrame; - const sendLastFrame = (final) => { - if (lastFrame) { - this.queue.put({ - requestId, - frame: lastFrame, - segmentId: requestId, - final - }); - lastFrame = void 0; - } - }; - for (const part of candidate.content.parts) { - if (((_b = part.inlineData) == null ? void 0 : _b.data) && ((_c = part.inlineData.mimeType) == null ? void 0 : _c.startsWith("audio/"))) { - const audioBuffer = Buffer.from(part.inlineData.data, "base64"); - for (const frame of bstream.write(audioBuffer)) { - sendLastFrame(false); - lastFrame = frame; - } - } - } - for (const frame of bstream.flush()) { - sendLastFrame(false); - lastFrame = frame; - } - sendLastFrame(true); - } -} -// Annotate the CommonJS export names for ESM import in node: -0 && (module.exports = { - ChunkedStream, - TTS -}); -//# sourceMappingURL=gemini_tts.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.cjs.map b/plugins/google/dist/beta/gemini_tts.cjs.map deleted file mode 100644 index 050d3cfd4..000000000 --- a/plugins/google/dist/beta/gemini_tts.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../src/beta/gemini_tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as types from '@google/genai';\nimport { GoogleGenAI } from '@google/genai';\nimport {\n type APIConnectOptions,\n APIConnectionError,\n APIStatusError,\n AudioByteStream,\n isAPIError,\n shortuuid,\n tts,\n} from '@livekit/agents';\nimport type { AudioFrame } from '@livekit/rtc-node';\n\nexport type GeminiTTSModels = 'gemini-2.5-flash-preview-tts' | 'gemini-2.5-pro-preview-tts';\n\nexport type GeminiVoices =\n | 'Zephyr'\n | 'Puck'\n | 'Charon'\n | 'Kore'\n | 'Fenrir'\n | 'Leda'\n | 'Orus'\n | 'Aoede'\n | 'Callirrhoe'\n | 'Autonoe'\n | 'Enceladus'\n | 'Iapetus'\n | 'Umbriel'\n | 'Algieba'\n | 'Despina'\n | 'Erinome'\n | 'Algenib'\n | 'Rasalgethi'\n | 'Laomedeia'\n | 'Achernar'\n | 'Alnilam'\n | 'Schedar'\n | 'Gacrux'\n | 'Pulcherrima'\n | 'Achird'\n | 'Zubenelgenubi'\n | 'Vindemiatrix'\n | 'Sadachbia'\n | 'Sadaltager'\n | 'Sulafat';\n\nconst DEFAULT_MODEL: GeminiTTSModels = 'gemini-2.5-flash-preview-tts';\nconst DEFAULT_VOICE: GeminiVoices = 'Kore';\nconst DEFAULT_SAMPLE_RATE = 24000; // not configurable\nconst NUM_CHANNELS = 1;\nconst DEFAULT_INSTRUCTIONS = \"Say the text with a proper tone, don't omit or add any words\";\n\nexport interface TTSOptions {\n model: GeminiTTSModels | string;\n voiceName: GeminiVoices | string;\n vertexai: boolean;\n project?: string;\n location?: string;\n instructions?: string;\n}\n\nexport class TTS extends tts.TTS {\n #opts: TTSOptions;\n #client: GoogleGenAI;\n label = 'google.gemini.TTS';\n\n /**\n * Create a new instance of Gemini TTS.\n *\n * Environment Requirements:\n * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file.\n * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable.\n *\n * @param opts - Configuration options for Gemini TTS\n */\n constructor({\n model = DEFAULT_MODEL,\n voiceName = DEFAULT_VOICE,\n apiKey,\n vertexai,\n project,\n location,\n instructions,\n }: Partial = {}) {\n super(DEFAULT_SAMPLE_RATE, NUM_CHANNELS, { streaming: false });\n\n const gcpProject: string | undefined = project || process.env.GOOGLE_CLOUD_PROJECT;\n const gcpLocation: string | undefined =\n location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const useVertexai = vertexai ?? process.env.GOOGLE_GENAI_USE_VERTEXAI === 'true';\n const geminiApiKey = apiKey || process.env.GOOGLE_API_KEY;\n\n let finalProject: string | undefined = gcpProject;\n let finalLocation: string | undefined = gcpLocation;\n let finalApiKey: string | undefined = geminiApiKey;\n\n if (useVertexai) {\n if (!finalProject) {\n throw new APIConnectionError({\n message:\n 'Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable',\n });\n }\n finalApiKey = undefined;\n } else {\n finalProject = undefined;\n finalLocation = undefined;\n if (!finalApiKey) {\n throw new APIConnectionError({\n message:\n 'API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable',\n });\n }\n }\n\n this.#opts = {\n model,\n voiceName,\n vertexai: useVertexai,\n project: finalProject,\n location: finalLocation,\n instructions: instructions ?? DEFAULT_INSTRUCTIONS,\n };\n\n const clientOptions: types.GoogleGenAIOptions = useVertexai\n ? {\n vertexai: true,\n project: finalProject,\n location: finalLocation,\n }\n : {\n apiKey: finalApiKey,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n }\n\n synthesize(\n text: string,\n connOptions?: APIConnectOptions,\n abortSignal?: AbortSignal,\n ): ChunkedStream {\n return new ChunkedStream(text, this, connOptions, abortSignal);\n }\n\n /**\n * Update the TTS options.\n *\n * @param opts - Options to update\n */\n updateOptions(opts: { voiceName?: GeminiVoices | string }) {\n if (opts.voiceName !== undefined) {\n this.#opts.voiceName = opts.voiceName;\n }\n }\n\n stream(): tts.SynthesizeStream {\n throw new Error('Streaming is not supported on Gemini TTS');\n }\n\n get opts(): TTSOptions {\n return this.#opts;\n }\n\n get client(): GoogleGenAI {\n return this.#client;\n }\n}\n\nexport class ChunkedStream extends tts.ChunkedStream {\n #tts: TTS;\n label = 'google.gemini.ChunkedStream';\n\n constructor(\n inputText: string,\n tts: TTS,\n connOptions?: APIConnectOptions,\n abortSignal?: AbortSignal,\n ) {\n super(inputText, tts, connOptions, abortSignal);\n this.#tts = tts;\n }\n\n protected async run() {\n const requestId = shortuuid();\n const bstream = new AudioByteStream(this.#tts.sampleRate, this.#tts.numChannels);\n\n const config: types.GenerateContentConfig = {\n responseModalities: ['AUDIO'],\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: this.#tts.opts.voiceName,\n },\n },\n },\n abortSignal: this.abortSignal,\n };\n\n let inputText = this.inputText;\n if (this.#tts.opts.instructions) {\n inputText = `${this.#tts.opts.instructions}:\\n\"${inputText}\"`;\n }\n\n const contents: types.Content[] = [\n {\n role: 'user',\n parts: [{ text: inputText }],\n },\n ];\n\n const responseStream = await this.#tts.client.models.generateContentStream({\n model: this.#tts.opts.model,\n contents,\n config,\n });\n\n try {\n for await (const response of responseStream) {\n await this.#processResponse(response, bstream, requestId);\n }\n } catch (error: unknown) {\n if (error instanceof Error && error.name === 'AbortError') {\n return;\n }\n if (isAPIError(error)) throw error;\n\n const err = error as {\n code?: number;\n message?: string;\n status?: string;\n type?: string;\n };\n\n if (err.code && err.code >= 400 && err.code < 500) {\n if (err.code === 429) {\n throw new APIStatusError({\n message: `Gemini TTS: Rate limit error - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: 429,\n retryable: true,\n },\n });\n } else {\n throw new APIStatusError({\n message: `Gemini TTS: Client error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: false,\n },\n });\n }\n }\n\n if (err.code && err.code >= 500) {\n throw new APIStatusError({\n message: `Gemini TTS: Server error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: true,\n },\n });\n }\n\n throw new APIConnectionError({\n message: `Gemini TTS: Connection error - ${err.message || 'Unknown error'}`,\n options: { retryable: true },\n });\n } finally {\n this.queue.close();\n }\n }\n\n async #processResponse(\n response: types.GenerateContentResponse,\n bstream: AudioByteStream,\n requestId: string,\n ) {\n if (!response.candidates || response.candidates.length === 0) {\n return;\n }\n\n const candidate = response.candidates[0];\n if (!candidate || !candidate.content?.parts) {\n return;\n }\n\n let lastFrame: AudioFrame | undefined;\n const sendLastFrame = (final: boolean) => {\n if (lastFrame) {\n this.queue.put({\n requestId,\n frame: lastFrame,\n segmentId: requestId,\n final,\n });\n lastFrame = undefined;\n }\n };\n\n for (const part of candidate.content.parts) {\n if (part.inlineData?.data && part.inlineData.mimeType?.startsWith('audio/')) {\n const audioBuffer = Buffer.from(part.inlineData.data, 'base64');\n\n for (const frame of bstream.write(audioBuffer)) {\n sendLastFrame(false);\n lastFrame = frame;\n }\n }\n }\n\n for (const frame of bstream.flush()) {\n sendLastFrame(false);\n lastFrame = frame;\n }\n\n sendLastFrame(true);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAA4B;AAC5B,oBAQO;AAqCP,MAAM,gBAAiC;AACvC,MAAM,gBAA8B;AACpC,MAAM,sBAAsB;AAC5B,MAAM,eAAe;AACrB,MAAM,uBAAuB;AAWtB,MAAM,YAAY,kBAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EACA,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWR,YAAY;AAAA,IACV,QAAQ;AAAA,IACR,YAAY;AAAA,IACZ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,IAA8C,CAAC,GAAG;AAChD,UAAM,qBAAqB,cAAc,EAAE,WAAW,MAAM,CAAC;AAE7D,UAAM,aAAiC,WAAW,QAAQ,IAAI;AAC9D,UAAM,cACJ,YAAY,QAAQ,IAAI,yBAAyB;AACnD,UAAM,cAAc,YAAY,QAAQ,IAAI,8BAA8B;AAC1E,UAAM,eAAe,UAAU,QAAQ,IAAI;AAE3C,QAAI,eAAmC;AACvC,QAAI,gBAAoC;AACxC,QAAI,cAAkC;AAEtC,QAAI,aAAa;AACf,UAAI,CAAC,cAAc;AACjB,cAAM,IAAI,iCAAmB;AAAA,UAC3B,SACE;AAAA,QACJ,CAAC;AAAA,MACH;AACA,oBAAc;AAAA,IAChB,OAAO;AACL,qBAAe;AACf,sBAAgB;AAChB,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,iCAAmB;AAAA,UAC3B,SACE;AAAA,QACJ,CAAC;AAAA,MACH;AAAA,IACF;AAEA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,MACV,cAAc,gBAAgB;AAAA,IAChC;AAEA,UAAM,gBAA0C,cAC5C;AAAA,MACE,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,IACZ,IACA;AAAA,MACE,QAAQ;AAAA,IACV;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAAA,EAC9C;AAAA,EAEA,WACE,MACA,aACA,aACe;AACf,WAAO,IAAI,cAAc,MAAM,MAAM,aAAa,WAAW;AAAA,EAC/D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,cAAc,MAA6C;AACzD,QAAI,KAAK,cAAc,QAAW;AAChC,WAAK,MAAM,YAAY,KAAK;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,SAA+B;AAC7B,UAAM,IAAI,MAAM,0CAA0C;AAAA,EAC5D;AAAA,EAEA,IAAI,OAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AACF;AAEO,MAAM,sBAAsB,kBAAI,cAAc;AAAA,EACnD;AAAA,EACA,QAAQ;AAAA,EAER,YACE,WACAA,MACA,aACA,aACA;AACA,UAAM,WAAWA,MAAK,aAAa,WAAW;AAC9C,SAAK,OAAOA;AAAA,EACd;AAAA,EAEA,MAAgB,MAAM;AACpB,UAAM,gBAAY,yBAAU;AAC5B,UAAM,UAAU,IAAI,8BAAgB,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW;AAE/E,UAAM,SAAsC;AAAA,MAC1C,oBAAoB,CAAC,OAAO;AAAA,MAC5B,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK,KAAK,KAAK;AAAA,UAC5B;AAAA,QACF;AAAA,MACF;AAAA,MACA,aAAa,KAAK;AAAA,IACpB;AAEA,QAAI,YAAY,KAAK;AACrB,QAAI,KAAK,KAAK,KAAK,cAAc;AAC/B,kBAAY,GAAG,KAAK,KAAK,KAAK,YAAY;AAAA,GAAO,SAAS;AAAA,IAC5D;AAEA,UAAM,WAA4B;AAAA,MAChC;AAAA,QACE,MAAM;AAAA,QACN,OAAO,CAAC,EAAE,MAAM,UAAU,CAAC;AAAA,MAC7B;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,KAAK,OAAO,OAAO,sBAAsB;AAAA,MACzE,OAAO,KAAK,KAAK,KAAK;AAAA,MACtB;AAAA,MACA;AAAA,IACF,CAAC;AAED,QAAI;AACF,uBAAiB,YAAY,gBAAgB;AAC3C,cAAM,KAAK,iBAAiB,UAAU,SAAS,SAAS;AAAA,MAC1D;AAAA,IACF,SAAS,OAAgB;AACvB,UAAI,iBAAiB,SAAS,MAAM,SAAS,cAAc;AACzD;AAAA,MACF;AACA,cAAI,0BAAW,KAAK,EAAG,OAAM;AAE7B,YAAM,MAAM;AAOZ,UAAI,IAAI,QAAQ,IAAI,QAAQ,OAAO,IAAI,OAAO,KAAK;AACjD,YAAI,IAAI,SAAS,KAAK;AACpB,gBAAM,IAAI,6BAAe;AAAA,YACvB,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,YACzE,SAAS;AAAA,cACP,YAAY;AAAA,cACZ,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH,OAAO;AACL,gBAAM,IAAI,6BAAe;AAAA,YACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,YACnF,SAAS;AAAA,cACP,YAAY,IAAI;AAAA,cAChB,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,IAAI,QAAQ,IAAI,QAAQ,KAAK;AAC/B,cAAM,IAAI,6BAAe;AAAA,UACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,UACnF,SAAS;AAAA,YACP,YAAY,IAAI;AAAA,YAChB,WAAW;AAAA,UACb;AAAA,QACF,CAAC;AAAA,MACH;AAEA,YAAM,IAAI,iCAAmB;AAAA,QAC3B,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,QACzE,SAAS,EAAE,WAAW,KAAK;AAAA,MAC7B,CAAC;AAAA,IACH,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAM,iBACJ,UACA,SACA,WACA;AAzRJ;AA0RI,QAAI,CAAC,SAAS,cAAc,SAAS,WAAW,WAAW,GAAG;AAC5D;AAAA,IACF;AAEA,UAAM,YAAY,SAAS,WAAW,CAAC;AACvC,QAAI,CAAC,aAAa,GAAC,eAAU,YAAV,mBAAmB,QAAO;AAC3C;AAAA,IACF;AAEA,QAAI;AACJ,UAAM,gBAAgB,CAAC,UAAmB;AACxC,UAAI,WAAW;AACb,aAAK,MAAM,IAAI;AAAA,UACb;AAAA,UACA,OAAO;AAAA,UACP,WAAW;AAAA,UACX;AAAA,QACF,CAAC;AACD,oBAAY;AAAA,MACd;AAAA,IACF;AAEA,eAAW,QAAQ,UAAU,QAAQ,OAAO;AAC1C,YAAI,UAAK,eAAL,mBAAiB,WAAQ,UAAK,WAAW,aAAhB,mBAA0B,WAAW,YAAW;AAC3E,cAAM,cAAc,OAAO,KAAK,KAAK,WAAW,MAAM,QAAQ;AAE9D,mBAAW,SAAS,QAAQ,MAAM,WAAW,GAAG;AAC9C,wBAAc,KAAK;AACnB,sBAAY;AAAA,QACd;AAAA,MACF;AAAA,IACF;AAEA,eAAW,SAAS,QAAQ,MAAM,GAAG;AACnC,oBAAc,KAAK;AACnB,kBAAY;AAAA,IACd;AAEA,kBAAc,IAAI;AAAA,EACpB;AACF;","names":["tts"]} \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.d.cts b/plugins/google/dist/beta/gemini_tts.d.cts deleted file mode 100644 index 3f228c356..000000000 --- a/plugins/google/dist/beta/gemini_tts.d.cts +++ /dev/null @@ -1,47 +0,0 @@ -import { GoogleGenAI } from '@google/genai'; -import { type APIConnectOptions, tts } from '@livekit/agents'; -export type GeminiTTSModels = 'gemini-2.5-flash-preview-tts' | 'gemini-2.5-pro-preview-tts'; -export type GeminiVoices = 'Zephyr' | 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Leda' | 'Orus' | 'Aoede' | 'Callirrhoe' | 'Autonoe' | 'Enceladus' | 'Iapetus' | 'Umbriel' | 'Algieba' | 'Despina' | 'Erinome' | 'Algenib' | 'Rasalgethi' | 'Laomedeia' | 'Achernar' | 'Alnilam' | 'Schedar' | 'Gacrux' | 'Pulcherrima' | 'Achird' | 'Zubenelgenubi' | 'Vindemiatrix' | 'Sadachbia' | 'Sadaltager' | 'Sulafat'; -export interface TTSOptions { - model: GeminiTTSModels | string; - voiceName: GeminiVoices | string; - vertexai: boolean; - project?: string; - location?: string; - instructions?: string; -} -export declare class TTS extends tts.TTS { - #private; - label: string; - /** - * Create a new instance of Gemini TTS. - * - * Environment Requirements: - * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file. - * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. - * - * @param opts - Configuration options for Gemini TTS - */ - constructor({ model, voiceName, apiKey, vertexai, project, location, instructions, }?: Partial); - synthesize(text: string, connOptions?: APIConnectOptions, abortSignal?: AbortSignal): ChunkedStream; - /** - * Update the TTS options. - * - * @param opts - Options to update - */ - updateOptions(opts: { - voiceName?: GeminiVoices | string; - }): void; - stream(): tts.SynthesizeStream; - get opts(): TTSOptions; - get client(): GoogleGenAI; -} -export declare class ChunkedStream extends tts.ChunkedStream { - #private; - label: string; - constructor(inputText: string, tts: TTS, connOptions?: APIConnectOptions, abortSignal?: AbortSignal); - protected run(): Promise; -} -//# sourceMappingURL=gemini_tts.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.d.ts b/plugins/google/dist/beta/gemini_tts.d.ts deleted file mode 100644 index 3f228c356..000000000 --- a/plugins/google/dist/beta/gemini_tts.d.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { GoogleGenAI } from '@google/genai'; -import { type APIConnectOptions, tts } from '@livekit/agents'; -export type GeminiTTSModels = 'gemini-2.5-flash-preview-tts' | 'gemini-2.5-pro-preview-tts'; -export type GeminiVoices = 'Zephyr' | 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Leda' | 'Orus' | 'Aoede' | 'Callirrhoe' | 'Autonoe' | 'Enceladus' | 'Iapetus' | 'Umbriel' | 'Algieba' | 'Despina' | 'Erinome' | 'Algenib' | 'Rasalgethi' | 'Laomedeia' | 'Achernar' | 'Alnilam' | 'Schedar' | 'Gacrux' | 'Pulcherrima' | 'Achird' | 'Zubenelgenubi' | 'Vindemiatrix' | 'Sadachbia' | 'Sadaltager' | 'Sulafat'; -export interface TTSOptions { - model: GeminiTTSModels | string; - voiceName: GeminiVoices | string; - vertexai: boolean; - project?: string; - location?: string; - instructions?: string; -} -export declare class TTS extends tts.TTS { - #private; - label: string; - /** - * Create a new instance of Gemini TTS. - * - * Environment Requirements: - * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file. - * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. - * - * @param opts - Configuration options for Gemini TTS - */ - constructor({ model, voiceName, apiKey, vertexai, project, location, instructions, }?: Partial); - synthesize(text: string, connOptions?: APIConnectOptions, abortSignal?: AbortSignal): ChunkedStream; - /** - * Update the TTS options. - * - * @param opts - Options to update - */ - updateOptions(opts: { - voiceName?: GeminiVoices | string; - }): void; - stream(): tts.SynthesizeStream; - get opts(): TTSOptions; - get client(): GoogleGenAI; -} -export declare class ChunkedStream extends tts.ChunkedStream { - #private; - label: string; - constructor(inputText: string, tts: TTS, connOptions?: APIConnectOptions, abortSignal?: AbortSignal); - protected run(): Promise; -} -//# sourceMappingURL=gemini_tts.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.d.ts.map b/plugins/google/dist/beta/gemini_tts.d.ts.map deleted file mode 100644 index a4f33ca24..000000000 --- a/plugins/google/dist/beta/gemini_tts.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"gemini_tts.d.ts","sourceRoot":"","sources":["../../src/beta/gemini_tts.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EACL,KAAK,iBAAiB,EAMtB,GAAG,EACJ,MAAM,iBAAiB,CAAC;AAGzB,MAAM,MAAM,eAAe,GAAG,8BAA8B,GAAG,4BAA4B,CAAC;AAE5F,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,MAAM,GACN,QAAQ,GACR,MAAM,GACN,QAAQ,GACR,MAAM,GACN,MAAM,GACN,OAAO,GACP,YAAY,GACZ,SAAS,GACT,WAAW,GACX,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,YAAY,GACZ,WAAW,GACX,UAAU,GACV,SAAS,GACT,SAAS,GACT,QAAQ,GACR,aAAa,GACb,QAAQ,GACR,eAAe,GACf,cAAc,GACd,WAAW,GACX,YAAY,GACZ,SAAS,CAAC;AAQd,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,eAAe,GAAG,MAAM,CAAC;IAChC,SAAS,EAAE,YAAY,GAAG,MAAM,CAAC;IACjC,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;;IAG9B,KAAK,SAAuB;IAE5B;;;;;;;;OAQG;gBACS,EACV,KAAqB,EACrB,SAAyB,EACzB,MAAM,EACN,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,YAAY,GACb,GAAE,OAAO,CAAC,UAAU,GAAG;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAM;IAsDhD,UAAU,CACR,IAAI,EAAE,MAAM,EACZ,WAAW,CAAC,EAAE,iBAAiB,EAC/B,WAAW,CAAC,EAAE,WAAW,GACxB,aAAa;IAIhB;;;;OAIG;IACH,aAAa,CAAC,IAAI,EAAE;QAAE,SAAS,CAAC,EAAE,YAAY,GAAG,MAAM,CAAA;KAAE;IAMzD,MAAM,IAAI,GAAG,CAAC,gBAAgB;IAI9B,IAAI,IAAI,IAAI,UAAU,CAErB;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;CACF;AAED,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;;IAElD,KAAK,SAAiC;gBAGpC,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,GAAG,EACR,WAAW,CAAC,EAAE,iBAAiB,EAC/B,WAAW,CAAC,EAAE,WAAW;cAMX,GAAG;CAuIpB"} \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.js b/plugins/google/dist/beta/gemini_tts.js deleted file mode 100644 index 747d356e5..000000000 --- a/plugins/google/dist/beta/gemini_tts.js +++ /dev/null @@ -1,225 +0,0 @@ -import { GoogleGenAI } from "@google/genai"; -import { - APIConnectionError, - APIStatusError, - AudioByteStream, - isAPIError, - shortuuid, - tts -} from "@livekit/agents"; -const DEFAULT_MODEL = "gemini-2.5-flash-preview-tts"; -const DEFAULT_VOICE = "Kore"; -const DEFAULT_SAMPLE_RATE = 24e3; -const NUM_CHANNELS = 1; -const DEFAULT_INSTRUCTIONS = "Say the text with a proper tone, don't omit or add any words"; -class TTS extends tts.TTS { - #opts; - #client; - label = "google.gemini.TTS"; - /** - * Create a new instance of Gemini TTS. - * - * Environment Requirements: - * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file. - * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. - * - * @param opts - Configuration options for Gemini TTS - */ - constructor({ - model = DEFAULT_MODEL, - voiceName = DEFAULT_VOICE, - apiKey, - vertexai, - project, - location, - instructions - } = {}) { - super(DEFAULT_SAMPLE_RATE, NUM_CHANNELS, { streaming: false }); - const gcpProject = project || process.env.GOOGLE_CLOUD_PROJECT; - const gcpLocation = location || process.env.GOOGLE_CLOUD_LOCATION || "us-central1"; - const useVertexai = vertexai ?? process.env.GOOGLE_GENAI_USE_VERTEXAI === "true"; - const geminiApiKey = apiKey || process.env.GOOGLE_API_KEY; - let finalProject = gcpProject; - let finalLocation = gcpLocation; - let finalApiKey = geminiApiKey; - if (useVertexai) { - if (!finalProject) { - throw new APIConnectionError({ - message: "Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable" - }); - } - finalApiKey = void 0; - } else { - finalProject = void 0; - finalLocation = void 0; - if (!finalApiKey) { - throw new APIConnectionError({ - message: "API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable" - }); - } - } - this.#opts = { - model, - voiceName, - vertexai: useVertexai, - project: finalProject, - location: finalLocation, - instructions: instructions ?? DEFAULT_INSTRUCTIONS - }; - const clientOptions = useVertexai ? { - vertexai: true, - project: finalProject, - location: finalLocation - } : { - apiKey: finalApiKey - }; - this.#client = new GoogleGenAI(clientOptions); - } - synthesize(text, connOptions, abortSignal) { - return new ChunkedStream(text, this, connOptions, abortSignal); - } - /** - * Update the TTS options. - * - * @param opts - Options to update - */ - updateOptions(opts) { - if (opts.voiceName !== void 0) { - this.#opts.voiceName = opts.voiceName; - } - } - stream() { - throw new Error("Streaming is not supported on Gemini TTS"); - } - get opts() { - return this.#opts; - } - get client() { - return this.#client; - } -} -class ChunkedStream extends tts.ChunkedStream { - #tts; - label = "google.gemini.ChunkedStream"; - constructor(inputText, tts2, connOptions, abortSignal) { - super(inputText, tts2, connOptions, abortSignal); - this.#tts = tts2; - } - async run() { - const requestId = shortuuid(); - const bstream = new AudioByteStream(this.#tts.sampleRate, this.#tts.numChannels); - const config = { - responseModalities: ["AUDIO"], - speechConfig: { - voiceConfig: { - prebuiltVoiceConfig: { - voiceName: this.#tts.opts.voiceName - } - } - }, - abortSignal: this.abortSignal - }; - let inputText = this.inputText; - if (this.#tts.opts.instructions) { - inputText = `${this.#tts.opts.instructions}: -"${inputText}"`; - } - const contents = [ - { - role: "user", - parts: [{ text: inputText }] - } - ]; - const responseStream = await this.#tts.client.models.generateContentStream({ - model: this.#tts.opts.model, - contents, - config - }); - try { - for await (const response of responseStream) { - await this.#processResponse(response, bstream, requestId); - } - } catch (error) { - if (error instanceof Error && error.name === "AbortError") { - return; - } - if (isAPIError(error)) throw error; - const err = error; - if (err.code && err.code >= 400 && err.code < 500) { - if (err.code === 429) { - throw new APIStatusError({ - message: `Gemini TTS: Rate limit error - ${err.message || "Unknown error"}`, - options: { - statusCode: 429, - retryable: true - } - }); - } else { - throw new APIStatusError({ - message: `Gemini TTS: Client error (${err.code}) - ${err.message || "Unknown error"}`, - options: { - statusCode: err.code, - retryable: false - } - }); - } - } - if (err.code && err.code >= 500) { - throw new APIStatusError({ - message: `Gemini TTS: Server error (${err.code}) - ${err.message || "Unknown error"}`, - options: { - statusCode: err.code, - retryable: true - } - }); - } - throw new APIConnectionError({ - message: `Gemini TTS: Connection error - ${err.message || "Unknown error"}`, - options: { retryable: true } - }); - } finally { - this.queue.close(); - } - } - async #processResponse(response, bstream, requestId) { - var _a, _b, _c; - if (!response.candidates || response.candidates.length === 0) { - return; - } - const candidate = response.candidates[0]; - if (!candidate || !((_a = candidate.content) == null ? void 0 : _a.parts)) { - return; - } - let lastFrame; - const sendLastFrame = (final) => { - if (lastFrame) { - this.queue.put({ - requestId, - frame: lastFrame, - segmentId: requestId, - final - }); - lastFrame = void 0; - } - }; - for (const part of candidate.content.parts) { - if (((_b = part.inlineData) == null ? void 0 : _b.data) && ((_c = part.inlineData.mimeType) == null ? void 0 : _c.startsWith("audio/"))) { - const audioBuffer = Buffer.from(part.inlineData.data, "base64"); - for (const frame of bstream.write(audioBuffer)) { - sendLastFrame(false); - lastFrame = frame; - } - } - } - for (const frame of bstream.flush()) { - sendLastFrame(false); - lastFrame = frame; - } - sendLastFrame(true); - } -} -export { - ChunkedStream, - TTS -}; -//# sourceMappingURL=gemini_tts.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.js.map b/plugins/google/dist/beta/gemini_tts.js.map deleted file mode 100644 index a4e3e1088..000000000 --- a/plugins/google/dist/beta/gemini_tts.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../src/beta/gemini_tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as types from '@google/genai';\nimport { GoogleGenAI } from '@google/genai';\nimport {\n type APIConnectOptions,\n APIConnectionError,\n APIStatusError,\n AudioByteStream,\n isAPIError,\n shortuuid,\n tts,\n} from '@livekit/agents';\nimport type { AudioFrame } from '@livekit/rtc-node';\n\nexport type GeminiTTSModels = 'gemini-2.5-flash-preview-tts' | 'gemini-2.5-pro-preview-tts';\n\nexport type GeminiVoices =\n | 'Zephyr'\n | 'Puck'\n | 'Charon'\n | 'Kore'\n | 'Fenrir'\n | 'Leda'\n | 'Orus'\n | 'Aoede'\n | 'Callirrhoe'\n | 'Autonoe'\n | 'Enceladus'\n | 'Iapetus'\n | 'Umbriel'\n | 'Algieba'\n | 'Despina'\n | 'Erinome'\n | 'Algenib'\n | 'Rasalgethi'\n | 'Laomedeia'\n | 'Achernar'\n | 'Alnilam'\n | 'Schedar'\n | 'Gacrux'\n | 'Pulcherrima'\n | 'Achird'\n | 'Zubenelgenubi'\n | 'Vindemiatrix'\n | 'Sadachbia'\n | 'Sadaltager'\n | 'Sulafat';\n\nconst DEFAULT_MODEL: GeminiTTSModels = 'gemini-2.5-flash-preview-tts';\nconst DEFAULT_VOICE: GeminiVoices = 'Kore';\nconst DEFAULT_SAMPLE_RATE = 24000; // not configurable\nconst NUM_CHANNELS = 1;\nconst DEFAULT_INSTRUCTIONS = \"Say the text with a proper tone, don't omit or add any words\";\n\nexport interface TTSOptions {\n model: GeminiTTSModels | string;\n voiceName: GeminiVoices | string;\n vertexai: boolean;\n project?: string;\n location?: string;\n instructions?: string;\n}\n\nexport class TTS extends tts.TTS {\n #opts: TTSOptions;\n #client: GoogleGenAI;\n label = 'google.gemini.TTS';\n\n /**\n * Create a new instance of Gemini TTS.\n *\n * Environment Requirements:\n * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file.\n * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable.\n *\n * @param opts - Configuration options for Gemini TTS\n */\n constructor({\n model = DEFAULT_MODEL,\n voiceName = DEFAULT_VOICE,\n apiKey,\n vertexai,\n project,\n location,\n instructions,\n }: Partial = {}) {\n super(DEFAULT_SAMPLE_RATE, NUM_CHANNELS, { streaming: false });\n\n const gcpProject: string | undefined = project || process.env.GOOGLE_CLOUD_PROJECT;\n const gcpLocation: string | undefined =\n location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const useVertexai = vertexai ?? process.env.GOOGLE_GENAI_USE_VERTEXAI === 'true';\n const geminiApiKey = apiKey || process.env.GOOGLE_API_KEY;\n\n let finalProject: string | undefined = gcpProject;\n let finalLocation: string | undefined = gcpLocation;\n let finalApiKey: string | undefined = geminiApiKey;\n\n if (useVertexai) {\n if (!finalProject) {\n throw new APIConnectionError({\n message:\n 'Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable',\n });\n }\n finalApiKey = undefined;\n } else {\n finalProject = undefined;\n finalLocation = undefined;\n if (!finalApiKey) {\n throw new APIConnectionError({\n message:\n 'API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable',\n });\n }\n }\n\n this.#opts = {\n model,\n voiceName,\n vertexai: useVertexai,\n project: finalProject,\n location: finalLocation,\n instructions: instructions ?? DEFAULT_INSTRUCTIONS,\n };\n\n const clientOptions: types.GoogleGenAIOptions = useVertexai\n ? {\n vertexai: true,\n project: finalProject,\n location: finalLocation,\n }\n : {\n apiKey: finalApiKey,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n }\n\n synthesize(\n text: string,\n connOptions?: APIConnectOptions,\n abortSignal?: AbortSignal,\n ): ChunkedStream {\n return new ChunkedStream(text, this, connOptions, abortSignal);\n }\n\n /**\n * Update the TTS options.\n *\n * @param opts - Options to update\n */\n updateOptions(opts: { voiceName?: GeminiVoices | string }) {\n if (opts.voiceName !== undefined) {\n this.#opts.voiceName = opts.voiceName;\n }\n }\n\n stream(): tts.SynthesizeStream {\n throw new Error('Streaming is not supported on Gemini TTS');\n }\n\n get opts(): TTSOptions {\n return this.#opts;\n }\n\n get client(): GoogleGenAI {\n return this.#client;\n }\n}\n\nexport class ChunkedStream extends tts.ChunkedStream {\n #tts: TTS;\n label = 'google.gemini.ChunkedStream';\n\n constructor(\n inputText: string,\n tts: TTS,\n connOptions?: APIConnectOptions,\n abortSignal?: AbortSignal,\n ) {\n super(inputText, tts, connOptions, abortSignal);\n this.#tts = tts;\n }\n\n protected async run() {\n const requestId = shortuuid();\n const bstream = new AudioByteStream(this.#tts.sampleRate, this.#tts.numChannels);\n\n const config: types.GenerateContentConfig = {\n responseModalities: ['AUDIO'],\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: this.#tts.opts.voiceName,\n },\n },\n },\n abortSignal: this.abortSignal,\n };\n\n let inputText = this.inputText;\n if (this.#tts.opts.instructions) {\n inputText = `${this.#tts.opts.instructions}:\\n\"${inputText}\"`;\n }\n\n const contents: types.Content[] = [\n {\n role: 'user',\n parts: [{ text: inputText }],\n },\n ];\n\n const responseStream = await this.#tts.client.models.generateContentStream({\n model: this.#tts.opts.model,\n contents,\n config,\n });\n\n try {\n for await (const response of responseStream) {\n await this.#processResponse(response, bstream, requestId);\n }\n } catch (error: unknown) {\n if (error instanceof Error && error.name === 'AbortError') {\n return;\n }\n if (isAPIError(error)) throw error;\n\n const err = error as {\n code?: number;\n message?: string;\n status?: string;\n type?: string;\n };\n\n if (err.code && err.code >= 400 && err.code < 500) {\n if (err.code === 429) {\n throw new APIStatusError({\n message: `Gemini TTS: Rate limit error - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: 429,\n retryable: true,\n },\n });\n } else {\n throw new APIStatusError({\n message: `Gemini TTS: Client error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: false,\n },\n });\n }\n }\n\n if (err.code && err.code >= 500) {\n throw new APIStatusError({\n message: `Gemini TTS: Server error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: true,\n },\n });\n }\n\n throw new APIConnectionError({\n message: `Gemini TTS: Connection error - ${err.message || 'Unknown error'}`,\n options: { retryable: true },\n });\n } finally {\n this.queue.close();\n }\n }\n\n async #processResponse(\n response: types.GenerateContentResponse,\n bstream: AudioByteStream,\n requestId: string,\n ) {\n if (!response.candidates || response.candidates.length === 0) {\n return;\n }\n\n const candidate = response.candidates[0];\n if (!candidate || !candidate.content?.parts) {\n return;\n }\n\n let lastFrame: AudioFrame | undefined;\n const sendLastFrame = (final: boolean) => {\n if (lastFrame) {\n this.queue.put({\n requestId,\n frame: lastFrame,\n segmentId: requestId,\n final,\n });\n lastFrame = undefined;\n }\n };\n\n for (const part of candidate.content.parts) {\n if (part.inlineData?.data && part.inlineData.mimeType?.startsWith('audio/')) {\n const audioBuffer = Buffer.from(part.inlineData.data, 'base64');\n\n for (const frame of bstream.write(audioBuffer)) {\n sendLastFrame(false);\n lastFrame = frame;\n }\n }\n }\n\n for (const frame of bstream.flush()) {\n sendLastFrame(false);\n lastFrame = frame;\n }\n\n sendLastFrame(true);\n }\n}\n"],"mappings":"AAIA,SAAS,mBAAmB;AAC5B;AAAA,EAEE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAqCP,MAAM,gBAAiC;AACvC,MAAM,gBAA8B;AACpC,MAAM,sBAAsB;AAC5B,MAAM,eAAe;AACrB,MAAM,uBAAuB;AAWtB,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EACA,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWR,YAAY;AAAA,IACV,QAAQ;AAAA,IACR,YAAY;AAAA,IACZ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,IAA8C,CAAC,GAAG;AAChD,UAAM,qBAAqB,cAAc,EAAE,WAAW,MAAM,CAAC;AAE7D,UAAM,aAAiC,WAAW,QAAQ,IAAI;AAC9D,UAAM,cACJ,YAAY,QAAQ,IAAI,yBAAyB;AACnD,UAAM,cAAc,YAAY,QAAQ,IAAI,8BAA8B;AAC1E,UAAM,eAAe,UAAU,QAAQ,IAAI;AAE3C,QAAI,eAAmC;AACvC,QAAI,gBAAoC;AACxC,QAAI,cAAkC;AAEtC,QAAI,aAAa;AACf,UAAI,CAAC,cAAc;AACjB,cAAM,IAAI,mBAAmB;AAAA,UAC3B,SACE;AAAA,QACJ,CAAC;AAAA,MACH;AACA,oBAAc;AAAA,IAChB,OAAO;AACL,qBAAe;AACf,sBAAgB;AAChB,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,mBAAmB;AAAA,UAC3B,SACE;AAAA,QACJ,CAAC;AAAA,MACH;AAAA,IACF;AAEA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,MACV,cAAc,gBAAgB;AAAA,IAChC;AAEA,UAAM,gBAA0C,cAC5C;AAAA,MACE,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,IACZ,IACA;AAAA,MACE,QAAQ;AAAA,IACV;AAEJ,SAAK,UAAU,IAAI,YAAY,aAAa;AAAA,EAC9C;AAAA,EAEA,WACE,MACA,aACA,aACe;AACf,WAAO,IAAI,cAAc,MAAM,MAAM,aAAa,WAAW;AAAA,EAC/D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,cAAc,MAA6C;AACzD,QAAI,KAAK,cAAc,QAAW;AAChC,WAAK,MAAM,YAAY,KAAK;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,SAA+B;AAC7B,UAAM,IAAI,MAAM,0CAA0C;AAAA,EAC5D;AAAA,EAEA,IAAI,OAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AACF;AAEO,MAAM,sBAAsB,IAAI,cAAc;AAAA,EACnD;AAAA,EACA,QAAQ;AAAA,EAER,YACE,WACAA,MACA,aACA,aACA;AACA,UAAM,WAAWA,MAAK,aAAa,WAAW;AAC9C,SAAK,OAAOA;AAAA,EACd;AAAA,EAEA,MAAgB,MAAM;AACpB,UAAM,YAAY,UAAU;AAC5B,UAAM,UAAU,IAAI,gBAAgB,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW;AAE/E,UAAM,SAAsC;AAAA,MAC1C,oBAAoB,CAAC,OAAO;AAAA,MAC5B,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK,KAAK,KAAK;AAAA,UAC5B;AAAA,QACF;AAAA,MACF;AAAA,MACA,aAAa,KAAK;AAAA,IACpB;AAEA,QAAI,YAAY,KAAK;AACrB,QAAI,KAAK,KAAK,KAAK,cAAc;AAC/B,kBAAY,GAAG,KAAK,KAAK,KAAK,YAAY;AAAA,GAAO,SAAS;AAAA,IAC5D;AAEA,UAAM,WAA4B;AAAA,MAChC;AAAA,QACE,MAAM;AAAA,QACN,OAAO,CAAC,EAAE,MAAM,UAAU,CAAC;AAAA,MAC7B;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,KAAK,OAAO,OAAO,sBAAsB;AAAA,MACzE,OAAO,KAAK,KAAK,KAAK;AAAA,MACtB;AAAA,MACA;AAAA,IACF,CAAC;AAED,QAAI;AACF,uBAAiB,YAAY,gBAAgB;AAC3C,cAAM,KAAK,iBAAiB,UAAU,SAAS,SAAS;AAAA,MAC1D;AAAA,IACF,SAAS,OAAgB;AACvB,UAAI,iBAAiB,SAAS,MAAM,SAAS,cAAc;AACzD;AAAA,MACF;AACA,UAAI,WAAW,KAAK,EAAG,OAAM;AAE7B,YAAM,MAAM;AAOZ,UAAI,IAAI,QAAQ,IAAI,QAAQ,OAAO,IAAI,OAAO,KAAK;AACjD,YAAI,IAAI,SAAS,KAAK;AACpB,gBAAM,IAAI,eAAe;AAAA,YACvB,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,YACzE,SAAS;AAAA,cACP,YAAY;AAAA,cACZ,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH,OAAO;AACL,gBAAM,IAAI,eAAe;AAAA,YACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,YACnF,SAAS;AAAA,cACP,YAAY,IAAI;AAAA,cAChB,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,IAAI,QAAQ,IAAI,QAAQ,KAAK;AAC/B,cAAM,IAAI,eAAe;AAAA,UACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,UACnF,SAAS;AAAA,YACP,YAAY,IAAI;AAAA,YAChB,WAAW;AAAA,UACb;AAAA,QACF,CAAC;AAAA,MACH;AAEA,YAAM,IAAI,mBAAmB;AAAA,QAC3B,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,QACzE,SAAS,EAAE,WAAW,KAAK;AAAA,MAC7B,CAAC;AAAA,IACH,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAM,iBACJ,UACA,SACA,WACA;AAzRJ;AA0RI,QAAI,CAAC,SAAS,cAAc,SAAS,WAAW,WAAW,GAAG;AAC5D;AAAA,IACF;AAEA,UAAM,YAAY,SAAS,WAAW,CAAC;AACvC,QAAI,CAAC,aAAa,GAAC,eAAU,YAAV,mBAAmB,QAAO;AAC3C;AAAA,IACF;AAEA,QAAI;AACJ,UAAM,gBAAgB,CAAC,UAAmB;AACxC,UAAI,WAAW;AACb,aAAK,MAAM,IAAI;AAAA,UACb;AAAA,UACA,OAAO;AAAA,UACP,WAAW;AAAA,UACX;AAAA,QACF,CAAC;AACD,oBAAY;AAAA,MACd;AAAA,IACF;AAEA,eAAW,QAAQ,UAAU,QAAQ,OAAO;AAC1C,YAAI,UAAK,eAAL,mBAAiB,WAAQ,UAAK,WAAW,aAAhB,mBAA0B,WAAW,YAAW;AAC3E,cAAM,cAAc,OAAO,KAAK,KAAK,WAAW,MAAM,QAAQ;AAE9D,mBAAW,SAAS,QAAQ,MAAM,WAAW,GAAG;AAC9C,wBAAc,KAAK;AACnB,sBAAY;AAAA,QACd;AAAA,MACF;AAAA,IACF;AAEA,eAAW,SAAS,QAAQ,MAAM,GAAG;AACnC,oBAAc,KAAK;AACnB,kBAAY;AAAA,IACd;AAEA,kBAAc,IAAI;AAAA,EACpB;AACF;","names":["tts"]} \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.cjs b/plugins/google/dist/beta/gemini_tts.test.cjs deleted file mode 100644 index 207bb62be..000000000 --- a/plugins/google/dist/beta/gemini_tts.test.cjs +++ /dev/null @@ -1,9 +0,0 @@ -"use strict"; -var import_agents_plugin_openai = require("@livekit/agents-plugin-openai"); -var import_agents_plugins_test = require("@livekit/agents-plugins-test"); -var import_vitest = require("vitest"); -var import_gemini_tts = require("./gemini_tts.cjs"); -import_vitest.describe.skip("Google Gemini TTS", async () => { - await (0, import_agents_plugins_test.tts)(new import_gemini_tts.TTS(), new import_agents_plugin_openai.STT()); -}); -//# sourceMappingURL=gemini_tts.test.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.cjs.map b/plugins/google/dist/beta/gemini_tts.test.cjs.map deleted file mode 100644 index f0d1500b9..000000000 --- a/plugins/google/dist/beta/gemini_tts.test.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../src/beta/gemini_tts.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { STT } from '@livekit/agents-plugin-openai';\nimport { tts } from '@livekit/agents-plugins-test';\nimport { describe } from 'vitest';\nimport { TTS } from './gemini_tts.js';\n\ndescribe.skip('Google Gemini TTS', async () => {\n await tts(new TTS(), new STT());\n});\n"],"mappings":";AAGA,kCAAoB;AACpB,iCAAoB;AACpB,oBAAyB;AACzB,wBAAoB;AAEpB,uBAAS,KAAK,qBAAqB,YAAY;AAC7C,YAAM,gCAAI,IAAI,sBAAI,GAAG,IAAI,gCAAI,CAAC;AAChC,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.d.cts b/plugins/google/dist/beta/gemini_tts.test.d.cts deleted file mode 100644 index 3578eadc7..000000000 --- a/plugins/google/dist/beta/gemini_tts.test.d.cts +++ /dev/null @@ -1,2 +0,0 @@ -export {}; -//# sourceMappingURL=gemini_tts.test.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.d.ts b/plugins/google/dist/beta/gemini_tts.test.d.ts deleted file mode 100644 index 3578eadc7..000000000 --- a/plugins/google/dist/beta/gemini_tts.test.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -export {}; -//# sourceMappingURL=gemini_tts.test.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.d.ts.map b/plugins/google/dist/beta/gemini_tts.test.d.ts.map deleted file mode 100644 index e4b03e46f..000000000 --- a/plugins/google/dist/beta/gemini_tts.test.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"gemini_tts.test.d.ts","sourceRoot":"","sources":["../../src/beta/gemini_tts.test.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.js b/plugins/google/dist/beta/gemini_tts.test.js deleted file mode 100644 index c01b1f996..000000000 --- a/plugins/google/dist/beta/gemini_tts.test.js +++ /dev/null @@ -1,8 +0,0 @@ -import { STT } from "@livekit/agents-plugin-openai"; -import { tts } from "@livekit/agents-plugins-test"; -import { describe } from "vitest"; -import { TTS } from "./gemini_tts.js"; -describe.skip("Google Gemini TTS", async () => { - await tts(new TTS(), new STT()); -}); -//# sourceMappingURL=gemini_tts.test.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/gemini_tts.test.js.map b/plugins/google/dist/beta/gemini_tts.test.js.map deleted file mode 100644 index 7e419a023..000000000 --- a/plugins/google/dist/beta/gemini_tts.test.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../src/beta/gemini_tts.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { STT } from '@livekit/agents-plugin-openai';\nimport { tts } from '@livekit/agents-plugins-test';\nimport { describe } from 'vitest';\nimport { TTS } from './gemini_tts.js';\n\ndescribe.skip('Google Gemini TTS', async () => {\n await tts(new TTS(), new STT());\n});\n"],"mappings":"AAGA,SAAS,WAAW;AACpB,SAAS,WAAW;AACpB,SAAS,gBAAgB;AACzB,SAAS,WAAW;AAEpB,SAAS,KAAK,qBAAqB,YAAY;AAC7C,QAAM,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,CAAC;AAChC,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/index.cjs b/plugins/google/dist/beta/index.cjs deleted file mode 100644 index c84230339..000000000 --- a/plugins/google/dist/beta/index.cjs +++ /dev/null @@ -1,42 +0,0 @@ -"use strict"; -var __create = Object.create; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __getProtoOf = Object.getPrototypeOf; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __export = (target, all) => { - for (var name in all) - __defProp(target, name, { get: all[name], enumerable: true }); -}; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( - // If the importer is in node compatibility mode or this is not an ESM - // file that has been converted to a CommonJS file using a Babel- - // compatible transform (i.e. "__esModule" has not been set), then set - // "default" to the CommonJS "module.exports" for node compatibility. - isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, - mod -)); -var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); -var beta_exports = {}; -__export(beta_exports, { - TTS: () => import_gemini_tts.TTS, - realtime: () => realtime -}); -module.exports = __toCommonJS(beta_exports); -var import_gemini_tts = require("./gemini_tts.cjs"); -var realtime = __toESM(require("./realtime/index.cjs"), 1); -// Annotate the CommonJS export names for ESM import in node: -0 && (module.exports = { - TTS, - realtime -}); -//# sourceMappingURL=index.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/index.cjs.map b/plugins/google/dist/beta/index.cjs.map deleted file mode 100644 index 0795a40b9..000000000 --- a/plugins/google/dist/beta/index.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../src/beta/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport { TTS, type GeminiTTSModels, type GeminiVoices, type TTSOptions } from './gemini_tts.js';\nexport * as realtime from './realtime/index.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,wBAA8E;AAC9E,eAA0B;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/index.d.cts b/plugins/google/dist/beta/index.d.cts deleted file mode 100644 index 2536cd29d..000000000 --- a/plugins/google/dist/beta/index.d.cts +++ /dev/null @@ -1,3 +0,0 @@ -export { TTS, type GeminiTTSModels, type GeminiVoices, type TTSOptions } from './gemini_tts.js'; -export * as realtime from './realtime/index.js'; -//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/index.d.ts b/plugins/google/dist/beta/index.d.ts deleted file mode 100644 index 2536cd29d..000000000 --- a/plugins/google/dist/beta/index.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { TTS, type GeminiTTSModels, type GeminiVoices, type TTSOptions } from './gemini_tts.js'; -export * as realtime from './realtime/index.js'; -//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/index.d.ts.map b/plugins/google/dist/beta/index.d.ts.map deleted file mode 100644 index d5701f348..000000000 --- a/plugins/google/dist/beta/index.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/beta/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,GAAG,EAAE,KAAK,eAAe,EAAE,KAAK,YAAY,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAChG,OAAO,KAAK,QAAQ,MAAM,qBAAqB,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/beta/index.js b/plugins/google/dist/beta/index.js deleted file mode 100644 index 31bcddc7e..000000000 --- a/plugins/google/dist/beta/index.js +++ /dev/null @@ -1,7 +0,0 @@ -import { TTS } from "./gemini_tts.js"; -import * as realtime from "./realtime/index.js"; -export { - TTS, - realtime -}; -//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/index.js.map b/plugins/google/dist/beta/index.js.map deleted file mode 100644 index d366ee8e1..000000000 --- a/plugins/google/dist/beta/index.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../src/beta/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport { TTS, type GeminiTTSModels, type GeminiVoices, type TTSOptions } from './gemini_tts.js';\nexport * as realtime from './realtime/index.js';\n"],"mappings":"AAIA,SAAS,WAAqE;AAC9E,YAAY,cAAc;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.cjs b/plugins/google/dist/beta/realtime/api_proto.cjs deleted file mode 100644 index d13608ec1..000000000 --- a/plugins/google/dist/beta/realtime/api_proto.cjs +++ /dev/null @@ -1,17 +0,0 @@ -"use strict"; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); -var api_proto_exports = {}; -module.exports = __toCommonJS(api_proto_exports); -//# sourceMappingURL=api_proto.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.cjs.map b/plugins/google/dist/beta/realtime/api_proto.cjs.map deleted file mode 100644 index 14ddc4d1a..000000000 --- a/plugins/google/dist/beta/realtime/api_proto.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../../src/beta/realtime/api_proto.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as types from '@google/genai';\n\n/**\n * Supported Google Live API models\n */\nexport type LiveAPIModels =\n | 'gemini-2.0-flash-exp'\n // models supported on Gemini API\n | 'gemini-2.0-flash-live-001'\n | 'gemini-2.5-flash-preview-native-audio-dialog'\n | 'gemini-2.5-flash-exp-native-audio-thinking-dialog';\n\n/**\n * Available voice options for Google Realtime API\n */\nexport type Voice = 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Aoede' | 'Leda' | 'Orus' | 'Zephyr';\n\n/**\n * Union type for all possible client events\n */\n\nexport type ClientEvents =\n | {\n type: 'content';\n value: types.LiveClientContent;\n }\n | {\n type: 'realtime_input';\n value: types.LiveClientRealtimeInput;\n }\n | {\n type: 'tool_response';\n value: types.LiveClientToolResponse;\n }\n | {\n type: 'function_response';\n value: types.FunctionResponse;\n };\n"],"mappings":";;;;;;;;;;;;;;AAAA;AAAA;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.d.cts b/plugins/google/dist/beta/realtime/api_proto.d.cts deleted file mode 100644 index 953af7083..000000000 --- a/plugins/google/dist/beta/realtime/api_proto.d.cts +++ /dev/null @@ -1,26 +0,0 @@ -import type * as types from '@google/genai'; -/** - * Supported Google Live API models - */ -export type LiveAPIModels = 'gemini-2.0-flash-exp' | 'gemini-2.0-flash-live-001' | 'gemini-2.5-flash-preview-native-audio-dialog' | 'gemini-2.5-flash-exp-native-audio-thinking-dialog'; -/** - * Available voice options for Google Realtime API - */ -export type Voice = 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Aoede' | 'Leda' | 'Orus' | 'Zephyr'; -/** - * Union type for all possible client events - */ -export type ClientEvents = { - type: 'content'; - value: types.LiveClientContent; -} | { - type: 'realtime_input'; - value: types.LiveClientRealtimeInput; -} | { - type: 'tool_response'; - value: types.LiveClientToolResponse; -} | { - type: 'function_response'; - value: types.FunctionResponse; -}; -//# sourceMappingURL=api_proto.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.d.ts b/plugins/google/dist/beta/realtime/api_proto.d.ts deleted file mode 100644 index 953af7083..000000000 --- a/plugins/google/dist/beta/realtime/api_proto.d.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type * as types from '@google/genai'; -/** - * Supported Google Live API models - */ -export type LiveAPIModels = 'gemini-2.0-flash-exp' | 'gemini-2.0-flash-live-001' | 'gemini-2.5-flash-preview-native-audio-dialog' | 'gemini-2.5-flash-exp-native-audio-thinking-dialog'; -/** - * Available voice options for Google Realtime API - */ -export type Voice = 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Aoede' | 'Leda' | 'Orus' | 'Zephyr'; -/** - * Union type for all possible client events - */ -export type ClientEvents = { - type: 'content'; - value: types.LiveClientContent; -} | { - type: 'realtime_input'; - value: types.LiveClientRealtimeInput; -} | { - type: 'tool_response'; - value: types.LiveClientToolResponse; -} | { - type: 'function_response'; - value: types.FunctionResponse; -}; -//# sourceMappingURL=api_proto.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.d.ts.map b/plugins/google/dist/beta/realtime/api_proto.d.ts.map deleted file mode 100644 index f49abf619..000000000 --- a/plugins/google/dist/beta/realtime/api_proto.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"api_proto.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/api_proto.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,KAAK,KAAK,MAAM,eAAe,CAAC;AAE5C;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,sBAAsB,GAEtB,2BAA2B,GAC3B,8CAA8C,GAC9C,mDAAmD,CAAC;AAExD;;GAEG;AACH,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEjG;;GAEG;AAEH,MAAM,MAAM,YAAY,GACpB;IACE,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,KAAK,CAAC,iBAAiB,CAAC;CAChC,GACD;IACE,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,KAAK,CAAC,uBAAuB,CAAC;CACtC,GACD;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC;CACrC,GACD;IACE,IAAI,EAAE,mBAAmB,CAAC;IAC1B,KAAK,EAAE,KAAK,CAAC,gBAAgB,CAAC;CAC/B,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.js b/plugins/google/dist/beta/realtime/api_proto.js deleted file mode 100644 index 041de4df6..000000000 --- a/plugins/google/dist/beta/realtime/api_proto.js +++ /dev/null @@ -1 +0,0 @@ -//# sourceMappingURL=api_proto.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/api_proto.js.map b/plugins/google/dist/beta/realtime/api_proto.js.map deleted file mode 100644 index 84c51b288..000000000 --- a/plugins/google/dist/beta/realtime/api_proto.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.cjs b/plugins/google/dist/beta/realtime/index.cjs deleted file mode 100644 index 449158563..000000000 --- a/plugins/google/dist/beta/realtime/index.cjs +++ /dev/null @@ -1,29 +0,0 @@ -"use strict"; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __export = (target, all) => { - for (var name in all) - __defProp(target, name, { get: all[name], enumerable: true }); -}; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); -var realtime_exports = {}; -__export(realtime_exports, { - RealtimeModel: () => import_realtime_api.RealtimeModel -}); -module.exports = __toCommonJS(realtime_exports); -var import_realtime_api = require("./realtime_api.cjs"); -// Annotate the CommonJS export names for ESM import in node: -0 && (module.exports = { - RealtimeModel -}); -//# sourceMappingURL=index.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.cjs.map b/plugins/google/dist/beta/realtime/index.cjs.map deleted file mode 100644 index cec90f754..000000000 --- a/plugins/google/dist/beta/realtime/index.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../../src/beta/realtime/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport type { ClientEvents, LiveAPIModels, Voice } from './api_proto.js';\nexport { RealtimeModel } from './realtime_api.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,0BAA8B;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.d.cts b/plugins/google/dist/beta/realtime/index.d.cts deleted file mode 100644 index d1c48a43c..000000000 --- a/plugins/google/dist/beta/realtime/index.d.cts +++ /dev/null @@ -1,3 +0,0 @@ -export type { ClientEvents, LiveAPIModels, Voice } from './api_proto.js'; -export { RealtimeModel } from './realtime_api.js'; -//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.d.ts b/plugins/google/dist/beta/realtime/index.d.ts deleted file mode 100644 index d1c48a43c..000000000 --- a/plugins/google/dist/beta/realtime/index.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type { ClientEvents, LiveAPIModels, Voice } from './api_proto.js'; -export { RealtimeModel } from './realtime_api.js'; -//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.d.ts.map b/plugins/google/dist/beta/realtime/index.d.ts.map deleted file mode 100644 index 5b3374be5..000000000 --- a/plugins/google/dist/beta/realtime/index.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/index.ts"],"names":[],"mappings":"AAGA,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.js b/plugins/google/dist/beta/realtime/index.js deleted file mode 100644 index 2fec47d74..000000000 --- a/plugins/google/dist/beta/realtime/index.js +++ /dev/null @@ -1,5 +0,0 @@ -import { RealtimeModel } from "./realtime_api.js"; -export { - RealtimeModel -}; -//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/index.js.map b/plugins/google/dist/beta/realtime/index.js.map deleted file mode 100644 index e46ec7404..000000000 --- a/plugins/google/dist/beta/realtime/index.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../../src/beta/realtime/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport type { ClientEvents, LiveAPIModels, Voice } from './api_proto.js';\nexport { RealtimeModel } from './realtime_api.js';\n"],"mappings":"AAIA,SAAS,qBAAqB;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.cjs b/plugins/google/dist/beta/realtime/realtime_api.cjs deleted file mode 100644 index a37b33cba..000000000 --- a/plugins/google/dist/beta/realtime/realtime_api.cjs +++ /dev/null @@ -1,1010 +0,0 @@ -"use strict"; -var __create = Object.create; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __getProtoOf = Object.getPrototypeOf; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __export = (target, all) => { - for (var name in all) - __defProp(target, name, { get: all[name], enumerable: true }); -}; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( - // If the importer is in node compatibility mode or this is not an ESM - // file that has been converted to a CommonJS file using a Babel- - // compatible transform (i.e. "__esModule" has not been set), then set - // "default" to the CommonJS "module.exports" for node compatibility. - isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, - mod -)); -var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); -var realtime_api_exports = {}; -__export(realtime_api_exports, { - DEFAULT_IMAGE_ENCODE_OPTIONS: () => DEFAULT_IMAGE_ENCODE_OPTIONS, - RealtimeModel: () => RealtimeModel, - RealtimeSession: () => RealtimeSession -}); -module.exports = __toCommonJS(realtime_api_exports); -var types = __toESM(require("@google/genai"), 1); -var import_genai = require("@google/genai"); -var import_agents = require("@livekit/agents"); -var import_mutex = require("@livekit/mutex"); -var import_rtc_node = require("@livekit/rtc-node"); -var import_tools = require("../../tools.cjs"); -var import_utils = require("../../utils.cjs"); -const INPUT_AUDIO_SAMPLE_RATE = 16e3; -const INPUT_AUDIO_CHANNELS = 1; -const OUTPUT_AUDIO_SAMPLE_RATE = 24e3; -const OUTPUT_AUDIO_CHANNELS = 1; -const DEFAULT_IMAGE_ENCODE_OPTIONS = { - format: "JPEG", - quality: 75, - resizeOptions: { - width: 1024, - height: 1024, - strategy: "scale_aspect_fit" - } -}; -function setsEqual(a, b) { - return a.size === b.size && [...a].every((x) => b.has(x)); -} -class RealtimeModel extends import_agents.llm.RealtimeModel { - /** @internal */ - _options; - constructor(options = {}) { - var _a, _b, _c; - const inputAudioTranscription = options.inputAudioTranscription === void 0 ? {} : options.inputAudioTranscription; - const outputAudioTranscription = options.outputAudioTranscription === void 0 ? {} : options.outputAudioTranscription; - let serverTurnDetection = true; - if ((_b = (_a = options.realtimeInputConfig) == null ? void 0 : _a.automaticActivityDetection) == null ? void 0 : _b.disabled) { - serverTurnDetection = false; - } - super({ - messageTruncation: false, - turnDetection: serverTurnDetection, - userTranscription: inputAudioTranscription !== null, - autoToolReplyGeneration: true, - audioOutput: ((_c = options.modalities) == null ? void 0 : _c.includes(import_genai.Modality.AUDIO)) ?? true - }); - const apiKey = options.apiKey || process.env.GOOGLE_API_KEY; - const project = options.project || process.env.GOOGLE_CLOUD_PROJECT; - const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || "us-central1"; - const vertexai = options.vertexai ?? false; - const defaultModel = vertexai ? "gemini-2.0-flash-exp" : "gemini-2.0-flash-live-001"; - this._options = { - model: options.model || defaultModel, - apiKey, - voice: options.voice || "Puck", - language: options.language, - responseModalities: options.modalities || [import_genai.Modality.AUDIO], - vertexai, - project, - location, - candidateCount: options.candidateCount || 1, - temperature: options.temperature, - maxOutputTokens: options.maxOutputTokens, - topP: options.topP, - topK: options.topK, - presencePenalty: options.presencePenalty, - frequencyPenalty: options.frequencyPenalty, - instructions: options.instructions, - inputAudioTranscription: inputAudioTranscription || void 0, - outputAudioTranscription: outputAudioTranscription || void 0, - imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS, - connOptions: options.connOptions || import_agents.DEFAULT_API_CONNECT_OPTIONS, - httpOptions: options.httpOptions, - enableAffectiveDialog: options.enableAffectiveDialog, - proactivity: options.proactivity, - realtimeInputConfig: options.realtimeInputConfig, - contextWindowCompression: options.contextWindowCompression, - apiVersion: options.apiVersion, - geminiTools: options.geminiTools, - thinkingConfig: options.thinkingConfig - }; - } - /** - * Create a new realtime session - */ - session() { - return new RealtimeSession(this); - } - /** - * Update model options - */ - updateOptions(options) { - if (options.voice !== void 0) { - this._options.voice = options.voice; - } - if (options.temperature !== void 0) { - this._options.temperature = options.temperature; - } - } - /** - * Close the model and cleanup resources - */ - async close() { - } -} -class RealtimeSession extends import_agents.llm.RealtimeSession { - _tools = {}; - _chatCtx = import_agents.llm.ChatContext.empty(); - options; - geminiDeclarations = []; - messageChannel = new import_agents.Queue(); - inputResampler; - inputResamplerInputRate; - instructions; - currentGeneration; - bstream; - // Google-specific properties - activeSession; - sessionShouldClose = new import_agents.Event(); - responseCreatedFutures = {}; - pendingGenerationFut; - sessionResumptionHandle; - inUserActivity = false; - sessionLock = new import_mutex.Mutex(); - numRetries = 0; - hasReceivedAudioInput = false; - #client; - #task; - #logger = (0, import_agents.log)(); - #closed = false; - constructor(realtimeModel) { - super(realtimeModel); - this.options = realtimeModel._options; - this.bstream = new import_agents.AudioByteStream( - INPUT_AUDIO_SAMPLE_RATE, - INPUT_AUDIO_CHANNELS, - INPUT_AUDIO_SAMPLE_RATE / 20 - ); - const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } = this.options; - const apiVersion = !this.options.apiVersion && (enableAffectiveDialog || proactivity) ? "v1alpha" : this.options.apiVersion; - const httpOptions = { - ...this.options.httpOptions, - apiVersion, - timeout: this.options.connOptions.timeoutMs - }; - const clientOptions = vertexai ? { - vertexai: true, - project, - location, - httpOptions - } : { - apiKey, - httpOptions - }; - this.#client = new import_genai.GoogleGenAI(clientOptions); - this.#task = this.#mainTask(); - } - async closeActiveSession() { - const unlock = await this.sessionLock.lock(); - const session = this.activeSession; - this.activeSession = void 0; - if (session) { - try { - await session.close(); - } catch (error) { - this.#logger.warn({ error }, "Error closing Gemini session"); - } - } - unlock(); - } - markRestartNeeded() { - if (!this.sessionShouldClose.isSet) { - this.sessionShouldClose.set(); - const oldQueue = this.messageChannel; - this.messageChannel = new import_agents.Queue(); - oldQueue.put({ type: "__internal_wakeup__" }); - } - } - getToolResultsForRealtime(ctx, vertexai) { - const toolResponses = []; - for (const item of ctx.items) { - if (item.type === "function_call_output") { - const response = { - id: item.callId, - name: item.name, - response: { output: item.output } - }; - if (!vertexai) { - response.id = item.callId; - } - toolResponses.push(response); - } - } - return toolResponses.length > 0 ? { functionResponses: toolResponses } : void 0; - } - updateOptions(options) { - let shouldRestart = false; - if (options.voice !== void 0 && this.options.voice !== options.voice) { - this.options.voice = options.voice; - shouldRestart = true; - } - if (options.temperature !== void 0 && this.options.temperature !== options.temperature) { - this.options.temperature = options.temperature; - shouldRestart = true; - } - if (shouldRestart) { - this.markRestartNeeded(); - } - } - async updateInstructions(instructions) { - if (this.options.instructions === void 0 || this.options.instructions !== instructions) { - this.options.instructions = instructions; - this.markRestartNeeded(); - } - } - async updateChatCtx(chatCtx) { - const unlock = await this.sessionLock.lock(); - try { - if (!this.activeSession) { - this._chatCtx = chatCtx.copy(); - return; - } - } finally { - unlock(); - } - const diffOps = import_agents.llm.computeChatCtxDiff(this._chatCtx, chatCtx); - if (diffOps.toRemove.length > 0) { - this.#logger.warn("Gemini Live does not support removing messages"); - } - const appendCtx = import_agents.llm.ChatContext.empty(); - for (const [, itemId] of diffOps.toCreate) { - const item = chatCtx.getById(itemId); - if (item) { - appendCtx.items.push(item); - } - } - if (appendCtx.items.length > 0) { - const [turns] = await appendCtx.copy({ - excludeFunctionCall: true - }).toProviderFormat("google", false); - const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai); - if (turns.length > 0) { - this.sendClientEvent({ - type: "content", - value: { - turns, - turnComplete: false - } - }); - } - if (toolResults) { - this.sendClientEvent({ - type: "tool_response", - value: toolResults - }); - } - } - this._chatCtx = chatCtx.copy(); - } - async updateTools(tools) { - const newDeclarations = (0, import_utils.toFunctionDeclarations)(tools); - const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name)); - const newToolNames = new Set(newDeclarations.map((f) => f.name)); - if (!setsEqual(currentToolNames, newToolNames)) { - this.geminiDeclarations = newDeclarations; - this._tools = tools; - this.markRestartNeeded(); - } - } - get chatCtx() { - return this._chatCtx.copy(); - } - get tools() { - return { ...this._tools }; - } - get manualActivityDetection() { - var _a, _b; - return ((_b = (_a = this.options.realtimeInputConfig) == null ? void 0 : _a.automaticActivityDetection) == null ? void 0 : _b.disabled) ?? false; - } - pushAudio(frame) { - this.hasReceivedAudioInput = true; - for (const f of this.resampleAudio(frame)) { - for (const nf of this.bstream.write(f.data.buffer)) { - const realtimeInput = { - mediaChunks: [ - { - mimeType: "audio/pcm", - data: Buffer.from(nf.data.buffer).toString("base64") - } - ] - }; - this.sendClientEvent({ - type: "realtime_input", - value: realtimeInput - }); - } - } - } - pushVideo(_) { - } - sendClientEvent(event) { - this.messageChannel.put(event); - } - async generateReply(instructions) { - if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { - this.#logger.warn( - "generateReply called while another generation is pending, cancelling previous." - ); - this.pendingGenerationFut.reject(new Error("Superseded by new generate_reply call")); - } - const fut = new import_agents.Future(); - this.pendingGenerationFut = fut; - if (this.inUserActivity) { - this.sendClientEvent({ - type: "realtime_input", - value: { - activityEnd: {} - } - }); - this.inUserActivity = false; - } - const turns = []; - if (instructions !== void 0) { - turns.push({ - parts: [{ text: instructions }], - role: "model" - }); - } - turns.push({ - parts: [{ text: "." }], - role: "user" - }); - this.sendClientEvent({ - type: "content", - value: { - turns, - turnComplete: true - } - }); - const timeoutHandle = setTimeout(() => { - if (!fut.done) { - fut.reject(new Error("generateReply timed out waiting for generation_created event.")); - if (this.pendingGenerationFut === fut) { - this.pendingGenerationFut = void 0; - } - } - }, 5e3); - fut.await.finally(() => clearTimeout(timeoutHandle)); - return fut.await; - } - startUserActivity() { - if (!this.manualActivityDetection) { - return; - } - if (!this.inUserActivity) { - this.inUserActivity = true; - this.sendClientEvent({ - type: "realtime_input", - value: { - activityStart: {} - } - }); - } - } - async interrupt() { - var _a; - if (((_a = this.options.realtimeInputConfig) == null ? void 0 : _a.activityHandling) === import_genai.ActivityHandling.NO_INTERRUPTION) { - return; - } - this.startUserActivity(); - } - async truncate(_options) { - this.#logger.warn("truncate is not supported by the Google Realtime API."); - } - async close() { - super.close(); - this.#closed = true; - this.sessionShouldClose.set(); - await this.closeActiveSession(); - if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { - this.pendingGenerationFut.reject(new Error("Session closed")); - } - for (const fut of Object.values(this.responseCreatedFutures)) { - if (!fut.done) { - fut.reject(new Error("Session closed before response created")); - } - } - this.responseCreatedFutures = {}; - if (this.currentGeneration) { - this.markCurrentGenerationDone(); - } - } - async #mainTask() { - const maxRetries = this.options.connOptions.maxRetry; - while (!this.#closed) { - await this.closeActiveSession(); - this.sessionShouldClose.clear(); - const config = this.buildConnectConfig(); - try { - this.#logger.debug("Connecting to Gemini Realtime API..."); - const sessionOpened = new import_agents.Event(); - const session = await this.#client.live.connect({ - model: this.options.model, - callbacks: { - onopen: () => sessionOpened.set(), - onmessage: (message) => { - this.onReceiveMessage(session, message); - }, - onerror: (error) => { - this.#logger.error({ error }, "Gemini Live session error"); - sessionOpened.set(); - if (!this.sessionShouldClose.isSet) { - this.markRestartNeeded(); - } - }, - onclose: (event) => { - if (this.activeSession !== session) { - return; - } - this.#logger.debug( - { code: event.code, reason: event.reason }, - "Gemini Live session closed" - ); - sessionOpened.set(); - this.markCurrentGenerationDone(); - if (!this.sessionShouldClose.isSet && !this.#closed) { - this.markRestartNeeded(); - } - } - }, - config - }); - await sessionOpened.wait(); - const unlock = await this.sessionLock.lock(); - try { - this.activeSession = session; - const [turns] = await this._chatCtx.copy({ - excludeFunctionCall: true - }).toProviderFormat("google", false); - if (turns.length > 0) { - await session.sendClientContent({ - turns, - turnComplete: false - }); - } - } finally { - unlock(); - } - const sendTask = import_agents.Task.from((controller) => this.sendTask(session, controller)); - const restartWaitTask = import_agents.Task.from(({ signal }) => { - const abortEvent = new import_agents.Event(); - signal.addEventListener("abort", () => abortEvent.set()); - return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]); - }); - await Promise.race([sendTask.result, restartWaitTask.result]); - if (!restartWaitTask.done && this.#closed) { - break; - } - await (0, import_agents.cancelAndWait)([sendTask, restartWaitTask], 2e3); - } catch (error) { - this.#logger.error(`Gemini Realtime API error: ${error}`); - if (this.#closed) break; - if (maxRetries === 0) { - this.emitError(error, false); - throw new import_agents.APIConnectionError({ - message: "Failed to connect to Gemini Live" - }); - } - if (this.numRetries >= maxRetries) { - this.emitError(error, false); - throw new import_agents.APIConnectionError({ - message: `Failed to connect to Gemini Live after ${maxRetries} attempts` - }); - } - const retryInterval = this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs; - this.#logger.warn( - { - attempt: this.numRetries, - maxRetries - }, - `Gemini Realtime API connection failed, retrying in ${retryInterval}ms` - ); - await (0, import_agents.delay)(retryInterval); - this.numRetries++; - } finally { - await this.closeActiveSession(); - } - } - } - async sendTask(session, controller) { - try { - while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) { - const msg = await this.messageChannel.get(); - if (controller.signal.aborted) break; - const unlock = await this.sessionLock.lock(); - try { - if (this.sessionShouldClose.isSet || this.activeSession !== session) { - break; - } - } finally { - unlock(); - } - switch (msg.type) { - case "content": - const { turns, turnComplete } = msg.value; - this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`); - await session.sendClientContent({ - turns, - turnComplete: turnComplete ?? true - }); - break; - case "tool_response": - const { functionResponses } = msg.value; - if (functionResponses) { - this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`); - await session.sendToolResponse({ - functionResponses - }); - } - break; - case "realtime_input": - const { mediaChunks, activityStart, activityEnd } = msg.value; - if (mediaChunks) { - for (const mediaChunk of mediaChunks) { - await session.sendRealtimeInput({ media: mediaChunk }); - } - } - if (activityStart) await session.sendRealtimeInput({ activityStart }); - if (activityEnd) await session.sendRealtimeInput({ activityEnd }); - break; - default: - this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`); - break; - } - } - } catch (e) { - if (!this.sessionShouldClose.isSet) { - this.#logger.error(`Error in send task: ${e}`); - this.markRestartNeeded(); - } - } finally { - this.#logger.debug( - { - closed: this.#closed, - sessionShouldClose: this.sessionShouldClose.isSet, - aborted: controller.signal.aborted - }, - "send task finished." - ); - } - } - async onReceiveMessage(session, response) { - var _a, _b, _c; - const hasAudioData = (_c = (_b = (_a = response.serverContent) == null ? void 0 : _a.modelTurn) == null ? void 0 : _b.parts) == null ? void 0 : _c.some( - (part) => { - var _a2; - return (_a2 = part.inlineData) == null ? void 0 : _a2.data; - } - ); - if (!hasAudioData) { - this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`); - } - const unlock = await this.sessionLock.lock(); - try { - if (this.sessionShouldClose.isSet || this.activeSession !== session) { - this.#logger.debug("onReceiveMessage: Session changed or closed, stopping receive."); - return; - } - } finally { - unlock(); - } - if ((!this.currentGeneration || this.currentGeneration._done) && (response.serverContent || response.toolCall)) { - this.startNewGeneration(); - } - if (response.sessionResumptionUpdate) { - if (response.sessionResumptionUpdate.resumable && response.sessionResumptionUpdate.newHandle) { - this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle; - } - } - try { - if (response.serverContent) { - this.handleServerContent(response.serverContent); - } - if (response.toolCall) { - this.handleToolCall(response.toolCall); - } - if (response.toolCallCancellation) { - this.handleToolCallCancellation(response.toolCallCancellation); - } - if (response.usageMetadata) { - this.handleUsageMetadata(response.usageMetadata); - } - if (response.goAway) { - this.handleGoAway(response.goAway); - } - if (this.numRetries > 0) { - this.numRetries = 0; - } - } catch (e) { - if (!this.sessionShouldClose.isSet) { - this.#logger.error(`Error in onReceiveMessage: ${e}`); - this.markRestartNeeded(); - } - } - } - /// Truncate large base64/audio payloads for logging to avoid flooding logs - truncateString(data, maxLength = 30) { - return data.length > maxLength ? `${data.slice(0, maxLength)}\u2026` : data; - } - loggableClientEvent(event, maxLength = 30) { - var _a; - const obj = { ...event }; - if (obj.type === "realtime_input" && ((_a = obj.value) == null ? void 0 : _a.mediaChunks)) { - obj.value = { - ...obj.value, - mediaChunks: obj.value.mediaChunks.map( - (mc) => ({ - ...mc, - data: typeof mc.data === "string" ? this.truncateString(mc.data, maxLength) : mc.data - }) - ) - }; - } - return obj; - } - loggableServerMessage(message, maxLength = 30) { - const obj = { ...message }; - if (obj.serverContent && obj.serverContent.modelTurn && Array.isArray(obj.serverContent.modelTurn.parts)) { - obj.serverContent = { ...obj.serverContent }; - obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn }; - obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part) => { - var _a; - if (((_a = part == null ? void 0 : part.inlineData) == null ? void 0 : _a.data) && typeof part.inlineData.data === "string") { - return { - ...part, - inlineData: { - ...part.inlineData, - data: this.truncateString(part.inlineData.data, maxLength) - } - }; - } - return part; - }); - } - return obj; - } - markCurrentGenerationDone() { - if (!this.currentGeneration || this.currentGeneration._done) { - return; - } - this.handleInputSpeechStopped(); - const gen = this.currentGeneration; - if (gen.inputTranscription) { - this.emit("input_audio_transcription_completed", { - itemId: gen.inputId, - transcript: gen.inputTranscription, - isFinal: true - }); - this._chatCtx.addMessage({ - role: "user", - content: gen.inputTranscription, - id: gen.inputId - }); - } - if (gen.outputText) { - this._chatCtx.addMessage({ - role: "assistant", - content: gen.outputText, - id: gen.responseId - }); - } - if (this.options.outputAudioTranscription === void 0) { - gen.textChannel.write(""); - } - gen.textChannel.close(); - gen.audioChannel.close(); - gen.functionChannel.close(); - gen.messageChannel.close(); - gen._done = true; - } - emitError(error, recoverable) { - this.emit("error", { - timestamp: Date.now(), - // TODO(brian): add label to realtime model - label: "google_realtime", - error, - recoverable - }); - } - buildConnectConfig() { - const opts = this.options; - const config = { - responseModalities: opts.responseModalities, - systemInstruction: opts.instructions ? { - parts: [{ text: opts.instructions }] - } : void 0, - speechConfig: { - voiceConfig: { - prebuiltVoiceConfig: { - voiceName: opts.voice - } - }, - languageCode: opts.language - }, - tools: [ - { - functionDeclarations: this.geminiDeclarations, - ...this.options.geminiTools - } - ], - inputAudioTranscription: opts.inputAudioTranscription, - outputAudioTranscription: opts.outputAudioTranscription, - sessionResumption: { - handle: this.sessionResumptionHandle - }, - thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : void 0 - }; - if (opts.temperature !== void 0) { - config.temperature = opts.temperature; - } - if (opts.maxOutputTokens !== void 0) { - config.maxOutputTokens = opts.maxOutputTokens; - } - if (opts.topP !== void 0) { - config.topP = opts.topP; - } - if (opts.topK !== void 0) { - config.topK = opts.topK; - } - if (opts.proactivity !== void 0) { - config.proactivity = { proactiveAudio: opts.proactivity }; - } - if (opts.enableAffectiveDialog !== void 0) { - config.enableAffectiveDialog = opts.enableAffectiveDialog; - } - if (opts.realtimeInputConfig !== void 0) { - config.realtimeInputConfig = opts.realtimeInputConfig; - } - if (opts.contextWindowCompression !== void 0) { - config.contextWindowCompression = opts.contextWindowCompression; - } - return config; - } - startNewGeneration() { - if (this.currentGeneration && !this.currentGeneration._done) { - this.#logger.warn("Starting new generation while another is active. Finalizing previous."); - this.markCurrentGenerationDone(); - } - const responseId = (0, import_agents.shortuuid)("GR_"); - this.currentGeneration = { - messageChannel: import_agents.stream.createStreamChannel(), - functionChannel: import_agents.stream.createStreamChannel(), - responseId, - inputId: (0, import_agents.shortuuid)("GI_"), - textChannel: import_agents.stream.createStreamChannel(), - audioChannel: import_agents.stream.createStreamChannel(), - inputTranscription: "", - outputText: "", - _createdTimestamp: Date.now(), - _done: false - }; - if (!this._realtimeModel.capabilities.audioOutput) { - this.currentGeneration.audioChannel.close(); - } - const modalities = this._realtimeModel.capabilities.audioOutput ? ["audio", "text"] : ["text"]; - this.currentGeneration.messageChannel.write({ - messageId: responseId, - textStream: this.currentGeneration.textChannel.stream(), - audioStream: this.currentGeneration.audioChannel.stream(), - modalities: Promise.resolve(modalities) - }); - const generationEvent = { - messageStream: this.currentGeneration.messageChannel.stream(), - functionStream: this.currentGeneration.functionChannel.stream(), - userInitiated: false - }; - if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { - generationEvent.userInitiated = true; - this.pendingGenerationFut.resolve(generationEvent); - this.pendingGenerationFut = void 0; - } else { - this.handleInputSpeechStarted(); - } - this.emit("generation_created", generationEvent); - } - handleInputSpeechStarted() { - this.emit("input_speech_started", {}); - } - handleInputSpeechStopped() { - this.emit("input_speech_stopped", { - userTranscriptionEnabled: false - }); - } - handleServerContent(serverContent) { - if (!this.currentGeneration) { - this.#logger.warn("received server content but no active generation."); - return; - } - const gen = this.currentGeneration; - if (serverContent.modelTurn) { - const turn = serverContent.modelTurn; - for (const part of turn.parts || []) { - if (part.text) { - gen.outputText += part.text; - gen.textChannel.write(part.text); - } - if (part.inlineData) { - if (!gen._firstTokenTimestamp) { - gen._firstTokenTimestamp = Date.now(); - } - try { - if (!part.inlineData.data) { - throw new Error("frameData is not bytes"); - } - const binaryString = atob(part.inlineData.data); - const len = binaryString.length; - const bytes = new Uint8Array(len); - for (let i = 0; i < len; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - const int16Array = new Int16Array(bytes.buffer); - const audioFrame = new import_rtc_node.AudioFrame( - int16Array, - OUTPUT_AUDIO_SAMPLE_RATE, - OUTPUT_AUDIO_CHANNELS, - int16Array.length / OUTPUT_AUDIO_CHANNELS - ); - gen.audioChannel.write(audioFrame); - } catch (error) { - this.#logger.error("Error processing audio data:", error); - } - } - } - } - if (serverContent.inputTranscription && serverContent.inputTranscription.text) { - let text = serverContent.inputTranscription.text; - if (gen.inputTranscription === "") { - text = text.trimStart(); - } - gen.inputTranscription += text; - this.emit("input_audio_transcription_completed", { - itemId: gen.inputId, - transcript: gen.inputTranscription, - isFinal: false - }); - } - if (serverContent.outputTranscription && serverContent.outputTranscription.text) { - const text = serverContent.outputTranscription.text; - gen.outputText += text; - gen.textChannel.write(text); - } - if (serverContent.generationComplete || serverContent.turnComplete) { - gen._completedTimestamp = Date.now(); - } - if (serverContent.interrupted) { - this.handleInputSpeechStarted(); - } - if (serverContent.turnComplete) { - this.markCurrentGenerationDone(); - } - } - handleToolCall(toolCall) { - if (!this.currentGeneration) { - this.#logger.warn("received tool call but no active generation."); - return; - } - const gen = this.currentGeneration; - for (const fc of toolCall.functionCalls || []) { - gen.functionChannel.write({ - callId: fc.id || (0, import_agents.shortuuid)("fnc-call-"), - name: fc.name, - args: fc.args ? JSON.stringify(fc.args) : "" - }); - } - this.markCurrentGenerationDone(); - } - handleToolCallCancellation(cancellation) { - this.#logger.warn( - { - functionCallIds: cancellation.ids - }, - "server cancelled tool calls" - ); - } - handleUsageMetadata(usage) { - if (!this.currentGeneration) { - this.#logger.debug("Received usage metadata but no active generation"); - return; - } - const gen = this.currentGeneration; - const createdTimestamp = gen._createdTimestamp; - const firstTokenTimestamp = gen._firstTokenTimestamp; - const completedTimestamp = gen._completedTimestamp || Date.now(); - const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1; - const durationMs = completedTimestamp - createdTimestamp; - const inputTokens = usage.promptTokenCount || 0; - const outputTokens = usage.responseTokenCount || 0; - const totalTokens = usage.totalTokenCount || 0; - const realtimeMetrics = { - type: "realtime_model_metrics", - timestamp: createdTimestamp, - requestId: gen.responseId, - ttftMs, - durationMs, - cancelled: gen._done && !gen._completedTimestamp, - label: "google_realtime", - inputTokens, - outputTokens, - totalTokens, - tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1e3) : 0, - inputTokenDetails: { - ...this.tokenDetailsMap(usage.promptTokensDetails), - cachedTokens: (usage.cacheTokensDetails || []).reduce( - (sum, detail) => sum + (detail.tokenCount || 0), - 0 - ), - cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails) - }, - outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails) - }; - this.emit("metrics_collected", realtimeMetrics); - } - tokenDetailsMap(tokenDetails) { - const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 }; - if (!tokenDetails) { - return tokenDetailsMap; - } - for (const tokenDetail of tokenDetails) { - if (!tokenDetail.tokenCount) { - continue; - } - if (tokenDetail.modality === types.MediaModality.AUDIO) { - tokenDetailsMap.audioTokens += tokenDetail.tokenCount; - } else if (tokenDetail.modality === types.MediaModality.TEXT) { - tokenDetailsMap.textTokens += tokenDetail.tokenCount; - } else if (tokenDetail.modality === types.MediaModality.IMAGE) { - tokenDetailsMap.imageTokens += tokenDetail.tokenCount; - } - } - return tokenDetailsMap; - } - handleGoAway(goAway) { - this.#logger.warn({ timeLeft: goAway.timeLeft }, "Gemini server indicates disconnection soon."); - this.sessionShouldClose.set(); - } - async commitAudio() { - } - async clearAudio() { - } - *resampleAudio(frame) { - if (this.inputResampler) { - if (frame.sampleRate !== this.inputResamplerInputRate) { - this.inputResampler = void 0; - this.inputResamplerInputRate = void 0; - } - } - if (this.inputResampler === void 0 && (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)) { - this.inputResampler = new import_rtc_node.AudioResampler( - frame.sampleRate, - INPUT_AUDIO_SAMPLE_RATE, - INPUT_AUDIO_CHANNELS - ); - this.inputResamplerInputRate = frame.sampleRate; - } - if (this.inputResampler) { - for (const resampledFrame of this.inputResampler.push(frame)) { - yield resampledFrame; - } - } else { - yield frame; - } - } -} -// Annotate the CommonJS export names for ESM import in node: -0 && (module.exports = { - DEFAULT_IMAGE_ENCODE_OPTIONS, - RealtimeModel, - RealtimeSession -}); -//# sourceMappingURL=realtime_api.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.cjs.map b/plugins/google/dist/beta/realtime/realtime_api.cjs.map deleted file mode 100644 index 241e29755..000000000 --- a/plugins/google/dist/beta/realtime/realtime_api.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n const session = this.activeSession;\n this.activeSession = undefined;\n\n if (session) {\n try {\n await session.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n // Replace the queue so new messages don't go to the old (closing) session.\n const oldQueue = this.messageChannel;\n this.messageChannel = new Queue();\n // Wake up the current sendTask if it's waiting for a message.\n oldQueue.put({ type: '__internal_wakeup__' } as any);\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error({ error }, 'Gemini Live session error');\n sessionOpened.set(); // Break the wait if we were waiting for open\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n if (this.activeSession !== session) {\n return;\n }\n\n this.#logger.debug(\n { code: event.code, reason: event.reason },\n 'Gemini Live session closed',\n );\n sessionOpened.set(); // Break the wait if we were waiting for open\n this.markCurrentGenerationDone();\n if (!this.sessionShouldClose.isSet && !this.#closed) {\n this.markRestartNeeded();\n }\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined,\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() {}\n\n async clearAudio() {}\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,YAAuB;AACvB,mBAQO;AAEP,oBAcO;AACP,mBAAsB;AACtB,sBAA4D;AAC5D,mBAA8B;AAC9B,mBAAuC;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,kBAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,sBAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,sBAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,kBAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,kBAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,oBAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,oBAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,mBAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,UAAM,UAAU,KAAK;AACrB,SAAK,gBAAgB;AAErB,QAAI,SAAS;AACX,UAAI;AACF,cAAM,QAAQ,MAAM;AAAA,MACtB,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAE5B,YAAM,WAAW,KAAK;AACtB,WAAK,iBAAiB,IAAI,oBAAM;AAEhC,eAAS,IAAI,EAAE,MAAM,sBAAsB,CAAQ;AAAA,IACrD;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,kBAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,kBAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,sBAAkB,qCAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AA9lBzC;AA+lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,qBAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AAzsBpB;AA2sBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,8BAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,oBAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,EAAE,MAAM,GAAG,2BAA2B;AACzD,4BAAc,IAAI;AAClB,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,kBAAI,KAAK,kBAAkB,SAAS;AAClC;AAAA,cACF;AAEA,mBAAK,QAAQ;AAAA,gBACX,EAAE,MAAM,MAAM,MAAM,QAAQ,MAAM,OAAO;AAAA,gBACzC;AAAA,cACF;AACA,4BAAc,IAAI;AAClB,mBAAK,0BAA0B;AAC/B,kBAAI,CAAC,KAAK,mBAAmB,SAAS,CAAC,KAAK,SAAS;AACnD,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,mBAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,mBAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,oBAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,kBAAM,6BAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,kBAAM,qBAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA76BnB;AA+6BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AAh7Bb,YAAAA;AAg7BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAx/B7B;AAy/BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAphC/F;AAqhCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,MACA,gBAAgB,KAAK,iBAAiB,KAAK,iBAAiB;AAAA,IAC9D;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,iBAAa,yBAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,qBAAO,oBAA2C;AAAA,MAClE,iBAAiB,qBAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,aAAS,yBAAU,KAAK;AAAA,MACxB,aAAa,qBAAO,oBAA4B;AAAA,MAChD,cAAc,qBAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,UAAM,yBAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAC;AAAA,EAErB,MAAM,aAAa;AAAA,EAAC;AAAA,EAEpB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.d.cts b/plugins/google/dist/beta/realtime/realtime_api.d.cts deleted file mode 100644 index 0b57b6a91..000000000 --- a/plugins/google/dist/beta/realtime/realtime_api.d.cts +++ /dev/null @@ -1,273 +0,0 @@ -import * as types from '@google/genai'; -import { type AudioTranscriptionConfig, type ContextWindowCompressionConfig, type HttpOptions, Modality, type RealtimeInputConfig } from '@google/genai'; -import type { APIConnectOptions } from '@livekit/agents'; -import { llm } from '@livekit/agents'; -import { AudioFrame, type VideoFrame } from '@livekit/rtc-node'; -import { type LLMTools } from '../../tools.js'; -import type { LiveAPIModels, Voice } from './api_proto.js'; -/** - * Default image encoding options for Google Realtime API - */ -export declare const DEFAULT_IMAGE_ENCODE_OPTIONS: { - format: "JPEG"; - quality: number; - resizeOptions: { - width: number; - height: number; - strategy: "scale_aspect_fit"; - }; -}; -/** - * Input transcription result - */ -export interface InputTranscription { - itemId: string; - transcript: string; -} -/** - * Internal realtime options for Google Realtime API - */ -interface RealtimeOptions { - model: LiveAPIModels | string; - apiKey?: string; - voice: Voice | string; - language?: string; - responseModalities: Modality[]; - vertexai: boolean; - project?: string; - location?: string; - candidateCount: number; - temperature?: number; - maxOutputTokens?: number; - topP?: number; - topK?: number; - presencePenalty?: number; - frequencyPenalty?: number; - instructions?: string; - inputAudioTranscription?: AudioTranscriptionConfig; - outputAudioTranscription?: AudioTranscriptionConfig; - imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS; - connOptions: APIConnectOptions; - httpOptions?: HttpOptions; - enableAffectiveDialog?: boolean; - proactivity?: boolean; - realtimeInputConfig?: RealtimeInputConfig; - contextWindowCompression?: ContextWindowCompressionConfig; - apiVersion?: string; - geminiTools?: LLMTools; - thinkingConfig?: types.ThinkingConfig; -} -/** - * Google Realtime Model for real-time voice conversations with Gemini models - */ -export declare class RealtimeModel extends llm.RealtimeModel { - /** @internal */ - _options: RealtimeOptions; - constructor(options?: { - /** - * Initial system instructions for the model - */ - instructions?: string; - /** - * The name of the model to use - */ - model?: LiveAPIModels | string; - /** - * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable - */ - apiKey?: string; - /** - * Voice setting for audio outputs - */ - voice?: Voice | string; - /** - * The language (BCP-47 Code) to use for the API - * See https://ai.google.dev/gemini-api/docs/live#supported-languages - */ - language?: string; - /** - * Modalities to use, such as [Modality.TEXT, Modality.AUDIO] - */ - modalities?: Modality[]; - /** - * Whether to use VertexAI for the API - */ - vertexai?: boolean; - /** - * The project ID to use for the API (for VertexAI) - */ - project?: string; - /** - * The location to use for the API (for VertexAI) - */ - location?: string; - /** - * The number of candidate responses to generate - */ - candidateCount?: number; - /** - * Sampling temperature for response generation - */ - temperature?: number; - /** - * Maximum number of tokens in the response - */ - maxOutputTokens?: number; - /** - * The top-p value for response generation - */ - topP?: number; - /** - * The top-k value for response generation - */ - topK?: number; - /** - * The presence penalty for response generation - */ - presencePenalty?: number; - /** - * The frequency penalty for response generation - */ - frequencyPenalty?: number; - /** - * The configuration for input audio transcription - */ - inputAudioTranscription?: AudioTranscriptionConfig | null; - /** - * The configuration for output audio transcription - */ - outputAudioTranscription?: AudioTranscriptionConfig | null; - /** - * The configuration for image encoding - */ - imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS; - /** - * Whether to enable affective dialog - */ - enableAffectiveDialog?: boolean; - /** - * Whether to enable proactive audio - */ - proactivity?: boolean; - /** - * The configuration for realtime input - */ - realtimeInputConfig?: RealtimeInputConfig; - /** - * The configuration for context window compression - */ - contextWindowCompression?: ContextWindowCompressionConfig; - /** - * API version to use - */ - apiVersion?: string; - /** - * The configuration for the API connection - */ - connOptions?: APIConnectOptions; - /** - * HTTP options for API requests - */ - httpOptions?: HttpOptions; - /** - * Gemini-specific tools to use for the session - */ - geminiTools?: LLMTools; - /** - * The thinking configuration for response generation - */ - thinkingConfig?: types.ThinkingConfig; - }); - /** - * Create a new realtime session - */ - session(): RealtimeSession; - /** - * Update model options - */ - updateOptions(options: { - voice?: Voice | string; - temperature?: number; - }): void; - /** - * Close the model and cleanup resources - */ - close(): Promise; -} -/** - * Google Realtime Session for real-time voice conversations - * - * This session provides real-time streaming capabilities with Google's Gemini models, - * supporting both text and audio modalities with function calling capabilities. - */ -export declare class RealtimeSession extends llm.RealtimeSession { - #private; - private _tools; - private _chatCtx; - private options; - private geminiDeclarations; - private messageChannel; - private inputResampler?; - private inputResamplerInputRate?; - private instructions?; - private currentGeneration?; - private bstream; - private activeSession?; - private sessionShouldClose; - private responseCreatedFutures; - private pendingGenerationFut?; - private sessionResumptionHandle?; - private inUserActivity; - private sessionLock; - private numRetries; - private hasReceivedAudioInput; - constructor(realtimeModel: RealtimeModel); - private closeActiveSession; - private markRestartNeeded; - private getToolResultsForRealtime; - updateOptions(options: { - voice?: Voice | string; - temperature?: number; - toolChoice?: llm.ToolChoice; - }): void; - updateInstructions(instructions: string): Promise; - updateChatCtx(chatCtx: llm.ChatContext): Promise; - updateTools(tools: llm.ToolContext): Promise; - get chatCtx(): llm.ChatContext; - get tools(): llm.ToolContext; - get manualActivityDetection(): boolean; - pushAudio(frame: AudioFrame): void; - pushVideo(_: VideoFrame): void; - private sendClientEvent; - generateReply(instructions?: string): Promise; - startUserActivity(): void; - interrupt(): Promise; - truncate(_options: { - messageId: string; - audioEndMs: number; - audioTranscript?: string; - }): Promise; - close(): Promise; - private sendTask; - private onReceiveMessage; - private truncateString; - private loggableClientEvent; - private loggableServerMessage; - private markCurrentGenerationDone; - private emitError; - private buildConnectConfig; - private startNewGeneration; - private handleInputSpeechStarted; - private handleInputSpeechStopped; - private handleServerContent; - private handleToolCall; - private handleToolCallCancellation; - private handleUsageMetadata; - private tokenDetailsMap; - private handleGoAway; - commitAudio(): Promise; - clearAudio(): Promise; - private resampleAudio; -} -export {}; -//# sourceMappingURL=realtime_api.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.d.ts b/plugins/google/dist/beta/realtime/realtime_api.d.ts deleted file mode 100644 index 0b57b6a91..000000000 --- a/plugins/google/dist/beta/realtime/realtime_api.d.ts +++ /dev/null @@ -1,273 +0,0 @@ -import * as types from '@google/genai'; -import { type AudioTranscriptionConfig, type ContextWindowCompressionConfig, type HttpOptions, Modality, type RealtimeInputConfig } from '@google/genai'; -import type { APIConnectOptions } from '@livekit/agents'; -import { llm } from '@livekit/agents'; -import { AudioFrame, type VideoFrame } from '@livekit/rtc-node'; -import { type LLMTools } from '../../tools.js'; -import type { LiveAPIModels, Voice } from './api_proto.js'; -/** - * Default image encoding options for Google Realtime API - */ -export declare const DEFAULT_IMAGE_ENCODE_OPTIONS: { - format: "JPEG"; - quality: number; - resizeOptions: { - width: number; - height: number; - strategy: "scale_aspect_fit"; - }; -}; -/** - * Input transcription result - */ -export interface InputTranscription { - itemId: string; - transcript: string; -} -/** - * Internal realtime options for Google Realtime API - */ -interface RealtimeOptions { - model: LiveAPIModels | string; - apiKey?: string; - voice: Voice | string; - language?: string; - responseModalities: Modality[]; - vertexai: boolean; - project?: string; - location?: string; - candidateCount: number; - temperature?: number; - maxOutputTokens?: number; - topP?: number; - topK?: number; - presencePenalty?: number; - frequencyPenalty?: number; - instructions?: string; - inputAudioTranscription?: AudioTranscriptionConfig; - outputAudioTranscription?: AudioTranscriptionConfig; - imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS; - connOptions: APIConnectOptions; - httpOptions?: HttpOptions; - enableAffectiveDialog?: boolean; - proactivity?: boolean; - realtimeInputConfig?: RealtimeInputConfig; - contextWindowCompression?: ContextWindowCompressionConfig; - apiVersion?: string; - geminiTools?: LLMTools; - thinkingConfig?: types.ThinkingConfig; -} -/** - * Google Realtime Model for real-time voice conversations with Gemini models - */ -export declare class RealtimeModel extends llm.RealtimeModel { - /** @internal */ - _options: RealtimeOptions; - constructor(options?: { - /** - * Initial system instructions for the model - */ - instructions?: string; - /** - * The name of the model to use - */ - model?: LiveAPIModels | string; - /** - * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable - */ - apiKey?: string; - /** - * Voice setting for audio outputs - */ - voice?: Voice | string; - /** - * The language (BCP-47 Code) to use for the API - * See https://ai.google.dev/gemini-api/docs/live#supported-languages - */ - language?: string; - /** - * Modalities to use, such as [Modality.TEXT, Modality.AUDIO] - */ - modalities?: Modality[]; - /** - * Whether to use VertexAI for the API - */ - vertexai?: boolean; - /** - * The project ID to use for the API (for VertexAI) - */ - project?: string; - /** - * The location to use for the API (for VertexAI) - */ - location?: string; - /** - * The number of candidate responses to generate - */ - candidateCount?: number; - /** - * Sampling temperature for response generation - */ - temperature?: number; - /** - * Maximum number of tokens in the response - */ - maxOutputTokens?: number; - /** - * The top-p value for response generation - */ - topP?: number; - /** - * The top-k value for response generation - */ - topK?: number; - /** - * The presence penalty for response generation - */ - presencePenalty?: number; - /** - * The frequency penalty for response generation - */ - frequencyPenalty?: number; - /** - * The configuration for input audio transcription - */ - inputAudioTranscription?: AudioTranscriptionConfig | null; - /** - * The configuration for output audio transcription - */ - outputAudioTranscription?: AudioTranscriptionConfig | null; - /** - * The configuration for image encoding - */ - imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS; - /** - * Whether to enable affective dialog - */ - enableAffectiveDialog?: boolean; - /** - * Whether to enable proactive audio - */ - proactivity?: boolean; - /** - * The configuration for realtime input - */ - realtimeInputConfig?: RealtimeInputConfig; - /** - * The configuration for context window compression - */ - contextWindowCompression?: ContextWindowCompressionConfig; - /** - * API version to use - */ - apiVersion?: string; - /** - * The configuration for the API connection - */ - connOptions?: APIConnectOptions; - /** - * HTTP options for API requests - */ - httpOptions?: HttpOptions; - /** - * Gemini-specific tools to use for the session - */ - geminiTools?: LLMTools; - /** - * The thinking configuration for response generation - */ - thinkingConfig?: types.ThinkingConfig; - }); - /** - * Create a new realtime session - */ - session(): RealtimeSession; - /** - * Update model options - */ - updateOptions(options: { - voice?: Voice | string; - temperature?: number; - }): void; - /** - * Close the model and cleanup resources - */ - close(): Promise; -} -/** - * Google Realtime Session for real-time voice conversations - * - * This session provides real-time streaming capabilities with Google's Gemini models, - * supporting both text and audio modalities with function calling capabilities. - */ -export declare class RealtimeSession extends llm.RealtimeSession { - #private; - private _tools; - private _chatCtx; - private options; - private geminiDeclarations; - private messageChannel; - private inputResampler?; - private inputResamplerInputRate?; - private instructions?; - private currentGeneration?; - private bstream; - private activeSession?; - private sessionShouldClose; - private responseCreatedFutures; - private pendingGenerationFut?; - private sessionResumptionHandle?; - private inUserActivity; - private sessionLock; - private numRetries; - private hasReceivedAudioInput; - constructor(realtimeModel: RealtimeModel); - private closeActiveSession; - private markRestartNeeded; - private getToolResultsForRealtime; - updateOptions(options: { - voice?: Voice | string; - temperature?: number; - toolChoice?: llm.ToolChoice; - }): void; - updateInstructions(instructions: string): Promise; - updateChatCtx(chatCtx: llm.ChatContext): Promise; - updateTools(tools: llm.ToolContext): Promise; - get chatCtx(): llm.ChatContext; - get tools(): llm.ToolContext; - get manualActivityDetection(): boolean; - pushAudio(frame: AudioFrame): void; - pushVideo(_: VideoFrame): void; - private sendClientEvent; - generateReply(instructions?: string): Promise; - startUserActivity(): void; - interrupt(): Promise; - truncate(_options: { - messageId: string; - audioEndMs: number; - audioTranscript?: string; - }): Promise; - close(): Promise; - private sendTask; - private onReceiveMessage; - private truncateString; - private loggableClientEvent; - private loggableServerMessage; - private markCurrentGenerationDone; - private emitError; - private buildConnectConfig; - private startNewGeneration; - private handleInputSpeechStarted; - private handleInputSpeechStopped; - private handleServerContent; - private handleToolCall; - private handleToolCallCancellation; - private handleUsageMetadata; - private tokenDetailsMap; - private handleGoAway; - commitAudio(): Promise; - clearAudio(): Promise; - private resampleAudio; -} -export {}; -//# sourceMappingURL=realtime_api.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.d.ts.map b/plugins/google/dist/beta/realtime/realtime_api.d.ts.map deleted file mode 100644 index 94128e672..000000000 --- a/plugins/google/dist/beta/realtime/realtime_api.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"realtime_api.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/realtime_api.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,eAAe,CAAC;AACvC,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EAEnC,KAAK,WAAW,EAChB,QAAQ,EACR,KAAK,mBAAmB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAUL,GAAG,EAIJ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,UAAU,EAAkB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAU3D;;GAEG;AACH,eAAO,MAAM,4BAA4B;;;;;;;;CAQxC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,UAAU,eAAe;IACvB,KAAK,EAAE,aAAa,GAAG,MAAM,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,QAAQ,EAAE,CAAC;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,wBAAwB,CAAC;IACnD,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;IACpD,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;IACzD,WAAW,EAAE,iBAAiB,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;CACvC;AA2BD;;GAEG;AACH,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,gBAAgB;IAChB,QAAQ,EAAE,eAAe,CAAC;gBAGxB,OAAO,GAAE;QACP;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;QAE/B;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAEvB;;;WAGG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;QAExB;;WAEG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAC;QAEnB;;WAEG;QACH,OAAO,CAAC,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,uBAAuB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE1D;;WAEG;QACH,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE3D;;WAEG;QACH,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;QAEzD;;WAEG;QACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,CAAC;QAEtB;;WAEG;QACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;QAE1C;;WAEG;QACH,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;QAE1D;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,WAAW,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,QAAQ,CAAC;QAEvB;;WAEG;QACH,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;KAClC;IA6DR;;OAEG;IACH,OAAO;IAIP;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAW9E;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,GAAG,CAAC,eAAe;;IACtD,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAA2B;IAE3C,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,cAAc,CAAuC;IAC7D,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAC,CAAqB;IAC/C,OAAO,CAAC,OAAO,CAAkB;IAGjC,OAAO,CAAC,aAAa,CAAC,CAAU;IAChC,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,sBAAsB,CAA4D;IAC1F,OAAO,CAAC,oBAAoB,CAAC,CAAqC;IAElE,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,qBAAqB,CAAS;gBAO1B,aAAa,EAAE,aAAa;YAwC1B,kBAAkB;IAiBhC,OAAO,CAAC,iBAAiB;IAWzB,OAAO,CAAC,yBAAyB;IAyBjC,aAAa,CAAC,OAAO,EAAE;QACrB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;KAC7B;IAkBK,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOvD,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDtD,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAYxD,IAAI,OAAO,IAAI,GAAG,CAAC,WAAW,CAE7B;IAED,IAAI,KAAK,IAAI,GAAG,CAAC,WAAW,CAE3B;IAED,IAAI,uBAAuB,IAAI,OAAO,CAErC;IAED,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAsBlC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI;IAI9B,OAAO,CAAC,eAAe;IAIjB,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAyD/E,iBAAiB,IAAI,IAAI;IAgBnB,SAAS;IAQT,QAAQ,CAAC,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE;IAItF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAoJd,QAAQ;YAiER,gBAAgB;IAuE9B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,mBAAmB;IAmB3B,OAAO,CAAC,qBAAqB;IA4B7B,OAAO,CAAC,yBAAyB;IA+CjC,OAAO,CAAC,SAAS;IAUjB,OAAO,CAAC,kBAAkB;IAiE1B,OAAO,CAAC,kBAAkB;IAwD1B,OAAO,CAAC,wBAAwB;IAIhC,OAAO,CAAC,wBAAwB;IAMhC,OAAO,CAAC,mBAAmB;IAoF3B,OAAO,CAAC,cAAc;IAmBtB,OAAO,CAAC,0BAA0B;IASlC,OAAO,CAAC,mBAAmB;IA6C3B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,YAAY;IAMd,WAAW;IAEX,UAAU;IAEhB,OAAO,CAAE,aAAa;CA8BvB"} \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.js b/plugins/google/dist/beta/realtime/realtime_api.js deleted file mode 100644 index 42842fa2f..000000000 --- a/plugins/google/dist/beta/realtime/realtime_api.js +++ /dev/null @@ -1,992 +0,0 @@ -import * as types from "@google/genai"; -import { - ActivityHandling, - GoogleGenAI, - Modality -} from "@google/genai"; -import { - APIConnectionError, - AudioByteStream, - DEFAULT_API_CONNECT_OPTIONS, - Event, - Future, - Queue, - Task, - cancelAndWait, - delay, - llm, - log, - shortuuid, - stream -} from "@livekit/agents"; -import { Mutex } from "@livekit/mutex"; -import { AudioFrame, AudioResampler } from "@livekit/rtc-node"; -import {} from "../../tools.js"; -import { toFunctionDeclarations } from "../../utils.js"; -const INPUT_AUDIO_SAMPLE_RATE = 16e3; -const INPUT_AUDIO_CHANNELS = 1; -const OUTPUT_AUDIO_SAMPLE_RATE = 24e3; -const OUTPUT_AUDIO_CHANNELS = 1; -const DEFAULT_IMAGE_ENCODE_OPTIONS = { - format: "JPEG", - quality: 75, - resizeOptions: { - width: 1024, - height: 1024, - strategy: "scale_aspect_fit" - } -}; -function setsEqual(a, b) { - return a.size === b.size && [...a].every((x) => b.has(x)); -} -class RealtimeModel extends llm.RealtimeModel { - /** @internal */ - _options; - constructor(options = {}) { - var _a, _b, _c; - const inputAudioTranscription = options.inputAudioTranscription === void 0 ? {} : options.inputAudioTranscription; - const outputAudioTranscription = options.outputAudioTranscription === void 0 ? {} : options.outputAudioTranscription; - let serverTurnDetection = true; - if ((_b = (_a = options.realtimeInputConfig) == null ? void 0 : _a.automaticActivityDetection) == null ? void 0 : _b.disabled) { - serverTurnDetection = false; - } - super({ - messageTruncation: false, - turnDetection: serverTurnDetection, - userTranscription: inputAudioTranscription !== null, - autoToolReplyGeneration: true, - audioOutput: ((_c = options.modalities) == null ? void 0 : _c.includes(Modality.AUDIO)) ?? true - }); - const apiKey = options.apiKey || process.env.GOOGLE_API_KEY; - const project = options.project || process.env.GOOGLE_CLOUD_PROJECT; - const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || "us-central1"; - const vertexai = options.vertexai ?? false; - const defaultModel = vertexai ? "gemini-2.0-flash-exp" : "gemini-2.0-flash-live-001"; - this._options = { - model: options.model || defaultModel, - apiKey, - voice: options.voice || "Puck", - language: options.language, - responseModalities: options.modalities || [Modality.AUDIO], - vertexai, - project, - location, - candidateCount: options.candidateCount || 1, - temperature: options.temperature, - maxOutputTokens: options.maxOutputTokens, - topP: options.topP, - topK: options.topK, - presencePenalty: options.presencePenalty, - frequencyPenalty: options.frequencyPenalty, - instructions: options.instructions, - inputAudioTranscription: inputAudioTranscription || void 0, - outputAudioTranscription: outputAudioTranscription || void 0, - imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS, - connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS, - httpOptions: options.httpOptions, - enableAffectiveDialog: options.enableAffectiveDialog, - proactivity: options.proactivity, - realtimeInputConfig: options.realtimeInputConfig, - contextWindowCompression: options.contextWindowCompression, - apiVersion: options.apiVersion, - geminiTools: options.geminiTools, - thinkingConfig: options.thinkingConfig - }; - } - /** - * Create a new realtime session - */ - session() { - return new RealtimeSession(this); - } - /** - * Update model options - */ - updateOptions(options) { - if (options.voice !== void 0) { - this._options.voice = options.voice; - } - if (options.temperature !== void 0) { - this._options.temperature = options.temperature; - } - } - /** - * Close the model and cleanup resources - */ - async close() { - } -} -class RealtimeSession extends llm.RealtimeSession { - _tools = {}; - _chatCtx = llm.ChatContext.empty(); - options; - geminiDeclarations = []; - messageChannel = new Queue(); - inputResampler; - inputResamplerInputRate; - instructions; - currentGeneration; - bstream; - // Google-specific properties - activeSession; - sessionShouldClose = new Event(); - responseCreatedFutures = {}; - pendingGenerationFut; - sessionResumptionHandle; - inUserActivity = false; - sessionLock = new Mutex(); - numRetries = 0; - hasReceivedAudioInput = false; - #client; - #task; - #logger = log(); - #closed = false; - constructor(realtimeModel) { - super(realtimeModel); - this.options = realtimeModel._options; - this.bstream = new AudioByteStream( - INPUT_AUDIO_SAMPLE_RATE, - INPUT_AUDIO_CHANNELS, - INPUT_AUDIO_SAMPLE_RATE / 20 - ); - const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } = this.options; - const apiVersion = !this.options.apiVersion && (enableAffectiveDialog || proactivity) ? "v1alpha" : this.options.apiVersion; - const httpOptions = { - ...this.options.httpOptions, - apiVersion, - timeout: this.options.connOptions.timeoutMs - }; - const clientOptions = vertexai ? { - vertexai: true, - project, - location, - httpOptions - } : { - apiKey, - httpOptions - }; - this.#client = new GoogleGenAI(clientOptions); - this.#task = this.#mainTask(); - } - async closeActiveSession() { - const unlock = await this.sessionLock.lock(); - const session = this.activeSession; - this.activeSession = void 0; - if (session) { - try { - await session.close(); - } catch (error) { - this.#logger.warn({ error }, "Error closing Gemini session"); - } - } - unlock(); - } - markRestartNeeded() { - if (!this.sessionShouldClose.isSet) { - this.sessionShouldClose.set(); - const oldQueue = this.messageChannel; - this.messageChannel = new Queue(); - oldQueue.put({ type: "__internal_wakeup__" }); - } - } - getToolResultsForRealtime(ctx, vertexai) { - const toolResponses = []; - for (const item of ctx.items) { - if (item.type === "function_call_output") { - const response = { - id: item.callId, - name: item.name, - response: { output: item.output } - }; - if (!vertexai) { - response.id = item.callId; - } - toolResponses.push(response); - } - } - return toolResponses.length > 0 ? { functionResponses: toolResponses } : void 0; - } - updateOptions(options) { - let shouldRestart = false; - if (options.voice !== void 0 && this.options.voice !== options.voice) { - this.options.voice = options.voice; - shouldRestart = true; - } - if (options.temperature !== void 0 && this.options.temperature !== options.temperature) { - this.options.temperature = options.temperature; - shouldRestart = true; - } - if (shouldRestart) { - this.markRestartNeeded(); - } - } - async updateInstructions(instructions) { - if (this.options.instructions === void 0 || this.options.instructions !== instructions) { - this.options.instructions = instructions; - this.markRestartNeeded(); - } - } - async updateChatCtx(chatCtx) { - const unlock = await this.sessionLock.lock(); - try { - if (!this.activeSession) { - this._chatCtx = chatCtx.copy(); - return; - } - } finally { - unlock(); - } - const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx); - if (diffOps.toRemove.length > 0) { - this.#logger.warn("Gemini Live does not support removing messages"); - } - const appendCtx = llm.ChatContext.empty(); - for (const [, itemId] of diffOps.toCreate) { - const item = chatCtx.getById(itemId); - if (item) { - appendCtx.items.push(item); - } - } - if (appendCtx.items.length > 0) { - const [turns] = await appendCtx.copy({ - excludeFunctionCall: true - }).toProviderFormat("google", false); - const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai); - if (turns.length > 0) { - this.sendClientEvent({ - type: "content", - value: { - turns, - turnComplete: false - } - }); - } - if (toolResults) { - this.sendClientEvent({ - type: "tool_response", - value: toolResults - }); - } - } - this._chatCtx = chatCtx.copy(); - } - async updateTools(tools) { - const newDeclarations = toFunctionDeclarations(tools); - const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name)); - const newToolNames = new Set(newDeclarations.map((f) => f.name)); - if (!setsEqual(currentToolNames, newToolNames)) { - this.geminiDeclarations = newDeclarations; - this._tools = tools; - this.markRestartNeeded(); - } - } - get chatCtx() { - return this._chatCtx.copy(); - } - get tools() { - return { ...this._tools }; - } - get manualActivityDetection() { - var _a, _b; - return ((_b = (_a = this.options.realtimeInputConfig) == null ? void 0 : _a.automaticActivityDetection) == null ? void 0 : _b.disabled) ?? false; - } - pushAudio(frame) { - this.hasReceivedAudioInput = true; - for (const f of this.resampleAudio(frame)) { - for (const nf of this.bstream.write(f.data.buffer)) { - const realtimeInput = { - mediaChunks: [ - { - mimeType: "audio/pcm", - data: Buffer.from(nf.data.buffer).toString("base64") - } - ] - }; - this.sendClientEvent({ - type: "realtime_input", - value: realtimeInput - }); - } - } - } - pushVideo(_) { - } - sendClientEvent(event) { - this.messageChannel.put(event); - } - async generateReply(instructions) { - if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { - this.#logger.warn( - "generateReply called while another generation is pending, cancelling previous." - ); - this.pendingGenerationFut.reject(new Error("Superseded by new generate_reply call")); - } - const fut = new Future(); - this.pendingGenerationFut = fut; - if (this.inUserActivity) { - this.sendClientEvent({ - type: "realtime_input", - value: { - activityEnd: {} - } - }); - this.inUserActivity = false; - } - const turns = []; - if (instructions !== void 0) { - turns.push({ - parts: [{ text: instructions }], - role: "model" - }); - } - turns.push({ - parts: [{ text: "." }], - role: "user" - }); - this.sendClientEvent({ - type: "content", - value: { - turns, - turnComplete: true - } - }); - const timeoutHandle = setTimeout(() => { - if (!fut.done) { - fut.reject(new Error("generateReply timed out waiting for generation_created event.")); - if (this.pendingGenerationFut === fut) { - this.pendingGenerationFut = void 0; - } - } - }, 5e3); - fut.await.finally(() => clearTimeout(timeoutHandle)); - return fut.await; - } - startUserActivity() { - if (!this.manualActivityDetection) { - return; - } - if (!this.inUserActivity) { - this.inUserActivity = true; - this.sendClientEvent({ - type: "realtime_input", - value: { - activityStart: {} - } - }); - } - } - async interrupt() { - var _a; - if (((_a = this.options.realtimeInputConfig) == null ? void 0 : _a.activityHandling) === ActivityHandling.NO_INTERRUPTION) { - return; - } - this.startUserActivity(); - } - async truncate(_options) { - this.#logger.warn("truncate is not supported by the Google Realtime API."); - } - async close() { - super.close(); - this.#closed = true; - this.sessionShouldClose.set(); - await this.closeActiveSession(); - if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { - this.pendingGenerationFut.reject(new Error("Session closed")); - } - for (const fut of Object.values(this.responseCreatedFutures)) { - if (!fut.done) { - fut.reject(new Error("Session closed before response created")); - } - } - this.responseCreatedFutures = {}; - if (this.currentGeneration) { - this.markCurrentGenerationDone(); - } - } - async #mainTask() { - const maxRetries = this.options.connOptions.maxRetry; - while (!this.#closed) { - await this.closeActiveSession(); - this.sessionShouldClose.clear(); - const config = this.buildConnectConfig(); - try { - this.#logger.debug("Connecting to Gemini Realtime API..."); - const sessionOpened = new Event(); - const session = await this.#client.live.connect({ - model: this.options.model, - callbacks: { - onopen: () => sessionOpened.set(), - onmessage: (message) => { - this.onReceiveMessage(session, message); - }, - onerror: (error) => { - this.#logger.error({ error }, "Gemini Live session error"); - sessionOpened.set(); - if (!this.sessionShouldClose.isSet) { - this.markRestartNeeded(); - } - }, - onclose: (event) => { - if (this.activeSession !== session) { - return; - } - this.#logger.debug( - { code: event.code, reason: event.reason }, - "Gemini Live session closed" - ); - sessionOpened.set(); - this.markCurrentGenerationDone(); - if (!this.sessionShouldClose.isSet && !this.#closed) { - this.markRestartNeeded(); - } - } - }, - config - }); - await sessionOpened.wait(); - const unlock = await this.sessionLock.lock(); - try { - this.activeSession = session; - const [turns] = await this._chatCtx.copy({ - excludeFunctionCall: true - }).toProviderFormat("google", false); - if (turns.length > 0) { - await session.sendClientContent({ - turns, - turnComplete: false - }); - } - } finally { - unlock(); - } - const sendTask = Task.from((controller) => this.sendTask(session, controller)); - const restartWaitTask = Task.from(({ signal }) => { - const abortEvent = new Event(); - signal.addEventListener("abort", () => abortEvent.set()); - return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]); - }); - await Promise.race([sendTask.result, restartWaitTask.result]); - if (!restartWaitTask.done && this.#closed) { - break; - } - await cancelAndWait([sendTask, restartWaitTask], 2e3); - } catch (error) { - this.#logger.error(`Gemini Realtime API error: ${error}`); - if (this.#closed) break; - if (maxRetries === 0) { - this.emitError(error, false); - throw new APIConnectionError({ - message: "Failed to connect to Gemini Live" - }); - } - if (this.numRetries >= maxRetries) { - this.emitError(error, false); - throw new APIConnectionError({ - message: `Failed to connect to Gemini Live after ${maxRetries} attempts` - }); - } - const retryInterval = this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs; - this.#logger.warn( - { - attempt: this.numRetries, - maxRetries - }, - `Gemini Realtime API connection failed, retrying in ${retryInterval}ms` - ); - await delay(retryInterval); - this.numRetries++; - } finally { - await this.closeActiveSession(); - } - } - } - async sendTask(session, controller) { - try { - while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) { - const msg = await this.messageChannel.get(); - if (controller.signal.aborted) break; - const unlock = await this.sessionLock.lock(); - try { - if (this.sessionShouldClose.isSet || this.activeSession !== session) { - break; - } - } finally { - unlock(); - } - switch (msg.type) { - case "content": - const { turns, turnComplete } = msg.value; - this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`); - await session.sendClientContent({ - turns, - turnComplete: turnComplete ?? true - }); - break; - case "tool_response": - const { functionResponses } = msg.value; - if (functionResponses) { - this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`); - await session.sendToolResponse({ - functionResponses - }); - } - break; - case "realtime_input": - const { mediaChunks, activityStart, activityEnd } = msg.value; - if (mediaChunks) { - for (const mediaChunk of mediaChunks) { - await session.sendRealtimeInput({ media: mediaChunk }); - } - } - if (activityStart) await session.sendRealtimeInput({ activityStart }); - if (activityEnd) await session.sendRealtimeInput({ activityEnd }); - break; - default: - this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`); - break; - } - } - } catch (e) { - if (!this.sessionShouldClose.isSet) { - this.#logger.error(`Error in send task: ${e}`); - this.markRestartNeeded(); - } - } finally { - this.#logger.debug( - { - closed: this.#closed, - sessionShouldClose: this.sessionShouldClose.isSet, - aborted: controller.signal.aborted - }, - "send task finished." - ); - } - } - async onReceiveMessage(session, response) { - var _a, _b, _c; - const hasAudioData = (_c = (_b = (_a = response.serverContent) == null ? void 0 : _a.modelTurn) == null ? void 0 : _b.parts) == null ? void 0 : _c.some( - (part) => { - var _a2; - return (_a2 = part.inlineData) == null ? void 0 : _a2.data; - } - ); - if (!hasAudioData) { - this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`); - } - const unlock = await this.sessionLock.lock(); - try { - if (this.sessionShouldClose.isSet || this.activeSession !== session) { - this.#logger.debug("onReceiveMessage: Session changed or closed, stopping receive."); - return; - } - } finally { - unlock(); - } - if ((!this.currentGeneration || this.currentGeneration._done) && (response.serverContent || response.toolCall)) { - this.startNewGeneration(); - } - if (response.sessionResumptionUpdate) { - if (response.sessionResumptionUpdate.resumable && response.sessionResumptionUpdate.newHandle) { - this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle; - } - } - try { - if (response.serverContent) { - this.handleServerContent(response.serverContent); - } - if (response.toolCall) { - this.handleToolCall(response.toolCall); - } - if (response.toolCallCancellation) { - this.handleToolCallCancellation(response.toolCallCancellation); - } - if (response.usageMetadata) { - this.handleUsageMetadata(response.usageMetadata); - } - if (response.goAway) { - this.handleGoAway(response.goAway); - } - if (this.numRetries > 0) { - this.numRetries = 0; - } - } catch (e) { - if (!this.sessionShouldClose.isSet) { - this.#logger.error(`Error in onReceiveMessage: ${e}`); - this.markRestartNeeded(); - } - } - } - /// Truncate large base64/audio payloads for logging to avoid flooding logs - truncateString(data, maxLength = 30) { - return data.length > maxLength ? `${data.slice(0, maxLength)}\u2026` : data; - } - loggableClientEvent(event, maxLength = 30) { - var _a; - const obj = { ...event }; - if (obj.type === "realtime_input" && ((_a = obj.value) == null ? void 0 : _a.mediaChunks)) { - obj.value = { - ...obj.value, - mediaChunks: obj.value.mediaChunks.map( - (mc) => ({ - ...mc, - data: typeof mc.data === "string" ? this.truncateString(mc.data, maxLength) : mc.data - }) - ) - }; - } - return obj; - } - loggableServerMessage(message, maxLength = 30) { - const obj = { ...message }; - if (obj.serverContent && obj.serverContent.modelTurn && Array.isArray(obj.serverContent.modelTurn.parts)) { - obj.serverContent = { ...obj.serverContent }; - obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn }; - obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part) => { - var _a; - if (((_a = part == null ? void 0 : part.inlineData) == null ? void 0 : _a.data) && typeof part.inlineData.data === "string") { - return { - ...part, - inlineData: { - ...part.inlineData, - data: this.truncateString(part.inlineData.data, maxLength) - } - }; - } - return part; - }); - } - return obj; - } - markCurrentGenerationDone() { - if (!this.currentGeneration || this.currentGeneration._done) { - return; - } - this.handleInputSpeechStopped(); - const gen = this.currentGeneration; - if (gen.inputTranscription) { - this.emit("input_audio_transcription_completed", { - itemId: gen.inputId, - transcript: gen.inputTranscription, - isFinal: true - }); - this._chatCtx.addMessage({ - role: "user", - content: gen.inputTranscription, - id: gen.inputId - }); - } - if (gen.outputText) { - this._chatCtx.addMessage({ - role: "assistant", - content: gen.outputText, - id: gen.responseId - }); - } - if (this.options.outputAudioTranscription === void 0) { - gen.textChannel.write(""); - } - gen.textChannel.close(); - gen.audioChannel.close(); - gen.functionChannel.close(); - gen.messageChannel.close(); - gen._done = true; - } - emitError(error, recoverable) { - this.emit("error", { - timestamp: Date.now(), - // TODO(brian): add label to realtime model - label: "google_realtime", - error, - recoverable - }); - } - buildConnectConfig() { - const opts = this.options; - const config = { - responseModalities: opts.responseModalities, - systemInstruction: opts.instructions ? { - parts: [{ text: opts.instructions }] - } : void 0, - speechConfig: { - voiceConfig: { - prebuiltVoiceConfig: { - voiceName: opts.voice - } - }, - languageCode: opts.language - }, - tools: [ - { - functionDeclarations: this.geminiDeclarations, - ...this.options.geminiTools - } - ], - inputAudioTranscription: opts.inputAudioTranscription, - outputAudioTranscription: opts.outputAudioTranscription, - sessionResumption: { - handle: this.sessionResumptionHandle - }, - thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : void 0 - }; - if (opts.temperature !== void 0) { - config.temperature = opts.temperature; - } - if (opts.maxOutputTokens !== void 0) { - config.maxOutputTokens = opts.maxOutputTokens; - } - if (opts.topP !== void 0) { - config.topP = opts.topP; - } - if (opts.topK !== void 0) { - config.topK = opts.topK; - } - if (opts.proactivity !== void 0) { - config.proactivity = { proactiveAudio: opts.proactivity }; - } - if (opts.enableAffectiveDialog !== void 0) { - config.enableAffectiveDialog = opts.enableAffectiveDialog; - } - if (opts.realtimeInputConfig !== void 0) { - config.realtimeInputConfig = opts.realtimeInputConfig; - } - if (opts.contextWindowCompression !== void 0) { - config.contextWindowCompression = opts.contextWindowCompression; - } - return config; - } - startNewGeneration() { - if (this.currentGeneration && !this.currentGeneration._done) { - this.#logger.warn("Starting new generation while another is active. Finalizing previous."); - this.markCurrentGenerationDone(); - } - const responseId = shortuuid("GR_"); - this.currentGeneration = { - messageChannel: stream.createStreamChannel(), - functionChannel: stream.createStreamChannel(), - responseId, - inputId: shortuuid("GI_"), - textChannel: stream.createStreamChannel(), - audioChannel: stream.createStreamChannel(), - inputTranscription: "", - outputText: "", - _createdTimestamp: Date.now(), - _done: false - }; - if (!this._realtimeModel.capabilities.audioOutput) { - this.currentGeneration.audioChannel.close(); - } - const modalities = this._realtimeModel.capabilities.audioOutput ? ["audio", "text"] : ["text"]; - this.currentGeneration.messageChannel.write({ - messageId: responseId, - textStream: this.currentGeneration.textChannel.stream(), - audioStream: this.currentGeneration.audioChannel.stream(), - modalities: Promise.resolve(modalities) - }); - const generationEvent = { - messageStream: this.currentGeneration.messageChannel.stream(), - functionStream: this.currentGeneration.functionChannel.stream(), - userInitiated: false - }; - if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { - generationEvent.userInitiated = true; - this.pendingGenerationFut.resolve(generationEvent); - this.pendingGenerationFut = void 0; - } else { - this.handleInputSpeechStarted(); - } - this.emit("generation_created", generationEvent); - } - handleInputSpeechStarted() { - this.emit("input_speech_started", {}); - } - handleInputSpeechStopped() { - this.emit("input_speech_stopped", { - userTranscriptionEnabled: false - }); - } - handleServerContent(serverContent) { - if (!this.currentGeneration) { - this.#logger.warn("received server content but no active generation."); - return; - } - const gen = this.currentGeneration; - if (serverContent.modelTurn) { - const turn = serverContent.modelTurn; - for (const part of turn.parts || []) { - if (part.text) { - gen.outputText += part.text; - gen.textChannel.write(part.text); - } - if (part.inlineData) { - if (!gen._firstTokenTimestamp) { - gen._firstTokenTimestamp = Date.now(); - } - try { - if (!part.inlineData.data) { - throw new Error("frameData is not bytes"); - } - const binaryString = atob(part.inlineData.data); - const len = binaryString.length; - const bytes = new Uint8Array(len); - for (let i = 0; i < len; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - const int16Array = new Int16Array(bytes.buffer); - const audioFrame = new AudioFrame( - int16Array, - OUTPUT_AUDIO_SAMPLE_RATE, - OUTPUT_AUDIO_CHANNELS, - int16Array.length / OUTPUT_AUDIO_CHANNELS - ); - gen.audioChannel.write(audioFrame); - } catch (error) { - this.#logger.error("Error processing audio data:", error); - } - } - } - } - if (serverContent.inputTranscription && serverContent.inputTranscription.text) { - let text = serverContent.inputTranscription.text; - if (gen.inputTranscription === "") { - text = text.trimStart(); - } - gen.inputTranscription += text; - this.emit("input_audio_transcription_completed", { - itemId: gen.inputId, - transcript: gen.inputTranscription, - isFinal: false - }); - } - if (serverContent.outputTranscription && serverContent.outputTranscription.text) { - const text = serverContent.outputTranscription.text; - gen.outputText += text; - gen.textChannel.write(text); - } - if (serverContent.generationComplete || serverContent.turnComplete) { - gen._completedTimestamp = Date.now(); - } - if (serverContent.interrupted) { - this.handleInputSpeechStarted(); - } - if (serverContent.turnComplete) { - this.markCurrentGenerationDone(); - } - } - handleToolCall(toolCall) { - if (!this.currentGeneration) { - this.#logger.warn("received tool call but no active generation."); - return; - } - const gen = this.currentGeneration; - for (const fc of toolCall.functionCalls || []) { - gen.functionChannel.write({ - callId: fc.id || shortuuid("fnc-call-"), - name: fc.name, - args: fc.args ? JSON.stringify(fc.args) : "" - }); - } - this.markCurrentGenerationDone(); - } - handleToolCallCancellation(cancellation) { - this.#logger.warn( - { - functionCallIds: cancellation.ids - }, - "server cancelled tool calls" - ); - } - handleUsageMetadata(usage) { - if (!this.currentGeneration) { - this.#logger.debug("Received usage metadata but no active generation"); - return; - } - const gen = this.currentGeneration; - const createdTimestamp = gen._createdTimestamp; - const firstTokenTimestamp = gen._firstTokenTimestamp; - const completedTimestamp = gen._completedTimestamp || Date.now(); - const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1; - const durationMs = completedTimestamp - createdTimestamp; - const inputTokens = usage.promptTokenCount || 0; - const outputTokens = usage.responseTokenCount || 0; - const totalTokens = usage.totalTokenCount || 0; - const realtimeMetrics = { - type: "realtime_model_metrics", - timestamp: createdTimestamp, - requestId: gen.responseId, - ttftMs, - durationMs, - cancelled: gen._done && !gen._completedTimestamp, - label: "google_realtime", - inputTokens, - outputTokens, - totalTokens, - tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1e3) : 0, - inputTokenDetails: { - ...this.tokenDetailsMap(usage.promptTokensDetails), - cachedTokens: (usage.cacheTokensDetails || []).reduce( - (sum, detail) => sum + (detail.tokenCount || 0), - 0 - ), - cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails) - }, - outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails) - }; - this.emit("metrics_collected", realtimeMetrics); - } - tokenDetailsMap(tokenDetails) { - const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 }; - if (!tokenDetails) { - return tokenDetailsMap; - } - for (const tokenDetail of tokenDetails) { - if (!tokenDetail.tokenCount) { - continue; - } - if (tokenDetail.modality === types.MediaModality.AUDIO) { - tokenDetailsMap.audioTokens += tokenDetail.tokenCount; - } else if (tokenDetail.modality === types.MediaModality.TEXT) { - tokenDetailsMap.textTokens += tokenDetail.tokenCount; - } else if (tokenDetail.modality === types.MediaModality.IMAGE) { - tokenDetailsMap.imageTokens += tokenDetail.tokenCount; - } - } - return tokenDetailsMap; - } - handleGoAway(goAway) { - this.#logger.warn({ timeLeft: goAway.timeLeft }, "Gemini server indicates disconnection soon."); - this.sessionShouldClose.set(); - } - async commitAudio() { - } - async clearAudio() { - } - *resampleAudio(frame) { - if (this.inputResampler) { - if (frame.sampleRate !== this.inputResamplerInputRate) { - this.inputResampler = void 0; - this.inputResamplerInputRate = void 0; - } - } - if (this.inputResampler === void 0 && (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)) { - this.inputResampler = new AudioResampler( - frame.sampleRate, - INPUT_AUDIO_SAMPLE_RATE, - INPUT_AUDIO_CHANNELS - ); - this.inputResamplerInputRate = frame.sampleRate; - } - if (this.inputResampler) { - for (const resampledFrame of this.inputResampler.push(frame)) { - yield resampledFrame; - } - } else { - yield frame; - } - } -} -export { - DEFAULT_IMAGE_ENCODE_OPTIONS, - RealtimeModel, - RealtimeSession -}; -//# sourceMappingURL=realtime_api.js.map \ No newline at end of file diff --git a/plugins/google/dist/beta/realtime/realtime_api.js.map b/plugins/google/dist/beta/realtime/realtime_api.js.map deleted file mode 100644 index e56f1bd20..000000000 --- a/plugins/google/dist/beta/realtime/realtime_api.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual(a: Set, b: Set): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel;\n functionChannel: stream.StreamChannel;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel;\n audioChannel: stream.StreamChannel;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * The thinking configuration for response generation\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai ? 'gemini-2.0-flash-exp' : 'gemini-2.0-flash-live-001';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future } = {};\n private pendingGenerationFut?: Future;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n\n #client: GoogleGenAI;\n #task: Promise;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise {\n const unlock = await this.sessionLock.lock();\n\n const session = this.activeSession;\n this.activeSession = undefined;\n\n if (session) {\n try {\n await session.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n }\n }\n\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n // Replace the queue so new messages don't go to the old (closing) session.\n const oldQueue = this.messageChannel;\n this.messageChannel = new Queue();\n // Wake up the current sendTask if it's waiting for a message.\n oldQueue.put({ type: '__internal_wakeup__' } as any);\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n return;\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n onerror: (error: ErrorEvent) => {\n this.#logger.error({ error }, 'Gemini Live session error');\n sessionOpened.set(); // Break the wait if we were waiting for open\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n if (this.activeSession !== session) {\n return;\n }\n\n this.#logger.debug(\n { code: event.code, reason: event.reason },\n 'Gemini Live session closed',\n );\n sessionOpened.set(); // Break the wait if we were waiting for open\n this.markCurrentGenerationDone();\n if (!this.sessionShouldClose.isSet && !this.#closed) {\n this.markRestartNeeded();\n }\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n await session.sendToolResponse({\n functionResponses,\n });\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd } = msg.value;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n if (\n (!this.currentGeneration || this.currentGeneration._done) &&\n (response.serverContent || response.toolCall)\n ) {\n this.startNewGeneration();\n }\n\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(): void {\n if (!this.currentGeneration || this.currentGeneration._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const gen = this.currentGeneration;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (gen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: gen.inputTranscription,\n id: gen.inputId,\n });\n }\n\n if (gen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: gen.outputText,\n id: gen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n gen.textChannel.write('');\n }\n\n gen.textChannel.close();\n gen.audioChannel.close();\n gen.functionChannel.close();\n gen.messageChannel.close();\n gen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined,\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel(),\n functionChannel: stream.createStreamChannel(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel(),\n audioChannel: stream.createStreamChannel(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (serverContent.modelTurn) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (serverContent.outputTranscription && serverContent.outputTranscription.text) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete) {\n this.markCurrentGenerationDone();\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n for (const fc of toolCall.functionCalls || []) {\n gen.functionChannel.write({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n } as llm.FunctionCall);\n }\n\n this.markCurrentGenerationDone();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() {}\n\n async clearAudio() {}\n\n private *resampleAudio(frame: AudioFrame): Generator {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n}\n"],"mappings":"AAIA,YAAY,WAAW;AACvB;AAAA,EACE;AAAA,EAGA;AAAA,EAEA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,aAAa;AACtB,SAAS,YAAY,sBAAuC;AAC5D,eAA8B;AAC9B,SAAS,8BAA8B;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAKvB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,IAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,YACE,UA6II,CAAC,GACL;AA1RJ;AA2RI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,SAAS,WAAU;AAAA,IAC/D,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WAAW,yBAAyB;AAEzD,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,oBAAoB,QAAQ,cAAc,CAAC,SAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,IAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,IAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,MAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,MAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,MAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EAEhC;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,YAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,UAAM,UAAU,KAAK;AACrB,SAAK,gBAAgB;AAErB,QAAI,SAAS;AACX,UAAI;AACF,cAAM,QAAQ,MAAM;AAAA,MACtB,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAE5B,YAAM,WAAW,KAAK;AACtB,WAAK,iBAAiB,IAAI,MAAM;AAEhC,eAAS,IAAI,EAAE,MAAM,sBAAsB,CAAQ;AAAA,IACrD;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,IAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,IAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,kBAAkB,uBAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AA9lBzC;AA+lBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AAEjC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,OAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YAAY;AAzsBpB;AA2sBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,iBAAiB,iBAAiB;AAC3F;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,MAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,EAAE,MAAM,GAAG,2BAA2B;AACzD,4BAAc,IAAI;AAClB,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAC9B,kBAAI,KAAK,kBAAkB,SAAS;AAClC;AAAA,cACF;AAEA,mBAAK,QAAQ;AAAA,gBACX,EAAE,MAAM,MAAM,MAAM,QAAQ,MAAM,OAAO;AAAA,gBACzC;AAAA,cACF;AACA,4BAAc,IAAI;AAClB,mBAAK,0BAA0B;AAC/B,kBAAI,CAAC,KAAK,mBAAmB,SAAS,CAAC,KAAK,SAAS;AACnD,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,KAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,MAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,cAAM,cAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,cAAM,MAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AACjF,oBAAM,QAAQ,iBAAiB;AAAA,gBAC7B;AAAA,cACF,CAAC;AAAA,YACH;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,YAAY,IAAI,IAAI;AACxD,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AA76BnB;AA+6BI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AAh7Bb,YAAAA;AAg7BgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,CAAC,cAAc;AACjB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,SACG,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,WAClD,SAAS,iBAAiB,SAAS,WACpC;AACA,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AAx/B7B;AAy/BI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAphC/F;AAqhCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,4BAAkC;AACxC,QAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,OAAO;AAC3D;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,MAAM,KAAK;AAIjB,QAAI,IAAI,oBAAoB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,IAAI,YAAY;AAClB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,IAAI;AAAA,QACb,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,UAAI,YAAY,MAAM,EAAE;AAAA,IAC1B;AAEA,QAAI,YAAY,MAAM;AACtB,QAAI,aAAa,MAAM;AACvB,QAAI,gBAAgB,MAAM;AAC1B,QAAI,eAAe,MAAM;AACzB,QAAI,QAAQ;AAAA,EACd;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,MACA,gBAAgB,KAAK,iBAAiB,KAAK,iBAAiB;AAAA,IAC9D;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,QAAQ,KAAK,uEAAuE;AACzF,WAAK,0BAA0B;AAAA,IACjC;AAEA,UAAM,aAAa,UAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,OAAO,oBAA2C;AAAA,MAClE,iBAAiB,OAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,SAAS,UAAU,KAAK;AAAA,MACxB,aAAa,OAAO,oBAA4B;AAAA,MAChD,cAAc,OAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AACnC,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QAAI,cAAc,uBAAuB,cAAc,oBAAoB,MAAM;AAC/E,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,aAAa;AAC7B,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,cAAc;AAC9B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,gBAAgB,MAAM;AAAA,QACxB,QAAQ,GAAG,MAAM,UAAU,WAAW;AAAA,QACtC,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,MAC5C,CAAqB;AAAA,IACvB;AAEA,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAC;AAAA,EAErB,MAAM,aAAa;AAAA,EAAC;AAAA,EAEpB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":["_a"]} \ No newline at end of file diff --git a/plugins/google/dist/index.cjs b/plugins/google/dist/index.cjs deleted file mode 100644 index 6f9040d48..000000000 --- a/plugins/google/dist/index.cjs +++ /dev/null @@ -1,58 +0,0 @@ -"use strict"; -var __create = Object.create; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __getProtoOf = Object.getPrototypeOf; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __export = (target, all) => { - for (var name in all) - __defProp(target, name, { get: all[name], enumerable: true }); -}; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default")); -var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( - // If the importer is in node compatibility mode or this is not an ESM - // file that has been converted to a CommonJS file using a Babel- - // compatible transform (i.e. "__esModule" has not been set), then set - // "default" to the CommonJS "module.exports" for node compatibility. - isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, - mod -)); -var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); -var index_exports = {}; -__export(index_exports, { - LLM: () => import_llm.LLM, - LLMStream: () => import_llm.LLMStream, - beta: () => beta -}); -module.exports = __toCommonJS(index_exports); -var import_agents = require("@livekit/agents"); -var beta = __toESM(require("./beta/index.cjs"), 1); -var import_llm = require("./llm.cjs"); -__reExport(index_exports, require("./models.cjs"), module.exports); -class GooglePlugin extends import_agents.Plugin { - constructor() { - super({ - title: "google", - version: "0.1.0", - package: "@livekit/agents-plugin-google" - }); - } -} -import_agents.Plugin.registerPlugin(new GooglePlugin()); -// Annotate the CommonJS export names for ESM import in node: -0 && (module.exports = { - LLM, - LLMStream, - beta, - ...require("./models.cjs") -}); -//# sourceMappingURL=index.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/index.cjs.map b/plugins/google/dist/index.cjs.map deleted file mode 100644 index 7d7d35f2a..000000000 --- a/plugins/google/dist/index.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Plugin } from '@livekit/agents';\n\nexport * as beta from './beta/index.js';\nexport { LLM, LLMStream, type LLMOptions } from './llm.js';\nexport * from './models.js';\n\nclass GooglePlugin extends Plugin {\n constructor() {\n super({\n title: 'google',\n version: '0.1.0',\n package: '@livekit/agents-plugin-google',\n });\n }\n}\n\nPlugin.registerPlugin(new GooglePlugin());\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAuB;AAEvB,WAAsB;AACtB,iBAAgD;AAChD,0BAAc,wBAPd;AASA,MAAM,qBAAqB,qBAAO;AAAA,EAChC,cAAc;AACZ,UAAM;AAAA,MACJ,OAAO;AAAA,MACP,SAAS;AAAA,MACT,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAEA,qBAAO,eAAe,IAAI,aAAa,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/index.d.cts b/plugins/google/dist/index.d.cts deleted file mode 100644 index fc4113b2e..000000000 --- a/plugins/google/dist/index.d.cts +++ /dev/null @@ -1,4 +0,0 @@ -export * as beta from './beta/index.js'; -export { LLM, LLMStream, type LLMOptions } from './llm.js'; -export * from './models.js'; -//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/index.d.ts b/plugins/google/dist/index.d.ts deleted file mode 100644 index fc4113b2e..000000000 --- a/plugins/google/dist/index.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * as beta from './beta/index.js'; -export { LLM, LLMStream, type LLMOptions } from './llm.js'; -export * from './models.js'; -//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/index.d.ts.map b/plugins/google/dist/index.d.ts.map deleted file mode 100644 index 44ab2e75e..000000000 --- a/plugins/google/dist/index.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,IAAI,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3D,cAAc,aAAa,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/index.js b/plugins/google/dist/index.js deleted file mode 100644 index a5ab61750..000000000 --- a/plugins/google/dist/index.js +++ /dev/null @@ -1,20 +0,0 @@ -import { Plugin } from "@livekit/agents"; -import * as beta from "./beta/index.js"; -import { LLM, LLMStream } from "./llm.js"; -export * from "./models.js"; -class GooglePlugin extends Plugin { - constructor() { - super({ - title: "google", - version: "0.1.0", - package: "@livekit/agents-plugin-google" - }); - } -} -Plugin.registerPlugin(new GooglePlugin()); -export { - LLM, - LLMStream, - beta -}; -//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/plugins/google/dist/index.js.map b/plugins/google/dist/index.js.map deleted file mode 100644 index 8f2b9d3ab..000000000 --- a/plugins/google/dist/index.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Plugin } from '@livekit/agents';\n\nexport * as beta from './beta/index.js';\nexport { LLM, LLMStream, type LLMOptions } from './llm.js';\nexport * from './models.js';\n\nclass GooglePlugin extends Plugin {\n constructor() {\n super({\n title: 'google',\n version: '0.1.0',\n package: '@livekit/agents-plugin-google',\n });\n }\n}\n\nPlugin.registerPlugin(new GooglePlugin());\n"],"mappings":"AAGA,SAAS,cAAc;AAEvB,YAAY,UAAU;AACtB,SAAS,KAAK,iBAAkC;AAChD,cAAc;AAEd,MAAM,qBAAqB,OAAO;AAAA,EAChC,cAAc;AACZ,UAAM;AAAA,MACJ,OAAO;AAAA,MACP,SAAS;AAAA,MACT,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAEA,OAAO,eAAe,IAAI,aAAa,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/llm.cjs b/plugins/google/dist/llm.cjs deleted file mode 100644 index 2856ec4c1..000000000 --- a/plugins/google/dist/llm.cjs +++ /dev/null @@ -1,381 +0,0 @@ -"use strict"; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __export = (target, all) => { - for (var name in all) - __defProp(target, name, { get: all[name], enumerable: true }); -}; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); -var llm_exports = {}; -__export(llm_exports, { - LLM: () => LLM, - LLMStream: () => LLMStream -}); -module.exports = __toCommonJS(llm_exports); -var import_genai = require("@google/genai"); -var import_agents = require("@livekit/agents"); -var import_utils = require("./utils.cjs"); -class LLM extends import_agents.llm.LLM { - #opts; - #client; - label() { - return "google.LLM"; - } - get model() { - return this.#opts.model; - } - /** - * Create a new instance of Google GenAI LLM. - * - * Environment Requirements: - * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods. - * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables - * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file, - * and the location defaults to "us-central1". - * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. - * - * @param model - The model name to use. Defaults to "gemini-2.0-flash-001". - * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable. - * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false. - * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined. - * @param location - The location to use for VertexAI API requests. Default value is "us-central1". - * @param temperature - Sampling temperature for response generation. Defaults to undefined. - * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined. - * @param topP - The nucleus sampling probability for response generation. Defaults to undefined. - * @param topK - The top-k sampling value for response generation. Defaults to undefined. - * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined. - * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined. - * @param toolChoice - Specifies whether to use tools during response generation. Defaults to "auto". - * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined. - * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined. - * @param geminiTools - The Gemini-specific tools to use for the session. - * @param httpOptions - The HTTP options to use for the session. - * @param seed - Random seed for reproducible results. Defaults to undefined. - */ - constructor({ - model, - apiKey, - vertexai, - project, - location, - temperature, - maxOutputTokens, - topP, - topK, - presencePenalty, - frequencyPenalty, - toolChoice, - thinkingConfig, - automaticFunctionCallingConfig, - geminiTools, - httpOptions, - seed - } = { - model: "gemini-2.0-flash-001" - }) { - super(); - const useVertexAI = vertexai ?? (process.env.GOOGLE_GENAI_USE_VERTEXAI === "true" || process.env.GOOGLE_GENAI_USE_VERTEXAI === "1"); - let gcpProject = project ?? process.env.GOOGLE_CLOUD_PROJECT; - let gcpLocation = location ?? process.env.GOOGLE_CLOUD_LOCATION; - let geminiApiKey = apiKey ?? process.env.GOOGLE_API_KEY; - if (useVertexAI) { - if (!gcpProject) { - throw new Error( - "Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable" - ); - } - geminiApiKey = void 0; - } else { - gcpProject = void 0; - gcpLocation = void 0; - if (!geminiApiKey) { - throw new Error( - "API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable" - ); - } - } - if ((thinkingConfig == null ? void 0 : thinkingConfig.thinkingBudget) !== void 0) { - const budget = thinkingConfig.thinkingBudget; - if (budget < 0 || budget > 24576) { - throw new Error("thinkingBudget inside thinkingConfig must be between 0 and 24576"); - } - } - const clientOptions = useVertexAI ? { - vertexai: true, - project: gcpProject, - location: gcpLocation - } : { - apiKey: geminiApiKey - }; - this.#client = new import_genai.GoogleGenAI(clientOptions); - this.#opts = { - model, - vertexai: useVertexAI, - project: gcpProject, - location: gcpLocation, - temperature, - maxOutputTokens, - topP, - topK, - presencePenalty, - frequencyPenalty, - toolChoice, - thinkingConfig, - automaticFunctionCallingConfig, - geminiTools, - httpOptions, - seed, - apiKey - }; - } - chat({ - chatCtx, - toolCtx, - connOptions = import_agents.DEFAULT_API_CONNECT_OPTIONS, - toolChoice, - extraKwargs, - geminiTools - }) { - const extras = { ...extraKwargs }; - toolChoice = toolChoice !== void 0 ? toolChoice : this.#opts.toolChoice; - if (toolChoice) { - let geminiToolConfig; - if (typeof toolChoice === "object" && toolChoice.type === "function") { - geminiToolConfig = { - functionCallingConfig: { - mode: import_genai.FunctionCallingConfigMode.ANY, - allowedFunctionNames: [toolChoice.function.name] - } - }; - } else if (toolChoice === "required") { - const toolNames = Object.entries(toolCtx || {}).map(([name]) => name); - geminiToolConfig = { - functionCallingConfig: { - mode: import_genai.FunctionCallingConfigMode.ANY, - allowedFunctionNames: toolNames.length > 0 ? toolNames : void 0 - } - }; - } else if (toolChoice === "auto") { - geminiToolConfig = { - functionCallingConfig: { - mode: import_genai.FunctionCallingConfigMode.AUTO - } - }; - } else if (toolChoice === "none") { - geminiToolConfig = { - functionCallingConfig: { - mode: import_genai.FunctionCallingConfigMode.NONE - } - }; - } else { - throw new Error(`Invalid tool choice: ${toolChoice}`); - } - extras.toolConfig = geminiToolConfig; - } - if (this.#opts.temperature !== void 0) { - extras.temperature = this.#opts.temperature; - } - if (this.#opts.maxOutputTokens !== void 0) { - extras.maxOutputTokens = this.#opts.maxOutputTokens; - } - if (this.#opts.topP !== void 0) { - extras.topP = this.#opts.topP; - } - if (this.#opts.topK !== void 0) { - extras.topK = this.#opts.topK; - } - if (this.#opts.presencePenalty !== void 0) { - extras.presencePenalty = this.#opts.presencePenalty; - } - if (this.#opts.frequencyPenalty !== void 0) { - extras.frequencyPenalty = this.#opts.frequencyPenalty; - } - if (this.#opts.seed !== void 0) { - extras.seed = this.#opts.seed; - } - if (this.#opts.thinkingConfig !== void 0) { - extras.thinkingConfig = this.#opts.thinkingConfig; - } - if (this.#opts.automaticFunctionCallingConfig !== void 0) { - extras.automaticFunctionCalling = this.#opts.automaticFunctionCallingConfig; - } - geminiTools = geminiTools !== void 0 ? geminiTools : this.#opts.geminiTools; - return new LLMStream(this, { - client: this.#client, - model: this.#opts.model, - chatCtx, - toolCtx, - connOptions, - geminiTools, - extraKwargs: extras - }); - } -} -class LLMStream extends import_agents.llm.LLMStream { - #client; - #model; - #geminiTools; - #extraKwargs; - constructor(llm2, { - client, - model, - chatCtx, - toolCtx, - connOptions, - geminiTools, - extraKwargs - }) { - super(llm2, { chatCtx, toolCtx, connOptions }); - this.#client = client; - this.#model = model; - this.#geminiTools = geminiTools; - this.#extraKwargs = extraKwargs; - } - async run() { - var _a, _b; - let retryable = true; - const requestId = `google_${Date.now()}`; - try { - const [turns, extraData] = await this.chatCtx.toProviderFormat("google"); - const contents = turns.map((turn) => ({ - role: turn.role, - parts: turn.parts - })); - const functionDeclarations = this.toolCtx ? (0, import_utils.toFunctionDeclarations)(this.toolCtx) : void 0; - const tools = functionDeclarations && functionDeclarations.length > 0 ? [{ functionDeclarations }] : void 0; - let systemInstruction = void 0; - if (extraData.systemMessages && extraData.systemMessages.length > 0) { - systemInstruction = { - parts: extraData.systemMessages.map((content) => ({ text: content })) - }; - } - const response = await this.#client.models.generateContentStream({ - model: this.#model, - contents, - config: { - ...this.#extraKwargs, - systemInstruction, - httpOptions: this.#extraKwargs.httpOptions ?? { - timeout: Math.floor(this.connOptions.timeoutMs) - }, - tools - } - }); - for await (const chunk of response) { - if (chunk.promptFeedback) { - throw new import_agents.APIStatusError({ - message: `Prompt feedback error: ${JSON.stringify(chunk.promptFeedback)}`, - options: { - retryable: false, - requestId - } - }); - } - if (!chunk.candidates || !((_b = (_a = chunk.candidates[0]) == null ? void 0 : _a.content) == null ? void 0 : _b.parts)) { - this.logger.warn(`No candidates in the response: ${JSON.stringify(chunk)}`); - continue; - } - if (chunk.candidates.length > 1) { - this.logger.warn( - "Google LLM: there are multiple candidates in the response, returning response from the first one." - ); - } - for (const part of chunk.candidates[0].content.parts) { - const chatChunk = this.#parsePart(requestId, part); - if (chatChunk) { - retryable = false; - this.queue.put(chatChunk); - } - } - if (chunk.usageMetadata) { - const usage = chunk.usageMetadata; - this.queue.put({ - id: requestId, - usage: { - completionTokens: usage.candidatesTokenCount || 0, - promptTokens: usage.promptTokenCount || 0, - promptCachedTokens: usage.cachedContentTokenCount || 0, - totalTokens: usage.totalTokenCount || 0 - } - }); - } - } - } catch (error) { - const err = error; - if (err.code && err.code >= 400 && err.code < 500) { - if (err.code === 429) { - throw new import_agents.APIStatusError({ - message: `Google LLM: Rate limit error - ${err.message || "Unknown error"}`, - options: { - statusCode: 429, - retryable: true - } - }); - } else { - throw new import_agents.APIStatusError({ - message: `Google LLM: Client error (${err.code}) - ${err.message || "Unknown error"}`, - options: { - statusCode: err.code, - retryable: false - } - }); - } - } - if (err.code && err.code >= 500) { - throw new import_agents.APIStatusError({ - message: `Google LLM: Server error (${err.code}) - ${err.message || "Unknown error"}`, - options: { - statusCode: err.code, - retryable - } - }); - } - throw new import_agents.APIConnectionError({ - message: `Google LLM: API error - ${err.message || "Unknown error"}`, - options: { - retryable - } - }); - } - } - #parsePart(id, part) { - if (part.functionCall) { - return { - id, - delta: { - role: "assistant", - toolCalls: [ - import_agents.llm.FunctionCall.create({ - callId: part.functionCall.id || (0, import_agents.shortuuid)("function_call_"), - name: part.functionCall.name, - args: JSON.stringify(part.functionCall.args) - }) - ] - } - }; - } - return { - id, - delta: { - content: part.text, - role: "assistant" - } - }; - } -} -// Annotate the CommonJS export names for ESM import in node: -0 && (module.exports = { - LLM, - LLMStream -}); -//# sourceMappingURL=llm.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/llm.cjs.map b/plugins/google/dist/llm.cjs.map deleted file mode 100644 index 501f0cdc2..000000000 --- a/plugins/google/dist/llm.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as types from '@google/genai';\nimport { FunctionCallingConfigMode, type GenerateContentConfig, GoogleGenAI } from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n APIStatusError,\n DEFAULT_API_CONNECT_OPTIONS,\n llm,\n shortuuid,\n} from '@livekit/agents';\nimport type { ChatModels } from './models.js';\nimport type { LLMTools } from './tools.js';\nimport { toFunctionDeclarations } from './utils.js';\n\ninterface GoogleFormatData {\n systemMessages: string[] | null;\n}\n\nexport interface LLMOptions {\n model: string | ChatModels;\n apiKey?: string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n vertexai?: boolean;\n project?: string;\n location?: string;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n thinkingConfig?: types.ThinkingConfig;\n automaticFunctionCallingConfig?: types.AutomaticFunctionCallingConfig;\n geminiTools?: LLMTools;\n httpOptions?: types.HttpOptions;\n seed?: number;\n}\n\nexport class LLM extends llm.LLM {\n #opts: LLMOptions;\n #client: GoogleGenAI;\n\n label(): string {\n return 'google.LLM';\n }\n\n get model(): string {\n return this.#opts.model;\n }\n\n /**\n * Create a new instance of Google GenAI LLM.\n *\n * Environment Requirements:\n * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods.\n * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables\n * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file,\n * and the location defaults to \"us-central1\".\n * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable.\n *\n * @param model - The model name to use. Defaults to \"gemini-2.0-flash-001\".\n * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable.\n * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false.\n * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined.\n * @param location - The location to use for VertexAI API requests. Default value is \"us-central1\".\n * @param temperature - Sampling temperature for response generation. Defaults to undefined.\n * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined.\n * @param topP - The nucleus sampling probability for response generation. Defaults to undefined.\n * @param topK - The top-k sampling value for response generation. Defaults to undefined.\n * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined.\n * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined.\n * @param toolChoice - Specifies whether to use tools during response generation. Defaults to \"auto\".\n * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined.\n * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined.\n * @param geminiTools - The Gemini-specific tools to use for the session.\n * @param httpOptions - The HTTP options to use for the session.\n * @param seed - Random seed for reproducible results. Defaults to undefined.\n */\n constructor(\n {\n model,\n apiKey,\n vertexai,\n project,\n location,\n temperature,\n maxOutputTokens,\n topP,\n topK,\n presencePenalty,\n frequencyPenalty,\n toolChoice,\n thinkingConfig,\n automaticFunctionCallingConfig,\n geminiTools,\n httpOptions,\n seed,\n }: LLMOptions = {\n model: 'gemini-2.0-flash-001',\n },\n ) {\n super();\n\n const useVertexAI =\n vertexai ??\n (process.env.GOOGLE_GENAI_USE_VERTEXAI === 'true' ||\n process.env.GOOGLE_GENAI_USE_VERTEXAI === '1');\n\n let gcpProject: string | undefined = project ?? process.env.GOOGLE_CLOUD_PROJECT;\n let gcpLocation: string | undefined = location ?? process.env.GOOGLE_CLOUD_LOCATION;\n let geminiApiKey: string | undefined = apiKey ?? process.env.GOOGLE_API_KEY;\n\n if (useVertexAI) {\n if (!gcpProject) {\n // TODO(brian): use default_async to get the project ID\n throw new Error(\n 'Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable',\n );\n }\n geminiApiKey = undefined;\n } else {\n gcpProject = undefined;\n gcpLocation = undefined;\n if (!geminiApiKey) {\n throw new Error(\n 'API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable',\n );\n }\n }\n\n // Validate thinkingConfig\n if (thinkingConfig?.thinkingBudget !== undefined) {\n const budget = thinkingConfig.thinkingBudget;\n if (budget < 0 || budget > 24576) {\n throw new Error('thinkingBudget inside thinkingConfig must be between 0 and 24576');\n }\n }\n\n const clientOptions: types.GoogleGenAIOptions = useVertexAI\n ? {\n vertexai: true,\n project: gcpProject,\n location: gcpLocation,\n }\n : {\n apiKey: geminiApiKey,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n\n this.#opts = {\n model,\n vertexai: useVertexAI,\n project: gcpProject,\n location: gcpLocation,\n temperature,\n maxOutputTokens,\n topP,\n topK,\n presencePenalty,\n frequencyPenalty,\n toolChoice,\n thinkingConfig,\n automaticFunctionCallingConfig,\n geminiTools,\n httpOptions,\n seed,\n apiKey,\n };\n }\n\n chat({\n chatCtx,\n toolCtx,\n connOptions = DEFAULT_API_CONNECT_OPTIONS,\n toolChoice,\n extraKwargs,\n geminiTools,\n }: {\n chatCtx: llm.ChatContext;\n toolCtx?: llm.ToolContext;\n connOptions?: APIConnectOptions;\n parallelToolCalls?: boolean;\n toolChoice?: llm.ToolChoice;\n extraKwargs?: Record;\n geminiTools?: LLMTools;\n }): LLMStream {\n const extras: GenerateContentConfig = { ...extraKwargs } as GenerateContentConfig;\n\n toolChoice = toolChoice !== undefined ? toolChoice : this.#opts.toolChoice;\n\n if (toolChoice) {\n let geminiToolConfig: types.ToolConfig;\n\n if (typeof toolChoice === 'object' && toolChoice.type === 'function') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.ANY,\n allowedFunctionNames: [toolChoice.function.name],\n },\n };\n } else if (toolChoice === 'required') {\n const toolNames = Object.entries(toolCtx || {}).map(([name]) => name);\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.ANY,\n allowedFunctionNames: toolNames.length > 0 ? toolNames : undefined,\n },\n };\n } else if (toolChoice === 'auto') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.AUTO,\n },\n };\n } else if (toolChoice === 'none') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.NONE,\n },\n };\n } else {\n throw new Error(`Invalid tool choice: ${toolChoice}`);\n }\n\n extras.toolConfig = geminiToolConfig;\n }\n\n if (this.#opts.temperature !== undefined) {\n extras.temperature = this.#opts.temperature;\n }\n if (this.#opts.maxOutputTokens !== undefined) {\n extras.maxOutputTokens = this.#opts.maxOutputTokens;\n }\n if (this.#opts.topP !== undefined) {\n extras.topP = this.#opts.topP;\n }\n if (this.#opts.topK !== undefined) {\n extras.topK = this.#opts.topK;\n }\n if (this.#opts.presencePenalty !== undefined) {\n extras.presencePenalty = this.#opts.presencePenalty;\n }\n if (this.#opts.frequencyPenalty !== undefined) {\n extras.frequencyPenalty = this.#opts.frequencyPenalty;\n }\n if (this.#opts.seed !== undefined) {\n extras.seed = this.#opts.seed;\n }\n\n if (this.#opts.thinkingConfig !== undefined) {\n extras.thinkingConfig = this.#opts.thinkingConfig;\n }\n\n if (this.#opts.automaticFunctionCallingConfig !== undefined) {\n extras.automaticFunctionCalling = this.#opts.automaticFunctionCallingConfig;\n }\n\n geminiTools = geminiTools !== undefined ? geminiTools : this.#opts.geminiTools;\n\n return new LLMStream(this, {\n client: this.#client,\n model: this.#opts.model,\n chatCtx,\n toolCtx,\n connOptions,\n geminiTools,\n extraKwargs: extras,\n });\n }\n}\n\nexport class LLMStream extends llm.LLMStream {\n #client: GoogleGenAI;\n #model: string;\n #geminiTools?: LLMTools;\n #extraKwargs: GenerateContentConfig;\n\n constructor(\n llm: LLM,\n {\n client,\n model,\n chatCtx,\n toolCtx,\n connOptions,\n geminiTools,\n extraKwargs,\n }: {\n client: GoogleGenAI;\n model: string;\n chatCtx: llm.ChatContext;\n toolCtx?: llm.ToolContext;\n connOptions: APIConnectOptions;\n geminiTools?: LLMTools;\n extraKwargs: GenerateContentConfig;\n },\n ) {\n // Call base constructor with dev 1.0 object parameter pattern\n super(llm, { chatCtx, toolCtx, connOptions });\n this.#client = client;\n this.#model = model;\n this.#geminiTools = geminiTools;\n this.#extraKwargs = extraKwargs;\n }\n\n protected async run(): Promise {\n let retryable = true;\n const requestId = `google_${Date.now()}`;\n\n try {\n const [turns, extraData] = (await this.chatCtx.toProviderFormat('google')) as [\n Record[],\n GoogleFormatData,\n ];\n\n const contents: types.Content[] = turns.map((turn: Record) => ({\n role: turn.role as types.Content['role'],\n parts: turn.parts as types.Part[],\n }));\n\n const functionDeclarations = this.toolCtx ? toFunctionDeclarations(this.toolCtx) : undefined;\n const tools =\n functionDeclarations && functionDeclarations.length > 0\n ? [{ functionDeclarations }]\n : undefined;\n\n let systemInstruction: types.Content | undefined = undefined;\n if (extraData.systemMessages && extraData.systemMessages.length > 0) {\n systemInstruction = {\n parts: extraData.systemMessages.map((content: string) => ({ text: content })),\n };\n }\n\n const response = await this.#client.models.generateContentStream({\n model: this.#model,\n contents,\n config: {\n ...this.#extraKwargs,\n systemInstruction,\n httpOptions: this.#extraKwargs.httpOptions ?? {\n timeout: Math.floor(this.connOptions.timeoutMs),\n },\n tools,\n },\n });\n\n for await (const chunk of response) {\n if (chunk.promptFeedback) {\n throw new APIStatusError({\n message: `Prompt feedback error: ${JSON.stringify(chunk.promptFeedback)}`,\n options: {\n retryable: false,\n requestId,\n },\n });\n }\n\n if (!chunk.candidates || !chunk.candidates[0]?.content?.parts) {\n this.logger.warn(`No candidates in the response: ${JSON.stringify(chunk)}`);\n continue;\n }\n\n if (chunk.candidates.length > 1) {\n this.logger.warn(\n 'Google LLM: there are multiple candidates in the response, returning response from the first one.',\n );\n }\n\n for (const part of chunk.candidates[0].content.parts) {\n const chatChunk = this.#parsePart(requestId, part);\n if (chatChunk) {\n retryable = false;\n this.queue.put(chatChunk);\n }\n }\n\n if (chunk.usageMetadata) {\n const usage = chunk.usageMetadata;\n this.queue.put({\n id: requestId,\n usage: {\n completionTokens: usage.candidatesTokenCount || 0,\n promptTokens: usage.promptTokenCount || 0,\n promptCachedTokens: usage.cachedContentTokenCount || 0,\n totalTokens: usage.totalTokenCount || 0,\n },\n });\n }\n }\n } catch (error: unknown) {\n const err = error as {\n code?: number;\n message?: string;\n status?: string;\n type?: string;\n };\n\n if (err.code && err.code >= 400 && err.code < 500) {\n if (err.code === 429) {\n throw new APIStatusError({\n message: `Google LLM: Rate limit error - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: 429,\n retryable: true,\n },\n });\n } else {\n throw new APIStatusError({\n message: `Google LLM: Client error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: false,\n },\n });\n }\n }\n\n if (err.code && err.code >= 500) {\n throw new APIStatusError({\n message: `Google LLM: Server error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable,\n },\n });\n }\n\n throw new APIConnectionError({\n message: `Google LLM: API error - ${err.message || 'Unknown error'}`,\n options: {\n retryable,\n },\n });\n }\n }\n\n #parsePart(id: string, part: types.Part): llm.ChatChunk | null {\n if (part.functionCall) {\n return {\n id,\n delta: {\n role: 'assistant',\n toolCalls: [\n llm.FunctionCall.create({\n callId: part.functionCall.id || shortuuid('function_call_'),\n name: part.functionCall.name!,\n args: JSON.stringify(part.functionCall.args!),\n }),\n ],\n },\n };\n }\n\n return {\n id,\n delta: {\n content: part.text,\n role: 'assistant',\n },\n };\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAmF;AAEnF,oBAMO;AAGP,mBAAuC;AA0BhC,MAAM,YAAY,kBAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EAEA,QAAgB;AACd,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA8BA,YACE;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,IAAgB;AAAA,IACd,OAAO;AAAA,EACT,GACA;AACA,UAAM;AAEN,UAAM,cACJ,aACC,QAAQ,IAAI,8BAA8B,UACzC,QAAQ,IAAI,8BAA8B;AAE9C,QAAI,aAAiC,WAAW,QAAQ,IAAI;AAC5D,QAAI,cAAkC,YAAY,QAAQ,IAAI;AAC9D,QAAI,eAAmC,UAAU,QAAQ,IAAI;AAE7D,QAAI,aAAa;AACf,UAAI,CAAC,YAAY;AAEf,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,qBAAe;AAAA,IACjB,OAAO;AACL,mBAAa;AACb,oBAAc;AACd,UAAI,CAAC,cAAc;AACjB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,SAAI,iDAAgB,oBAAmB,QAAW;AAChD,YAAM,SAAS,eAAe;AAC9B,UAAI,SAAS,KAAK,SAAS,OAAO;AAChC,cAAM,IAAI,MAAM,kEAAkE;AAAA,MACpF;AAAA,IACF;AAEA,UAAM,gBAA0C,cAC5C;AAAA,MACE,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,IACZ,IACA;AAAA,MACE,QAAQ;AAAA,IACV;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAE5C,SAAK,QAAQ;AAAA,MACX;AAAA,MACA,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAQc;AACZ,UAAM,SAAgC,EAAE,GAAG,YAAY;AAEvD,iBAAa,eAAe,SAAY,aAAa,KAAK,MAAM;AAEhE,QAAI,YAAY;AACd,UAAI;AAEJ,UAAI,OAAO,eAAe,YAAY,WAAW,SAAS,YAAY;AACpE,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,uCAA0B;AAAA,YAChC,sBAAsB,CAAC,WAAW,SAAS,IAAI;AAAA,UACjD;AAAA,QACF;AAAA,MACF,WAAW,eAAe,YAAY;AACpC,cAAM,YAAY,OAAO,QAAQ,WAAW,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,MAAM,IAAI;AACpE,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,uCAA0B;AAAA,YAChC,sBAAsB,UAAU,SAAS,IAAI,YAAY;AAAA,UAC3D;AAAA,QACF;AAAA,MACF,WAAW,eAAe,QAAQ;AAChC,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,uCAA0B;AAAA,UAClC;AAAA,QACF;AAAA,MACF,WAAW,eAAe,QAAQ;AAChC,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,uCAA0B;AAAA,UAClC;AAAA,QACF;AAAA,MACF,OAAO;AACL,cAAM,IAAI,MAAM,wBAAwB,UAAU,EAAE;AAAA,MACtD;AAEA,aAAO,aAAa;AAAA,IACtB;AAEA,QAAI,KAAK,MAAM,gBAAgB,QAAW;AACxC,aAAO,cAAc,KAAK,MAAM;AAAA,IAClC;AACA,QAAI,KAAK,MAAM,oBAAoB,QAAW;AAC5C,aAAO,kBAAkB,KAAK,MAAM;AAAA,IACtC;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AACA,QAAI,KAAK,MAAM,oBAAoB,QAAW;AAC5C,aAAO,kBAAkB,KAAK,MAAM;AAAA,IACtC;AACA,QAAI,KAAK,MAAM,qBAAqB,QAAW;AAC7C,aAAO,mBAAmB,KAAK,MAAM;AAAA,IACvC;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AAEA,QAAI,KAAK,MAAM,mBAAmB,QAAW;AAC3C,aAAO,iBAAiB,KAAK,MAAM;AAAA,IACrC;AAEA,QAAI,KAAK,MAAM,mCAAmC,QAAW;AAC3D,aAAO,2BAA2B,KAAK,MAAM;AAAA,IAC/C;AAEA,kBAAc,gBAAgB,SAAY,cAAc,KAAK,MAAM;AAEnE,WAAO,IAAI,UAAU,MAAM;AAAA,MACzB,QAAQ,KAAK;AAAA,MACb,OAAO,KAAK,MAAM;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa;AAAA,IACf,CAAC;AAAA,EACH;AACF;AAEO,MAAM,kBAAkB,kBAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACEA,MACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GASA;AAEA,UAAMA,MAAK,EAAE,SAAS,SAAS,YAAY,CAAC;AAC5C,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,eAAe;AACpB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,MAAqB;AArTvC;AAsTI,QAAI,YAAY;AAChB,UAAM,YAAY,UAAU,KAAK,IAAI,CAAC;AAEtC,QAAI;AACF,YAAM,CAAC,OAAO,SAAS,IAAK,MAAM,KAAK,QAAQ,iBAAiB,QAAQ;AAKxE,YAAM,WAA4B,MAAM,IAAI,CAAC,UAAmC;AAAA,QAC9E,MAAM,KAAK;AAAA,QACX,OAAO,KAAK;AAAA,MACd,EAAE;AAEF,YAAM,uBAAuB,KAAK,cAAU,qCAAuB,KAAK,OAAO,IAAI;AACnF,YAAM,QACJ,wBAAwB,qBAAqB,SAAS,IAClD,CAAC,EAAE,qBAAqB,CAAC,IACzB;AAEN,UAAI,oBAA+C;AACnD,UAAI,UAAU,kBAAkB,UAAU,eAAe,SAAS,GAAG;AACnE,4BAAoB;AAAA,UAClB,OAAO,UAAU,eAAe,IAAI,CAAC,aAAqB,EAAE,MAAM,QAAQ,EAAE;AAAA,QAC9E;AAAA,MACF;AAEA,YAAM,WAAW,MAAM,KAAK,QAAQ,OAAO,sBAAsB;AAAA,QAC/D,OAAO,KAAK;AAAA,QACZ;AAAA,QACA,QAAQ;AAAA,UACN,GAAG,KAAK;AAAA,UACR;AAAA,UACA,aAAa,KAAK,aAAa,eAAe;AAAA,YAC5C,SAAS,KAAK,MAAM,KAAK,YAAY,SAAS;AAAA,UAChD;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAED,uBAAiB,SAAS,UAAU;AAClC,YAAI,MAAM,gBAAgB;AACxB,gBAAM,IAAI,6BAAe;AAAA,YACvB,SAAS,0BAA0B,KAAK,UAAU,MAAM,cAAc,CAAC;AAAA,YACvE,SAAS;AAAA,cACP,WAAW;AAAA,cACX;AAAA,YACF;AAAA,UACF,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,MAAM,cAAc,GAAC,iBAAM,WAAW,CAAC,MAAlB,mBAAqB,YAArB,mBAA8B,QAAO;AAC7D,eAAK,OAAO,KAAK,kCAAkC,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1E;AAAA,QACF;AAEA,YAAI,MAAM,WAAW,SAAS,GAAG;AAC/B,eAAK,OAAO;AAAA,YACV;AAAA,UACF;AAAA,QACF;AAEA,mBAAW,QAAQ,MAAM,WAAW,CAAC,EAAE,QAAQ,OAAO;AACpD,gBAAM,YAAY,KAAK,WAAW,WAAW,IAAI;AACjD,cAAI,WAAW;AACb,wBAAY;AACZ,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAAA,QACF;AAEA,YAAI,MAAM,eAAe;AACvB,gBAAM,QAAQ,MAAM;AACpB,eAAK,MAAM,IAAI;AAAA,YACb,IAAI;AAAA,YACJ,OAAO;AAAA,cACL,kBAAkB,MAAM,wBAAwB;AAAA,cAChD,cAAc,MAAM,oBAAoB;AAAA,cACxC,oBAAoB,MAAM,2BAA2B;AAAA,cACrD,aAAa,MAAM,mBAAmB;AAAA,YACxC;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,SAAS,OAAgB;AACvB,YAAM,MAAM;AAOZ,UAAI,IAAI,QAAQ,IAAI,QAAQ,OAAO,IAAI,OAAO,KAAK;AACjD,YAAI,IAAI,SAAS,KAAK;AACpB,gBAAM,IAAI,6BAAe;AAAA,YACvB,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,YACzE,SAAS;AAAA,cACP,YAAY;AAAA,cACZ,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH,OAAO;AACL,gBAAM,IAAI,6BAAe;AAAA,YACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,YACnF,SAAS;AAAA,cACP,YAAY,IAAI;AAAA,cAChB,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,IAAI,QAAQ,IAAI,QAAQ,KAAK;AAC/B,cAAM,IAAI,6BAAe;AAAA,UACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,UACnF,SAAS;AAAA,YACP,YAAY,IAAI;AAAA,YAChB;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH;AAEA,YAAM,IAAI,iCAAmB;AAAA,QAC3B,SAAS,2BAA2B,IAAI,WAAW,eAAe;AAAA,QAClE,SAAS;AAAA,UACP;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,WAAW,IAAY,MAAwC;AAC7D,QAAI,KAAK,cAAc;AACrB,aAAO;AAAA,QACL;AAAA,QACA,OAAO;AAAA,UACL,MAAM;AAAA,UACN,WAAW;AAAA,YACT,kBAAI,aAAa,OAAO;AAAA,cACtB,QAAQ,KAAK,aAAa,UAAM,yBAAU,gBAAgB;AAAA,cAC1D,MAAM,KAAK,aAAa;AAAA,cACxB,MAAM,KAAK,UAAU,KAAK,aAAa,IAAK;AAAA,YAC9C,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL;AAAA,MACA,OAAO;AAAA,QACL,SAAS,KAAK;AAAA,QACd,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;","names":["llm"]} \ No newline at end of file diff --git a/plugins/google/dist/llm.d.cts b/plugins/google/dist/llm.d.cts deleted file mode 100644 index 8961efcae..000000000 --- a/plugins/google/dist/llm.d.cts +++ /dev/null @@ -1,82 +0,0 @@ -import type * as types from '@google/genai'; -import { type GenerateContentConfig, GoogleGenAI } from '@google/genai'; -import type { APIConnectOptions } from '@livekit/agents'; -import { llm } from '@livekit/agents'; -import type { ChatModels } from './models.js'; -import type { LLMTools } from './tools.js'; -export interface LLMOptions { - model: string | ChatModels; - apiKey?: string; - temperature?: number; - toolChoice?: llm.ToolChoice; - vertexai?: boolean; - project?: string; - location?: string; - maxOutputTokens?: number; - topP?: number; - topK?: number; - presencePenalty?: number; - frequencyPenalty?: number; - thinkingConfig?: types.ThinkingConfig; - automaticFunctionCallingConfig?: types.AutomaticFunctionCallingConfig; - geminiTools?: LLMTools; - httpOptions?: types.HttpOptions; - seed?: number; -} -export declare class LLM extends llm.LLM { - #private; - label(): string; - get model(): string; - /** - * Create a new instance of Google GenAI LLM. - * - * Environment Requirements: - * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods. - * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables - * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file, - * and the location defaults to "us-central1". - * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. - * - * @param model - The model name to use. Defaults to "gemini-2.0-flash-001". - * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable. - * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false. - * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined. - * @param location - The location to use for VertexAI API requests. Default value is "us-central1". - * @param temperature - Sampling temperature for response generation. Defaults to undefined. - * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined. - * @param topP - The nucleus sampling probability for response generation. Defaults to undefined. - * @param topK - The top-k sampling value for response generation. Defaults to undefined. - * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined. - * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined. - * @param toolChoice - Specifies whether to use tools during response generation. Defaults to "auto". - * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined. - * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined. - * @param geminiTools - The Gemini-specific tools to use for the session. - * @param httpOptions - The HTTP options to use for the session. - * @param seed - Random seed for reproducible results. Defaults to undefined. - */ - constructor({ model, apiKey, vertexai, project, location, temperature, maxOutputTokens, topP, topK, presencePenalty, frequencyPenalty, toolChoice, thinkingConfig, automaticFunctionCallingConfig, geminiTools, httpOptions, seed, }?: LLMOptions); - chat({ chatCtx, toolCtx, connOptions, toolChoice, extraKwargs, geminiTools, }: { - chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; - connOptions?: APIConnectOptions; - parallelToolCalls?: boolean; - toolChoice?: llm.ToolChoice; - extraKwargs?: Record; - geminiTools?: LLMTools; - }): LLMStream; -} -export declare class LLMStream extends llm.LLMStream { - #private; - constructor(llm: LLM, { client, model, chatCtx, toolCtx, connOptions, geminiTools, extraKwargs, }: { - client: GoogleGenAI; - model: string; - chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; - connOptions: APIConnectOptions; - geminiTools?: LLMTools; - extraKwargs: GenerateContentConfig; - }); - protected run(): Promise; -} -//# sourceMappingURL=llm.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/llm.d.ts b/plugins/google/dist/llm.d.ts deleted file mode 100644 index 8961efcae..000000000 --- a/plugins/google/dist/llm.d.ts +++ /dev/null @@ -1,82 +0,0 @@ -import type * as types from '@google/genai'; -import { type GenerateContentConfig, GoogleGenAI } from '@google/genai'; -import type { APIConnectOptions } from '@livekit/agents'; -import { llm } from '@livekit/agents'; -import type { ChatModels } from './models.js'; -import type { LLMTools } from './tools.js'; -export interface LLMOptions { - model: string | ChatModels; - apiKey?: string; - temperature?: number; - toolChoice?: llm.ToolChoice; - vertexai?: boolean; - project?: string; - location?: string; - maxOutputTokens?: number; - topP?: number; - topK?: number; - presencePenalty?: number; - frequencyPenalty?: number; - thinkingConfig?: types.ThinkingConfig; - automaticFunctionCallingConfig?: types.AutomaticFunctionCallingConfig; - geminiTools?: LLMTools; - httpOptions?: types.HttpOptions; - seed?: number; -} -export declare class LLM extends llm.LLM { - #private; - label(): string; - get model(): string; - /** - * Create a new instance of Google GenAI LLM. - * - * Environment Requirements: - * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods. - * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables - * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file, - * and the location defaults to "us-central1". - * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. - * - * @param model - The model name to use. Defaults to "gemini-2.0-flash-001". - * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable. - * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false. - * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined. - * @param location - The location to use for VertexAI API requests. Default value is "us-central1". - * @param temperature - Sampling temperature for response generation. Defaults to undefined. - * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined. - * @param topP - The nucleus sampling probability for response generation. Defaults to undefined. - * @param topK - The top-k sampling value for response generation. Defaults to undefined. - * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined. - * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined. - * @param toolChoice - Specifies whether to use tools during response generation. Defaults to "auto". - * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined. - * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined. - * @param geminiTools - The Gemini-specific tools to use for the session. - * @param httpOptions - The HTTP options to use for the session. - * @param seed - Random seed for reproducible results. Defaults to undefined. - */ - constructor({ model, apiKey, vertexai, project, location, temperature, maxOutputTokens, topP, topK, presencePenalty, frequencyPenalty, toolChoice, thinkingConfig, automaticFunctionCallingConfig, geminiTools, httpOptions, seed, }?: LLMOptions); - chat({ chatCtx, toolCtx, connOptions, toolChoice, extraKwargs, geminiTools, }: { - chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; - connOptions?: APIConnectOptions; - parallelToolCalls?: boolean; - toolChoice?: llm.ToolChoice; - extraKwargs?: Record; - geminiTools?: LLMTools; - }): LLMStream; -} -export declare class LLMStream extends llm.LLMStream { - #private; - constructor(llm: LLM, { client, model, chatCtx, toolCtx, connOptions, geminiTools, extraKwargs, }: { - client: GoogleGenAI; - model: string; - chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; - connOptions: APIConnectOptions; - geminiTools?: LLMTools; - extraKwargs: GenerateContentConfig; - }); - protected run(): Promise; -} -//# sourceMappingURL=llm.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/llm.d.ts.map b/plugins/google/dist/llm.d.ts.map deleted file mode 100644 index 7f9b262b9..000000000 --- a/plugins/google/dist/llm.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,KAAK,KAAK,MAAM,eAAe,CAAC;AAC5C,OAAO,EAA6B,KAAK,qBAAqB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACnG,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAIL,GAAG,EAEJ,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAO3C,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,GAAG,UAAU,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;IACtC,8BAA8B,CAAC,EAAE,KAAK,CAAC,8BAA8B,CAAC;IACtE,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,WAAW,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;;IAI9B,KAAK,IAAI,MAAM;IAIf,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;gBAED,EACE,KAAK,EACL,MAAM,EACN,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,WAAW,EACX,eAAe,EACf,IAAI,EACJ,IAAI,EACJ,eAAe,EACf,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,8BAA8B,EAC9B,WAAW,EACX,WAAW,EACX,IAAI,GACL,GAAE,UAEF;IAwEH,IAAI,CAAC,EACH,OAAO,EACP,OAAO,EACP,WAAyC,EACzC,UAAU,EACV,WAAW,EACX,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;QAC5B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,WAAW,CAAC,EAAE,QAAQ,CAAC;KACxB,GAAG,SAAS;CAoFd;AAED,qBAAa,SAAU,SAAQ,GAAG,CAAC,SAAS;;gBAOxC,GAAG,EAAE,GAAG,EACR,EACE,MAAM,EACN,KAAK,EACL,OAAO,EACP,OAAO,EACP,WAAW,EACX,WAAW,EACX,WAAW,GACZ,EAAE;QACD,MAAM,EAAE,WAAW,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,WAAW,EAAE,iBAAiB,CAAC;QAC/B,WAAW,CAAC,EAAE,QAAQ,CAAC;QACvB,WAAW,EAAE,qBAAqB,CAAC;KACpC;cAUa,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CA4JrC"} \ No newline at end of file diff --git a/plugins/google/dist/llm.js b/plugins/google/dist/llm.js deleted file mode 100644 index ed4702692..000000000 --- a/plugins/google/dist/llm.js +++ /dev/null @@ -1,362 +0,0 @@ -import { FunctionCallingConfigMode, GoogleGenAI } from "@google/genai"; -import { - APIConnectionError, - APIStatusError, - DEFAULT_API_CONNECT_OPTIONS, - llm, - shortuuid -} from "@livekit/agents"; -import { toFunctionDeclarations } from "./utils.js"; -class LLM extends llm.LLM { - #opts; - #client; - label() { - return "google.LLM"; - } - get model() { - return this.#opts.model; - } - /** - * Create a new instance of Google GenAI LLM. - * - * Environment Requirements: - * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods. - * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables - * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file, - * and the location defaults to "us-central1". - * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable. - * - * @param model - The model name to use. Defaults to "gemini-2.0-flash-001". - * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable. - * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false. - * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined. - * @param location - The location to use for VertexAI API requests. Default value is "us-central1". - * @param temperature - Sampling temperature for response generation. Defaults to undefined. - * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined. - * @param topP - The nucleus sampling probability for response generation. Defaults to undefined. - * @param topK - The top-k sampling value for response generation. Defaults to undefined. - * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined. - * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined. - * @param toolChoice - Specifies whether to use tools during response generation. Defaults to "auto". - * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined. - * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined. - * @param geminiTools - The Gemini-specific tools to use for the session. - * @param httpOptions - The HTTP options to use for the session. - * @param seed - Random seed for reproducible results. Defaults to undefined. - */ - constructor({ - model, - apiKey, - vertexai, - project, - location, - temperature, - maxOutputTokens, - topP, - topK, - presencePenalty, - frequencyPenalty, - toolChoice, - thinkingConfig, - automaticFunctionCallingConfig, - geminiTools, - httpOptions, - seed - } = { - model: "gemini-2.0-flash-001" - }) { - super(); - const useVertexAI = vertexai ?? (process.env.GOOGLE_GENAI_USE_VERTEXAI === "true" || process.env.GOOGLE_GENAI_USE_VERTEXAI === "1"); - let gcpProject = project ?? process.env.GOOGLE_CLOUD_PROJECT; - let gcpLocation = location ?? process.env.GOOGLE_CLOUD_LOCATION; - let geminiApiKey = apiKey ?? process.env.GOOGLE_API_KEY; - if (useVertexAI) { - if (!gcpProject) { - throw new Error( - "Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable" - ); - } - geminiApiKey = void 0; - } else { - gcpProject = void 0; - gcpLocation = void 0; - if (!geminiApiKey) { - throw new Error( - "API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable" - ); - } - } - if ((thinkingConfig == null ? void 0 : thinkingConfig.thinkingBudget) !== void 0) { - const budget = thinkingConfig.thinkingBudget; - if (budget < 0 || budget > 24576) { - throw new Error("thinkingBudget inside thinkingConfig must be between 0 and 24576"); - } - } - const clientOptions = useVertexAI ? { - vertexai: true, - project: gcpProject, - location: gcpLocation - } : { - apiKey: geminiApiKey - }; - this.#client = new GoogleGenAI(clientOptions); - this.#opts = { - model, - vertexai: useVertexAI, - project: gcpProject, - location: gcpLocation, - temperature, - maxOutputTokens, - topP, - topK, - presencePenalty, - frequencyPenalty, - toolChoice, - thinkingConfig, - automaticFunctionCallingConfig, - geminiTools, - httpOptions, - seed, - apiKey - }; - } - chat({ - chatCtx, - toolCtx, - connOptions = DEFAULT_API_CONNECT_OPTIONS, - toolChoice, - extraKwargs, - geminiTools - }) { - const extras = { ...extraKwargs }; - toolChoice = toolChoice !== void 0 ? toolChoice : this.#opts.toolChoice; - if (toolChoice) { - let geminiToolConfig; - if (typeof toolChoice === "object" && toolChoice.type === "function") { - geminiToolConfig = { - functionCallingConfig: { - mode: FunctionCallingConfigMode.ANY, - allowedFunctionNames: [toolChoice.function.name] - } - }; - } else if (toolChoice === "required") { - const toolNames = Object.entries(toolCtx || {}).map(([name]) => name); - geminiToolConfig = { - functionCallingConfig: { - mode: FunctionCallingConfigMode.ANY, - allowedFunctionNames: toolNames.length > 0 ? toolNames : void 0 - } - }; - } else if (toolChoice === "auto") { - geminiToolConfig = { - functionCallingConfig: { - mode: FunctionCallingConfigMode.AUTO - } - }; - } else if (toolChoice === "none") { - geminiToolConfig = { - functionCallingConfig: { - mode: FunctionCallingConfigMode.NONE - } - }; - } else { - throw new Error(`Invalid tool choice: ${toolChoice}`); - } - extras.toolConfig = geminiToolConfig; - } - if (this.#opts.temperature !== void 0) { - extras.temperature = this.#opts.temperature; - } - if (this.#opts.maxOutputTokens !== void 0) { - extras.maxOutputTokens = this.#opts.maxOutputTokens; - } - if (this.#opts.topP !== void 0) { - extras.topP = this.#opts.topP; - } - if (this.#opts.topK !== void 0) { - extras.topK = this.#opts.topK; - } - if (this.#opts.presencePenalty !== void 0) { - extras.presencePenalty = this.#opts.presencePenalty; - } - if (this.#opts.frequencyPenalty !== void 0) { - extras.frequencyPenalty = this.#opts.frequencyPenalty; - } - if (this.#opts.seed !== void 0) { - extras.seed = this.#opts.seed; - } - if (this.#opts.thinkingConfig !== void 0) { - extras.thinkingConfig = this.#opts.thinkingConfig; - } - if (this.#opts.automaticFunctionCallingConfig !== void 0) { - extras.automaticFunctionCalling = this.#opts.automaticFunctionCallingConfig; - } - geminiTools = geminiTools !== void 0 ? geminiTools : this.#opts.geminiTools; - return new LLMStream(this, { - client: this.#client, - model: this.#opts.model, - chatCtx, - toolCtx, - connOptions, - geminiTools, - extraKwargs: extras - }); - } -} -class LLMStream extends llm.LLMStream { - #client; - #model; - #geminiTools; - #extraKwargs; - constructor(llm2, { - client, - model, - chatCtx, - toolCtx, - connOptions, - geminiTools, - extraKwargs - }) { - super(llm2, { chatCtx, toolCtx, connOptions }); - this.#client = client; - this.#model = model; - this.#geminiTools = geminiTools; - this.#extraKwargs = extraKwargs; - } - async run() { - var _a, _b; - let retryable = true; - const requestId = `google_${Date.now()}`; - try { - const [turns, extraData] = await this.chatCtx.toProviderFormat("google"); - const contents = turns.map((turn) => ({ - role: turn.role, - parts: turn.parts - })); - const functionDeclarations = this.toolCtx ? toFunctionDeclarations(this.toolCtx) : void 0; - const tools = functionDeclarations && functionDeclarations.length > 0 ? [{ functionDeclarations }] : void 0; - let systemInstruction = void 0; - if (extraData.systemMessages && extraData.systemMessages.length > 0) { - systemInstruction = { - parts: extraData.systemMessages.map((content) => ({ text: content })) - }; - } - const response = await this.#client.models.generateContentStream({ - model: this.#model, - contents, - config: { - ...this.#extraKwargs, - systemInstruction, - httpOptions: this.#extraKwargs.httpOptions ?? { - timeout: Math.floor(this.connOptions.timeoutMs) - }, - tools - } - }); - for await (const chunk of response) { - if (chunk.promptFeedback) { - throw new APIStatusError({ - message: `Prompt feedback error: ${JSON.stringify(chunk.promptFeedback)}`, - options: { - retryable: false, - requestId - } - }); - } - if (!chunk.candidates || !((_b = (_a = chunk.candidates[0]) == null ? void 0 : _a.content) == null ? void 0 : _b.parts)) { - this.logger.warn(`No candidates in the response: ${JSON.stringify(chunk)}`); - continue; - } - if (chunk.candidates.length > 1) { - this.logger.warn( - "Google LLM: there are multiple candidates in the response, returning response from the first one." - ); - } - for (const part of chunk.candidates[0].content.parts) { - const chatChunk = this.#parsePart(requestId, part); - if (chatChunk) { - retryable = false; - this.queue.put(chatChunk); - } - } - if (chunk.usageMetadata) { - const usage = chunk.usageMetadata; - this.queue.put({ - id: requestId, - usage: { - completionTokens: usage.candidatesTokenCount || 0, - promptTokens: usage.promptTokenCount || 0, - promptCachedTokens: usage.cachedContentTokenCount || 0, - totalTokens: usage.totalTokenCount || 0 - } - }); - } - } - } catch (error) { - const err = error; - if (err.code && err.code >= 400 && err.code < 500) { - if (err.code === 429) { - throw new APIStatusError({ - message: `Google LLM: Rate limit error - ${err.message || "Unknown error"}`, - options: { - statusCode: 429, - retryable: true - } - }); - } else { - throw new APIStatusError({ - message: `Google LLM: Client error (${err.code}) - ${err.message || "Unknown error"}`, - options: { - statusCode: err.code, - retryable: false - } - }); - } - } - if (err.code && err.code >= 500) { - throw new APIStatusError({ - message: `Google LLM: Server error (${err.code}) - ${err.message || "Unknown error"}`, - options: { - statusCode: err.code, - retryable - } - }); - } - throw new APIConnectionError({ - message: `Google LLM: API error - ${err.message || "Unknown error"}`, - options: { - retryable - } - }); - } - } - #parsePart(id, part) { - if (part.functionCall) { - return { - id, - delta: { - role: "assistant", - toolCalls: [ - llm.FunctionCall.create({ - callId: part.functionCall.id || shortuuid("function_call_"), - name: part.functionCall.name, - args: JSON.stringify(part.functionCall.args) - }) - ] - } - }; - } - return { - id, - delta: { - content: part.text, - role: "assistant" - } - }; - } -} -export { - LLM, - LLMStream -}; -//# sourceMappingURL=llm.js.map \ No newline at end of file diff --git a/plugins/google/dist/llm.js.map b/plugins/google/dist/llm.js.map deleted file mode 100644 index eea21e4b3..000000000 --- a/plugins/google/dist/llm.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as types from '@google/genai';\nimport { FunctionCallingConfigMode, type GenerateContentConfig, GoogleGenAI } from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n APIStatusError,\n DEFAULT_API_CONNECT_OPTIONS,\n llm,\n shortuuid,\n} from '@livekit/agents';\nimport type { ChatModels } from './models.js';\nimport type { LLMTools } from './tools.js';\nimport { toFunctionDeclarations } from './utils.js';\n\ninterface GoogleFormatData {\n systemMessages: string[] | null;\n}\n\nexport interface LLMOptions {\n model: string | ChatModels;\n apiKey?: string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n vertexai?: boolean;\n project?: string;\n location?: string;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n thinkingConfig?: types.ThinkingConfig;\n automaticFunctionCallingConfig?: types.AutomaticFunctionCallingConfig;\n geminiTools?: LLMTools;\n httpOptions?: types.HttpOptions;\n seed?: number;\n}\n\nexport class LLM extends llm.LLM {\n #opts: LLMOptions;\n #client: GoogleGenAI;\n\n label(): string {\n return 'google.LLM';\n }\n\n get model(): string {\n return this.#opts.model;\n }\n\n /**\n * Create a new instance of Google GenAI LLM.\n *\n * Environment Requirements:\n * - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file or use any of the other Google Cloud auth methods.\n * The Google Cloud project and location can be set via `project` and `location` arguments or the environment variables\n * `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`. By default, the project is inferred from the service account key file,\n * and the location defaults to \"us-central1\".\n * - For Google Gemini API: Set the `apiKey` argument or the `GOOGLE_API_KEY` environment variable.\n *\n * @param model - The model name to use. Defaults to \"gemini-2.0-flash-001\".\n * @param apiKey - The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable.\n * @param vertexai - Whether to use VertexAI. If not provided, it attempts to read from the `GOOGLE_GENAI_USE_VERTEXAI` environment variable. Defaults to false.\n * @param project - The Google Cloud project to use (only for VertexAI). Defaults to undefined.\n * @param location - The location to use for VertexAI API requests. Default value is \"us-central1\".\n * @param temperature - Sampling temperature for response generation. Defaults to undefined.\n * @param maxOutputTokens - Maximum number of tokens to generate in the output. Defaults to undefined.\n * @param topP - The nucleus sampling probability for response generation. Defaults to undefined.\n * @param topK - The top-k sampling value for response generation. Defaults to undefined.\n * @param presencePenalty - Penalizes the model for generating previously mentioned concepts. Defaults to undefined.\n * @param frequencyPenalty - Penalizes the model for repeating words. Defaults to undefined.\n * @param toolChoice - Specifies whether to use tools during response generation. Defaults to \"auto\".\n * @param thinkingConfig - The thinking configuration for response generation. Defaults to undefined.\n * @param automaticFunctionCallingConfig - The automatic function calling configuration for response generation. Defaults to undefined.\n * @param geminiTools - The Gemini-specific tools to use for the session.\n * @param httpOptions - The HTTP options to use for the session.\n * @param seed - Random seed for reproducible results. Defaults to undefined.\n */\n constructor(\n {\n model,\n apiKey,\n vertexai,\n project,\n location,\n temperature,\n maxOutputTokens,\n topP,\n topK,\n presencePenalty,\n frequencyPenalty,\n toolChoice,\n thinkingConfig,\n automaticFunctionCallingConfig,\n geminiTools,\n httpOptions,\n seed,\n }: LLMOptions = {\n model: 'gemini-2.0-flash-001',\n },\n ) {\n super();\n\n const useVertexAI =\n vertexai ??\n (process.env.GOOGLE_GENAI_USE_VERTEXAI === 'true' ||\n process.env.GOOGLE_GENAI_USE_VERTEXAI === '1');\n\n let gcpProject: string | undefined = project ?? process.env.GOOGLE_CLOUD_PROJECT;\n let gcpLocation: string | undefined = location ?? process.env.GOOGLE_CLOUD_LOCATION;\n let geminiApiKey: string | undefined = apiKey ?? process.env.GOOGLE_API_KEY;\n\n if (useVertexAI) {\n if (!gcpProject) {\n // TODO(brian): use default_async to get the project ID\n throw new Error(\n 'Project ID is required for Vertex AI. Set via project option or GOOGLE_CLOUD_PROJECT environment variable',\n );\n }\n geminiApiKey = undefined;\n } else {\n gcpProject = undefined;\n gcpLocation = undefined;\n if (!geminiApiKey) {\n throw new Error(\n 'API key is required for Google API either via apiKey or GOOGLE_API_KEY environment variable',\n );\n }\n }\n\n // Validate thinkingConfig\n if (thinkingConfig?.thinkingBudget !== undefined) {\n const budget = thinkingConfig.thinkingBudget;\n if (budget < 0 || budget > 24576) {\n throw new Error('thinkingBudget inside thinkingConfig must be between 0 and 24576');\n }\n }\n\n const clientOptions: types.GoogleGenAIOptions = useVertexAI\n ? {\n vertexai: true,\n project: gcpProject,\n location: gcpLocation,\n }\n : {\n apiKey: geminiApiKey,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n\n this.#opts = {\n model,\n vertexai: useVertexAI,\n project: gcpProject,\n location: gcpLocation,\n temperature,\n maxOutputTokens,\n topP,\n topK,\n presencePenalty,\n frequencyPenalty,\n toolChoice,\n thinkingConfig,\n automaticFunctionCallingConfig,\n geminiTools,\n httpOptions,\n seed,\n apiKey,\n };\n }\n\n chat({\n chatCtx,\n toolCtx,\n connOptions = DEFAULT_API_CONNECT_OPTIONS,\n toolChoice,\n extraKwargs,\n geminiTools,\n }: {\n chatCtx: llm.ChatContext;\n toolCtx?: llm.ToolContext;\n connOptions?: APIConnectOptions;\n parallelToolCalls?: boolean;\n toolChoice?: llm.ToolChoice;\n extraKwargs?: Record;\n geminiTools?: LLMTools;\n }): LLMStream {\n const extras: GenerateContentConfig = { ...extraKwargs } as GenerateContentConfig;\n\n toolChoice = toolChoice !== undefined ? toolChoice : this.#opts.toolChoice;\n\n if (toolChoice) {\n let geminiToolConfig: types.ToolConfig;\n\n if (typeof toolChoice === 'object' && toolChoice.type === 'function') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.ANY,\n allowedFunctionNames: [toolChoice.function.name],\n },\n };\n } else if (toolChoice === 'required') {\n const toolNames = Object.entries(toolCtx || {}).map(([name]) => name);\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.ANY,\n allowedFunctionNames: toolNames.length > 0 ? toolNames : undefined,\n },\n };\n } else if (toolChoice === 'auto') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.AUTO,\n },\n };\n } else if (toolChoice === 'none') {\n geminiToolConfig = {\n functionCallingConfig: {\n mode: FunctionCallingConfigMode.NONE,\n },\n };\n } else {\n throw new Error(`Invalid tool choice: ${toolChoice}`);\n }\n\n extras.toolConfig = geminiToolConfig;\n }\n\n if (this.#opts.temperature !== undefined) {\n extras.temperature = this.#opts.temperature;\n }\n if (this.#opts.maxOutputTokens !== undefined) {\n extras.maxOutputTokens = this.#opts.maxOutputTokens;\n }\n if (this.#opts.topP !== undefined) {\n extras.topP = this.#opts.topP;\n }\n if (this.#opts.topK !== undefined) {\n extras.topK = this.#opts.topK;\n }\n if (this.#opts.presencePenalty !== undefined) {\n extras.presencePenalty = this.#opts.presencePenalty;\n }\n if (this.#opts.frequencyPenalty !== undefined) {\n extras.frequencyPenalty = this.#opts.frequencyPenalty;\n }\n if (this.#opts.seed !== undefined) {\n extras.seed = this.#opts.seed;\n }\n\n if (this.#opts.thinkingConfig !== undefined) {\n extras.thinkingConfig = this.#opts.thinkingConfig;\n }\n\n if (this.#opts.automaticFunctionCallingConfig !== undefined) {\n extras.automaticFunctionCalling = this.#opts.automaticFunctionCallingConfig;\n }\n\n geminiTools = geminiTools !== undefined ? geminiTools : this.#opts.geminiTools;\n\n return new LLMStream(this, {\n client: this.#client,\n model: this.#opts.model,\n chatCtx,\n toolCtx,\n connOptions,\n geminiTools,\n extraKwargs: extras,\n });\n }\n}\n\nexport class LLMStream extends llm.LLMStream {\n #client: GoogleGenAI;\n #model: string;\n #geminiTools?: LLMTools;\n #extraKwargs: GenerateContentConfig;\n\n constructor(\n llm: LLM,\n {\n client,\n model,\n chatCtx,\n toolCtx,\n connOptions,\n geminiTools,\n extraKwargs,\n }: {\n client: GoogleGenAI;\n model: string;\n chatCtx: llm.ChatContext;\n toolCtx?: llm.ToolContext;\n connOptions: APIConnectOptions;\n geminiTools?: LLMTools;\n extraKwargs: GenerateContentConfig;\n },\n ) {\n // Call base constructor with dev 1.0 object parameter pattern\n super(llm, { chatCtx, toolCtx, connOptions });\n this.#client = client;\n this.#model = model;\n this.#geminiTools = geminiTools;\n this.#extraKwargs = extraKwargs;\n }\n\n protected async run(): Promise {\n let retryable = true;\n const requestId = `google_${Date.now()}`;\n\n try {\n const [turns, extraData] = (await this.chatCtx.toProviderFormat('google')) as [\n Record[],\n GoogleFormatData,\n ];\n\n const contents: types.Content[] = turns.map((turn: Record) => ({\n role: turn.role as types.Content['role'],\n parts: turn.parts as types.Part[],\n }));\n\n const functionDeclarations = this.toolCtx ? toFunctionDeclarations(this.toolCtx) : undefined;\n const tools =\n functionDeclarations && functionDeclarations.length > 0\n ? [{ functionDeclarations }]\n : undefined;\n\n let systemInstruction: types.Content | undefined = undefined;\n if (extraData.systemMessages && extraData.systemMessages.length > 0) {\n systemInstruction = {\n parts: extraData.systemMessages.map((content: string) => ({ text: content })),\n };\n }\n\n const response = await this.#client.models.generateContentStream({\n model: this.#model,\n contents,\n config: {\n ...this.#extraKwargs,\n systemInstruction,\n httpOptions: this.#extraKwargs.httpOptions ?? {\n timeout: Math.floor(this.connOptions.timeoutMs),\n },\n tools,\n },\n });\n\n for await (const chunk of response) {\n if (chunk.promptFeedback) {\n throw new APIStatusError({\n message: `Prompt feedback error: ${JSON.stringify(chunk.promptFeedback)}`,\n options: {\n retryable: false,\n requestId,\n },\n });\n }\n\n if (!chunk.candidates || !chunk.candidates[0]?.content?.parts) {\n this.logger.warn(`No candidates in the response: ${JSON.stringify(chunk)}`);\n continue;\n }\n\n if (chunk.candidates.length > 1) {\n this.logger.warn(\n 'Google LLM: there are multiple candidates in the response, returning response from the first one.',\n );\n }\n\n for (const part of chunk.candidates[0].content.parts) {\n const chatChunk = this.#parsePart(requestId, part);\n if (chatChunk) {\n retryable = false;\n this.queue.put(chatChunk);\n }\n }\n\n if (chunk.usageMetadata) {\n const usage = chunk.usageMetadata;\n this.queue.put({\n id: requestId,\n usage: {\n completionTokens: usage.candidatesTokenCount || 0,\n promptTokens: usage.promptTokenCount || 0,\n promptCachedTokens: usage.cachedContentTokenCount || 0,\n totalTokens: usage.totalTokenCount || 0,\n },\n });\n }\n }\n } catch (error: unknown) {\n const err = error as {\n code?: number;\n message?: string;\n status?: string;\n type?: string;\n };\n\n if (err.code && err.code >= 400 && err.code < 500) {\n if (err.code === 429) {\n throw new APIStatusError({\n message: `Google LLM: Rate limit error - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: 429,\n retryable: true,\n },\n });\n } else {\n throw new APIStatusError({\n message: `Google LLM: Client error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable: false,\n },\n });\n }\n }\n\n if (err.code && err.code >= 500) {\n throw new APIStatusError({\n message: `Google LLM: Server error (${err.code}) - ${err.message || 'Unknown error'}`,\n options: {\n statusCode: err.code,\n retryable,\n },\n });\n }\n\n throw new APIConnectionError({\n message: `Google LLM: API error - ${err.message || 'Unknown error'}`,\n options: {\n retryable,\n },\n });\n }\n }\n\n #parsePart(id: string, part: types.Part): llm.ChatChunk | null {\n if (part.functionCall) {\n return {\n id,\n delta: {\n role: 'assistant',\n toolCalls: [\n llm.FunctionCall.create({\n callId: part.functionCall.id || shortuuid('function_call_'),\n name: part.functionCall.name!,\n args: JSON.stringify(part.functionCall.args!),\n }),\n ],\n },\n };\n }\n\n return {\n id,\n delta: {\n content: part.text,\n role: 'assistant',\n },\n };\n }\n}\n"],"mappings":"AAIA,SAAS,2BAAuD,mBAAmB;AAEnF;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAGP,SAAS,8BAA8B;AA0BhC,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EAEA,QAAgB;AACd,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA8BA,YACE;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,IAAgB;AAAA,IACd,OAAO;AAAA,EACT,GACA;AACA,UAAM;AAEN,UAAM,cACJ,aACC,QAAQ,IAAI,8BAA8B,UACzC,QAAQ,IAAI,8BAA8B;AAE9C,QAAI,aAAiC,WAAW,QAAQ,IAAI;AAC5D,QAAI,cAAkC,YAAY,QAAQ,IAAI;AAC9D,QAAI,eAAmC,UAAU,QAAQ,IAAI;AAE7D,QAAI,aAAa;AACf,UAAI,CAAC,YAAY;AAEf,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,qBAAe;AAAA,IACjB,OAAO;AACL,mBAAa;AACb,oBAAc;AACd,UAAI,CAAC,cAAc;AACjB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,SAAI,iDAAgB,oBAAmB,QAAW;AAChD,YAAM,SAAS,eAAe;AAC9B,UAAI,SAAS,KAAK,SAAS,OAAO;AAChC,cAAM,IAAI,MAAM,kEAAkE;AAAA,MACpF;AAAA,IACF;AAEA,UAAM,gBAA0C,cAC5C;AAAA,MACE,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,IACZ,IACA;AAAA,MACE,QAAQ;AAAA,IACV;AAEJ,SAAK,UAAU,IAAI,YAAY,aAAa;AAE5C,SAAK,QAAQ;AAAA,MACX;AAAA,MACA,UAAU;AAAA,MACV,SAAS;AAAA,MACT,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAQc;AACZ,UAAM,SAAgC,EAAE,GAAG,YAAY;AAEvD,iBAAa,eAAe,SAAY,aAAa,KAAK,MAAM;AAEhE,QAAI,YAAY;AACd,UAAI;AAEJ,UAAI,OAAO,eAAe,YAAY,WAAW,SAAS,YAAY;AACpE,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,0BAA0B;AAAA,YAChC,sBAAsB,CAAC,WAAW,SAAS,IAAI;AAAA,UACjD;AAAA,QACF;AAAA,MACF,WAAW,eAAe,YAAY;AACpC,cAAM,YAAY,OAAO,QAAQ,WAAW,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,MAAM,IAAI;AACpE,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,0BAA0B;AAAA,YAChC,sBAAsB,UAAU,SAAS,IAAI,YAAY;AAAA,UAC3D;AAAA,QACF;AAAA,MACF,WAAW,eAAe,QAAQ;AAChC,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,0BAA0B;AAAA,UAClC;AAAA,QACF;AAAA,MACF,WAAW,eAAe,QAAQ;AAChC,2BAAmB;AAAA,UACjB,uBAAuB;AAAA,YACrB,MAAM,0BAA0B;AAAA,UAClC;AAAA,QACF;AAAA,MACF,OAAO;AACL,cAAM,IAAI,MAAM,wBAAwB,UAAU,EAAE;AAAA,MACtD;AAEA,aAAO,aAAa;AAAA,IACtB;AAEA,QAAI,KAAK,MAAM,gBAAgB,QAAW;AACxC,aAAO,cAAc,KAAK,MAAM;AAAA,IAClC;AACA,QAAI,KAAK,MAAM,oBAAoB,QAAW;AAC5C,aAAO,kBAAkB,KAAK,MAAM;AAAA,IACtC;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AACA,QAAI,KAAK,MAAM,oBAAoB,QAAW;AAC5C,aAAO,kBAAkB,KAAK,MAAM;AAAA,IACtC;AACA,QAAI,KAAK,MAAM,qBAAqB,QAAW;AAC7C,aAAO,mBAAmB,KAAK,MAAM;AAAA,IACvC;AACA,QAAI,KAAK,MAAM,SAAS,QAAW;AACjC,aAAO,OAAO,KAAK,MAAM;AAAA,IAC3B;AAEA,QAAI,KAAK,MAAM,mBAAmB,QAAW;AAC3C,aAAO,iBAAiB,KAAK,MAAM;AAAA,IACrC;AAEA,QAAI,KAAK,MAAM,mCAAmC,QAAW;AAC3D,aAAO,2BAA2B,KAAK,MAAM;AAAA,IAC/C;AAEA,kBAAc,gBAAgB,SAAY,cAAc,KAAK,MAAM;AAEnE,WAAO,IAAI,UAAU,MAAM;AAAA,MACzB,QAAQ,KAAK;AAAA,MACb,OAAO,KAAK,MAAM;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa;AAAA,IACf,CAAC;AAAA,EACH;AACF;AAEO,MAAM,kBAAkB,IAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACEA,MACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GASA;AAEA,UAAMA,MAAK,EAAE,SAAS,SAAS,YAAY,CAAC;AAC5C,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,eAAe;AACpB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,MAAqB;AArTvC;AAsTI,QAAI,YAAY;AAChB,UAAM,YAAY,UAAU,KAAK,IAAI,CAAC;AAEtC,QAAI;AACF,YAAM,CAAC,OAAO,SAAS,IAAK,MAAM,KAAK,QAAQ,iBAAiB,QAAQ;AAKxE,YAAM,WAA4B,MAAM,IAAI,CAAC,UAAmC;AAAA,QAC9E,MAAM,KAAK;AAAA,QACX,OAAO,KAAK;AAAA,MACd,EAAE;AAEF,YAAM,uBAAuB,KAAK,UAAU,uBAAuB,KAAK,OAAO,IAAI;AACnF,YAAM,QACJ,wBAAwB,qBAAqB,SAAS,IAClD,CAAC,EAAE,qBAAqB,CAAC,IACzB;AAEN,UAAI,oBAA+C;AACnD,UAAI,UAAU,kBAAkB,UAAU,eAAe,SAAS,GAAG;AACnE,4BAAoB;AAAA,UAClB,OAAO,UAAU,eAAe,IAAI,CAAC,aAAqB,EAAE,MAAM,QAAQ,EAAE;AAAA,QAC9E;AAAA,MACF;AAEA,YAAM,WAAW,MAAM,KAAK,QAAQ,OAAO,sBAAsB;AAAA,QAC/D,OAAO,KAAK;AAAA,QACZ;AAAA,QACA,QAAQ;AAAA,UACN,GAAG,KAAK;AAAA,UACR;AAAA,UACA,aAAa,KAAK,aAAa,eAAe;AAAA,YAC5C,SAAS,KAAK,MAAM,KAAK,YAAY,SAAS;AAAA,UAChD;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAED,uBAAiB,SAAS,UAAU;AAClC,YAAI,MAAM,gBAAgB;AACxB,gBAAM,IAAI,eAAe;AAAA,YACvB,SAAS,0BAA0B,KAAK,UAAU,MAAM,cAAc,CAAC;AAAA,YACvE,SAAS;AAAA,cACP,WAAW;AAAA,cACX;AAAA,YACF;AAAA,UACF,CAAC;AAAA,QACH;AAEA,YAAI,CAAC,MAAM,cAAc,GAAC,iBAAM,WAAW,CAAC,MAAlB,mBAAqB,YAArB,mBAA8B,QAAO;AAC7D,eAAK,OAAO,KAAK,kCAAkC,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1E;AAAA,QACF;AAEA,YAAI,MAAM,WAAW,SAAS,GAAG;AAC/B,eAAK,OAAO;AAAA,YACV;AAAA,UACF;AAAA,QACF;AAEA,mBAAW,QAAQ,MAAM,WAAW,CAAC,EAAE,QAAQ,OAAO;AACpD,gBAAM,YAAY,KAAK,WAAW,WAAW,IAAI;AACjD,cAAI,WAAW;AACb,wBAAY;AACZ,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAAA,QACF;AAEA,YAAI,MAAM,eAAe;AACvB,gBAAM,QAAQ,MAAM;AACpB,eAAK,MAAM,IAAI;AAAA,YACb,IAAI;AAAA,YACJ,OAAO;AAAA,cACL,kBAAkB,MAAM,wBAAwB;AAAA,cAChD,cAAc,MAAM,oBAAoB;AAAA,cACxC,oBAAoB,MAAM,2BAA2B;AAAA,cACrD,aAAa,MAAM,mBAAmB;AAAA,YACxC;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,SAAS,OAAgB;AACvB,YAAM,MAAM;AAOZ,UAAI,IAAI,QAAQ,IAAI,QAAQ,OAAO,IAAI,OAAO,KAAK;AACjD,YAAI,IAAI,SAAS,KAAK;AACpB,gBAAM,IAAI,eAAe;AAAA,YACvB,SAAS,kCAAkC,IAAI,WAAW,eAAe;AAAA,YACzE,SAAS;AAAA,cACP,YAAY;AAAA,cACZ,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH,OAAO;AACL,gBAAM,IAAI,eAAe;AAAA,YACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,YACnF,SAAS;AAAA,cACP,YAAY,IAAI;AAAA,cAChB,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,IAAI,QAAQ,IAAI,QAAQ,KAAK;AAC/B,cAAM,IAAI,eAAe;AAAA,UACvB,SAAS,6BAA6B,IAAI,IAAI,OAAO,IAAI,WAAW,eAAe;AAAA,UACnF,SAAS;AAAA,YACP,YAAY,IAAI;AAAA,YAChB;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH;AAEA,YAAM,IAAI,mBAAmB;AAAA,QAC3B,SAAS,2BAA2B,IAAI,WAAW,eAAe;AAAA,QAClE,SAAS;AAAA,UACP;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,WAAW,IAAY,MAAwC;AAC7D,QAAI,KAAK,cAAc;AACrB,aAAO;AAAA,QACL;AAAA,QACA,OAAO;AAAA,UACL,MAAM;AAAA,UACN,WAAW;AAAA,YACT,IAAI,aAAa,OAAO;AAAA,cACtB,QAAQ,KAAK,aAAa,MAAM,UAAU,gBAAgB;AAAA,cAC1D,MAAM,KAAK,aAAa;AAAA,cACxB,MAAM,KAAK,UAAU,KAAK,aAAa,IAAK;AAAA,YAC9C,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL;AAAA,MACA,OAAO;AAAA,QACL,SAAS,KAAK;AAAA,QACd,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;","names":["llm"]} \ No newline at end of file diff --git a/plugins/google/dist/llm.test.cjs b/plugins/google/dist/llm.test.cjs deleted file mode 100644 index 79eb0be52..000000000 --- a/plugins/google/dist/llm.test.cjs +++ /dev/null @@ -1,14 +0,0 @@ -"use strict"; -var import_agents_plugins_test = require("@livekit/agents-plugins-test"); -var import_vitest = require("vitest"); -var import_llm = require("./llm.cjs"); -(0, import_vitest.describe)("Google", async () => { - await (0, import_agents_plugins_test.llm)( - new import_llm.LLM({ - model: "gemini-2.5-flash", - temperature: 0 - }), - true - ); -}); -//# sourceMappingURL=llm.test.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/llm.test.cjs.map b/plugins/google/dist/llm.test.cjs.map deleted file mode 100644 index 40d74b59f..000000000 --- a/plugins/google/dist/llm.test.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/llm.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm } from '@livekit/agents-plugins-test';\nimport { describe } from 'vitest';\nimport { LLM } from './llm.js';\n\ndescribe('Google', async () => {\n await llm(\n new LLM({\n model: 'gemini-2.5-flash',\n temperature: 0,\n }),\n true,\n );\n});\n"],"mappings":";AAGA,iCAAoB;AACpB,oBAAyB;AACzB,iBAAoB;AAAA,IAEpB,wBAAS,UAAU,YAAY;AAC7B,YAAM;AAAA,IACJ,IAAI,eAAI;AAAA,MACN,OAAO;AAAA,MACP,aAAa;AAAA,IACf,CAAC;AAAA,IACD;AAAA,EACF;AACF,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/llm.test.d.cts b/plugins/google/dist/llm.test.d.cts deleted file mode 100644 index e16fd8dca..000000000 --- a/plugins/google/dist/llm.test.d.cts +++ /dev/null @@ -1,2 +0,0 @@ -export {}; -//# sourceMappingURL=llm.test.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/llm.test.d.ts b/plugins/google/dist/llm.test.d.ts deleted file mode 100644 index e16fd8dca..000000000 --- a/plugins/google/dist/llm.test.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -export {}; -//# sourceMappingURL=llm.test.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/llm.test.d.ts.map b/plugins/google/dist/llm.test.d.ts.map deleted file mode 100644 index 7666c651f..000000000 --- a/plugins/google/dist/llm.test.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"llm.test.d.ts","sourceRoot":"","sources":["../src/llm.test.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/plugins/google/dist/llm.test.js b/plugins/google/dist/llm.test.js deleted file mode 100644 index 06c96c3b1..000000000 --- a/plugins/google/dist/llm.test.js +++ /dev/null @@ -1,13 +0,0 @@ -import { llm } from "@livekit/agents-plugins-test"; -import { describe } from "vitest"; -import { LLM } from "./llm.js"; -describe("Google", async () => { - await llm( - new LLM({ - model: "gemini-2.5-flash", - temperature: 0 - }), - true - ); -}); -//# sourceMappingURL=llm.test.js.map \ No newline at end of file diff --git a/plugins/google/dist/llm.test.js.map b/plugins/google/dist/llm.test.js.map deleted file mode 100644 index 1b9abe082..000000000 --- a/plugins/google/dist/llm.test.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/llm.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm } from '@livekit/agents-plugins-test';\nimport { describe } from 'vitest';\nimport { LLM } from './llm.js';\n\ndescribe('Google', async () => {\n await llm(\n new LLM({\n model: 'gemini-2.5-flash',\n temperature: 0,\n }),\n true,\n );\n});\n"],"mappings":"AAGA,SAAS,WAAW;AACpB,SAAS,gBAAgB;AACzB,SAAS,WAAW;AAEpB,SAAS,UAAU,YAAY;AAC7B,QAAM;AAAA,IACJ,IAAI,IAAI;AAAA,MACN,OAAO;AAAA,MACP,aAAa;AAAA,IACf,CAAC;AAAA,IACD;AAAA,EACF;AACF,CAAC;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/models.cjs b/plugins/google/dist/models.cjs deleted file mode 100644 index d97268935..000000000 --- a/plugins/google/dist/models.cjs +++ /dev/null @@ -1,17 +0,0 @@ -"use strict"; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); -var models_exports = {}; -module.exports = __toCommonJS(models_exports); -//# sourceMappingURL=models.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/models.cjs.map b/plugins/google/dist/models.cjs.map deleted file mode 100644 index 5d92bc5c1..000000000 --- a/plugins/google/dist/models.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/models.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n// Speech to Text v2\nexport type SpeechModels =\n | 'long'\n | 'short'\n | 'telephony'\n | 'medical_dictation'\n | 'medical_conversation'\n | 'chirp'\n | 'chirp_2'\n | 'latest_long'\n | 'latest_short';\n\nexport type SpeechLanguages =\n | 'en-US'\n | 'ja-JP'\n | 'en-IN'\n | 'en-GB'\n | 'hi-IN'\n | 'af-ZA'\n | 'sq-AL'\n | 'am-ET'\n | 'ar-EG'\n | 'hy-AM'\n | 'ast-ES'\n | 'az-AZ'\n | 'eu-ES'\n | 'be-BY'\n | 'bs-BA'\n | 'bg-BG'\n | 'my-MM'\n | 'ca-ES'\n | 'ceb-PH'\n | 'ckb-IQ'\n | 'zh-Hans-CN'\n | 'yue-Hant-HK'\n | 'zh-TW'\n | 'hr-HR'\n | 'cs-CZ'\n | 'da-DK'\n | 'nl-NL'\n | 'en-AU'\n | 'et-EE'\n | 'fil-PH'\n | 'fi-FI'\n | 'fr-CA'\n | 'fr-FR'\n | 'gl-ES'\n | 'ka-GE'\n | 'de-DE'\n | 'el-GR'\n | 'gu-IN'\n | 'ha-NG'\n | 'iw-IL'\n | 'hu-HU'\n | 'is-IS'\n | 'id-ID'\n | 'it-IT'\n | 'jv-ID'\n | 'kea-CV'\n | 'kam-KE'\n | 'kn-IN'\n | 'kk-KZ'\n | 'km-KH'\n | 'ko-KR'\n | 'ky-KG'\n | 'lo-LA'\n | 'lv-LV'\n | 'ln-CD'\n | 'lt-LT'\n | 'luo-KE'\n | 'lb-LU'\n | 'mk-MK'\n | 'no-NO'\n | 'pl-PL'\n | 'pt-BR'\n | 'pt-PT'\n | 'ro-RO'\n | 'ru-RU'\n | 'es-CO'\n | 'es-MX'\n | 'es-US'\n | 'th-TH'\n | 'tr-TR'\n | 'uk-UA'\n | 'vi-VN';\n\nexport type Gender = 'male' | 'female' | 'neutral';\n\nexport type ChatModels =\n | 'gemini-2.5-pro-preview-05-06'\n | 'gemini-2.5-flash-preview-04-17'\n | 'gemini-2.5-flash-preview-05-20'\n | 'gemini-2.0-flash-001'\n | 'gemini-2.0-flash-lite-preview-02-05'\n | 'gemini-2.0-pro-exp-02-05'\n | 'gemini-1.5-pro';\n"],"mappings":";;;;;;;;;;;;;;AAAA;AAAA;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/models.d.cts b/plugins/google/dist/models.d.cts deleted file mode 100644 index 1534100fd..000000000 --- a/plugins/google/dist/models.d.cts +++ /dev/null @@ -1,5 +0,0 @@ -export type SpeechModels = 'long' | 'short' | 'telephony' | 'medical_dictation' | 'medical_conversation' | 'chirp' | 'chirp_2' | 'latest_long' | 'latest_short'; -export type SpeechLanguages = 'en-US' | 'ja-JP' | 'en-IN' | 'en-GB' | 'hi-IN' | 'af-ZA' | 'sq-AL' | 'am-ET' | 'ar-EG' | 'hy-AM' | 'ast-ES' | 'az-AZ' | 'eu-ES' | 'be-BY' | 'bs-BA' | 'bg-BG' | 'my-MM' | 'ca-ES' | 'ceb-PH' | 'ckb-IQ' | 'zh-Hans-CN' | 'yue-Hant-HK' | 'zh-TW' | 'hr-HR' | 'cs-CZ' | 'da-DK' | 'nl-NL' | 'en-AU' | 'et-EE' | 'fil-PH' | 'fi-FI' | 'fr-CA' | 'fr-FR' | 'gl-ES' | 'ka-GE' | 'de-DE' | 'el-GR' | 'gu-IN' | 'ha-NG' | 'iw-IL' | 'hu-HU' | 'is-IS' | 'id-ID' | 'it-IT' | 'jv-ID' | 'kea-CV' | 'kam-KE' | 'kn-IN' | 'kk-KZ' | 'km-KH' | 'ko-KR' | 'ky-KG' | 'lo-LA' | 'lv-LV' | 'ln-CD' | 'lt-LT' | 'luo-KE' | 'lb-LU' | 'mk-MK' | 'no-NO' | 'pl-PL' | 'pt-BR' | 'pt-PT' | 'ro-RO' | 'ru-RU' | 'es-CO' | 'es-MX' | 'es-US' | 'th-TH' | 'tr-TR' | 'uk-UA' | 'vi-VN'; -export type Gender = 'male' | 'female' | 'neutral'; -export type ChatModels = 'gemini-2.5-pro-preview-05-06' | 'gemini-2.5-flash-preview-04-17' | 'gemini-2.5-flash-preview-05-20' | 'gemini-2.0-flash-001' | 'gemini-2.0-flash-lite-preview-02-05' | 'gemini-2.0-pro-exp-02-05' | 'gemini-1.5-pro'; -//# sourceMappingURL=models.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/models.d.ts b/plugins/google/dist/models.d.ts deleted file mode 100644 index 1534100fd..000000000 --- a/plugins/google/dist/models.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type SpeechModels = 'long' | 'short' | 'telephony' | 'medical_dictation' | 'medical_conversation' | 'chirp' | 'chirp_2' | 'latest_long' | 'latest_short'; -export type SpeechLanguages = 'en-US' | 'ja-JP' | 'en-IN' | 'en-GB' | 'hi-IN' | 'af-ZA' | 'sq-AL' | 'am-ET' | 'ar-EG' | 'hy-AM' | 'ast-ES' | 'az-AZ' | 'eu-ES' | 'be-BY' | 'bs-BA' | 'bg-BG' | 'my-MM' | 'ca-ES' | 'ceb-PH' | 'ckb-IQ' | 'zh-Hans-CN' | 'yue-Hant-HK' | 'zh-TW' | 'hr-HR' | 'cs-CZ' | 'da-DK' | 'nl-NL' | 'en-AU' | 'et-EE' | 'fil-PH' | 'fi-FI' | 'fr-CA' | 'fr-FR' | 'gl-ES' | 'ka-GE' | 'de-DE' | 'el-GR' | 'gu-IN' | 'ha-NG' | 'iw-IL' | 'hu-HU' | 'is-IS' | 'id-ID' | 'it-IT' | 'jv-ID' | 'kea-CV' | 'kam-KE' | 'kn-IN' | 'kk-KZ' | 'km-KH' | 'ko-KR' | 'ky-KG' | 'lo-LA' | 'lv-LV' | 'ln-CD' | 'lt-LT' | 'luo-KE' | 'lb-LU' | 'mk-MK' | 'no-NO' | 'pl-PL' | 'pt-BR' | 'pt-PT' | 'ro-RO' | 'ru-RU' | 'es-CO' | 'es-MX' | 'es-US' | 'th-TH' | 'tr-TR' | 'uk-UA' | 'vi-VN'; -export type Gender = 'male' | 'female' | 'neutral'; -export type ChatModels = 'gemini-2.5-pro-preview-05-06' | 'gemini-2.5-flash-preview-04-17' | 'gemini-2.5-flash-preview-05-20' | 'gemini-2.0-flash-001' | 'gemini-2.0-flash-lite-preview-02-05' | 'gemini-2.0-pro-exp-02-05' | 'gemini-1.5-pro'; -//# sourceMappingURL=models.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/models.d.ts.map b/plugins/google/dist/models.d.ts.map deleted file mode 100644 index 8204511ae..000000000 --- a/plugins/google/dist/models.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAKA,MAAM,MAAM,YAAY,GACpB,MAAM,GACN,OAAO,GACP,WAAW,GACX,mBAAmB,GACnB,sBAAsB,GACtB,OAAO,GACP,SAAS,GACT,aAAa,GACb,cAAc,CAAC;AAEnB,MAAM,MAAM,eAAe,GACvB,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,YAAY,GACZ,aAAa,GACb,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,CAAC;AAEZ,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEnD,MAAM,MAAM,UAAU,GAClB,8BAA8B,GAC9B,gCAAgC,GAChC,gCAAgC,GAChC,sBAAsB,GACtB,qCAAqC,GACrC,0BAA0B,GAC1B,gBAAgB,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/models.js b/plugins/google/dist/models.js deleted file mode 100644 index 5e38071cf..000000000 --- a/plugins/google/dist/models.js +++ /dev/null @@ -1 +0,0 @@ -//# sourceMappingURL=models.js.map \ No newline at end of file diff --git a/plugins/google/dist/models.js.map b/plugins/google/dist/models.js.map deleted file mode 100644 index 84c51b288..000000000 --- a/plugins/google/dist/models.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/tools.cjs b/plugins/google/dist/tools.cjs deleted file mode 100644 index f14d3006c..000000000 --- a/plugins/google/dist/tools.cjs +++ /dev/null @@ -1,17 +0,0 @@ -"use strict"; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); -var tools_exports = {}; -module.exports = __toCommonJS(tools_exports); -//# sourceMappingURL=tools.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/tools.cjs.map b/plugins/google/dist/tools.cjs.map deleted file mode 100644 index 174549578..000000000 --- a/plugins/google/dist/tools.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/tools.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Tool } from '@google/genai';\n\nexport type LLMTools = Omit;\n"],"mappings":";;;;;;;;;;;;;;AAAA;AAAA;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/tools.d.cts b/plugins/google/dist/tools.d.cts deleted file mode 100644 index 583504a9f..000000000 --- a/plugins/google/dist/tools.d.cts +++ /dev/null @@ -1,3 +0,0 @@ -import type { Tool } from '@google/genai'; -export type LLMTools = Omit; -//# sourceMappingURL=tools.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/tools.d.ts b/plugins/google/dist/tools.d.ts deleted file mode 100644 index 583504a9f..000000000 --- a/plugins/google/dist/tools.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type { Tool } from '@google/genai'; -export type LLMTools = Omit; -//# sourceMappingURL=tools.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/tools.d.ts.map b/plugins/google/dist/tools.d.ts.map deleted file mode 100644 index c82959cd7..000000000 --- a/plugins/google/dist/tools.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAE1C,MAAM,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,sBAAsB,CAAC,CAAC"} \ No newline at end of file diff --git a/plugins/google/dist/tools.js b/plugins/google/dist/tools.js deleted file mode 100644 index 4a36c860e..000000000 --- a/plugins/google/dist/tools.js +++ /dev/null @@ -1 +0,0 @@ -//# sourceMappingURL=tools.js.map \ No newline at end of file diff --git a/plugins/google/dist/tools.js.map b/plugins/google/dist/tools.js.map deleted file mode 100644 index 84c51b288..000000000 --- a/plugins/google/dist/tools.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/utils.cjs b/plugins/google/dist/utils.cjs deleted file mode 100644 index e64730020..000000000 --- a/plugins/google/dist/utils.cjs +++ /dev/null @@ -1,137 +0,0 @@ -"use strict"; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __export = (target, all) => { - for (var name in all) - __defProp(target, name, { get: all[name], enumerable: true }); -}; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); -var utils_exports = {}; -__export(utils_exports, { - convertJSONSchemaToOpenAPISchema: () => convertJSONSchemaToOpenAPISchema, - toFunctionDeclarations: () => toFunctionDeclarations -}); -module.exports = __toCommonJS(utils_exports); -var import_agents = require("@livekit/agents"); -function convertJSONSchemaToOpenAPISchema(jsonSchema) { - if (jsonSchema == null || isEmptyObjectSchema(jsonSchema)) { - return void 0; - } - if (typeof jsonSchema === "boolean") { - return { type: "boolean", properties: {} }; - } - const { - type, - description, - required, - properties, - items, - allOf, - anyOf, - oneOf, - format, - const: constValue, - minLength, - enum: enumValues - } = jsonSchema; - const result = {}; - if (description) result.description = description; - if (required) result.required = required; - if (format) result.format = format; - if (constValue !== void 0) { - result.enum = [constValue]; - } - if (type) { - if (Array.isArray(type)) { - if (type.includes("null")) { - result.type = type.filter((t) => t !== "null")[0]; - result.nullable = true; - } else { - result.type = type; - } - } else if (type === "null") { - result.type = "null"; - } else { - result.type = type; - } - } - if (enumValues !== void 0) { - result.enum = enumValues; - } - if (properties != null) { - result.properties = Object.entries(properties).reduce( - (acc, [key, value]) => { - acc[key] = convertJSONSchemaToOpenAPISchema(value); - return acc; - }, - {} - ); - } - if (items) { - result.items = Array.isArray(items) ? items.map(convertJSONSchemaToOpenAPISchema) : convertJSONSchemaToOpenAPISchema(items); - } - if (allOf) { - result.allOf = allOf.map(convertJSONSchemaToOpenAPISchema); - } - if (anyOf) { - if (anyOf.some((schema) => typeof schema === "object" && (schema == null ? void 0 : schema.type) === "null")) { - const nonNullSchemas = anyOf.filter( - (schema) => !(typeof schema === "object" && (schema == null ? void 0 : schema.type) === "null") - ); - if (nonNullSchemas.length === 1) { - const converted = convertJSONSchemaToOpenAPISchema( - nonNullSchemas[0] - ); - if (typeof converted === "object") { - result.nullable = true; - Object.assign(result, converted); - } - } else { - result.anyOf = nonNullSchemas.map(convertJSONSchemaToOpenAPISchema); - result.nullable = true; - } - } else { - result.anyOf = anyOf.map(convertJSONSchemaToOpenAPISchema); - } - } - if (oneOf) { - result.oneOf = oneOf.map(convertJSONSchemaToOpenAPISchema); - } - if (minLength !== void 0) { - result.minLength = minLength; - } - return result; -} -function isEmptyObjectSchema(jsonSchema) { - return jsonSchema != null && typeof jsonSchema === "object" && jsonSchema.type === "object" && (jsonSchema.properties == null || Object.keys(jsonSchema.properties).length === 0) && !jsonSchema.additionalProperties; -} -function toFunctionDeclarations(toolCtx) { - const functionDeclarations = []; - for (const [name, tool] of Object.entries(toolCtx)) { - const { description, parameters } = tool; - const jsonSchema = import_agents.llm.toJsonSchema(parameters, false); - const schemaCopy = JSON.parse(JSON.stringify(jsonSchema)); - functionDeclarations.push({ - name, - description, - parameters: convertJSONSchemaToOpenAPISchema(schemaCopy) - }); - } - return functionDeclarations; -} -// Annotate the CommonJS export names for ESM import in node: -0 && (module.exports = { - convertJSONSchemaToOpenAPISchema, - toFunctionDeclarations -}); -//# sourceMappingURL=utils.cjs.map \ No newline at end of file diff --git a/plugins/google/dist/utils.cjs.map b/plugins/google/dist/utils.cjs.map deleted file mode 100644 index 80fb8001b..000000000 --- a/plugins/google/dist/utils.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { FunctionDeclaration, Schema } from '@google/genai';\nimport { llm } from '@livekit/agents';\nimport type { JSONSchema7 } from 'json-schema';\n\n/**\n * JSON Schema v7\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01\n */\nexport type JSONSchema7Definition = JSONSchema7 | boolean;\n\n/**\n * Converts JSON Schema 7 to OpenAPI Schema 3.0\n */\n// Adapted from https://github.com/vercel/ai/blob/main/packages/google/src/convert-json-schema-to-openapi-schema.ts\nexport function convertJSONSchemaToOpenAPISchema(jsonSchema: JSONSchema7Definition): unknown {\n // parameters need to be undefined if they are empty objects:\n if (jsonSchema == null || isEmptyObjectSchema(jsonSchema)) {\n return undefined;\n }\n\n if (typeof jsonSchema === 'boolean') {\n return { type: 'boolean', properties: {} };\n }\n\n const {\n type,\n description,\n required,\n properties,\n items,\n allOf,\n anyOf,\n oneOf,\n format,\n const: constValue,\n minLength,\n enum: enumValues,\n } = jsonSchema;\n\n const result: Record = {};\n\n if (description) result.description = description;\n if (required) result.required = required;\n if (format) result.format = format;\n\n if (constValue !== undefined) {\n result.enum = [constValue];\n }\n\n // Handle type\n if (type) {\n if (Array.isArray(type)) {\n if (type.includes('null')) {\n result.type = type.filter((t) => t !== 'null')[0];\n result.nullable = true;\n } else {\n result.type = type;\n }\n } else if (type === 'null') {\n result.type = 'null';\n } else {\n result.type = type;\n }\n }\n\n // Handle enum\n if (enumValues !== undefined) {\n result.enum = enumValues;\n }\n\n if (properties != null) {\n result.properties = Object.entries(properties).reduce(\n (acc, [key, value]) => {\n acc[key] = convertJSONSchemaToOpenAPISchema(value);\n return acc;\n },\n {} as Record,\n );\n }\n\n if (items) {\n result.items = Array.isArray(items)\n ? items.map(convertJSONSchemaToOpenAPISchema)\n : convertJSONSchemaToOpenAPISchema(items);\n }\n\n if (allOf) {\n result.allOf = allOf.map(convertJSONSchemaToOpenAPISchema);\n }\n if (anyOf) {\n // Handle cases where anyOf includes a null type\n if (anyOf.some((schema) => typeof schema === 'object' && schema?.type === 'null')) {\n const nonNullSchemas = anyOf.filter(\n (schema) => !(typeof schema === 'object' && schema?.type === 'null'),\n );\n\n if (nonNullSchemas.length === 1) {\n // If there's only one non-null schema, convert it and make it nullable\n const converted = convertJSONSchemaToOpenAPISchema(\n nonNullSchemas[0] as JSONSchema7Definition,\n );\n if (typeof converted === 'object') {\n result.nullable = true;\n Object.assign(result, converted);\n }\n } else {\n // If there are multiple non-null schemas, keep them in anyOf\n result.anyOf = nonNullSchemas.map(convertJSONSchemaToOpenAPISchema);\n result.nullable = true;\n }\n } else {\n result.anyOf = anyOf.map(convertJSONSchemaToOpenAPISchema);\n }\n }\n if (oneOf) {\n result.oneOf = oneOf.map(convertJSONSchemaToOpenAPISchema);\n }\n\n if (minLength !== undefined) {\n result.minLength = minLength;\n }\n\n return result;\n}\n\nfunction isEmptyObjectSchema(jsonSchema: JSONSchema7Definition): boolean {\n return (\n jsonSchema != null &&\n typeof jsonSchema === 'object' &&\n jsonSchema.type === 'object' &&\n (jsonSchema.properties == null || Object.keys(jsonSchema.properties).length === 0) &&\n !jsonSchema.additionalProperties\n );\n}\n\nexport function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[] {\n const functionDeclarations: FunctionDeclaration[] = [];\n\n for (const [name, tool] of Object.entries(toolCtx)) {\n const { description, parameters } = tool;\n const jsonSchema = llm.toJsonSchema(parameters, false);\n\n // Create a deep copy to prevent the Google GenAI library from mutating the schema\n const schemaCopy = JSON.parse(JSON.stringify(jsonSchema));\n\n functionDeclarations.push({\n name,\n description,\n parameters: convertJSONSchemaToOpenAPISchema(schemaCopy) as Schema,\n });\n }\n\n return functionDeclarations;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,oBAAoB;AAab,SAAS,iCAAiC,YAA4C;AAE3F,MAAI,cAAc,QAAQ,oBAAoB,UAAU,GAAG;AACzD,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,eAAe,WAAW;AACnC,WAAO,EAAE,MAAM,WAAW,YAAY,CAAC,EAAE;AAAA,EAC3C;AAEA,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP;AAAA,IACA,MAAM;AAAA,EACR,IAAI;AAEJ,QAAM,SAAkC,CAAC;AAEzC,MAAI,YAAa,QAAO,cAAc;AACtC,MAAI,SAAU,QAAO,WAAW;AAChC,MAAI,OAAQ,QAAO,SAAS;AAE5B,MAAI,eAAe,QAAW;AAC5B,WAAO,OAAO,CAAC,UAAU;AAAA,EAC3B;AAGA,MAAI,MAAM;AACR,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,UAAI,KAAK,SAAS,MAAM,GAAG;AACzB,eAAO,OAAO,KAAK,OAAO,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC;AAChD,eAAO,WAAW;AAAA,MACpB,OAAO;AACL,eAAO,OAAO;AAAA,MAChB;AAAA,IACF,WAAW,SAAS,QAAQ;AAC1B,aAAO,OAAO;AAAA,IAChB,OAAO;AACL,aAAO,OAAO;AAAA,IAChB;AAAA,EACF;AAGA,MAAI,eAAe,QAAW;AAC5B,WAAO,OAAO;AAAA,EAChB;AAEA,MAAI,cAAc,MAAM;AACtB,WAAO,aAAa,OAAO,QAAQ,UAAU,EAAE;AAAA,MAC7C,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM;AACrB,YAAI,GAAG,IAAI,iCAAiC,KAAK;AACjD,eAAO;AAAA,MACT;AAAA,MACA,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,QAAQ,KAAK,IAC9B,MAAM,IAAI,gCAAgC,IAC1C,iCAAiC,KAAK;AAAA,EAC5C;AAEA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,EAC3D;AACA,MAAI,OAAO;AAET,QAAI,MAAM,KAAK,CAAC,WAAW,OAAO,WAAW,aAAY,iCAAQ,UAAS,MAAM,GAAG;AACjF,YAAM,iBAAiB,MAAM;AAAA,QAC3B,CAAC,WAAW,EAAE,OAAO,WAAW,aAAY,iCAAQ,UAAS;AAAA,MAC/D;AAEA,UAAI,eAAe,WAAW,GAAG;AAE/B,cAAM,YAAY;AAAA,UAChB,eAAe,CAAC;AAAA,QAClB;AACA,YAAI,OAAO,cAAc,UAAU;AACjC,iBAAO,WAAW;AAClB,iBAAO,OAAO,QAAQ,SAAS;AAAA,QACjC;AAAA,MACF,OAAO;AAEL,eAAO,QAAQ,eAAe,IAAI,gCAAgC;AAClE,eAAO,WAAW;AAAA,MACpB;AAAA,IACF,OAAO;AACL,aAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,IAC3D;AAAA,EACF;AACA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,EAC3D;AAEA,MAAI,cAAc,QAAW;AAC3B,WAAO,YAAY;AAAA,EACrB;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,YAA4C;AACvE,SACE,cAAc,QACd,OAAO,eAAe,YACtB,WAAW,SAAS,aACnB,WAAW,cAAc,QAAQ,OAAO,KAAK,WAAW,UAAU,EAAE,WAAW,MAChF,CAAC,WAAW;AAEhB;AAEO,SAAS,uBAAuB,SAAiD;AACtF,QAAM,uBAA8C,CAAC;AAErD,aAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AAClD,UAAM,EAAE,aAAa,WAAW,IAAI;AACpC,UAAM,aAAa,kBAAI,aAAa,YAAY,KAAK;AAGrD,UAAM,aAAa,KAAK,MAAM,KAAK,UAAU,UAAU,CAAC;AAExD,yBAAqB,KAAK;AAAA,MACxB;AAAA,MACA;AAAA,MACA,YAAY,iCAAiC,UAAU;AAAA,IACzD,CAAC;AAAA,EACH;AAEA,SAAO;AACT;","names":[]} \ No newline at end of file diff --git a/plugins/google/dist/utils.d.cts b/plugins/google/dist/utils.d.cts deleted file mode 100644 index a4826d8a4..000000000 --- a/plugins/google/dist/utils.d.cts +++ /dev/null @@ -1,14 +0,0 @@ -import type { FunctionDeclaration } from '@google/genai'; -import { llm } from '@livekit/agents'; -import type { JSONSchema7 } from 'json-schema'; -/** - * JSON Schema v7 - * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01 - */ -export type JSONSchema7Definition = JSONSchema7 | boolean; -/** - * Converts JSON Schema 7 to OpenAPI Schema 3.0 - */ -export declare function convertJSONSchemaToOpenAPISchema(jsonSchema: JSONSchema7Definition): unknown; -export declare function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[]; -//# sourceMappingURL=utils.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/utils.d.ts b/plugins/google/dist/utils.d.ts deleted file mode 100644 index a4826d8a4..000000000 --- a/plugins/google/dist/utils.d.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { FunctionDeclaration } from '@google/genai'; -import { llm } from '@livekit/agents'; -import type { JSONSchema7 } from 'json-schema'; -/** - * JSON Schema v7 - * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01 - */ -export type JSONSchema7Definition = JSONSchema7 | boolean; -/** - * Converts JSON Schema 7 to OpenAPI Schema 3.0 - */ -export declare function convertJSONSchemaToOpenAPISchema(jsonSchema: JSONSchema7Definition): unknown; -export declare function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[]; -//# sourceMappingURL=utils.d.ts.map \ No newline at end of file diff --git a/plugins/google/dist/utils.d.ts.map b/plugins/google/dist/utils.d.ts.map deleted file mode 100644 index 9b3bef578..000000000 --- a/plugins/google/dist/utils.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,mBAAmB,EAAU,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,WAAW,GAAG,OAAO,CAAC;AAE1D;;GAEG;AAEH,wBAAgB,gCAAgC,CAAC,UAAU,EAAE,qBAAqB,GAAG,OAAO,CA6G3F;AAYD,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,mBAAmB,EAAE,CAkBtF"} \ No newline at end of file diff --git a/plugins/google/dist/utils.js b/plugins/google/dist/utils.js deleted file mode 100644 index f551ada4d..000000000 --- a/plugins/google/dist/utils.js +++ /dev/null @@ -1,112 +0,0 @@ -import { llm } from "@livekit/agents"; -function convertJSONSchemaToOpenAPISchema(jsonSchema) { - if (jsonSchema == null || isEmptyObjectSchema(jsonSchema)) { - return void 0; - } - if (typeof jsonSchema === "boolean") { - return { type: "boolean", properties: {} }; - } - const { - type, - description, - required, - properties, - items, - allOf, - anyOf, - oneOf, - format, - const: constValue, - minLength, - enum: enumValues - } = jsonSchema; - const result = {}; - if (description) result.description = description; - if (required) result.required = required; - if (format) result.format = format; - if (constValue !== void 0) { - result.enum = [constValue]; - } - if (type) { - if (Array.isArray(type)) { - if (type.includes("null")) { - result.type = type.filter((t) => t !== "null")[0]; - result.nullable = true; - } else { - result.type = type; - } - } else if (type === "null") { - result.type = "null"; - } else { - result.type = type; - } - } - if (enumValues !== void 0) { - result.enum = enumValues; - } - if (properties != null) { - result.properties = Object.entries(properties).reduce( - (acc, [key, value]) => { - acc[key] = convertJSONSchemaToOpenAPISchema(value); - return acc; - }, - {} - ); - } - if (items) { - result.items = Array.isArray(items) ? items.map(convertJSONSchemaToOpenAPISchema) : convertJSONSchemaToOpenAPISchema(items); - } - if (allOf) { - result.allOf = allOf.map(convertJSONSchemaToOpenAPISchema); - } - if (anyOf) { - if (anyOf.some((schema) => typeof schema === "object" && (schema == null ? void 0 : schema.type) === "null")) { - const nonNullSchemas = anyOf.filter( - (schema) => !(typeof schema === "object" && (schema == null ? void 0 : schema.type) === "null") - ); - if (nonNullSchemas.length === 1) { - const converted = convertJSONSchemaToOpenAPISchema( - nonNullSchemas[0] - ); - if (typeof converted === "object") { - result.nullable = true; - Object.assign(result, converted); - } - } else { - result.anyOf = nonNullSchemas.map(convertJSONSchemaToOpenAPISchema); - result.nullable = true; - } - } else { - result.anyOf = anyOf.map(convertJSONSchemaToOpenAPISchema); - } - } - if (oneOf) { - result.oneOf = oneOf.map(convertJSONSchemaToOpenAPISchema); - } - if (minLength !== void 0) { - result.minLength = minLength; - } - return result; -} -function isEmptyObjectSchema(jsonSchema) { - return jsonSchema != null && typeof jsonSchema === "object" && jsonSchema.type === "object" && (jsonSchema.properties == null || Object.keys(jsonSchema.properties).length === 0) && !jsonSchema.additionalProperties; -} -function toFunctionDeclarations(toolCtx) { - const functionDeclarations = []; - for (const [name, tool] of Object.entries(toolCtx)) { - const { description, parameters } = tool; - const jsonSchema = llm.toJsonSchema(parameters, false); - const schemaCopy = JSON.parse(JSON.stringify(jsonSchema)); - functionDeclarations.push({ - name, - description, - parameters: convertJSONSchemaToOpenAPISchema(schemaCopy) - }); - } - return functionDeclarations; -} -export { - convertJSONSchemaToOpenAPISchema, - toFunctionDeclarations -}; -//# sourceMappingURL=utils.js.map \ No newline at end of file diff --git a/plugins/google/dist/utils.js.map b/plugins/google/dist/utils.js.map deleted file mode 100644 index a5ec8f6c1..000000000 --- a/plugins/google/dist/utils.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { FunctionDeclaration, Schema } from '@google/genai';\nimport { llm } from '@livekit/agents';\nimport type { JSONSchema7 } from 'json-schema';\n\n/**\n * JSON Schema v7\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01\n */\nexport type JSONSchema7Definition = JSONSchema7 | boolean;\n\n/**\n * Converts JSON Schema 7 to OpenAPI Schema 3.0\n */\n// Adapted from https://github.com/vercel/ai/blob/main/packages/google/src/convert-json-schema-to-openapi-schema.ts\nexport function convertJSONSchemaToOpenAPISchema(jsonSchema: JSONSchema7Definition): unknown {\n // parameters need to be undefined if they are empty objects:\n if (jsonSchema == null || isEmptyObjectSchema(jsonSchema)) {\n return undefined;\n }\n\n if (typeof jsonSchema === 'boolean') {\n return { type: 'boolean', properties: {} };\n }\n\n const {\n type,\n description,\n required,\n properties,\n items,\n allOf,\n anyOf,\n oneOf,\n format,\n const: constValue,\n minLength,\n enum: enumValues,\n } = jsonSchema;\n\n const result: Record = {};\n\n if (description) result.description = description;\n if (required) result.required = required;\n if (format) result.format = format;\n\n if (constValue !== undefined) {\n result.enum = [constValue];\n }\n\n // Handle type\n if (type) {\n if (Array.isArray(type)) {\n if (type.includes('null')) {\n result.type = type.filter((t) => t !== 'null')[0];\n result.nullable = true;\n } else {\n result.type = type;\n }\n } else if (type === 'null') {\n result.type = 'null';\n } else {\n result.type = type;\n }\n }\n\n // Handle enum\n if (enumValues !== undefined) {\n result.enum = enumValues;\n }\n\n if (properties != null) {\n result.properties = Object.entries(properties).reduce(\n (acc, [key, value]) => {\n acc[key] = convertJSONSchemaToOpenAPISchema(value);\n return acc;\n },\n {} as Record,\n );\n }\n\n if (items) {\n result.items = Array.isArray(items)\n ? items.map(convertJSONSchemaToOpenAPISchema)\n : convertJSONSchemaToOpenAPISchema(items);\n }\n\n if (allOf) {\n result.allOf = allOf.map(convertJSONSchemaToOpenAPISchema);\n }\n if (anyOf) {\n // Handle cases where anyOf includes a null type\n if (anyOf.some((schema) => typeof schema === 'object' && schema?.type === 'null')) {\n const nonNullSchemas = anyOf.filter(\n (schema) => !(typeof schema === 'object' && schema?.type === 'null'),\n );\n\n if (nonNullSchemas.length === 1) {\n // If there's only one non-null schema, convert it and make it nullable\n const converted = convertJSONSchemaToOpenAPISchema(\n nonNullSchemas[0] as JSONSchema7Definition,\n );\n if (typeof converted === 'object') {\n result.nullable = true;\n Object.assign(result, converted);\n }\n } else {\n // If there are multiple non-null schemas, keep them in anyOf\n result.anyOf = nonNullSchemas.map(convertJSONSchemaToOpenAPISchema);\n result.nullable = true;\n }\n } else {\n result.anyOf = anyOf.map(convertJSONSchemaToOpenAPISchema);\n }\n }\n if (oneOf) {\n result.oneOf = oneOf.map(convertJSONSchemaToOpenAPISchema);\n }\n\n if (minLength !== undefined) {\n result.minLength = minLength;\n }\n\n return result;\n}\n\nfunction isEmptyObjectSchema(jsonSchema: JSONSchema7Definition): boolean {\n return (\n jsonSchema != null &&\n typeof jsonSchema === 'object' &&\n jsonSchema.type === 'object' &&\n (jsonSchema.properties == null || Object.keys(jsonSchema.properties).length === 0) &&\n !jsonSchema.additionalProperties\n );\n}\n\nexport function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[] {\n const functionDeclarations: FunctionDeclaration[] = [];\n\n for (const [name, tool] of Object.entries(toolCtx)) {\n const { description, parameters } = tool;\n const jsonSchema = llm.toJsonSchema(parameters, false);\n\n // Create a deep copy to prevent the Google GenAI library from mutating the schema\n const schemaCopy = JSON.parse(JSON.stringify(jsonSchema));\n\n functionDeclarations.push({\n name,\n description,\n parameters: convertJSONSchemaToOpenAPISchema(schemaCopy) as Schema,\n });\n }\n\n return functionDeclarations;\n}\n"],"mappings":"AAIA,SAAS,WAAW;AAab,SAAS,iCAAiC,YAA4C;AAE3F,MAAI,cAAc,QAAQ,oBAAoB,UAAU,GAAG;AACzD,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,eAAe,WAAW;AACnC,WAAO,EAAE,MAAM,WAAW,YAAY,CAAC,EAAE;AAAA,EAC3C;AAEA,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP;AAAA,IACA,MAAM;AAAA,EACR,IAAI;AAEJ,QAAM,SAAkC,CAAC;AAEzC,MAAI,YAAa,QAAO,cAAc;AACtC,MAAI,SAAU,QAAO,WAAW;AAChC,MAAI,OAAQ,QAAO,SAAS;AAE5B,MAAI,eAAe,QAAW;AAC5B,WAAO,OAAO,CAAC,UAAU;AAAA,EAC3B;AAGA,MAAI,MAAM;AACR,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,UAAI,KAAK,SAAS,MAAM,GAAG;AACzB,eAAO,OAAO,KAAK,OAAO,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC;AAChD,eAAO,WAAW;AAAA,MACpB,OAAO;AACL,eAAO,OAAO;AAAA,MAChB;AAAA,IACF,WAAW,SAAS,QAAQ;AAC1B,aAAO,OAAO;AAAA,IAChB,OAAO;AACL,aAAO,OAAO;AAAA,IAChB;AAAA,EACF;AAGA,MAAI,eAAe,QAAW;AAC5B,WAAO,OAAO;AAAA,EAChB;AAEA,MAAI,cAAc,MAAM;AACtB,WAAO,aAAa,OAAO,QAAQ,UAAU,EAAE;AAAA,MAC7C,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM;AACrB,YAAI,GAAG,IAAI,iCAAiC,KAAK;AACjD,eAAO;AAAA,MACT;AAAA,MACA,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,QAAQ,KAAK,IAC9B,MAAM,IAAI,gCAAgC,IAC1C,iCAAiC,KAAK;AAAA,EAC5C;AAEA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,EAC3D;AACA,MAAI,OAAO;AAET,QAAI,MAAM,KAAK,CAAC,WAAW,OAAO,WAAW,aAAY,iCAAQ,UAAS,MAAM,GAAG;AACjF,YAAM,iBAAiB,MAAM;AAAA,QAC3B,CAAC,WAAW,EAAE,OAAO,WAAW,aAAY,iCAAQ,UAAS;AAAA,MAC/D;AAEA,UAAI,eAAe,WAAW,GAAG;AAE/B,cAAM,YAAY;AAAA,UAChB,eAAe,CAAC;AAAA,QAClB;AACA,YAAI,OAAO,cAAc,UAAU;AACjC,iBAAO,WAAW;AAClB,iBAAO,OAAO,QAAQ,SAAS;AAAA,QACjC;AAAA,MACF,OAAO;AAEL,eAAO,QAAQ,eAAe,IAAI,gCAAgC;AAClE,eAAO,WAAW;AAAA,MACpB;AAAA,IACF,OAAO;AACL,aAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,IAC3D;AAAA,EACF;AACA,MAAI,OAAO;AACT,WAAO,QAAQ,MAAM,IAAI,gCAAgC;AAAA,EAC3D;AAEA,MAAI,cAAc,QAAW;AAC3B,WAAO,YAAY;AAAA,EACrB;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,YAA4C;AACvE,SACE,cAAc,QACd,OAAO,eAAe,YACtB,WAAW,SAAS,aACnB,WAAW,cAAc,QAAQ,OAAO,KAAK,WAAW,UAAU,EAAE,WAAW,MAChF,CAAC,WAAW;AAEhB;AAEO,SAAS,uBAAuB,SAAiD;AACtF,QAAM,uBAA8C,CAAC;AAErD,aAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AAClD,UAAM,EAAE,aAAa,WAAW,IAAI;AACpC,UAAM,aAAa,IAAI,aAAa,YAAY,KAAK;AAGrD,UAAM,aAAa,KAAK,MAAM,KAAK,UAAU,UAAU,CAAC;AAExD,yBAAqB,KAAK;AAAA,MACxB;AAAA,MACA;AAAA,MACA,YAAY,iCAAiC,UAAU;AAAA,IACzD,CAAC;AAAA,EACH;AAEA,SAAO;AACT;","names":[]} \ No newline at end of file From 44fbfbf68bcc16c13f8b7bca680544b4b6864985 Mon Sep 17 00:00:00 2001 From: oOPa <5878650+oOPa@users.noreply.github.com> Date: Sun, 21 Dec 2025 19:24:19 +0000 Subject: [PATCH 6/9] rever addtional logging changes --- .../google/src/beta/realtime/realtime_api.ts | 31 ++++--------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index 5d2d3dbe0..e97030b70 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -102,7 +102,6 @@ interface RealtimeOptions { contextWindowCompression?: ContextWindowCompressionConfig; apiVersion?: string; geminiTools?: LLMTools; - thinkingConfig?: types.ThinkingConfig; } /** @@ -448,14 +447,13 @@ export class RealtimeSession extends llm.RealtimeSession { private async closeActiveSession(): Promise { const unlock = await this.sessionLock.lock(); - const session = this.activeSession; - this.activeSession = undefined; - - if (session) { + if (this.activeSession) { try { - await session.close(); + await this.activeSession.close(); } catch (error) { this.#logger.warn({ error }, 'Error closing Gemini session'); + } finally { + this.activeSession = undefined; } } @@ -465,11 +463,7 @@ export class RealtimeSession extends llm.RealtimeSession { private markRestartNeeded(): void { if (!this.sessionShouldClose.isSet) { this.sessionShouldClose.set(); - // Replace the queue so new messages don't go to the old (closing) session. - const oldQueue = this.messageChannel; this.messageChannel = new Queue(); - // Wake up the current sendTask if it's waiting for a message. - oldQueue.put({ type: '__internal_wakeup__' } as any); } } @@ -769,26 +763,14 @@ export class RealtimeSession extends llm.RealtimeSession { this.onReceiveMessage(session, message); }, onerror: (error: ErrorEvent) => { - this.#logger.error({ error }, 'Gemini Live session error'); - sessionOpened.set(); // Break the wait if we were waiting for open + this.#logger.error('Gemini Live session error:', error); if (!this.sessionShouldClose.isSet) { this.markRestartNeeded(); } }, onclose: (event: CloseEvent) => { - if (this.activeSession !== session) { - return; - } - - this.#logger.debug( - { code: event.code, reason: event.reason }, - 'Gemini Live session closed', - ); - sessionOpened.set(); // Break the wait if we were waiting for open + this.#logger.debug('Gemini Live session closed:', event.code, event.reason); this.markCurrentGenerationDone(); - if (!this.sessionShouldClose.isSet && !this.#closed) { - this.markRestartNeeded(); - } }, }, config, @@ -1144,7 +1126,6 @@ export class RealtimeSession extends llm.RealtimeSession { sessionResumption: { handle: this.sessionResumptionHandle, }, - thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined, }; // Add generation fields at TOP LEVEL (NO generationConfig!) From 66eb38787d142dfabfd386feea54aa7f1c4969cd Mon Sep 17 00:00:00 2001 From: oOPa <5878650+oOPa@users.noreply.github.com> Date: Sun, 21 Dec 2025 19:30:01 +0000 Subject: [PATCH 7/9] reintroducing loc that i accidentally removed --- plugins/google/src/beta/realtime/realtime_api.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index e97030b70..ab5aaeabe 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -102,6 +102,7 @@ interface RealtimeOptions { contextWindowCompression?: ContextWindowCompressionConfig; apiVersion?: string; geminiTools?: LLMTools; + thinkingConfig?: types.ThinkingConfig; } /** @@ -1126,6 +1127,7 @@ export class RealtimeSession extends llm.RealtimeSession { sessionResumption: { handle: this.sessionResumptionHandle, }, + thinkingConfig: opts.thinkingConfig ? opts.thinkingConfig : undefined, }; // Add generation fields at TOP LEVEL (NO generationConfig!) From 052f67c49c196399ba5d56e41737b2244bbaad6e Mon Sep 17 00:00:00 2001 From: oOPa <5878650+oOPa@users.noreply.github.com> Date: Sun, 21 Dec 2025 19:51:24 +0000 Subject: [PATCH 8/9] Add changeset --- .changeset/google-thinking-config.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/google-thinking-config.md diff --git a/.changeset/google-thinking-config.md b/.changeset/google-thinking-config.md new file mode 100644 index 000000000..25f9d6450 --- /dev/null +++ b/.changeset/google-thinking-config.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents-plugin-google": minor +--- + +Add thinkingConfig support to Google realtime API and update @google/genai. From 9ce287745124b12c177f90579c1581a7f8371b82 Mon Sep 17 00:00:00 2001 From: oOPa <5878650+oOPa@users.noreply.github.com> Date: Thu, 8 Jan 2026 18:31:00 +0000 Subject: [PATCH 9/9] Change changset from minor to patch --- .changeset/google-thinking-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/google-thinking-config.md b/.changeset/google-thinking-config.md index 25f9d6450..2b7cfffdc 100644 --- a/.changeset/google-thinking-config.md +++ b/.changeset/google-thinking-config.md @@ -1,5 +1,5 @@ --- -"@livekit/agents-plugin-google": minor +"@livekit/agents-plugin-google": patch --- Add thinkingConfig support to Google realtime API and update @google/genai.