From fadb9d37dacbae9899511633f34b0c77d7c8b837 Mon Sep 17 00:00:00 2001 From: 8x22b <062340513224@student.polsri.ac.id> Date: Tue, 7 Jul 2026 02:31:36 +0300 Subject: [PATCH] feat(stt): add JSON input_audio request format Add STT_REQUEST_FORMAT env var (multipart default | json). The json format sends base64 audio in an input_audio JSON body instead of multipart form-data, required by providers like OpenRouter. Ports a local STT protocol patch onto the current source. --- .env.example | 4 ++ README.md | 1 + src/app/services/stt-service.ts | 79 ++++++++++++++++++++------ src/config.ts | 24 ++++++++ tests/app/services/stt-service.test.ts | 58 +++++++++++++++++++ 5 files changed, 149 insertions(+), 17 deletions(-) diff --git a/.env.example b/.env.example index b3916907..263c209e 100644 --- a/.env.example +++ b/.env.example @@ -125,6 +125,10 @@ OPENCODE_MODEL_ID=big-pickle # STT_API_KEY= # STT_MODEL= # STT_LANGUAGE= +# STT request format (default: multipart) +# multipart = standard OpenAI/Groq Whisper form-data upload +# json = base64 audio in an `input_audio` JSON body (e.g. OpenRouter) +# STT_REQUEST_FORMAT=multipart # STT_NOTE_PROMPT="The following text is transcribed from voice. It may contain homophone or phonetic errors. Infer the intended meaning from context." # Text-to-Speech credentials (optional) diff --git a/README.md b/README.md index 3edd6ee7..ba03677a 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,7 @@ When installed via npm, the configuration wizard handles the initial setup. The | `STT_API_KEY` | API key for your STT provider | No | — | | `STT_MODEL` | STT model name passed to `/audio/transcriptions` | No | `whisper-large-v3-turbo` | | `STT_LANGUAGE` | Optional language hint (empty = provider auto-detect) | No | — | +| `STT_REQUEST_FORMAT` | STT request format: `multipart` (standard OpenAI/Groq Whisper) or `json` (base64 `input_audio` body, e.g. OpenRouter) | No | `multipart` | | `STT_NOTE_PROMPT` | Optional note prepended to the LLM prompt as `[Note: ...]` for voice transcriptions; empty / `false` / `0` disable it | No | — | | `TTS_PROVIDER` | TTS provider: `openai` for OpenAI-compatible APIs, `elevenlabs` for ElevenLabs, or `google` for Google Cloud TTS | No | `openai` | | `TTS_API_URL` | TTS API base URL for OpenAI-compatible APIs or ElevenLabs | No | — | diff --git a/src/app/services/stt-service.ts b/src/app/services/stt-service.ts index 39796c29..cef93f10 100644 --- a/src/app/services/stt-service.ts +++ b/src/app/services/stt-service.ts @@ -7,6 +7,17 @@ export interface SttResult { text: string; } +const AUDIO_FORMAT_BY_EXTENSION: Record = { + oga: "ogg", + ogg: "ogg", + mp3: "mp3", + wav: "wav", + m4a: "m4a", + flac: "flac", + aac: "aac", + webm: "webm", +}; + /** * Returns true if STT is configured (API URL and API key are set). */ @@ -14,13 +25,20 @@ export function isSttConfigured(): boolean { return Boolean(config.stt.apiUrl && config.stt.apiKey); } +function getAudioFormat(filename: string): string { + const extension = (filename.split(".").pop() || "").toLowerCase(); + return AUDIO_FORMAT_BY_EXTENSION[extension] || "ogg"; +} + /** - * Transcribes an audio buffer using a Whisper-compatible API (OpenAI / Groq / etc.). + * Transcribes an audio buffer using a Whisper-compatible API. * - * Sends a multipart/form-data POST to `{STT_API_URL}/audio/transcriptions`. + * Two request formats are supported via `STT_REQUEST_FORMAT`: + * - `multipart` (default): standard OpenAI/Groq `multipart/form-data` upload. + * - `json`: base64 audio in an `input_audio` JSON body (e.g. OpenRouter). * * @param audioBuffer - Raw audio file bytes (ogg, mp3, wav, m4a, webm, etc.) - * @param filename - Original filename with extension (used by the API to detect format) + * @param filename - Original filename with extension (used to detect format) * @returns Transcribed text * @throws Error if STT is not configured, the request fails, or the response is invalid */ @@ -30,19 +48,48 @@ export async function transcribeAudio(audioBuffer: Buffer, filename: string): Pr } const url = `${config.stt.apiUrl}/audio/transcriptions`; + const useJsonFormat = config.stt.requestFormat === "json"; + + const headers: Record = { + Authorization: `Bearer ${config.stt.apiKey}`, + }; + let body: FormData | string; + + if (useJsonFormat) { + const payload: Record = { + model: config.stt.model, + input_audio: { + data: Buffer.from(audioBuffer).toString("base64"), + format: getAudioFormat(filename), + }, + }; - const formData = new FormData(); - formData.append("file", new Blob([new Uint8Array(audioBuffer)]), filename); - formData.append("model", config.stt.model); - formData.append("response_format", "json"); + if (config.stt.language) { + payload.language = config.stt.language; + } - if (config.stt.language) { - formData.append("language", config.stt.language); - } + headers["Content-Type"] = "application/json"; + body = JSON.stringify(payload); - logger.debug( - `[STT] Sending transcription request: url=${url}, model=${config.stt.model}, filename=${filename}, size=${audioBuffer.length} bytes`, - ); + logger.debug( + `[STT] Sending transcription request (json): url=${url}, model=${config.stt.model}, format=${getAudioFormat(filename)}, size=${audioBuffer.length} bytes`, + ); + } else { + const formData = new FormData(); + formData.append("file", new Blob([new Uint8Array(audioBuffer)]), filename); + formData.append("model", config.stt.model); + formData.append("response_format", "json"); + + if (config.stt.language) { + formData.append("language", config.stt.language); + } + + body = formData; + + logger.debug( + `[STT] Sending transcription request (multipart): url=${url}, model=${config.stt.model}, filename=${filename}, size=${audioBuffer.length} bytes`, + ); + } const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), STT_REQUEST_TIMEOUT_MS); @@ -50,10 +97,8 @@ export async function transcribeAudio(audioBuffer: Buffer, filename: string): Pr try { const response = await fetch(url, { method: "POST", - headers: { - Authorization: `Bearer ${config.stt.apiKey}`, - }, - body: formData, + headers, + body, signal: controller.signal, }); diff --git a/src/config.ts b/src/config.ts index 12051fc6..969435c1 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,6 +7,7 @@ dotenv.config({ path: runtimePaths.envFilePath, quiet: true }); export type MessageFormatMode = "raw" | "markdown"; export type TtsProvider = "openai" | "google" | "elevenlabs" | "edge"; +export type SttRequestFormat = "multipart" | "json"; function getEnvVar(key: string, required: boolean = true): string { const value = process.env[key]; @@ -93,6 +94,26 @@ function getOptionalTtsProviderEnvVar(key: string, defaultValue: TtsProvider): T return defaultValue; } +const VALID_STT_REQUEST_FORMATS: SttRequestFormat[] = ["multipart", "json"]; + +function getOptionalSttRequestFormatEnvVar( + key: string, + defaultValue: SttRequestFormat, +): SttRequestFormat { + const value = getEnvVar(key, false); + + if (!value) { + return defaultValue; + } + + const normalized = value.trim().toLowerCase(); + if (VALID_STT_REQUEST_FORMATS.includes(normalized as SttRequestFormat)) { + return normalized as SttRequestFormat; + } + + return defaultValue; +} + export function buildTelegramConfig(): { token: string; allowedUserId: number; @@ -180,6 +201,9 @@ export const config = { model: getEnvVar("STT_MODEL", false) || "whisper-large-v3-turbo", language: getEnvVar("STT_LANGUAGE", false), notePrompt: getEnvVar("STT_NOTE_PROMPT", false), + // "multipart" (default) = standard OpenAI/Groq Whisper form-data upload. + // "json" = base64 audio in an `input_audio` JSON body (e.g. OpenRouter). + requestFormat: getOptionalSttRequestFormatEnvVar("STT_REQUEST_FORMAT", "multipart"), }, tts: (() => { const provider = getOptionalTtsProviderEnvVar("TTS_PROVIDER", "openai"); diff --git a/tests/app/services/stt-service.test.ts b/tests/app/services/stt-service.test.ts index 9695ee55..3ca1480d 100644 --- a/tests/app/services/stt-service.test.ts +++ b/tests/app/services/stt-service.test.ts @@ -15,6 +15,7 @@ const mockStt = vi.hoisted(() => ({ apiKey: "", model: "whisper-large-v3-turbo", language: "", + requestFormat: "multipart" as "multipart" | "json", })); vi.mock("../../../src/config.js", () => ({ @@ -48,6 +49,7 @@ describe("isSttConfigured", () => { mockStt.apiKey = ""; mockStt.model = "whisper-large-v3-turbo"; mockStt.language = ""; + mockStt.requestFormat = "multipart"; }); it("returns false when both apiUrl and apiKey are empty", () => { @@ -77,6 +79,7 @@ describe("transcribeAudio", () => { mockStt.apiKey = "sk-test-key"; mockStt.model = "whisper-large-v3-turbo"; mockStt.language = ""; + mockStt.requestFormat = "multipart"; vi.restoreAllMocks(); }); @@ -174,4 +177,59 @@ describe("transcribeAudio", () => { "STT API response does not contain a text field", ); }); + + it("sends a base64 input_audio JSON body when requestFormat is json", async () => { + mockStt.requestFormat = "json"; + + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ text: "Hello world" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + ); + + const audioBuffer = Buffer.from("fake-audio-data"); + const result = await transcribeAudio(audioBuffer, "voice.oga"); + + expect(result).toEqual({ text: "Hello world" }); + + expect(fetchSpy).toHaveBeenCalledOnce(); + const [url, options] = fetchSpy.mock.calls[0]; + expect(url).toBe("https://api.groq.com/openai/v1/audio/transcriptions"); + expect(options?.method).toBe("POST"); + const headers = options?.headers as Record; + expect(headers["Authorization"]).toBe("Bearer sk-test-key"); + expect(headers["Content-Type"]).toBe("application/json"); + + const parsedBody = JSON.parse(options?.body as string) as { + model: string; + input_audio: { data: string; format: string }; + language?: string; + }; + expect(parsedBody.model).toBe("whisper-large-v3-turbo"); + expect(parsedBody.input_audio.format).toBe("ogg"); + expect(parsedBody.input_audio.data).toBe(audioBuffer.toString("base64")); + expect(parsedBody.language).toBeUndefined(); + }); + + it("includes language in the JSON body when requestFormat is json and language is set", async () => { + mockStt.requestFormat = "json"; + mockStt.language = "ru"; + + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ text: "Привет" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + ); + + await transcribeAudio(Buffer.from("fake-audio-data"), "voice.mp3"); + + const parsedBody = JSON.parse(fetchSpy.mock.calls[0][1]?.body as string) as { + input_audio: { format: string }; + language: string; + }; + expect(parsedBody.input_audio.format).toBe("mp3"); + expect(parsedBody.language).toBe("ru"); + }); });