diff --git a/backend/python/whisperx/backend.py b/backend/python/whisperx/backend.py index 7318e10b7726..dc62022862b2 100644 --- a/backend/python/whisperx/backend.py +++ b/backend/python/whisperx/backend.py @@ -16,6 +16,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) from grpc_auth import get_auth_interceptors +from transcript_utils import require_diarization_token, seconds_to_nanoseconds @@ -81,6 +82,11 @@ def AudioTranscription(self, request, context): import whisperx from whisperx.diarize import DiarizationPipeline + try: + require_diarization_token(request.diarize, self.hf_token) + except ValueError as err: + context.abort(grpc.StatusCode.FAILED_PRECONDITION, str(err)) + resultSegments = [] text = "" try: @@ -117,8 +123,8 @@ def AudioTranscription(self, request, context): # Build result segments for idx, seg in enumerate(transcript["segments"]): seg_text = seg.get("text", "") - start = int(seg.get("start", 0)) - end = int(seg.get("end", 0)) + start = seconds_to_nanoseconds(seg.get("start", 0)) + end = seconds_to_nanoseconds(seg.get("end", 0)) speaker = seg.get("speaker", "") resultSegments.append(backend_pb2.TranscriptSegment( diff --git a/backend/python/whisperx/test_transcript_utils.py b/backend/python/whisperx/test_transcript_utils.py new file mode 100644 index 000000000000..debe2ea6ecf7 --- /dev/null +++ b/backend/python/whisperx/test_transcript_utils.py @@ -0,0 +1,25 @@ +import unittest + +import transcript_utils + + +class TestTranscriptUtils(unittest.TestCase): + def test_diarization_requires_hugging_face_token(self): + with self.assertRaisesRegex( + ValueError, + "HF_TOKEN is required for WhisperX diarization", + ): + transcript_utils.require_diarization_token(True, None) + + def test_diarization_does_not_require_token_when_disabled(self): + transcript_utils.require_diarization_token(False, None) + + def test_seconds_are_serialized_as_nanoseconds(self): + self.assertEqual( + transcript_utils.seconds_to_nanoseconds(3.25), + 3_250_000_000, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/python/whisperx/transcript_utils.py b/backend/python/whisperx/transcript_utils.py new file mode 100644 index 000000000000..a8ac5751057e --- /dev/null +++ b/backend/python/whisperx/transcript_utils.py @@ -0,0 +1,12 @@ +"""Helpers for WhisperX transcript responses.""" + + +def require_diarization_token(diarize, token): + """Reject diarization when WhisperX cannot load its gated pipeline.""" + if diarize and not token: + raise ValueError("HF_TOKEN is required for WhisperX diarization") + + +def seconds_to_nanoseconds(seconds): + """Convert WhisperX timestamps to the duration unit used by LocalAI.""" + return int(seconds * 1_000_000_000) diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 0312d392a98e..5a5e833cf3a3 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -11,6 +11,7 @@ The transcription endpoint allows to convert audio files to text. The endpoint s - **[whisper.cpp](https://github.com/ggerganov/whisper.cpp)**: A C++ library for audio transcription (default) - **moonshine**: Ultra-fast transcription engine optimized for low-end devices - **faster-whisper**: Fast Whisper implementation with CTranslate2 +- **WhisperX**: Whisper transcription with word alignment and optional speaker diarization. Set `HF_TOKEN` and pass `diarize=true` to load WhisperX's gated pyannote diarization pipeline. - **[parakeet-cpp](https://github.com/mudler/parakeet.cpp)**: A C++/ggml port of NVIDIA NeMo Parakeet (FastConformer TDT/CTC/RNNT/hybrid). Runs quantized GGUFs on CPU or GPU, emits word-level timestamps, and supports cache-aware streaming (the `realtime_eou` model surfaces end-of-utterance events). - **llama-cpp**: Route transcription to any multimodal-audio GGUF model served by the `llama-cpp` backend (e.g. [Qwen3-ASR](https://huggingface.co/ggml-org/Qwen3-ASR-0.6B-GGUF), Voxtral, Qwen2-Audio). Under the hood the request is converted into a chat completion with the audio attached via the model's audio encoder - the same path the upstream llama.cpp server uses. Set `backend: llama-cpp` in the model YAML and point `mmproj` at the matching audio encoder. - **voxtral**: Voxtral-family models served by a dedicated backend @@ -109,7 +110,7 @@ In addition to `file` and `model`, the endpoint accepts the following multipart | `timestamp_granularities[]` | Multi-value form field: `word` and/or `segment`. Honored when the backend produces the requested granularity. | | `response_format` | One of `json` (default for backwards-compat), `verbose_json`, `text`, `srt`, `vtt`, `lrc`. | | `stream` | When `true`, the endpoint emits an SSE stream of `transcript.text.delta` events followed by a final `transcript.text.done` event. | -| `diarize` | LocalAI extension - speaker diarization (whisper.cpp only). | +| `diarize` | LocalAI extension - speaker diarization. WhisperX requires `HF_TOKEN`; requests fail with `FailedPrecondition` when it is missing. | The response body for `verbose_json` includes `text`, `language`, `duration`, and `segments[]` (with `speaker` populated when diarization is enabled).