feat(voice): expose GA streamed transcription options

src/agents/voice/model.py

@@ -141,7 +141,24 @@ class STTModelSettings:
     """The temperature of the model."""
 
     turn_detection: dict[str, Any] | None = None
-    """The turn detection settings for the model when using streamed audio input."""
+    """The turn detection settings for the model when using streamed audio input. `None` uses
+    the default (`{"type": "semantic_vad"}`). Pass `{"type": "none"}` to disable turn detection,
+    which is sent as `null` and is required by `gpt-realtime-whisper`."""
+
+    languages: list[str] | None = None
+    """Possible languages of the audio input, in ISO-639-1 format, when using streamed audio
+    input. Supported by `gpt-transcribe` and `gpt-live-transcribe`. Takes precedence over
+    `language`."""
+
+    keywords: list[str] | None = None
+    """Words or phrases to guide transcription of the audio input when using streamed audio
+    input. Supported by `gpt-transcribe` and `gpt-live-transcribe`."""
+
+    delay: Literal["minimal", "low", "medium", "high", "xhigh"] | None = None
+    """How long the model waits before emitting transcription text when using streamed audio
+    input. Higher values can improve transcription accuracy at the cost of latency. Only
+    supported with `gpt-realtime-whisper`, which also requires disabling turn detection with
+    `turn_detection={"type": "none"}`."""
 
 
 class STTModel(abc.ABC):

src/agents/voice/models/openai_stt.py

@@ -123,7 +123,9 @@ def __init__(
         self._client = client
         self._model = model
         self._settings = settings
-        self._turn_detection = settings.turn_detection or DEFAULT_TURN_DETECTION
+        turn_detection = settings.turn_detection or DEFAULT_TURN_DETECTION
+        # ``{"type": "none"}`` disables turn detection, which the wire schema spells as ``null``.
+        self._turn_detection = None if turn_detection.get("type") == "none" else turn_detection
         self._trace_include_sensitive_data = trace_include_sensitive_data
         self._trace_include_sensitive_audio_data = trace_include_sensitive_audio_data
 
@@ -152,6 +154,9 @@ def _start_turn(self) -> None:
             model_config={
                 "temperature": self._settings.temperature,
                 "language": self._settings.language,
+                "languages": self._settings.languages,
+                "keywords": self._settings.keywords,
+                "delay": self._settings.delay,
                 "prompt": self._settings.prompt,
                 "turn_detection": self._turn_detection,
             },
@@ -202,13 +207,19 @@ async def _event_listener(self) -> None:
     async def _configure_session(self) -> None:
         assert self._websocket is not None, "Websocket not initialized"
         transcription_config: dict[str, Any] = {"model": self._model}
-        if self._settings.language is not None:
+        if self._settings.languages is not None:
+            transcription_config["languages"] = self._settings.languages
+        elif self._settings.language is not None:
             if self._model in {"gpt-transcribe", "gpt-live-transcribe"}:
                 transcription_config["languages"] = [self._settings.language]
             else:
                 transcription_config["language"] = self._settings.language
         if self._settings.prompt is not None:
             transcription_config["prompt"] = self._settings.prompt
+        if self._settings.keywords is not None:
+            transcription_config["keywords"] = self._settings.keywords
+        if self._settings.delay is not None:
+            transcription_config["delay"] = self._settings.delay
 
         await self._websocket.send(
             json.dumps(
@@ -310,9 +321,22 @@ async def _stream_audio(
     ) -> None:
         assert self._websocket is not None, "Websocket not initialized"
         self._start_turn()
+        appended_audio = False
         while True:
             buffer = await audio_queue.get()
             if buffer is None:
+                # With turn detection disabled the server never finalizes a turn on its
+                # own, so commit the appended audio to receive its transcription.
+                if self._turn_detection is None and appended_audio:
+                    try:
+                        await self._websocket.send(
+                            json.dumps({"type": "input_audio_buffer.commit"})
+                        )
+                    except websockets.ConnectionClosed:
+                        pass
+                    except Exception as e:
+                        await self._output_queue.put(ErrorSentinel(e))
+                        raise
                 break
 
             if self._trace_include_sensitive_audio_data:
@@ -333,6 +357,8 @@ async def _stream_audio(
             except Exception as e:
                 await self._output_queue.put(ErrorSentinel(e))
                 raise
+            if buffer.size > 0:
+                appended_audio = True
 
             await asyncio.sleep(0)  # yield control
 

tests/voice/test_openai_stt_session_config.py

@@ -1,6 +1,8 @@
+import asyncio
 import json
 from unittest.mock import AsyncMock
 
+import numpy as np
 import pytest
 
 from agents.voice import StreamedAudioInput, STTModelSettings
@@ -59,3 +61,144 @@ async def test_streaming_stt_omits_unset_language_and_prompt() -> None:
 
     payload = json.loads(websocket.send.await_args.args[0])
     assert payload["session"]["audio"]["input"]["transcription"] == {"model": "gpt-4o-transcribe"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("model", ["gpt-4o-transcribe", "gpt-transcribe", "gpt-live-transcribe"])
+async def test_streaming_stt_sends_languages_over_language(model: str) -> None:
+    session = OpenAISTTTranscriptionSession(
+        input=StreamedAudioInput(),
+        client=AsyncMock(api_key="FAKE_KEY"),
+        model=model,
+        settings=STTModelSettings(language="fr", languages=["fr", "en"]),
+        trace_include_sensitive_data=False,
+        trace_include_sensitive_audio_data=False,
+    )
+    websocket = AsyncMock()
+    session._websocket = websocket
+
+    await session._configure_session()
+
+    payload = json.loads(websocket.send.await_args.args[0])
+    assert payload["session"]["audio"]["input"]["transcription"] == {
+        "model": model,
+        "languages": ["fr", "en"],
+    }
+
+
+@pytest.mark.asyncio
+async def test_streaming_stt_sends_keywords_and_delay() -> None:
+    session = OpenAISTTTranscriptionSession(
+        input=StreamedAudioInput(),
+        client=AsyncMock(api_key="FAKE_KEY"),
+        model="gpt-live-transcribe",
+        settings=STTModelSettings(keywords=["agents", "sdk"], delay="low"),
+        trace_include_sensitive_data=False,
+        trace_include_sensitive_audio_data=False,
+    )
+    websocket = AsyncMock()
+    session._websocket = websocket
+
+    await session._configure_session()
+
+    payload = json.loads(websocket.send.await_args.args[0])
+    assert payload["session"]["audio"]["input"]["transcription"] == {
+        "model": "gpt-live-transcribe",
+        "keywords": ["agents", "sdk"],
+        "delay": "low",
+    }
+
+
+@pytest.mark.asyncio
+async def test_streaming_stt_delay_with_disabled_turn_detection() -> None:
+    """gpt-realtime-whisper requires turn_detection: null alongside the delay option."""
+    session = OpenAISTTTranscriptionSession(
+        input=StreamedAudioInput(),
+        client=AsyncMock(api_key="FAKE_KEY"),
+        model="gpt-realtime-whisper",
+        settings=STTModelSettings(delay="high", turn_detection={"type": "none"}),
+        trace_include_sensitive_data=False,
+        trace_include_sensitive_audio_data=False,
+    )
+    websocket = AsyncMock()
+    session._websocket = websocket
+
+    await session._configure_session()
+
+    payload = json.loads(websocket.send.await_args.args[0])
+    audio_input = payload["session"]["audio"]["input"]
+    assert audio_input["transcription"] == {"model": "gpt-realtime-whisper", "delay": "high"}
+    assert audio_input["turn_detection"] is None
+
+
+@pytest.mark.asyncio
+async def test_streaming_stt_default_turn_detection_unchanged() -> None:
+    session = OpenAISTTTranscriptionSession(
+        input=StreamedAudioInput(),
+        client=AsyncMock(api_key="FAKE_KEY"),
+        model="gpt-4o-transcribe",
+        settings=STTModelSettings(),
+        trace_include_sensitive_data=False,
+        trace_include_sensitive_audio_data=False,
+    )
+    websocket = AsyncMock()
+    session._websocket = websocket
+
+    await session._configure_session()
+
+    payload = json.loads(websocket.send.await_args.args[0])
+    assert payload["session"]["audio"]["input"]["turn_detection"] == {"type": "semantic_vad"}
+
+
+def _sent_types(websocket: AsyncMock) -> list[str]:
+    return [json.loads(call.args[0])["type"] for call in websocket.send.await_args_list]
+
+
+async def _run_stream_audio(
+    settings: STTModelSettings,
+    buffers: list[np.ndarray | None],
+) -> AsyncMock:
+    session = OpenAISTTTranscriptionSession(
+        input=StreamedAudioInput(),
+        client=AsyncMock(api_key="FAKE_KEY"),
+        model="gpt-realtime-whisper",
+        settings=settings,
+        trace_include_sensitive_data=False,
+        trace_include_sensitive_audio_data=False,
+    )
+    websocket = AsyncMock()
+    session._websocket = websocket
+    queue: asyncio.Queue[np.ndarray | None] = asyncio.Queue()
+    for buffer in buffers:
+        queue.put_nowait(buffer)
+    await session._stream_audio(queue)
+    session._end_turn("")
+    return websocket
+
+
+@pytest.mark.asyncio
+async def test_streaming_stt_commits_audio_when_turn_detection_disabled() -> None:
+    """Without server VAD the client has to commit the buffer to finish the turn."""
+    websocket = await _run_stream_audio(
+        STTModelSettings(delay="high", turn_detection={"type": "none"}),
+        [np.zeros(2400, dtype=np.int16), None],
+    )
+    assert _sent_types(websocket) == ["input_audio_buffer.append", "input_audio_buffer.commit"]
+
+
+@pytest.mark.asyncio
+async def test_streaming_stt_does_not_commit_with_default_turn_detection() -> None:
+    websocket = await _run_stream_audio(
+        STTModelSettings(),
+        [np.zeros(2400, dtype=np.int16), None],
+    )
+    assert _sent_types(websocket) == ["input_audio_buffer.append"]
+
+
+@pytest.mark.asyncio
+async def test_streaming_stt_does_not_commit_empty_buffer() -> None:
+    websocket = await _run_stream_audio(
+        STTModelSettings(turn_detection={"type": "none"}),
+        [None],
+    )
+    assert _sent_types(websocket) == []