diff --git a/src/agents/voice/model.py b/src/agents/voice/model.py index 8ed3c5b62f..3b4a8e85b5 100644 --- a/src/agents/voice/model.py +++ b/src/agents/voice/model.py @@ -2,7 +2,7 @@ import abc from collections.abc import AsyncIterator, Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Literal from typing_extensions import TypedDict @@ -143,6 +143,15 @@ class STTModelSettings: turn_detection: dict[str, Any] | None = None """The turn detection settings for the model when using streamed audio input.""" + languages: list[str] | None = field(default=None, kw_only=True) + """Possible languages of the audio input, expressed as API-supported language codes, when + using streamed audio input. Supported by `gpt-transcribe` and `gpt-live-transcribe`. Takes + precedence over `language`.""" + + keywords: list[str] | None = field(default=None, kw_only=True) + """Words or phrases to guide transcription of the audio input when using streamed audio + input. Supported by `gpt-transcribe` and `gpt-live-transcribe`.""" + class STTModel(abc.ABC): """A speech-to-text model that can convert audio input into text.""" diff --git a/src/agents/voice/models/openai_stt.py b/src/agents/voice/models/openai_stt.py index 3a9edce3c4..089999789f 100644 --- a/src/agents/voice/models/openai_stt.py +++ b/src/agents/voice/models/openai_stt.py @@ -138,6 +138,7 @@ def __init__( self._state_queue: asyncio.Queue[dict[str, Any] | ErrorSentinel] = asyncio.Queue() self._turn_audio_buffer: list[npt.NDArray[np.int16 | np.float32]] = [] self._tracing_span: Span[TranscriptionSpanData] | None = None + self._transcription_config: dict[str, Any] | None = None # tasks self._listener_task: asyncio.Task[Any] | None = None @@ -146,13 +147,37 @@ def __init__( self._connection_task: asyncio.Task[Any] | None = None self._stored_exception: Exception | None = None + def _get_transcription_config(self) -> dict[str, Any]: + transcription_config: dict[str, Any] = {"model": self._model} + if self._settings.languages is not None: + transcription_config["languages"] = list(self._settings.languages) + elif self._settings.language is not None: + if self._model in {"gpt-transcribe", "gpt-live-transcribe"}: + transcription_config["languages"] = [self._settings.language] + else: + transcription_config["language"] = self._settings.language + if self._settings.prompt is not None: + transcription_config["prompt"] = self._settings.prompt + if self._settings.keywords is not None: + transcription_config["keywords"] = list(self._settings.keywords) + return transcription_config + def _start_turn(self) -> None: + # A listener failure can surface a buffered transcript before session.update completes. + # Once configured, every normal turn reuses the exact detached request snapshot. + transcription_config = self._transcription_config or self._get_transcription_config() self._tracing_span = transcription_span( model=self._model, model_config={ "temperature": self._settings.temperature, - "language": self._settings.language, - "prompt": self._settings.prompt, + "language": transcription_config.get("language"), + "languages": transcription_config.get("languages"), + "keywords": ( + transcription_config.get("keywords") + if self._trace_include_sensitive_data + else None + ), + "prompt": transcription_config.get("prompt"), "turn_detection": self._turn_detection, }, ) @@ -201,32 +226,25 @@ async def _event_listener(self) -> None: async def _configure_session(self) -> None: assert self._websocket is not None, "Websocket not initialized" - transcription_config: dict[str, Any] = {"model": self._model} - if self._settings.language is not None: - if self._model in {"gpt-transcribe", "gpt-live-transcribe"}: - transcription_config["languages"] = [self._settings.language] - else: - transcription_config["language"] = self._settings.language - if self._settings.prompt is not None: - transcription_config["prompt"] = self._settings.prompt - - await self._websocket.send( - json.dumps( - { - "type": "session.update", - "session": { - "type": "transcription", - "audio": { - "input": { - "format": {"type": "audio/pcm", "rate": 24000}, - "transcription": transcription_config, - "turn_detection": self._turn_detection, - } - }, + transcription_config = self._get_transcription_config() + session_update = json.dumps( + { + "type": "session.update", + "session": { + "type": "transcription", + "audio": { + "input": { + "format": {"type": "audio/pcm", "rate": 24000}, + "transcription": transcription_config, + "turn_detection": self._turn_detection, + } }, - } - ) + }, + } ) + self._transcription_config = transcription_config + + await self._websocket.send(session_update) async def _setup_connection(self, ws: websockets.ClientConnection) -> None: self._websocket = ws diff --git a/tests/voice/test_openai_stt_session_config.py b/tests/voice/test_openai_stt_session_config.py index 65388f6c7c..246ef83baa 100644 --- a/tests/voice/test_openai_stt_session_config.py +++ b/tests/voice/test_openai_stt_session_config.py @@ -1,5 +1,6 @@ import json -from unittest.mock import AsyncMock +from dataclasses import dataclass, fields +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -7,6 +8,37 @@ from agents.voice.models.openai_stt import OpenAISTTTranscriptionSession +def test_stt_model_settings_appends_streaming_options() -> None: + assert [field.name for field in fields(STTModelSettings)] == [ + "prompt", + "language", + "temperature", + "turn_detection", + "languages", + "keywords", + ] + + +def test_stt_model_settings_preserves_provider_subclass_positional_fields() -> None: + @dataclass + class ProviderSTTModelSettings(STTModelSettings): + provider_language: str | None = None + + settings = ProviderSTTModelSettings( + None, + None, + None, + None, + "provider-ja", + languages=["ja"], + keywords=["Agents SDK"], + ) + + assert settings.provider_language == "provider-ja" + assert settings.languages == ["ja"] + assert settings.keywords == ["Agents SDK"] + + @pytest.mark.asyncio @pytest.mark.parametrize( ("model", "language_field", "language_value"), @@ -59,3 +91,128 @@ async def test_streaming_stt_omits_unset_language_and_prompt() -> None: payload = json.loads(websocket.send.await_args.args[0]) assert payload["session"]["audio"]["input"]["transcription"] == {"model": "gpt-4o-transcribe"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gpt-transcribe", "gpt-live-transcribe"]) +async def test_streaming_stt_sends_languages_over_language(model: str) -> None: + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model=model, + settings=STTModelSettings(language="fr", languages=["fr", "eng", "zh-tw"]), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + + await session._configure_session() + + payload = json.loads(websocket.send.await_args.args[0]) + assert payload["session"]["audio"]["input"]["transcription"] == { + "model": model, + "languages": ["fr", "eng", "zh-tw"], + } + + +@pytest.mark.asyncio +async def test_streaming_stt_sends_keywords() -> None: + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-live-transcribe", + settings=STTModelSettings(keywords=["agents", "sdk"]), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + + await session._configure_session() + + payload = json.loads(websocket.send.await_args.args[0]) + assert payload["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-live-transcribe", + "keywords": ["agents", "sdk"], + } + + +@pytest.mark.asyncio +async def test_streaming_stt_trace_records_transcription_options_with_sensitive_data() -> None: + languages = ["en", "fr"] + keywords = ["Agents SDK"] + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-live-transcribe", + settings=STTModelSettings( + prompt="customer support", + language="en", + temperature=0.2, + languages=languages, + keywords=keywords, + ), + trace_include_sensitive_data=True, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + await session._configure_session() + + languages[:] = ["de"] + keywords[:] = ["Changed after configuration"] + span = MagicMock() + + with patch( + "agents.voice.models.openai_stt.transcription_span", + return_value=span, + ) as create_span: + session._start_turn() + languages[:] = ["it"] + keywords[:] = ["Changed during the turn"] + session._end_turn("") + + create_span.assert_called_once_with( + model="gpt-live-transcribe", + model_config={ + "temperature": 0.2, + "language": None, + "languages": ["en", "fr"], + "keywords": ["Agents SDK"], + "prompt": "customer support", + "turn_detection": {"type": "semantic_vad"}, + }, + ) + span.start.assert_called_once_with() + span.finish.assert_called_once_with() + + +@pytest.mark.asyncio +async def test_streaming_stt_trace_redacts_keywords_without_sensitive_data() -> None: + sensitive_keywords = ["CUSTOMER_SECRET_NAME"] + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-live-transcribe", + settings=STTModelSettings(keywords=sensitive_keywords), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + await session._configure_session() + span = MagicMock() + + with patch( + "agents.voice.models.openai_stt.transcription_span", + return_value=span, + ) as create_span: + session._start_turn() + session._end_turn("") + + model_config = create_span.call_args.kwargs["model_config"] + assert model_config["keywords"] is None + assert all(value is not sensitive_keywords for value in model_config.values()) + span.start.assert_called_once_with() + span.finish.assert_called_once_with()