diff --git a/src/agents/voice/model.py b/src/agents/voice/model.py index 8ed3c5b62f..ac05e012f0 100644 --- a/src/agents/voice/model.py +++ b/src/agents/voice/model.py @@ -141,7 +141,24 @@ class STTModelSettings: """The temperature of the model.""" turn_detection: dict[str, Any] | None = None - """The turn detection settings for the model when using streamed audio input.""" + """The turn detection settings for the model when using streamed audio input. `None` uses + the default (`{"type": "semantic_vad"}`). Pass `{"type": "none"}` to disable turn detection, + which is sent as `null` and is required by `gpt-realtime-whisper`.""" + + languages: list[str] | None = None + """Possible languages of the audio input, in ISO-639-1 format, when using streamed audio + input. Supported by `gpt-transcribe` and `gpt-live-transcribe`. Takes precedence over + `language`.""" + + keywords: list[str] | None = None + """Words or phrases to guide transcription of the audio input when using streamed audio + input. Supported by `gpt-transcribe` and `gpt-live-transcribe`.""" + + delay: Literal["minimal", "low", "medium", "high", "xhigh"] | None = None + """How long the model waits before emitting transcription text when using streamed audio + input. Higher values can improve transcription accuracy at the cost of latency. Only + supported with `gpt-realtime-whisper`, which also requires disabling turn detection with + `turn_detection={"type": "none"}`.""" class STTModel(abc.ABC): diff --git a/src/agents/voice/models/openai_stt.py b/src/agents/voice/models/openai_stt.py index 3a9edce3c4..8542f60169 100644 --- a/src/agents/voice/models/openai_stt.py +++ b/src/agents/voice/models/openai_stt.py @@ -123,7 +123,9 @@ def __init__( self._client = client self._model = model self._settings = settings - self._turn_detection = settings.turn_detection or DEFAULT_TURN_DETECTION + turn_detection = settings.turn_detection or DEFAULT_TURN_DETECTION + # ``{"type": "none"}`` disables turn detection, which the wire schema spells as ``null``. + self._turn_detection = None if turn_detection.get("type") == "none" else turn_detection self._trace_include_sensitive_data = trace_include_sensitive_data self._trace_include_sensitive_audio_data = trace_include_sensitive_audio_data @@ -152,6 +154,9 @@ def _start_turn(self) -> None: model_config={ "temperature": self._settings.temperature, "language": self._settings.language, + "languages": self._settings.languages, + "keywords": self._settings.keywords, + "delay": self._settings.delay, "prompt": self._settings.prompt, "turn_detection": self._turn_detection, }, @@ -202,13 +207,19 @@ async def _event_listener(self) -> None: async def _configure_session(self) -> None: assert self._websocket is not None, "Websocket not initialized" transcription_config: dict[str, Any] = {"model": self._model} - if self._settings.language is not None: + if self._settings.languages is not None: + transcription_config["languages"] = self._settings.languages + elif self._settings.language is not None: if self._model in {"gpt-transcribe", "gpt-live-transcribe"}: transcription_config["languages"] = [self._settings.language] else: transcription_config["language"] = self._settings.language if self._settings.prompt is not None: transcription_config["prompt"] = self._settings.prompt + if self._settings.keywords is not None: + transcription_config["keywords"] = self._settings.keywords + if self._settings.delay is not None: + transcription_config["delay"] = self._settings.delay await self._websocket.send( json.dumps( @@ -310,9 +321,22 @@ async def _stream_audio( ) -> None: assert self._websocket is not None, "Websocket not initialized" self._start_turn() + appended_audio = False while True: buffer = await audio_queue.get() if buffer is None: + # With turn detection disabled the server never finalizes a turn on its + # own, so commit the appended audio to receive its transcription. + if self._turn_detection is None and appended_audio: + try: + await self._websocket.send( + json.dumps({"type": "input_audio_buffer.commit"}) + ) + except websockets.ConnectionClosed: + pass + except Exception as e: + await self._output_queue.put(ErrorSentinel(e)) + raise break if self._trace_include_sensitive_audio_data: @@ -333,6 +357,8 @@ async def _stream_audio( except Exception as e: await self._output_queue.put(ErrorSentinel(e)) raise + if buffer.size > 0: + appended_audio = True await asyncio.sleep(0) # yield control diff --git a/tests/voice/test_openai_stt_session_config.py b/tests/voice/test_openai_stt_session_config.py index 65388f6c7c..161afb97cd 100644 --- a/tests/voice/test_openai_stt_session_config.py +++ b/tests/voice/test_openai_stt_session_config.py @@ -1,6 +1,8 @@ +import asyncio import json from unittest.mock import AsyncMock +import numpy as np import pytest from agents.voice import StreamedAudioInput, STTModelSettings @@ -59,3 +61,144 @@ async def test_streaming_stt_omits_unset_language_and_prompt() -> None: payload = json.loads(websocket.send.await_args.args[0]) assert payload["session"]["audio"]["input"]["transcription"] == {"model": "gpt-4o-transcribe"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gpt-4o-transcribe", "gpt-transcribe", "gpt-live-transcribe"]) +async def test_streaming_stt_sends_languages_over_language(model: str) -> None: + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model=model, + settings=STTModelSettings(language="fr", languages=["fr", "en"]), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + + await session._configure_session() + + payload = json.loads(websocket.send.await_args.args[0]) + assert payload["session"]["audio"]["input"]["transcription"] == { + "model": model, + "languages": ["fr", "en"], + } + + +@pytest.mark.asyncio +async def test_streaming_stt_sends_keywords_and_delay() -> None: + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-live-transcribe", + settings=STTModelSettings(keywords=["agents", "sdk"], delay="low"), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + + await session._configure_session() + + payload = json.loads(websocket.send.await_args.args[0]) + assert payload["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-live-transcribe", + "keywords": ["agents", "sdk"], + "delay": "low", + } + + +@pytest.mark.asyncio +async def test_streaming_stt_delay_with_disabled_turn_detection() -> None: + """gpt-realtime-whisper requires turn_detection: null alongside the delay option.""" + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-realtime-whisper", + settings=STTModelSettings(delay="high", turn_detection={"type": "none"}), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + + await session._configure_session() + + payload = json.loads(websocket.send.await_args.args[0]) + audio_input = payload["session"]["audio"]["input"] + assert audio_input["transcription"] == {"model": "gpt-realtime-whisper", "delay": "high"} + assert audio_input["turn_detection"] is None + + +@pytest.mark.asyncio +async def test_streaming_stt_default_turn_detection_unchanged() -> None: + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-4o-transcribe", + settings=STTModelSettings(), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + + await session._configure_session() + + payload = json.loads(websocket.send.await_args.args[0]) + assert payload["session"]["audio"]["input"]["turn_detection"] == {"type": "semantic_vad"} + + +def _sent_types(websocket: AsyncMock) -> list[str]: + return [json.loads(call.args[0])["type"] for call in websocket.send.await_args_list] + + +async def _run_stream_audio( + settings: STTModelSettings, + buffers: list[np.ndarray | None], +) -> AsyncMock: + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-realtime-whisper", + settings=settings, + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + queue: asyncio.Queue[np.ndarray | None] = asyncio.Queue() + for buffer in buffers: + queue.put_nowait(buffer) + await session._stream_audio(queue) + session._end_turn("") + return websocket + + +@pytest.mark.asyncio +async def test_streaming_stt_commits_audio_when_turn_detection_disabled() -> None: + """Without server VAD the client has to commit the buffer to finish the turn.""" + websocket = await _run_stream_audio( + STTModelSettings(delay="high", turn_detection={"type": "none"}), + [np.zeros(2400, dtype=np.int16), None], + ) + assert _sent_types(websocket) == ["input_audio_buffer.append", "input_audio_buffer.commit"] + + +@pytest.mark.asyncio +async def test_streaming_stt_does_not_commit_with_default_turn_detection() -> None: + websocket = await _run_stream_audio( + STTModelSettings(), + [np.zeros(2400, dtype=np.int16), None], + ) + assert _sent_types(websocket) == ["input_audio_buffer.append"] + + +@pytest.mark.asyncio +async def test_streaming_stt_does_not_commit_empty_buffer() -> None: + websocket = await _run_stream_audio( + STTModelSettings(turn_detection={"type": "none"}), + [None], + ) + assert _sent_types(websocket) == []