From 8cf8824ae58d3918e267a3b3725bfe7e96272568 Mon Sep 17 00:00:00 2001 From: abhay-codes07 Date: Mon, 24 Aug 2026 13:31:43 +0530 Subject: [PATCH 1/3] feat(voice): expose GA streamed transcription options Add languages, keywords, and delay to STTModelSettings and forward them in the streamed STT session configuration. These GA transcription options exist on the wire schema but had no SDK surface: languages and keywords guide gpt-transcribe / gpt-live-transcribe, and delay controls transcription latency for gpt-realtime-whisper. An explicit languages list takes precedence over the single language setting, which otherwise keeps its existing per-model spelling. The transcription span's model_config records the new options alongside the existing ones. All three fields are optional and appended after the existing fields, so the released dataclass contract is unchanged. --- src/agents/voice/model.py | 14 ++++++ src/agents/voice/models/openai_stt.py | 11 ++++- tests/voice/test_openai_stt_session_config.py | 46 +++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/agents/voice/model.py b/src/agents/voice/model.py index 8ed3c5b62f..9ec7240371 100644 --- a/src/agents/voice/model.py +++ b/src/agents/voice/model.py @@ -143,6 +143,20 @@ class STTModelSettings: turn_detection: dict[str, Any] | None = None """The turn detection settings for the model when using streamed audio input.""" + languages: list[str] | None = None + """Possible languages of the audio input, in ISO-639-1 format, when using streamed audio + input. Supported by `gpt-transcribe` and `gpt-live-transcribe`. Takes precedence over + `language`.""" + + keywords: list[str] | None = None + """Words or phrases to guide transcription of the audio input when using streamed audio + input. Supported by `gpt-transcribe` and `gpt-live-transcribe`.""" + + delay: Literal["minimal", "low", "medium", "high", "xhigh"] | None = None + """How long the model waits before emitting transcription text when using streamed audio + input. Higher values can improve transcription accuracy at the cost of latency. Only + supported with `gpt-realtime-whisper`.""" + class STTModel(abc.ABC): """A speech-to-text model that can convert audio input into text.""" diff --git a/src/agents/voice/models/openai_stt.py b/src/agents/voice/models/openai_stt.py index 3a9edce3c4..ef06225a0f 100644 --- a/src/agents/voice/models/openai_stt.py +++ b/src/agents/voice/models/openai_stt.py @@ -152,6 +152,9 @@ def _start_turn(self) -> None: model_config={ "temperature": self._settings.temperature, "language": self._settings.language, + "languages": self._settings.languages, + "keywords": self._settings.keywords, + "delay": self._settings.delay, "prompt": self._settings.prompt, "turn_detection": self._turn_detection, }, @@ -202,13 +205,19 @@ async def _event_listener(self) -> None: async def _configure_session(self) -> None: assert self._websocket is not None, "Websocket not initialized" transcription_config: dict[str, Any] = {"model": self._model} - if self._settings.language is not None: + if self._settings.languages is not None: + transcription_config["languages"] = self._settings.languages + elif self._settings.language is not None: if self._model in {"gpt-transcribe", "gpt-live-transcribe"}: transcription_config["languages"] = [self._settings.language] else: transcription_config["language"] = self._settings.language if self._settings.prompt is not None: transcription_config["prompt"] = self._settings.prompt + if self._settings.keywords is not None: + transcription_config["keywords"] = self._settings.keywords + if self._settings.delay is not None: + transcription_config["delay"] = self._settings.delay await self._websocket.send( json.dumps( diff --git a/tests/voice/test_openai_stt_session_config.py b/tests/voice/test_openai_stt_session_config.py index 65388f6c7c..e8298e3959 100644 --- a/tests/voice/test_openai_stt_session_config.py +++ b/tests/voice/test_openai_stt_session_config.py @@ -59,3 +59,49 @@ async def test_streaming_stt_omits_unset_language_and_prompt() -> None: payload = json.loads(websocket.send.await_args.args[0]) assert payload["session"]["audio"]["input"]["transcription"] == {"model": "gpt-4o-transcribe"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gpt-4o-transcribe", "gpt-transcribe", "gpt-live-transcribe"]) +async def test_streaming_stt_sends_languages_over_language(model: str) -> None: + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model=model, + settings=STTModelSettings(language="fr", languages=["fr", "en"]), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + + await session._configure_session() + + payload = json.loads(websocket.send.await_args.args[0]) + assert payload["session"]["audio"]["input"]["transcription"] == { + "model": model, + "languages": ["fr", "en"], + } + + +@pytest.mark.asyncio +async def test_streaming_stt_sends_keywords_and_delay() -> None: + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-live-transcribe", + settings=STTModelSettings(keywords=["agents", "sdk"], delay="low"), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + + await session._configure_session() + + payload = json.loads(websocket.send.await_args.args[0]) + assert payload["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-live-transcribe", + "keywords": ["agents", "sdk"], + "delay": "low", + } From ebdc1010b307156caff40a5dbc11d9604180f6a5 Mon Sep 17 00:00:00 2001 From: abhay-codes07 Date: Mon, 24 Aug 2026 18:41:15 +0530 Subject: [PATCH 2/3] fix(voice): allow disabling turn detection for gpt-realtime-whisper The delay option targets gpt-realtime-whisper, which requires turn_detection: null in the session configuration, but None in STTModelSettings.turn_detection falls back to the semantic_vad default so null was unreachable. Accept {"type": "none"} as an explicit disable marker, sent as null, mirroring the realtime module's None-disables contract while keeping the released None-means-default behavior. Cross-reference the requirement from the delay docstring. --- src/agents/voice/model.py | 7 +++- src/agents/voice/models/openai_stt.py | 4 +- tests/voice/test_openai_stt_session_config.py | 41 +++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/agents/voice/model.py b/src/agents/voice/model.py index 9ec7240371..ac05e012f0 100644 --- a/src/agents/voice/model.py +++ b/src/agents/voice/model.py @@ -141,7 +141,9 @@ class STTModelSettings: """The temperature of the model.""" turn_detection: dict[str, Any] | None = None - """The turn detection settings for the model when using streamed audio input.""" + """The turn detection settings for the model when using streamed audio input. `None` uses + the default (`{"type": "semantic_vad"}`). Pass `{"type": "none"}` to disable turn detection, + which is sent as `null` and is required by `gpt-realtime-whisper`.""" languages: list[str] | None = None """Possible languages of the audio input, in ISO-639-1 format, when using streamed audio @@ -155,7 +157,8 @@ class STTModelSettings: delay: Literal["minimal", "low", "medium", "high", "xhigh"] | None = None """How long the model waits before emitting transcription text when using streamed audio input. Higher values can improve transcription accuracy at the cost of latency. Only - supported with `gpt-realtime-whisper`.""" + supported with `gpt-realtime-whisper`, which also requires disabling turn detection with + `turn_detection={"type": "none"}`.""" class STTModel(abc.ABC): diff --git a/src/agents/voice/models/openai_stt.py b/src/agents/voice/models/openai_stt.py index ef06225a0f..a77ca1610b 100644 --- a/src/agents/voice/models/openai_stt.py +++ b/src/agents/voice/models/openai_stt.py @@ -123,7 +123,9 @@ def __init__( self._client = client self._model = model self._settings = settings - self._turn_detection = settings.turn_detection or DEFAULT_TURN_DETECTION + turn_detection = settings.turn_detection or DEFAULT_TURN_DETECTION + # ``{"type": "none"}`` disables turn detection, which the wire schema spells as ``null``. + self._turn_detection = None if turn_detection.get("type") == "none" else turn_detection self._trace_include_sensitive_data = trace_include_sensitive_data self._trace_include_sensitive_audio_data = trace_include_sensitive_audio_data diff --git a/tests/voice/test_openai_stt_session_config.py b/tests/voice/test_openai_stt_session_config.py index e8298e3959..9181548ad3 100644 --- a/tests/voice/test_openai_stt_session_config.py +++ b/tests/voice/test_openai_stt_session_config.py @@ -105,3 +105,44 @@ async def test_streaming_stt_sends_keywords_and_delay() -> None: "keywords": ["agents", "sdk"], "delay": "low", } + + +@pytest.mark.asyncio +async def test_streaming_stt_delay_with_disabled_turn_detection() -> None: + """gpt-realtime-whisper requires turn_detection: null alongside the delay option.""" + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-realtime-whisper", + settings=STTModelSettings(delay="high", turn_detection={"type": "none"}), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + + await session._configure_session() + + payload = json.loads(websocket.send.await_args.args[0]) + audio_input = payload["session"]["audio"]["input"] + assert audio_input["transcription"] == {"model": "gpt-realtime-whisper", "delay": "high"} + assert audio_input["turn_detection"] is None + + +@pytest.mark.asyncio +async def test_streaming_stt_default_turn_detection_unchanged() -> None: + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-4o-transcribe", + settings=STTModelSettings(), + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + + await session._configure_session() + + payload = json.loads(websocket.send.await_args.args[0]) + assert payload["session"]["audio"]["input"]["turn_detection"] == {"type": "semantic_vad"} From 4f83d55ed562a96516cadb7db9308a6e66ab70dc Mon Sep 17 00:00:00 2001 From: abhay-codes07 Date: Mon, 24 Aug 2026 19:44:51 +0530 Subject: [PATCH 3/3] fix(voice): commit streamed audio when turn detection is disabled With turn_detection: null the server never finalizes a turn on its own, so a session that only appends audio ends without ever producing a transcript. Send input_audio_buffer.commit for the appended audio when the input stream ends and turn detection is disabled. Sessions with server-side VAD are unchanged, and nothing is committed when no audio was appended so an empty buffer cannot be finalized. --- src/agents/voice/models/openai_stt.py | 15 +++++ tests/voice/test_openai_stt_session_config.py | 56 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/src/agents/voice/models/openai_stt.py b/src/agents/voice/models/openai_stt.py index a77ca1610b..8542f60169 100644 --- a/src/agents/voice/models/openai_stt.py +++ b/src/agents/voice/models/openai_stt.py @@ -321,9 +321,22 @@ async def _stream_audio( ) -> None: assert self._websocket is not None, "Websocket not initialized" self._start_turn() + appended_audio = False while True: buffer = await audio_queue.get() if buffer is None: + # With turn detection disabled the server never finalizes a turn on its + # own, so commit the appended audio to receive its transcription. + if self._turn_detection is None and appended_audio: + try: + await self._websocket.send( + json.dumps({"type": "input_audio_buffer.commit"}) + ) + except websockets.ConnectionClosed: + pass + except Exception as e: + await self._output_queue.put(ErrorSentinel(e)) + raise break if self._trace_include_sensitive_audio_data: @@ -344,6 +357,8 @@ async def _stream_audio( except Exception as e: await self._output_queue.put(ErrorSentinel(e)) raise + if buffer.size > 0: + appended_audio = True await asyncio.sleep(0) # yield control diff --git a/tests/voice/test_openai_stt_session_config.py b/tests/voice/test_openai_stt_session_config.py index 9181548ad3..161afb97cd 100644 --- a/tests/voice/test_openai_stt_session_config.py +++ b/tests/voice/test_openai_stt_session_config.py @@ -1,6 +1,8 @@ +import asyncio import json from unittest.mock import AsyncMock +import numpy as np import pytest from agents.voice import StreamedAudioInput, STTModelSettings @@ -146,3 +148,57 @@ async def test_streaming_stt_default_turn_detection_unchanged() -> None: payload = json.loads(websocket.send.await_args.args[0]) assert payload["session"]["audio"]["input"]["turn_detection"] == {"type": "semantic_vad"} + + +def _sent_types(websocket: AsyncMock) -> list[str]: + return [json.loads(call.args[0])["type"] for call in websocket.send.await_args_list] + + +async def _run_stream_audio( + settings: STTModelSettings, + buffers: list[np.ndarray | None], +) -> AsyncMock: + session = OpenAISTTTranscriptionSession( + input=StreamedAudioInput(), + client=AsyncMock(api_key="FAKE_KEY"), + model="gpt-realtime-whisper", + settings=settings, + trace_include_sensitive_data=False, + trace_include_sensitive_audio_data=False, + ) + websocket = AsyncMock() + session._websocket = websocket + queue: asyncio.Queue[np.ndarray | None] = asyncio.Queue() + for buffer in buffers: + queue.put_nowait(buffer) + await session._stream_audio(queue) + session._end_turn("") + return websocket + + +@pytest.mark.asyncio +async def test_streaming_stt_commits_audio_when_turn_detection_disabled() -> None: + """Without server VAD the client has to commit the buffer to finish the turn.""" + websocket = await _run_stream_audio( + STTModelSettings(delay="high", turn_detection={"type": "none"}), + [np.zeros(2400, dtype=np.int16), None], + ) + assert _sent_types(websocket) == ["input_audio_buffer.append", "input_audio_buffer.commit"] + + +@pytest.mark.asyncio +async def test_streaming_stt_does_not_commit_with_default_turn_detection() -> None: + websocket = await _run_stream_audio( + STTModelSettings(), + [np.zeros(2400, dtype=np.int16), None], + ) + assert _sent_types(websocket) == ["input_audio_buffer.append"] + + +@pytest.mark.asyncio +async def test_streaming_stt_does_not_commit_empty_buffer() -> None: + websocket = await _run_stream_audio( + STTModelSettings(turn_detection={"type": "none"}), + [None], + ) + assert _sent_types(websocket) == []