From e44b6014f1cb25cd197cc3435d7def3a3e51a4ea Mon Sep 17 00:00:00 2001 From: Adir Amsalem Date: Thu, 8 Oct 2026 07:04:57 +0000 Subject: [PATCH] fix(realtime): derive the remote-audio subscription from the local stream; drop the remoteAudio option Stacked on #219. The server's audio track is the client's own audio played back delayed to stay in sync with the transformed video, so the only correct subscription rule is "subscribe exactly when we publish audio", which #219 already uses as the default. The explicit `remoteAudio` override had no caller: `true` would serve a model that emits audio the client never sent (none is served), `false` would serve an app that sends its microphone but does not want the synced voice back. Removing it keeps the fix transparent to apps and avoids a public knob for a hypothetical. The connect-time decision is stable for the life of the session: the SDK's only post-connect change to the local stream is replaceVideoTrack, which preserves the audio tracks. Tests: the two option tests go; a null-localStream case pins the video-only default for sessions without a camera stream. --- packages/sdk/README.md | 22 ++++-------- packages/sdk/src/realtime/client.ts | 11 ------ packages/sdk/src/realtime/media-channel.ts | 17 ++++----- packages/sdk/src/realtime/stream-session.ts | 2 -- packages/sdk/tests/realtime.unit.test.ts | 38 +++------------------ 5 files changed, 18 insertions(+), 72 deletions(-) diff --git a/packages/sdk/README.md b/packages/sdk/README.md index a6cd00e..63e775d 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -84,22 +84,12 @@ Options: #### Remote audio -The server's audio track is a passthrough of the audio you publish. The SDK subscribes to it only -when your input stream carries an audio track, so a video-only session never receives (or plays) -audio. Override with `remoteAudio`: - -```ts -const realtimeClient = await client.realtime.connect(stream, { - model, - remoteAudio: false, // never subscribe to the server's audio track - // ... -}); -``` - -- unset (default) — subscribe when the input stream has an audio track. -- `true` — always subscribe to the server's audio track. -- `false` — never subscribe. On iOS this is what keeps a video-only app from showing the microphone - permission prompt, which the WebRTC audio engine triggers as soon as any remote audio plays out. +The server's audio track is the audio you publish, played back in sync with the transformed video +(the model adds latency, so the server delays your audio to match). The SDK subscribes to it only +when your input stream carries an audio track; the remote stream then has both a video and an audio +track, and you should play the remote audio rather than your local microphone so voice and picture +stay in sync. A video-only input stream never subscribes, so the remote stream has no audio track and +iOS does not show the microphone permission prompt that playing out any remote audio would trigger. #### Fast mode diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index 3ff963d..bef2e30 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -109,15 +109,6 @@ const realTimeClientConnectOptionsSchema = z.object({ retries: z.number().int().min(0).optional(), /** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */ preferredVideoCodec: z.enum(["h264", "vp8", "vp9"]).optional(), - /** - * Subscribe to the server's audio track. Defaults to `true` only when the input - * stream carries an audio track: the server's audio is a passthrough of the - * client's, so a video-only session has nothing to hear, and subscribing would - * start the device's audio engine (on iOS that alone shows the microphone - * permission prompt). Set `true` to always receive the remote audio track, - * `false` to never receive it. - */ - remoteAudio: z.boolean().optional(), /** * @deprecated Glass-to-glass measurement now runs automatically in browsers * when LiveKit frame metadata is available. This legacy flag is accepted for @@ -197,7 +188,6 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { speed, retries, preferredVideoCodec, - remoteAudio, } = parsedOptions.data; const mirror = parsedOptions.data.mirror ?? false; @@ -287,7 +277,6 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { connectRetries: retries, logger, videoCodec: preparedConnection.videoCodec, - remoteAudio, createMediaChannel: preparedConnection.createMediaChannel, }); diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index 8f1d4f2..74342b3 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -92,14 +92,6 @@ export interface MediaChannelConfig { logger?: Logger; videoCodec?: VideoCodec; createFrameMetadataWorker?: () => Worker; - /** - * Subscribe to the inference server's audio track. Defaults to whether the - * local stream carries an audio track: the server's audio is a passthrough of - * the client's, so a video-only client gets nothing but a silent track, and - * on iOS merely playing out a remote audio track starts the WebRTC audio - * engine and triggers the microphone permission prompt. - */ - remoteAudio?: boolean; } export type MediaConnectOptions = { @@ -269,9 +261,14 @@ export class LiveKitMediaChannel implements MediaChannel { } } - /** Explicit `remoteAudio` wins; otherwise mirror the local stream (the server's audio is a passthrough of ours). */ + /** + * The server's audio track is our own audio played back, delayed to stay in + * sync with the transformed video, so subscribe to it exactly when we publish + * audio. A video-only client has nothing to hear, and merely playing out a + * remote audio track starts the device's audio engine (on iOS that alone shows + * the microphone permission prompt). + */ private wantsRemoteAudio(): boolean { - if (this.config.remoteAudio !== undefined) return this.config.remoteAudio; return (this.config.localStream?.getAudioTracks().length ?? 0) > 0; } diff --git a/packages/sdk/src/realtime/stream-session.ts b/packages/sdk/src/realtime/stream-session.ts index 33091c8..05ca691 100644 --- a/packages/sdk/src/realtime/stream-session.ts +++ b/packages/sdk/src/realtime/stream-session.ts @@ -124,7 +124,6 @@ interface StreamSessionConfig { connectRetries?: number; logger?: Logger; videoCodec?: VideoCodec; - remoteAudio?: boolean; createMediaChannel: MediaChannelFactory; } @@ -497,7 +496,6 @@ export class StreamSession { localStream: this.config.localStream, logger: this.logger, videoCodec: this.config.videoCodec, - remoteAudio: this.config.remoteAudio, }); this.wireSignalingEvents(); this.wireMediaEvents(); diff --git a/packages/sdk/tests/realtime.unit.test.ts b/packages/sdk/tests/realtime.unit.test.ts index d15ef8f..d274f95 100644 --- a/packages/sdk/tests/realtime.unit.test.ts +++ b/packages/sdk/tests/realtime.unit.test.ts @@ -1717,16 +1717,11 @@ describe("StreamSession startup orchestration", () => { expect(audio.setSubscribed).toHaveBeenCalledWith(true); }); - it("lets remoteAudio override the local-stream default in both directions", async () => { - const muted = createLiveKitMediaChannel({ localStream: createLocalStream(), logger, remoteAudio: false }); - const mutedTracks = await connectWithServer(muted); - expect(mutedTracks.video.setSubscribed).toHaveBeenCalledWith(true); - expect(mutedTracks.audio.setSubscribed).not.toHaveBeenCalled(); - - liveKitMock.roomInstances.length = 0; - const forced = createLiveKitMediaChannel({ localStream: videoOnlyStream(), logger, remoteAudio: true }); - const forcedTracks = await connectWithServer(forced); - expect(forcedTracks.audio.setSubscribed).toHaveBeenCalledWith(true); + it("subscribes to the server's video only when there is no local stream at all", async () => { + const channel = createLiveKitMediaChannel({ localStream: null, logger }); + const { audio, video } = await connectWithServer(channel); + expect(video.setSubscribed).toHaveBeenCalledWith(true); + expect(audio.setSubscribed).not.toHaveBeenCalled(); }); it("re-subscribes after a livekit-internal full reconnect", async () => { @@ -1753,29 +1748,6 @@ describe("StreamSession startup orchestration", () => { room.emit(liveKitMock.RoomEvent.TrackPublished, pub, other); expect(pub.setSubscribed).not.toHaveBeenCalled(); }); - - it("threads remoteAudio from StreamSession into the media channel", async () => { - const { StreamSession } = await import("../src/realtime/stream-session.js"); - const configs: Array<{ remoteAudio?: boolean }> = []; - const session = new StreamSession({ - url: "wss://example.test/realtime", - localStream: createLocalStream(), - remoteAudio: false, - createMediaChannel: (config) => { - configs.push(config); - return createLiveKitMediaChannel(config); - }, - }); - const connectPromise = session.connect(); - const ws = FakeWebSocket.instances[0]; - ws.onopen?.(); - await flushMicrotasks(); - sendRoomInfo(ws); - await flushMicrotasks(); - await expect(connectPromise).resolves.toBeUndefined(); - expect(configs).toEqual([expect.objectContaining({ remoteAudio: false })]); - session.disconnect(); - }); }); it("sends only a lean passthrough join for a bare localStream connect (no set_image bootstrap)", async () => {