diff --git a/packages/sdk/README.md b/packages/sdk/README.md index a6cd00e..63e775d 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -84,22 +84,12 @@ Options: #### Remote audio -The server's audio track is a passthrough of the audio you publish. The SDK subscribes to it only -when your input stream carries an audio track, so a video-only session never receives (or plays) -audio. Override with `remoteAudio`: - -```ts -const realtimeClient = await client.realtime.connect(stream, { - model, - remoteAudio: false, // never subscribe to the server's audio track - // ... -}); -``` - -- unset (default) — subscribe when the input stream has an audio track. -- `true` — always subscribe to the server's audio track. -- `false` — never subscribe. On iOS this is what keeps a video-only app from showing the microphone - permission prompt, which the WebRTC audio engine triggers as soon as any remote audio plays out. +The server's audio track is the audio you publish, played back in sync with the transformed video +(the model adds latency, so the server delays your audio to match). The SDK subscribes to it only +when your input stream carries an audio track; the remote stream then has both a video and an audio +track, and you should play the remote audio rather than your local microphone so voice and picture +stay in sync. A video-only input stream never subscribes, so the remote stream has no audio track and +iOS does not show the microphone permission prompt that playing out any remote audio would trigger. #### Fast mode diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index 3ff963d..bef2e30 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -109,15 +109,6 @@ const realTimeClientConnectOptionsSchema = z.object({ retries: z.number().int().min(0).optional(), /** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */ preferredVideoCodec: z.enum(["h264", "vp8", "vp9"]).optional(), - /** - * Subscribe to the server's audio track. Defaults to `true` only when the input - * stream carries an audio track: the server's audio is a passthrough of the - * client's, so a video-only session has nothing to hear, and subscribing would - * start the device's audio engine (on iOS that alone shows the microphone - * permission prompt). Set `true` to always receive the remote audio track, - * `false` to never receive it. - */ - remoteAudio: z.boolean().optional(), /** * @deprecated Glass-to-glass measurement now runs automatically in browsers * when LiveKit frame metadata is available. This legacy flag is accepted for @@ -197,7 +188,6 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { speed, retries, preferredVideoCodec, - remoteAudio, } = parsedOptions.data; const mirror = parsedOptions.data.mirror ?? false; @@ -287,7 +277,6 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { connectRetries: retries, logger, videoCodec: preparedConnection.videoCodec, - remoteAudio, createMediaChannel: preparedConnection.createMediaChannel, }); diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index 8f1d4f2..74342b3 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -92,14 +92,6 @@ export interface MediaChannelConfig { logger?: Logger; videoCodec?: VideoCodec; createFrameMetadataWorker?: () => Worker; - /** - * Subscribe to the inference server's audio track. Defaults to whether the - * local stream carries an audio track: the server's audio is a passthrough of - * the client's, so a video-only client gets nothing but a silent track, and - * on iOS merely playing out a remote audio track starts the WebRTC audio - * engine and triggers the microphone permission prompt. - */ - remoteAudio?: boolean; } export type MediaConnectOptions = { @@ -269,9 +261,14 @@ export class LiveKitMediaChannel implements MediaChannel { } } - /** Explicit `remoteAudio` wins; otherwise mirror the local stream (the server's audio is a passthrough of ours). */ + /** + * The server's audio track is our own audio played back, delayed to stay in + * sync with the transformed video, so subscribe to it exactly when we publish + * audio. A video-only client has nothing to hear, and merely playing out a + * remote audio track starts the device's audio engine (on iOS that alone shows + * the microphone permission prompt). + */ private wantsRemoteAudio(): boolean { - if (this.config.remoteAudio !== undefined) return this.config.remoteAudio; return (this.config.localStream?.getAudioTracks().length ?? 0) > 0; } diff --git a/packages/sdk/src/realtime/stream-session.ts b/packages/sdk/src/realtime/stream-session.ts index 33091c8..05ca691 100644 --- a/packages/sdk/src/realtime/stream-session.ts +++ b/packages/sdk/src/realtime/stream-session.ts @@ -124,7 +124,6 @@ interface StreamSessionConfig { connectRetries?: number; logger?: Logger; videoCodec?: VideoCodec; - remoteAudio?: boolean; createMediaChannel: MediaChannelFactory; } @@ -497,7 +496,6 @@ export class StreamSession { localStream: this.config.localStream, logger: this.logger, videoCodec: this.config.videoCodec, - remoteAudio: this.config.remoteAudio, }); this.wireSignalingEvents(); this.wireMediaEvents(); diff --git a/packages/sdk/tests/realtime.unit.test.ts b/packages/sdk/tests/realtime.unit.test.ts index d15ef8f..d274f95 100644 --- a/packages/sdk/tests/realtime.unit.test.ts +++ b/packages/sdk/tests/realtime.unit.test.ts @@ -1717,16 +1717,11 @@ describe("StreamSession startup orchestration", () => { expect(audio.setSubscribed).toHaveBeenCalledWith(true); }); - it("lets remoteAudio override the local-stream default in both directions", async () => { - const muted = createLiveKitMediaChannel({ localStream: createLocalStream(), logger, remoteAudio: false }); - const mutedTracks = await connectWithServer(muted); - expect(mutedTracks.video.setSubscribed).toHaveBeenCalledWith(true); - expect(mutedTracks.audio.setSubscribed).not.toHaveBeenCalled(); - - liveKitMock.roomInstances.length = 0; - const forced = createLiveKitMediaChannel({ localStream: videoOnlyStream(), logger, remoteAudio: true }); - const forcedTracks = await connectWithServer(forced); - expect(forcedTracks.audio.setSubscribed).toHaveBeenCalledWith(true); + it("subscribes to the server's video only when there is no local stream at all", async () => { + const channel = createLiveKitMediaChannel({ localStream: null, logger }); + const { audio, video } = await connectWithServer(channel); + expect(video.setSubscribed).toHaveBeenCalledWith(true); + expect(audio.setSubscribed).not.toHaveBeenCalled(); }); it("re-subscribes after a livekit-internal full reconnect", async () => { @@ -1753,29 +1748,6 @@ describe("StreamSession startup orchestration", () => { room.emit(liveKitMock.RoomEvent.TrackPublished, pub, other); expect(pub.setSubscribed).not.toHaveBeenCalled(); }); - - it("threads remoteAudio from StreamSession into the media channel", async () => { - const { StreamSession } = await import("../src/realtime/stream-session.js"); - const configs: Array<{ remoteAudio?: boolean }> = []; - const session = new StreamSession({ - url: "wss://example.test/realtime", - localStream: createLocalStream(), - remoteAudio: false, - createMediaChannel: (config) => { - configs.push(config); - return createLiveKitMediaChannel(config); - }, - }); - const connectPromise = session.connect(); - const ws = FakeWebSocket.instances[0]; - ws.onopen?.(); - await flushMicrotasks(); - sendRoomInfo(ws); - await flushMicrotasks(); - await expect(connectPromise).resolves.toBeUndefined(); - expect(configs).toEqual([expect.objectContaining({ remoteAudio: false })]); - session.disconnect(); - }); }); it("sends only a lean passthrough join for a bare localStream connect (no set_image bootstrap)", async () => {