fix(realtime): derive the remote-audio subscription from the local stream; drop the remoteAudio option

packages/sdk/README.md

@@ -84,22 +84,12 @@ Options:
 
 #### Remote audio
 
-The server's audio track is a passthrough of the audio you publish. The SDK subscribes to it only
-when your input stream carries an audio track, so a video-only session never receives (or plays)
-audio. Override with `remoteAudio`:
-
-```ts
-const realtimeClient = await client.realtime.connect(stream, {
-  model,
-  remoteAudio: false, // never subscribe to the server's audio track
-  // ...
-});
-```
-
-- unset (default) — subscribe when the input stream has an audio track.
-- `true` — always subscribe to the server's audio track.
-- `false` — never subscribe. On iOS this is what keeps a video-only app from showing the microphone
-  permission prompt, which the WebRTC audio engine triggers as soon as any remote audio plays out.
+The server's audio track is the audio you publish, played back in sync with the transformed video
+(the model adds latency, so the server delays your audio to match). The SDK subscribes to it only
+when your input stream carries an audio track; the remote stream then has both a video and an audio
+track, and you should play the remote audio rather than your local microphone so voice and picture
+stay in sync. A video-only input stream never subscribes, so the remote stream has no audio track and
+iOS does not show the microphone permission prompt that playing out any remote audio would trigger.
 
 #### Fast mode
 

packages/sdk/src/realtime/client.ts

@@ -109,15 +109,6 @@ const realTimeClientConnectOptionsSchema = z.object({
   retries: z.number().int().min(0).optional(),
   /** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */
   preferredVideoCodec: z.enum(["h264", "vp8", "vp9"]).optional(),
-  /**
-   * Subscribe to the server's audio track. Defaults to `true` only when the input
-   * stream carries an audio track: the server's audio is a passthrough of the
-   * client's, so a video-only session has nothing to hear, and subscribing would
-   * start the device's audio engine (on iOS that alone shows the microphone
-   * permission prompt). Set `true` to always receive the remote audio track,
-   * `false` to never receive it.
-   */
-  remoteAudio: z.boolean().optional(),
   /**
    * @deprecated Glass-to-glass measurement now runs automatically in browsers
    * when LiveKit frame metadata is available. This legacy flag is accepted for
@@ -197,7 +188,6 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
       speed,
       retries,
       preferredVideoCodec,
-      remoteAudio,
     } = parsedOptions.data;
     const mirror = parsedOptions.data.mirror ?? false;
 
@@ -287,7 +277,6 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
         connectRetries: retries,
         logger,
         videoCodec: preparedConnection.videoCodec,
-        remoteAudio,
         createMediaChannel: preparedConnection.createMediaChannel,
       });
 

packages/sdk/src/realtime/media-channel.ts

@@ -92,14 +92,6 @@ export interface MediaChannelConfig {
   logger?: Logger;
   videoCodec?: VideoCodec;
   createFrameMetadataWorker?: () => Worker;
-  /**
-   * Subscribe to the inference server's audio track. Defaults to whether the
-   * local stream carries an audio track: the server's audio is a passthrough of
-   * the client's, so a video-only client gets nothing but a silent track, and
-   * on iOS merely playing out a remote audio track starts the WebRTC audio
-   * engine and triggers the microphone permission prompt.
-   */
-  remoteAudio?: boolean;
 }
 
 export type MediaConnectOptions = {
@@ -269,9 +261,14 @@ export class LiveKitMediaChannel implements MediaChannel {
     }
   }
 
-  /** Explicit `remoteAudio` wins; otherwise mirror the local stream (the server's audio is a passthrough of ours). */
+  /**
+   * The server's audio track is our own audio played back, delayed to stay in
+   * sync with the transformed video, so subscribe to it exactly when we publish
+   * audio. A video-only client has nothing to hear, and merely playing out a
+   * remote audio track starts the device's audio engine (on iOS that alone shows
+   * the microphone permission prompt).
+   */
   private wantsRemoteAudio(): boolean {
-    if (this.config.remoteAudio !== undefined) return this.config.remoteAudio;
     return (this.config.localStream?.getAudioTracks().length ?? 0) > 0;
   }
 

packages/sdk/src/realtime/stream-session.ts

@@ -124,7 +124,6 @@ interface StreamSessionConfig {
   connectRetries?: number;
   logger?: Logger;
   videoCodec?: VideoCodec;
-  remoteAudio?: boolean;
   createMediaChannel: MediaChannelFactory;
 }
 
@@ -497,7 +496,6 @@ export class StreamSession {
       localStream: this.config.localStream,
       logger: this.logger,
       videoCodec: this.config.videoCodec,
-      remoteAudio: this.config.remoteAudio,
     });
     this.wireSignalingEvents();
     this.wireMediaEvents();

packages/sdk/tests/realtime.unit.test.ts

@@ -1717,16 +1717,11 @@ describe("StreamSession startup orchestration", () => {
       expect(audio.setSubscribed).toHaveBeenCalledWith(true);
     });
 
-    it("lets remoteAudio override the local-stream default in both directions", async () => {
-      const muted = createLiveKitMediaChannel({ localStream: createLocalStream(), logger, remoteAudio: false });
-      const mutedTracks = await connectWithServer(muted);
-      expect(mutedTracks.video.setSubscribed).toHaveBeenCalledWith(true);
-      expect(mutedTracks.audio.setSubscribed).not.toHaveBeenCalled();
-
-      liveKitMock.roomInstances.length = 0;
-      const forced = createLiveKitMediaChannel({ localStream: videoOnlyStream(), logger, remoteAudio: true });
-      const forcedTracks = await connectWithServer(forced);
-      expect(forcedTracks.audio.setSubscribed).toHaveBeenCalledWith(true);
+    it("subscribes to the server's video only when there is no local stream at all", async () => {
+      const channel = createLiveKitMediaChannel({ localStream: null, logger });
+      const { audio, video } = await connectWithServer(channel);
+      expect(video.setSubscribed).toHaveBeenCalledWith(true);
+      expect(audio.setSubscribed).not.toHaveBeenCalled();
     });
 
     it("re-subscribes after a livekit-internal full reconnect", async () => {
@@ -1753,29 +1748,6 @@ describe("StreamSession startup orchestration", () => {
       room.emit(liveKitMock.RoomEvent.TrackPublished, pub, other);
       expect(pub.setSubscribed).not.toHaveBeenCalled();
     });
-
-    it("threads remoteAudio from StreamSession into the media channel", async () => {
-      const { StreamSession } = await import("../src/realtime/stream-session.js");
-      const configs: Array<{ remoteAudio?: boolean }> = [];
-      const session = new StreamSession({
-        url: "wss://example.test/realtime",
-        localStream: createLocalStream(),
-        remoteAudio: false,
-        createMediaChannel: (config) => {
-          configs.push(config);
-          return createLiveKitMediaChannel(config);
-        },
-      });
-      const connectPromise = session.connect();
-      const ws = FakeWebSocket.instances[0];
-      ws.onopen?.();
-      await flushMicrotasks();
-      sendRoomInfo(ws);
-      await flushMicrotasks();
-      await expect(connectPromise).resolves.toBeUndefined();
-      expect(configs).toEqual([expect.objectContaining({ remoteAudio: false })]);
-      session.disconnect();
-    });
   });
 
   it("sends only a lean passthrough join for a bare localStream connect (no set_image bootstrap)", async () => {