fix(realtime): subscribe to the server's audio track only when the client publishes audio

packages/sdk/README.md

@@ -82,6 +82,15 @@ Options:
 - `"auto"` — mirror when the input track reports `facingMode: "user"` (mobile front cameras).
 - `true` — always mirror (e.g. desktop webcams).
 
+#### Remote audio
+
+The server's audio track is the audio you publish, played back in sync with the transformed video
+(the model adds latency, so the server delays your audio to match). The SDK subscribes to it only
+when your input stream carries an audio track; the remote stream then has both a video and an audio
+track, and you should play the remote audio rather than your local microphone so voice and picture
+stay in sync. A video-only input stream never subscribes, so the remote stream has no audio track and
+iOS does not show the microphone permission prompt that playing out any remote audio would trigger.
+
 #### Fast mode
 
 Fast mode (`speed: "fast"`) serves the session from a higher-compute tier for lower latency and higher throughput; output quality is unchanged. It is currently available for `lucy-2.5` / `lucy-latest` and `lucy-vton-3.5` / `lucy-vton-latest`, in the US region only, and is billed at 2x the standard realtime rate for those models. Other models ignore the option. Omit it (the default) for standard mode.

packages/sdk/src/realtime/media-channel.ts

@@ -2,6 +2,7 @@ import type {
   DisconnectReason,
   RemoteParticipant,
   RemoteTrack,
+  RemoteTrackPublication,
   RemoteVideoTrack,
   Room,
   TrackPublishOptions,
@@ -165,10 +166,37 @@ export class LiveKitMediaChannel implements MediaChannel {
       }
     }
     const room = this.room;
+    const remoteAudio = this.wantsRemoteAudio();
+    const isInferenceServer = (participant: RemoteParticipant) =>
+      participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix);
+    const subscribeIfWanted = (publication: RemoteTrackPublication, participant: RemoteParticipant) => {
+      if (!isInferenceServer(participant)) return;
+      if (publication.kind === "video" || (publication.kind === "audio" && remoteAudio)) {
+        publication.setSubscribed(true);
+      }
+    };
+
+    const subscribeExisting = () => {
+      for (const participant of room.remoteParticipants.values()) {
+        for (const publication of participant.trackPublications.values()) {
+          subscribeIfWanted(publication as RemoteTrackPublication, participant);
+        }
+      }
+    };
+
+    // The room joins with `autoSubscribe: false` so the client decides per track.
+    // Publications that arrive after the join surface here; the ones already in
+    // the room when we join (the server publishes its audio track before handing
+    // out the client token) never emit TrackPublished and are swept after
+    // connect. A livekit-internal full reconnect lands on a fresh SFU session
+    // with no subscriptions either, so sweep again; `setSubscribed(true)` is
+    // idempotent.
+    room.on(RoomEvent.TrackPublished, subscribeIfWanted);
+    room.on(RoomEvent.Reconnected, subscribeExisting);
 
     room.on(RoomEvent.TrackSubscribed, (track: RemoteTrack, _pub, participant: RemoteParticipant) => {
-      if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
-      if (track.kind !== "video" && track.kind !== "audio") return;
+      if (!isInferenceServer(participant)) return;
+      if (track.kind !== "video" && !(track.kind === "audio" && remoteAudio)) return;
 
       const mediaStreamTrack = track.mediaStreamTrack;
       if (mediaStreamTrack) {
@@ -198,8 +226,9 @@ export class LiveKitMediaChannel implements MediaChannel {
     });
 
     this.config.observability?.startPhase("webrtc-handshake");
-    await room.connect(opts.url, opts.token);
+    await room.connect(opts.url, opts.token, { autoSubscribe: false });
     this.config.observability?.endPhase("webrtc-handshake", { success: true });
+    subscribeExisting();
     this.seedStartBitrate(room);
     this.config.observability?.setLiveKitRoom(room);
   }
@@ -232,6 +261,17 @@ export class LiveKitMediaChannel implements MediaChannel {
     }
   }
 
+  /**
+   * The server's audio track is our own audio played back, delayed to stay in
+   * sync with the transformed video, so subscribe to it exactly when we publish
+   * audio. A video-only client has nothing to hear, and merely playing out a
+   * remote audio track starts the device's audio engine (on iOS that alone shows
+   * the microphone permission prompt).
+   */
+  private wantsRemoteAudio(): boolean {
+    return (this.config.localStream?.getAudioTracks().length ?? 0) > 0;
+  }
+
   /**
    * livekit-client has no start-bitrate option for H264/VP8, so seed the estimator through the
    * SFU answer. Installed once per room: if livekit-client itself performs a full reconnect it

packages/sdk/tests/react-native-compat.unit.test.ts

@@ -26,6 +26,7 @@ vi.mock("livekit-client", () => {
       publishTrack: this.publishTrack,
       videoTrackPublications: new Map(),
     };
+    readonly remoteParticipants = new Map();
 
     constructor() {
       liveKitMockState.rooms.push(this);
@@ -186,12 +187,14 @@ describe("React Native compatibility", () => {
       import("../src/utils/logger.js"),
     ]);
     const videoTrack = { kind: "video" } as MediaStreamTrack;
-    const localStream = { getTracks: () => [videoTrack] } as MediaStream;
+    const localStream = { getTracks: () => [videoTrack], getAudioTracks: () => [] } as unknown as MediaStream;
     const mediaChannel = new LiveKitMediaChannel({ localStream, videoCodec: "vp8" });
 
     await mediaChannel.connect({ url: "wss://livekit.test", token: "publish-token" });
     await mediaChannel.publishLocalTracks();
-    expect(liveKitMockState.rooms[0]?.connect).toHaveBeenCalledWith("wss://livekit.test", "publish-token");
+    expect(liveKitMockState.rooms[0]?.connect).toHaveBeenCalledWith("wss://livekit.test", "publish-token", {
+      autoSubscribe: false,
+    });
     expect(liveKitMockState.rooms[0]?.publishTrack).toHaveBeenCalledWith(
       videoTrack,
       expect.objectContaining({ videoCodec: "vp8" }),

packages/sdk/tests/realtime.unit.test.ts

@@ -12,7 +12,9 @@ const liveKitMock = vi.hoisted(() => {
   const connectMocks: Array<() => Promise<void>> = [];
 
   const RoomEvent = {
+    TrackPublished: "trackPublished",
     TrackSubscribed: "trackSubscribed",
+    Reconnected: "reconnected",
     Disconnected: "disconnected",
     ConnectionStateChanged: "connectionStateChanged",
   } as const;
@@ -29,13 +31,23 @@ const liveKitMock = vi.hoisted(() => {
     Disconnected: "disconnected",
   } as const;
 
+  class MockPublication {
+    setSubscribed = vi.fn();
+    constructor(public readonly kind: "video" | "audio") {}
+  }
+  class MockRemoteParticipant {
+    trackPublications = new Map<string, MockPublication>();
+    constructor(public readonly identity: string) {}
+  }
+
   class MockRoom {
     handlers = new Map<string, Array<(...args: unknown[]) => void>>();
     state = ConnectionState.Connected;
     localParticipant = {
       publishTrack: vi.fn(),
       videoTrackPublications: new Map<string, { videoTrack: { replaceTrack: ReturnType<typeof vi.fn> } }>(),
     };
+    remoteParticipants = new Map<string, MockRemoteParticipant>();
     connect = vi.fn().mockImplementation(() => connectMocks.shift()?.() ?? Promise.resolve());
     disconnect = vi.fn().mockResolvedValue(undefined);
     /** Inner publisher `setRemoteDescription` (livekit-client internal) — kept so tests can see what reached it. */
@@ -69,7 +81,17 @@ const liveKitMock = vi.hoisted(() => {
     }
   }
 
-  return { roomInstances, connectMocks, RoomEvent, Track, TrackEvent, ConnectionState, MockRoom };
+  return {
+    roomInstances,
+    connectMocks,
+    RoomEvent,
+    Track,
+    TrackEvent,
+    ConnectionState,
+    MockRoom,
+    MockPublication,
+    MockRemoteParticipant,
+  };
 });
 
 vi.mock("livekit-client", () => ({
@@ -1431,7 +1453,7 @@ describe("StreamSession startup orchestration", () => {
     expect(ws.sentMessages).toEqual([leanJoin, initialPrompt]);
 
     const room = liveKitMock.roomInstances[0] as InstanceType<typeof liveKitMock.MockRoom>;
-    expect(room.connect).toHaveBeenCalledWith("wss://livekit.example.test", "token");
+    expect(room.connect).toHaveBeenCalledWith("wss://livekit.example.test", "token", { autoSubscribe: false });
     expect(states).toEqual(["connecting"]);
 
     await expect(connectPromise).resolves.toBeUndefined();
@@ -1550,7 +1572,7 @@ describe("StreamSession startup orchestration", () => {
     await flushMicrotasks();
 
     const room = liveKitMock.roomInstances[0] as InstanceType<typeof liveKitMock.MockRoom>;
-    expect(room.connect).toHaveBeenCalledWith("wss://livekit.example.test", "token");
+    expect(room.connect).toHaveBeenCalledWith("wss://livekit.example.test", "token", { autoSubscribe: false });
     await expect(connectPromise).resolves.toBeUndefined();
     expect(room.localParticipant.publishTrack).toHaveBeenCalledTimes(2);
     expect(room.localParticipant.publishTrack).toHaveBeenNthCalledWith(
@@ -1626,6 +1648,108 @@ describe("StreamSession startup orchestration", () => {
     expect(liveKitMock.roomInstances).toHaveLength(0);
   });
 
+  describe("remote track subscription", () => {
+    type MockRoom = InstanceType<typeof liveKitMock.MockRoom>;
+    const videoOnlyStream = () => new MediaStream([{ id: "local-video", kind: "video" }] as unknown[]) as MediaStream;
+
+    /** Server already in the room with both tracks when we join, plus a late video publication. */
+    const seedServer = (room: MockRoom) => {
+      const server = new liveKitMock.MockRemoteParticipant("inference-server-1");
+      const audio = new liveKitMock.MockPublication("audio");
+      const video = new liveKitMock.MockPublication("video");
+      server.trackPublications.set("audio", audio);
+      server.trackPublications.set("video", video);
+      room.remoteParticipants.set(server.identity, server);
+      return { server, audio, video };
+    };
+
+    const connectWithServer = async (channel: MediaChannel) => {
+      liveKitMock.connectMocks.push(() => {
+        seedServer(liveKitMock.roomInstances[0] as MockRoom);
+        return Promise.resolve();
+      });
+      await channel.connect({ url: "wss://livekit.example.test", token: "token" });
+      const room = liveKitMock.roomInstances[0] as MockRoom;
+      const server = room.remoteParticipants.get("inference-server-1") as InstanceType<
+        typeof liveKitMock.MockRemoteParticipant
+      >;
+      return {
+        room,
+        server,
+        audio: server.trackPublications.get("audio") as InstanceType<typeof liveKitMock.MockPublication>,
+        video: server.trackPublications.get("video") as InstanceType<typeof liveKitMock.MockPublication>,
+      };
+    };
+
+    it("joins without auto-subscribe and subscribes only to the server's video for a video-only client", async () => {
+      const channel = createLiveKitMediaChannel({ localStream: videoOnlyStream(), logger });
+      const remoteStreams: MediaStream[] = [];
+      channel.on("remoteStream", (stream) => remoteStreams.push(stream));
+
+      const { room, server, audio, video } = await connectWithServer(channel);
+
+      expect(room.connect).toHaveBeenCalledWith("wss://livekit.example.test", "token", { autoSubscribe: false });
+      expect(video.setSubscribed).toHaveBeenCalledWith(true);
+      expect(audio.setSubscribed).not.toHaveBeenCalled();
+
+      // A publication that shows up after the join goes through the same gate.
+      const lateAudio = new liveKitMock.MockPublication("audio");
+      const lateVideo = new liveKitMock.MockPublication("video");
+      room.emit(liveKitMock.RoomEvent.TrackPublished, lateAudio, server);
+      room.emit(liveKitMock.RoomEvent.TrackPublished, lateVideo, server);
+      expect(lateAudio.setSubscribed).not.toHaveBeenCalled();
+      expect(lateVideo.setSubscribed).toHaveBeenCalledWith(true);
+
+      // Even if the SFU hands us an audio track anyway, it never reaches the consumer.
+      room.emit(
+        liveKitMock.RoomEvent.TrackSubscribed,
+        { kind: "audio", mediaStreamTrack: { id: "remote-audio", kind: "audio" } },
+        {},
+        server,
+      );
+      expect(remoteStreams).toHaveLength(0);
+    });
+
+    it("subscribes to the server's audio when the client publishes audio", async () => {
+      const channel = createLiveKitMediaChannel({ localStream: createLocalStream(), logger });
+      const { audio, video } = await connectWithServer(channel);
+      expect(video.setSubscribed).toHaveBeenCalledWith(true);
+      expect(audio.setSubscribed).toHaveBeenCalledWith(true);
+    });
+
+    it("subscribes to the server's video only when there is no local stream at all", async () => {
+      const channel = createLiveKitMediaChannel({ localStream: null, logger });
+      const { audio, video } = await connectWithServer(channel);
+      expect(video.setSubscribed).toHaveBeenCalledWith(true);
+      expect(audio.setSubscribed).not.toHaveBeenCalled();
+    });
+
+    it("re-subscribes after a livekit-internal full reconnect", async () => {
+      const channel = createLiveKitMediaChannel({ localStream: videoOnlyStream(), logger });
+      const { room, server, audio, video } = await connectWithServer(channel);
+      video.setSubscribed.mockClear();
+
+      // A full reconnect lands on a fresh SFU session; publications already known to the
+      // Room never re-emit TrackPublished, so the sweep must run again on Reconnected.
+      const republished = new liveKitMock.MockPublication("video");
+      server.trackPublications.set("video-2", republished);
+      room.emit(liveKitMock.RoomEvent.Reconnected);
+
+      expect(video.setSubscribed).toHaveBeenCalledWith(true);
+      expect(republished.setSubscribed).toHaveBeenCalledWith(true);
+      expect(audio.setSubscribed).not.toHaveBeenCalled();
+    });
+
+    it("ignores publications from participants other than the inference server", async () => {
+      const channel = createLiveKitMediaChannel({ localStream: createLocalStream(), logger });
+      const { room } = await connectWithServer(channel);
+      const other = new liveKitMock.MockRemoteParticipant("viewer-1");
+      const pub = new liveKitMock.MockPublication("video");
+      room.emit(liveKitMock.RoomEvent.TrackPublished, pub, other);
+      expect(pub.setSubscribed).not.toHaveBeenCalled();
+    });
+  });
+
   it("sends only a lean passthrough join for a bare localStream connect (no set_image bootstrap)", async () => {
     const { StreamSession } = await import("../src/realtime/stream-session.js");
     const localStream = createLocalStream();