diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 2037381..63e775d 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -82,6 +82,15 @@ Options: - `"auto"` — mirror when the input track reports `facingMode: "user"` (mobile front cameras). - `true` — always mirror (e.g. desktop webcams). +#### Remote audio + +The server's audio track is the audio you publish, played back in sync with the transformed video +(the model adds latency, so the server delays your audio to match). The SDK subscribes to it only +when your input stream carries an audio track; the remote stream then has both a video and an audio +track, and you should play the remote audio rather than your local microphone so voice and picture +stay in sync. A video-only input stream never subscribes, so the remote stream has no audio track and +iOS does not show the microphone permission prompt that playing out any remote audio would trigger. + #### Fast mode Fast mode (`speed: "fast"`) serves the session from a higher-compute tier for lower latency and higher throughput; output quality is unchanged. It is currently available for `lucy-2.5` / `lucy-latest` and `lucy-vton-3.5` / `lucy-vton-latest`, in the US region only, and is billed at 2x the standard realtime rate for those models. Other models ignore the option. Omit it (the default) for standard mode. diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index 2940482..74342b3 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -2,6 +2,7 @@ import type { DisconnectReason, RemoteParticipant, RemoteTrack, + RemoteTrackPublication, RemoteVideoTrack, Room, TrackPublishOptions, @@ -165,10 +166,37 @@ export class LiveKitMediaChannel implements MediaChannel { } } const room = this.room; + const remoteAudio = this.wantsRemoteAudio(); + const isInferenceServer = (participant: RemoteParticipant) => + participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix); + const subscribeIfWanted = (publication: RemoteTrackPublication, participant: RemoteParticipant) => { + if (!isInferenceServer(participant)) return; + if (publication.kind === "video" || (publication.kind === "audio" && remoteAudio)) { + publication.setSubscribed(true); + } + }; + + const subscribeExisting = () => { + for (const participant of room.remoteParticipants.values()) { + for (const publication of participant.trackPublications.values()) { + subscribeIfWanted(publication as RemoteTrackPublication, participant); + } + } + }; + + // The room joins with `autoSubscribe: false` so the client decides per track. + // Publications that arrive after the join surface here; the ones already in + // the room when we join (the server publishes its audio track before handing + // out the client token) never emit TrackPublished and are swept after + // connect. A livekit-internal full reconnect lands on a fresh SFU session + // with no subscriptions either, so sweep again; `setSubscribed(true)` is + // idempotent. + room.on(RoomEvent.TrackPublished, subscribeIfWanted); + room.on(RoomEvent.Reconnected, subscribeExisting); room.on(RoomEvent.TrackSubscribed, (track: RemoteTrack, _pub, participant: RemoteParticipant) => { - if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return; - if (track.kind !== "video" && track.kind !== "audio") return; + if (!isInferenceServer(participant)) return; + if (track.kind !== "video" && !(track.kind === "audio" && remoteAudio)) return; const mediaStreamTrack = track.mediaStreamTrack; if (mediaStreamTrack) { @@ -198,8 +226,9 @@ export class LiveKitMediaChannel implements MediaChannel { }); this.config.observability?.startPhase("webrtc-handshake"); - await room.connect(opts.url, opts.token); + await room.connect(opts.url, opts.token, { autoSubscribe: false }); this.config.observability?.endPhase("webrtc-handshake", { success: true }); + subscribeExisting(); this.seedStartBitrate(room); this.config.observability?.setLiveKitRoom(room); } @@ -232,6 +261,17 @@ export class LiveKitMediaChannel implements MediaChannel { } } + /** + * The server's audio track is our own audio played back, delayed to stay in + * sync with the transformed video, so subscribe to it exactly when we publish + * audio. A video-only client has nothing to hear, and merely playing out a + * remote audio track starts the device's audio engine (on iOS that alone shows + * the microphone permission prompt). + */ + private wantsRemoteAudio(): boolean { + return (this.config.localStream?.getAudioTracks().length ?? 0) > 0; + } + /** * livekit-client has no start-bitrate option for H264/VP8, so seed the estimator through the * SFU answer. Installed once per room: if livekit-client itself performs a full reconnect it diff --git a/packages/sdk/tests/react-native-compat.unit.test.ts b/packages/sdk/tests/react-native-compat.unit.test.ts index 9cc2e64..57ef0e0 100644 --- a/packages/sdk/tests/react-native-compat.unit.test.ts +++ b/packages/sdk/tests/react-native-compat.unit.test.ts @@ -26,6 +26,7 @@ vi.mock("livekit-client", () => { publishTrack: this.publishTrack, videoTrackPublications: new Map(), }; + readonly remoteParticipants = new Map(); constructor() { liveKitMockState.rooms.push(this); @@ -186,12 +187,14 @@ describe("React Native compatibility", () => { import("../src/utils/logger.js"), ]); const videoTrack = { kind: "video" } as MediaStreamTrack; - const localStream = { getTracks: () => [videoTrack] } as MediaStream; + const localStream = { getTracks: () => [videoTrack], getAudioTracks: () => [] } as unknown as MediaStream; const mediaChannel = new LiveKitMediaChannel({ localStream, videoCodec: "vp8" }); await mediaChannel.connect({ url: "wss://livekit.test", token: "publish-token" }); await mediaChannel.publishLocalTracks(); - expect(liveKitMockState.rooms[0]?.connect).toHaveBeenCalledWith("wss://livekit.test", "publish-token"); + expect(liveKitMockState.rooms[0]?.connect).toHaveBeenCalledWith("wss://livekit.test", "publish-token", { + autoSubscribe: false, + }); expect(liveKitMockState.rooms[0]?.publishTrack).toHaveBeenCalledWith( videoTrack, expect.objectContaining({ videoCodec: "vp8" }), diff --git a/packages/sdk/tests/realtime.unit.test.ts b/packages/sdk/tests/realtime.unit.test.ts index 7d44ebd..d274f95 100644 --- a/packages/sdk/tests/realtime.unit.test.ts +++ b/packages/sdk/tests/realtime.unit.test.ts @@ -12,7 +12,9 @@ const liveKitMock = vi.hoisted(() => { const connectMocks: Array<() => Promise> = []; const RoomEvent = { + TrackPublished: "trackPublished", TrackSubscribed: "trackSubscribed", + Reconnected: "reconnected", Disconnected: "disconnected", ConnectionStateChanged: "connectionStateChanged", } as const; @@ -29,6 +31,15 @@ const liveKitMock = vi.hoisted(() => { Disconnected: "disconnected", } as const; + class MockPublication { + setSubscribed = vi.fn(); + constructor(public readonly kind: "video" | "audio") {} + } + class MockRemoteParticipant { + trackPublications = new Map(); + constructor(public readonly identity: string) {} + } + class MockRoom { handlers = new Map void>>(); state = ConnectionState.Connected; @@ -36,6 +47,7 @@ const liveKitMock = vi.hoisted(() => { publishTrack: vi.fn(), videoTrackPublications: new Map } }>(), }; + remoteParticipants = new Map(); connect = vi.fn().mockImplementation(() => connectMocks.shift()?.() ?? Promise.resolve()); disconnect = vi.fn().mockResolvedValue(undefined); /** Inner publisher `setRemoteDescription` (livekit-client internal) — kept so tests can see what reached it. */ @@ -69,7 +81,17 @@ const liveKitMock = vi.hoisted(() => { } } - return { roomInstances, connectMocks, RoomEvent, Track, TrackEvent, ConnectionState, MockRoom }; + return { + roomInstances, + connectMocks, + RoomEvent, + Track, + TrackEvent, + ConnectionState, + MockRoom, + MockPublication, + MockRemoteParticipant, + }; }); vi.mock("livekit-client", () => ({ @@ -1431,7 +1453,7 @@ describe("StreamSession startup orchestration", () => { expect(ws.sentMessages).toEqual([leanJoin, initialPrompt]); const room = liveKitMock.roomInstances[0] as InstanceType; - expect(room.connect).toHaveBeenCalledWith("wss://livekit.example.test", "token"); + expect(room.connect).toHaveBeenCalledWith("wss://livekit.example.test", "token", { autoSubscribe: false }); expect(states).toEqual(["connecting"]); await expect(connectPromise).resolves.toBeUndefined(); @@ -1550,7 +1572,7 @@ describe("StreamSession startup orchestration", () => { await flushMicrotasks(); const room = liveKitMock.roomInstances[0] as InstanceType; - expect(room.connect).toHaveBeenCalledWith("wss://livekit.example.test", "token"); + expect(room.connect).toHaveBeenCalledWith("wss://livekit.example.test", "token", { autoSubscribe: false }); await expect(connectPromise).resolves.toBeUndefined(); expect(room.localParticipant.publishTrack).toHaveBeenCalledTimes(2); expect(room.localParticipant.publishTrack).toHaveBeenNthCalledWith( @@ -1626,6 +1648,108 @@ describe("StreamSession startup orchestration", () => { expect(liveKitMock.roomInstances).toHaveLength(0); }); + describe("remote track subscription", () => { + type MockRoom = InstanceType; + const videoOnlyStream = () => new MediaStream([{ id: "local-video", kind: "video" }] as unknown[]) as MediaStream; + + /** Server already in the room with both tracks when we join, plus a late video publication. */ + const seedServer = (room: MockRoom) => { + const server = new liveKitMock.MockRemoteParticipant("inference-server-1"); + const audio = new liveKitMock.MockPublication("audio"); + const video = new liveKitMock.MockPublication("video"); + server.trackPublications.set("audio", audio); + server.trackPublications.set("video", video); + room.remoteParticipants.set(server.identity, server); + return { server, audio, video }; + }; + + const connectWithServer = async (channel: MediaChannel) => { + liveKitMock.connectMocks.push(() => { + seedServer(liveKitMock.roomInstances[0] as MockRoom); + return Promise.resolve(); + }); + await channel.connect({ url: "wss://livekit.example.test", token: "token" }); + const room = liveKitMock.roomInstances[0] as MockRoom; + const server = room.remoteParticipants.get("inference-server-1") as InstanceType< + typeof liveKitMock.MockRemoteParticipant + >; + return { + room, + server, + audio: server.trackPublications.get("audio") as InstanceType, + video: server.trackPublications.get("video") as InstanceType, + }; + }; + + it("joins without auto-subscribe and subscribes only to the server's video for a video-only client", async () => { + const channel = createLiveKitMediaChannel({ localStream: videoOnlyStream(), logger }); + const remoteStreams: MediaStream[] = []; + channel.on("remoteStream", (stream) => remoteStreams.push(stream)); + + const { room, server, audio, video } = await connectWithServer(channel); + + expect(room.connect).toHaveBeenCalledWith("wss://livekit.example.test", "token", { autoSubscribe: false }); + expect(video.setSubscribed).toHaveBeenCalledWith(true); + expect(audio.setSubscribed).not.toHaveBeenCalled(); + + // A publication that shows up after the join goes through the same gate. + const lateAudio = new liveKitMock.MockPublication("audio"); + const lateVideo = new liveKitMock.MockPublication("video"); + room.emit(liveKitMock.RoomEvent.TrackPublished, lateAudio, server); + room.emit(liveKitMock.RoomEvent.TrackPublished, lateVideo, server); + expect(lateAudio.setSubscribed).not.toHaveBeenCalled(); + expect(lateVideo.setSubscribed).toHaveBeenCalledWith(true); + + // Even if the SFU hands us an audio track anyway, it never reaches the consumer. + room.emit( + liveKitMock.RoomEvent.TrackSubscribed, + { kind: "audio", mediaStreamTrack: { id: "remote-audio", kind: "audio" } }, + {}, + server, + ); + expect(remoteStreams).toHaveLength(0); + }); + + it("subscribes to the server's audio when the client publishes audio", async () => { + const channel = createLiveKitMediaChannel({ localStream: createLocalStream(), logger }); + const { audio, video } = await connectWithServer(channel); + expect(video.setSubscribed).toHaveBeenCalledWith(true); + expect(audio.setSubscribed).toHaveBeenCalledWith(true); + }); + + it("subscribes to the server's video only when there is no local stream at all", async () => { + const channel = createLiveKitMediaChannel({ localStream: null, logger }); + const { audio, video } = await connectWithServer(channel); + expect(video.setSubscribed).toHaveBeenCalledWith(true); + expect(audio.setSubscribed).not.toHaveBeenCalled(); + }); + + it("re-subscribes after a livekit-internal full reconnect", async () => { + const channel = createLiveKitMediaChannel({ localStream: videoOnlyStream(), logger }); + const { room, server, audio, video } = await connectWithServer(channel); + video.setSubscribed.mockClear(); + + // A full reconnect lands on a fresh SFU session; publications already known to the + // Room never re-emit TrackPublished, so the sweep must run again on Reconnected. + const republished = new liveKitMock.MockPublication("video"); + server.trackPublications.set("video-2", republished); + room.emit(liveKitMock.RoomEvent.Reconnected); + + expect(video.setSubscribed).toHaveBeenCalledWith(true); + expect(republished.setSubscribed).toHaveBeenCalledWith(true); + expect(audio.setSubscribed).not.toHaveBeenCalled(); + }); + + it("ignores publications from participants other than the inference server", async () => { + const channel = createLiveKitMediaChannel({ localStream: createLocalStream(), logger }); + const { room } = await connectWithServer(channel); + const other = new liveKitMock.MockRemoteParticipant("viewer-1"); + const pub = new liveKitMock.MockPublication("video"); + room.emit(liveKitMock.RoomEvent.TrackPublished, pub, other); + expect(pub.setSubscribed).not.toHaveBeenCalled(); + }); + }); + it("sends only a lean passthrough join for a bare localStream connect (no set_image bootstrap)", async () => { const { StreamSession } = await import("../src/realtime/stream-session.js"); const localStream = createLocalStream();