diff --git a/.changeset/default-video-degradation-preferences.md b/.changeset/default-video-degradation-preferences.md new file mode 100644 index 00000000..a2ef7356 --- /dev/null +++ b/.changeset/default-video-degradation-preferences.md @@ -0,0 +1,7 @@ +--- +"client-sdk-android": patch +--- + +Use source-specific default video degradation preferences: camera tracks default to maintaining framerate, screen share tracks default to maintaining resolution, and other video sources default to balanced. This matches client-sdk-js. Video tracks published with an explicit `source` other than camera or screen share now use balanced rather than WebRTC's implicit choice; set `degradationPreference` on the publish options to override. + +The resolved preference is now also applied to the backup codec's sender. Previously only the primary encoder was configured and the backup encoder let libwebrtc derive a preference implicitly, so the two encoders could adapt along different axes off the same video source. diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index 5abbc922..f3d7454d 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -569,11 +569,7 @@ internal constructor( requestConfig = { width = track.dimensions.width height = track.dimensions.height - source = options.source?.toProto() ?: if (track.options.isScreencast) { - LivekitModels.TrackSource.SCREEN_SHARE - } else { - LivekitModels.TrackSource.CAMERA - } + source = resolveVideoTrackSource(track, options).toProto() addAllLayers(videoLayers) addSimulcastCodecs( @@ -732,9 +728,7 @@ internal constructor( transceiver.sortVideoCodecPreferences(finalOptions.videoCodec, capabilitiesGetter) (track as LocalVideoTrack).codec = finalOptions.videoCodec - val rtpParameters = transceiver.sender.parameters - rtpParameters.degradationPreference = finalOptions.degradationPreference - transceiver.sender.parameters = rtpParameters + transceiver.applyDegradationPreference(finalOptions.degradationPreference, trackSource) } // PublisherTransportObserver.onRenegotiationNeeded() gets triggered automatically @@ -1250,6 +1244,15 @@ internal constructor( transceiver.sortVideoCodecPreferences(newOptions.videoCodec, capabilitiesGetter) simulcastTrack.sender = transceiver.sender + // The backup codec has its own sender, so it needs the same degradation + // preference as the primary applied explicitly. Resolve the source the same + // way the primary publish did, rather than reading it back off the + // publication, so the two encoders can't disagree. + transceiver.applyDegradationPreference( + newOptions.degradationPreference, + resolveVideoTrackSource(track, newOptions), + ) + engine.negotiatePublisher() } val publishJob = async { @@ -1484,10 +1487,21 @@ abstract class BaseVideoTrackPublishOptions { abstract val backupCodec: BackupVideoCodec? /** - * When bandwidth is constrained, this preference indicates which is preferred - * between degrading resolution vs. framerate. + * Controls how the encoder trades off between resolution and framerate + * when bandwidth is constrained. * - * null value indicates default value (maintain framerate). + * - MAINTAIN_FRAMERATE: Prioritizes framerate, reduces resolution if needed + * - MAINTAIN_RESOLUTION: Prioritizes resolution, drops frames if needed + * - BALANCED: Balances between both + * + * If not set (null), the SDK uses defaults based on track source: + * - Camera: MAINTAIN_FRAMERATE (smoother video for real-time communication) + * - Screen share: MAINTAIN_RESOLUTION (clarity is critical for text/UI) + * - Other/unknown: BALANCED + * + * Note that a preference is always applied to video senders, so leaving this null + * selects the source-based default above rather than deferring to WebRTC's own + * implicit choice. */ abstract val degradationPreference: RtpParameters.DegradationPreference? @@ -1689,6 +1703,63 @@ internal fun VideoTrackPublishOptions.hasBackupCodec(): Boolean { private val backupCodecs = listOf(VideoCodec.VP8.codecName, VideoCodec.H264.codecName) private fun isBackupCodec(codecName: String) = backupCodecs.contains(codecName) +/** + * Resolves the [Track.Source] a video track is published under: the explicitly requested + * source if any, otherwise inferred from whether the track is backed by a screencast source. + */ +private fun resolveVideoTrackSource(track: LocalVideoTrack, options: VideoTrackPublishOptions): Track.Source { + return options.source ?: if (track.options.isScreencast) { + Track.Source.SCREEN_SHARE + } else { + Track.Source.CAMERA + } +} + +/** + * Returns the appropriate degradation preference for a video track based on its source. + * + * - Camera: MAINTAIN_FRAMERATE (smoother video for real-time communication) + * - Screen share: MAINTAIN_RESOLUTION (clarity is critical for reading text/UI) + * - Other/unknown: BALANCED + * + * Any other source means the application declined to declare a motion-vs-detail intent, + * so this falls back to BALANCED, the preference the WebRTC spec mandates as the default. + * This deliberately does not defer to libwebrtc's implicit derivation, which keys off the + * native source's is_screencast flag: custom feeds report is_screencast = false regardless + * of content (see VideoFrameCapturer/BitmapFrameCapturer), so deferring would resolve to + * MAINTAIN_FRAMERATE for every custom feed rather than recovering any real intent. + * + * This is the intended behavior across LiveKit client SDKs; client-sdk-js + * (`getDefaultDegradationPreference` in publishUtils.ts) and the Rust SDK + * (`get_default_degradation_preference` in room/options.rs) use the same mapping. + */ +private fun getDefaultDegradationPreference(source: Track.Source): RtpParameters.DegradationPreference { + return when (source) { + Track.Source.CAMERA -> RtpParameters.DegradationPreference.MAINTAIN_FRAMERATE + Track.Source.SCREEN_SHARE -> RtpParameters.DegradationPreference.MAINTAIN_RESOLUTION + else -> RtpParameters.DegradationPreference.BALANCED + } +} + +/** + * Applies [preference] to this transceiver's sender, falling back to the + * source-based default from [getDefaultDegradationPreference]. + * + * Degradation preference is a property of the sender, not of the track, so every + * sender feeding from a track needs it applied separately. In particular the backup + * codec gets its own transceiver over the same rtc track, and would otherwise let + * libwebrtc resolve a preference implicitly from the native source's is_screencast + * flag, diverging from the primary encoder. + */ +private fun RtpTransceiver.applyDegradationPreference( + preference: RtpParameters.DegradationPreference?, + source: Track.Source, +) { + val rtpParameters = sender.parameters + rtpParameters.degradationPreference = preference ?: getDefaultDegradationPreference(source) + sender.parameters = rtpParameters +} + /** * A handler that processes an RPC request and returns a string * that will be sent back to the requester. The payload must diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/participant/LocalParticipantMockE2ETest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/participant/LocalParticipantMockE2ETest.kt index 5270bb70..0498b8ed 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/participant/LocalParticipantMockE2ETest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/participant/LocalParticipantMockE2ETest.kt @@ -841,6 +841,163 @@ class LocalParticipantMockE2ETest : MockE2ETest() { assertEquals(preference, transceiver.sender.parameters.degradationPreference) } + @Test + fun publishCameraUsesDefaultDegradationPreference() = runTest { + connect() + + room.localParticipant.publishVideoTrack(track = createLocalTrack()) + + val peerConnection = getPublisherPeerConnection() + val transceiver = peerConnection.transceivers.first() + + assertEquals( + RtpParameters.DegradationPreference.MAINTAIN_FRAMERATE, + transceiver.sender.parameters.degradationPreference, + ) + } + + @Test + fun publishScreenShareUsesDefaultDegradationPreference() = runTest { + connect() + + room.localParticipant.publishVideoTrack(track = createLocalTrack(isScreencast = true)) + + val peerConnection = getPublisherPeerConnection() + val transceiver = peerConnection.transceivers.first() + + assertEquals( + RtpParameters.DegradationPreference.MAINTAIN_RESOLUTION, + transceiver.sender.parameters.degradationPreference, + ) + } + + @Test + fun publishOtherSourceUsesBalancedDegradationPreference() = runTest { + connect() + + room.localParticipant.publishVideoTrack( + track = createLocalTrack(), + options = VideoTrackPublishOptions( + null, + room.videoTrackPublishDefaults, + source = Track.Source.UNKNOWN, + ), + ) + + val peerConnection = getPublisherPeerConnection() + val transceiver = peerConnection.transceivers.first() + + assertEquals( + RtpParameters.DegradationPreference.BALANCED, + transceiver.sender.parameters.degradationPreference, + ) + } + + @Test + fun backupCodecUsesSameDefaultDegradationPreferenceAsPrimary() = runTest { + room.videoTrackPublishDefaults = room.videoTrackPublishDefaults.copy( + videoCodec = VideoCodec.VP9.codecName, + scalabilityMode = "L3T3", + backupCodec = BackupVideoCodec(codec = VideoCodec.VP8.codecName), + ) + + connect() + room.localParticipant.publishVideoTrack(track = createLocalTrack()) + + receiveSubscribedQualityUpdate(room.localParticipant.videoTrackPublications.first().first.sid) + + val transceivers = getPublisherPeerConnection().transceivers + assertEquals(2, transceivers.size) + + // Both the primary and the backup codec sender must resolve to the same preference, + // otherwise the two encoders adapt along different axes off a shared video source. + transceivers.forEach { transceiver -> + assertEquals( + RtpParameters.DegradationPreference.MAINTAIN_FRAMERATE, + transceiver.sender.parameters.degradationPreference, + ) + } + } + + @Test + fun backupCodecUsesScreenShareDefaultDegradationPreference() = runTest { + room.screenShareTrackPublishDefaults = room.screenShareTrackPublishDefaults.copy( + videoCodec = VideoCodec.VP9.codecName, + backupCodec = BackupVideoCodec(codec = VideoCodec.VP8.codecName), + ) + + connect() + room.localParticipant.publishVideoTrack(track = createLocalTrack(isScreencast = true)) + + receiveSubscribedQualityUpdate(room.localParticipant.videoTrackPublications.first().first.sid) + + val transceivers = getPublisherPeerConnection().transceivers + assertEquals(2, transceivers.size) + + transceivers.forEach { transceiver -> + assertEquals( + RtpParameters.DegradationPreference.MAINTAIN_RESOLUTION, + transceiver.sender.parameters.degradationPreference, + ) + } + } + + @Test + fun backupCodecUsesExplicitDegradationPreference() = runTest { + val preference = RtpParameters.DegradationPreference.DISABLED + room.videoTrackPublishDefaults = room.videoTrackPublishDefaults.copy( + videoCodec = VideoCodec.VP9.codecName, + scalabilityMode = "L3T3", + backupCodec = BackupVideoCodec(codec = VideoCodec.VP8.codecName), + degradationPreference = preference, + ) + + connect() + room.localParticipant.publishVideoTrack(track = createLocalTrack()) + + receiveSubscribedQualityUpdate(room.localParticipant.videoTrackPublications.first().first.sid) + + val transceivers = getPublisherPeerConnection().transceivers + assertEquals(2, transceivers.size) + + transceivers.forEach { transceiver -> + assertEquals(preference, transceiver.sender.parameters.degradationPreference) + } + } + + @Test + fun backupCodecDegradationPreferenceFollowsExplicitSourceNotScreencastFlag() = runTest { + room.videoTrackPublishDefaults = room.videoTrackPublishDefaults.copy( + videoCodec = VideoCodec.VP9.codecName, + backupCodec = BackupVideoCodec(codec = VideoCodec.VP8.codecName), + ) + + connect() + // A screencast-backed track deliberately published as a camera source: both senders + // must follow the declared source rather than letting the backup fall back to the + // native source's is_screencast flag. + room.localParticipant.publishVideoTrack( + track = createLocalTrack(isScreencast = true), + options = VideoTrackPublishOptions( + null, + room.videoTrackPublishDefaults, + source = Track.Source.CAMERA, + ), + ) + + receiveSubscribedQualityUpdate(room.localParticipant.videoTrackPublications.first().first.sid) + + val transceivers = getPublisherPeerConnection().transceivers + assertEquals(2, transceivers.size) + + transceivers.forEach { transceiver -> + assertEquals( + RtpParameters.DegradationPreference.MAINTAIN_FRAMERATE, + transceiver.sender.parameters.degradationPreference, + ) + } + } + @Test fun lackOfPublishPermissionReturnsFalse() = runTest { val noCanPublishJoin = with(TestData.JOIN.toBuilder()) {