From b219713cd20e5b6f75b75b2c3891ce76664b4898 Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Fri, 16 Oct 2026 15:34:46 -0500 Subject: [PATCH 1/3] Recover from non-monotonic presentationTimeUs in MP4 video tracks without ctts Squash of #3352 (325f2c5..410db95, previously based on release) applied to main. Conflicts were resolved mechanically: main has since removed Format#hasPrerollSamples, and its MediaCodecRenderer additions sit next to the ones in this change. MP4 video tracks that reorder frames but have no ctts box give every sample a presentation time equal to its decode time, for example a raw H.265 elementary stream muxed with ffmpeg -fflags +genpts -c:v copy. MediaCodec echoes those timestamps back on frames that come out in display order, so the renderer sees non-monotonic timestamps and drops ~30% of frames as late. See #3347. - Format#hasReliablePresentationTimestamps, set false by BoxParser for a video track with no ctts and maxNumReorderSamples != 0. - MediaCodecRenderer relabels output buffers of such formats with the queued timestamps, in output order. - MediaCodecVideoRenderer uses the very-late threshold in shouldDropOutputBuffer and disables shouldDropBuffersToKeyframe for them. --- .../java/androidx/media3/common/Format.java | 27 ++ .../mediacodec/MediaCodecRenderer.java | 37 +++ .../video/MediaCodecVideoRenderer.java | 17 ++ .../mediacodec/MediaCodecRendererTest.java | 135 +++++++++ .../video/MediaCodecVideoRendererTest.java | 271 ++++++++++++++++++ .../media3/extractor/mp4/BoxParser.java | 14 + 6 files changed, 501 insertions(+) diff --git a/libraries/common/src/main/java/androidx/media3/common/Format.java b/libraries/common/src/main/java/androidx/media3/common/Format.java index e1f58007c1a..3ba05ce012d 100644 --- a/libraries/common/src/main/java/androidx/media3/common/Format.java +++ b/libraries/common/src/main/java/androidx/media3/common/Format.java @@ -177,6 +177,7 @@ public static final class Builder { @Nullable private List initializationData; @Nullable private DrmInitData drmInitData; private long subsampleOffsetUs; + private boolean hasReliablePresentationTimestamps; // Video specific. @@ -227,6 +228,7 @@ public Builder() { maxInputSize = NO_VALUE; maxNumReorderSamples = NO_VALUE; subsampleOffsetUs = OFFSET_SAMPLE_RELATIVE; + hasReliablePresentationTimestamps = true; // most formats have accurate per-sample timestamps // Video specific. width = NO_VALUE; height = NO_VALUE; @@ -281,6 +283,7 @@ private Builder(Format format) { this.initializationData = format.initializationData; this.drmInitData = format.drmInitData; this.subsampleOffsetUs = format.subsampleOffsetUs; + this.hasReliablePresentationTimestamps = format.hasReliablePresentationTimestamps; // Video specific. this.width = format.width; this.height = format.height; @@ -604,6 +607,20 @@ public Builder setSubsampleOffsetUs(long subsampleOffsetUs) { return this; } + /** + * Sets {@link Format#hasReliablePresentationTimestamps}. The default value is {@code true}. + * + * @param hasReliablePresentationTimestamps The {@link + * Format#hasReliablePresentationTimestamps}. + * @return The builder. + */ + @CanIgnoreReturnValue + public Builder setHasReliablePresentationTimestamps( + boolean hasReliablePresentationTimestamps) { + this.hasReliablePresentationTimestamps = hasReliablePresentationTimestamps; + return this; + } + // Video specific. /** @@ -1117,6 +1134,15 @@ public Format build() { */ @UnstableApi public final long subsampleOffsetUs; + /** + * Indicates whether per-sample presentation timestamps for this track are known to be accurate. + * + *

When {@code false}, the source couldn't determine each sample's true presentation time + * (for example an MP4 track with no {@code ctts} box). Consumers shouldn't treat the + * timestamps as ground truth for scheduling decisions. Defaults to {@code true}. + */ + @UnstableApi public final boolean hasReliablePresentationTimestamps; + // Video specific. /** The width of the video in pixels, or {@link #NO_VALUE} if unknown or not applicable. */ @@ -1317,6 +1343,7 @@ private Format(Builder builder) { builder.initializationData == null ? Collections.emptyList() : builder.initializationData; drmInitData = builder.drmInitData; subsampleOffsetUs = builder.subsampleOffsetUs; + hasReliablePresentationTimestamps = builder.hasReliablePresentationTimestamps; // Video specific. width = builder.width; height = builder.height; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index c0a22e0d301..5f3bb589362 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -46,6 +46,7 @@ import androidx.annotation.IntDef; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; +import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.Flags; import androidx.media3.common.Format; @@ -398,6 +399,16 @@ private static String buildCustomDiagnosticInfo(int errorCode) { private CodecParameters activeCodecParameters; private CodecParameters lastDispatchedCodecParameters; private ImmutableSet subscribedCodecParameterKeys; + // Used only when Format#hasReliablePresentationTimestamps is false: replays each sample's own + // queued presentationTimeUs in output-arrival order instead of trusting MediaCodec's echoed + // (possibly non-monotonic) timestamp. Preserves true per-sample duration, unlike deriving from + // a single assumed frame rate. + private final ArrayDeque arrivalOrderPtsQueue = new ArrayDeque<>(); + + @VisibleForTesting + /* package */ int getArrivalOrderPtsQueueSizeForTesting() { + return arrivalOrderPtsQueue.size(); + } /** * @param context A context. @@ -1107,6 +1118,7 @@ protected void resetCodecStateForFlush() { hasSkippedFlushAndWaitingForQueueInputBuffer = false; skippedFlushOffsetUs = 0; largestStaleModifiedPresentationTimeUs = C.TIME_UNSET; + arrivalOrderPtsQueue.clear(); } /** @@ -1655,6 +1667,20 @@ private boolean feedInputBuffer() throws ExoPlaybackException { handleInputBufferSupplementalData(buffer); } + if (getTrackType() == C.TRACK_TYPE_VIDEO + && codecInputFormat != null + && !codecInputFormat.hasReliablePresentationTimestamps + && !getConfiguration().tunneling) { + // codecInputFormat, not inputFormat: this must describe the format actually governing the + // buffer being queued right now, which during a full codec reinit can briefly lag behind + // inputFormat (see drainOutputBuffer's matching check). Skipped entirely in tunneling mode, + // where drainOutputBuffer -- and so the corresponding pop -- never runs at all; queueing + // here without ever draining would just grow this unboundedly. + // + // Must run before the += skippedFlushOffsetUs below, to match the scale drainOutputBuffer + // converts dequeued timestamps back to. + arrivalOrderPtsQueue.addLast(presentationTimeUs); + } onQueueInputBuffer(buffer); int flags = getCodecBufferFlags(buffer); presentationTimeUs += skippedFlushOffsetUs; @@ -2260,6 +2286,17 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) return false; } + if (getTrackType() == C.TRACK_TYPE_VIDEO + && codecInputFormat != null + && !codecInputFormat.hasReliablePresentationTimestamps + && !arrivalOrderPtsQueue.isEmpty()) { + // codecInputFormat, not inputFormat -- see the matching check in feedInputBuffer. No + // tunneling check needed here: in tunneling mode this method isn't called at all (see + // updateOutputFormatForTime's javadoc), so the push side already prevents anything from + // reaching this queue to begin with. + outputBufferInfo.presentationTimeUs = arrivalOrderPtsQueue.removeFirst(); + } + this.outputIndex = outputIndex; outputBuffer = codec.getOutputBuffer(outputIndex); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 30d77966963..f4501edf0d0 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -2221,6 +2221,16 @@ protected void onProcessedStreamChange() { */ protected boolean shouldDropOutputBuffer( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { + @Nullable Format codecInputFormat = getCodecInputFormat(); + if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { + // earlyUs is derived from a reconstructed timestamp here, not ground truth, so the normal + // threshold is too trigger-happy: pipeline depth from the decoder's own reorder buffering + // can plausibly look this late even with no real backlog. But it's still a genuine + // pacing signal (see arrivalOrderPtsQueue's declaration), so don't disable dropping + // outright either -- that would leave no recovery path if the decoder is genuinely falling + // behind. Require VERY_LATE-level lateness instead of the normal threshold as a backstop. + return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer; + } return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && !isLastBuffer; } @@ -2236,6 +2246,13 @@ protected boolean shouldDropOutputBuffer( */ protected boolean shouldDropBuffersToKeyframe( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { + @Nullable Format codecInputFormat = getCodecInputFormat(); + if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { + // Unlike shouldDropOutputBuffer, stays disabled here: this skips to a keyframe based on + // *how many* buffers to discard, which needs real per-sample identity we don't have in this + // case -- getting that wrong risks skipping past buffers that were never actually late. + return false; + } return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer; } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java index 9d0aeb98920..7dc6c411584 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -1027,6 +1027,41 @@ public void codecReinitialized_withSubscribedKeys_resubscribesToVendorParameters assertThat(stringListCaptor.getValue()).containsExactly("key1", "key2"); } + @Test + public void feedInputBuffer_withUnreliablePtsAndTunneling_doesNotGrowArrivalOrderPtsQueue() + throws Exception { + // In tunneling mode drainOutputBuffer -- and so the queue's pop side -- never runs (see its + // javadoc), so feedInputBuffer must skip pushing altogether or the queue grows unboundedly for + // the life of the session. Track type must be video: the guard is a no-op for other types. + Format unreliablePtsVideo = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_H264) + .setHasReliablePresentationTimestamps(false) + .build(); + FakeSampleStream fakeSampleStream = + createFakeSampleStream(unreliablePtsVideo, /* sampleTimesUs...= */ 0, 100, 200); + VideoTestRenderer renderer = new VideoTestRenderer(); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.enable( + new RendererConfiguration(/* tunneling= */ true), + new Format[] {unreliablePtsVideo}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + assertThat(renderer.getArrivalOrderPtsQueueSizeForTesting()).isEqualTo(0); + } + private TestRenderer setUpAndEnableRenderer(Format format) throws Exception { MediaCodecAdapter mockCodecAdapter = mock(MediaCodecAdapter.class); MediaCodecAdapter.Factory mockCodecAdapterFactory = configuration -> mockCodecAdapter; @@ -1191,6 +1226,106 @@ protected DecoderReuseEvaluation canReuseCodec( } } + /** Same as {@link TestRenderer}, but with {@link C#TRACK_TYPE_VIDEO} for video-only guards. */ + private static class VideoTestRenderer extends MediaCodecRenderer { + + VideoTestRenderer() { + this(MediaCodecAdapter.Factory.getDefault(ApplicationProvider.getApplicationContext())); + } + + VideoTestRenderer(MediaCodecAdapter.Factory mediaCodecAdapterFactory) { + super( + ApplicationProvider.getApplicationContext(), + C.TRACK_TYPE_VIDEO, + mediaCodecAdapterFactory, + /* mediaCodecSelector= */ (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> + Collections.singletonList( + MediaCodecInfo.newInstance( + /* name= */ "name", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ null, + /* hardwareAccelerated= */ false, + /* softwareOnly= */ true, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)), + /* enableDecoderFallback= */ false, + /* assumedMinimumCodecOperatingRate= */ 44100); + } + + @Override + public String getName() { + return "videoTest"; + } + + @Override + protected @Capabilities int supportsFormat(MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected List getDecoderInfos( + MediaCodecSelector mediaCodecSelector, Format format, boolean requiresSecureDecoder) + throws MediaCodecUtil.DecoderQueryException { + return mediaCodecSelector.getDecoderInfos( + format.sampleMimeType, + /* requiresSecureDecoder= */ false, + /* requiresTunnelingDecoder= */ false); + } + + @Override + protected MediaCodecAdapter.Configuration getMediaCodecConfiguration( + MediaCodecInfo codecInfo, + Format format, + @Nullable MediaCrypto crypto, + float codecOperatingRate) { + MediaFormat mediaFormat = new MediaFormat(); + applyCodecParametersToMediaFormat(mediaFormat); + return MediaCodecAdapter.Configuration.createForVideoDecoding( + codecInfo, mediaFormat, format, /* surface= */ null, crypto); + } + + @Override + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + @Nullable MediaCodecAdapter codec, + @Nullable ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + int sampleCount, + long bufferPresentationTimeUs, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, + Format format) + throws ExoPlaybackException { + if (bufferPresentationTimeUs <= positionUs) { + // Only release buffers when the position advances far enough for realistic behavior where + // input of buffers to the codec is faster than output. + if (codec != null) { + codec.releaseOutputBuffer(bufferIndex, /* render= */ true); + } + return true; + } + return false; + } + + @Override + protected void onCodecParametersChanged(CodecParameters codecParameters) { + // No-op for this test renderer. + } + + @Override + protected DecoderReuseEvaluation canReuseCodec( + MediaCodecInfo codecInfo, + Format oldFormat, + Format newFormat, + boolean isAdaptiveFormatChange) { + return codecInfo.canReuseCodec(oldFormat, newFormat); + } + } + /** * A {@link MediaCodecAdapter} that throws a pre-specified exception from every decoding-related * interaction. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index b45730f25ef..c6d2ea942e9 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -44,6 +44,7 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assume.assumeTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.clearInvocations; @@ -397,6 +398,103 @@ public void render_withLateBuffer_dropsBuffer() throws Exception { verify(eventListener).onDroppedFrames(eq(1), anyLong()); } + @Test + public void render_withUnreliablePtsAndModeratelyLateBuffer_doesNotDrop() throws Exception { + // Point 3: unreliable timestamps must not disable drop detection outright (see + // shouldDropOutputBuffer), but do need much more slack than normal since earlyUs isn't ground + // truth here. 100ms late clears the normal 30ms threshold but not the widened 500ms one. + Format unreliablePts = + VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePts, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 50_000), // Moderately late buffer. + oneByteSample(/* timeUs= */ 100_000), // Last buffer. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePts}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + for (int i = 0; i < 5; i++) { + mediaCodecVideoRenderer.render(40_000, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + } + mediaCodecVideoRenderer.setCurrentStreamFinal(); + int posUs = 150_001; // ~100ms late relative to the 50_000 sample. + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + posUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener, never()).onDroppedFrames(anyInt(), anyLong()); + } + + @Test + public void render_withUnreliablePtsAndVeryLateBuffer_stillDrops() throws Exception { + // Same as render_withUnreliablePtsAndModeratelyLateBuffer_doesNotDrop, but ~600ms late -- + // past the widened threshold, proving the backstop actually engages. + Format unreliablePts = + VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePts, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 50_000), // Very late buffer. + oneByteSample(/* timeUs= */ 100_000), // Last buffer. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePts}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + for (int i = 0; i < 5; i++) { + mediaCodecVideoRenderer.render(40_000, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + } + mediaCodecVideoRenderer.setCurrentStreamFinal(); + int posUs = 650_001; // ~600ms late relative to the 50_000 sample. + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + posUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener).onDroppedFrames(eq(1), anyLong()); + } + @Test public void render_withVeryLateBuffer_dropsBuffersUpstream() throws Exception { FakeSampleStream fakeSampleStream = @@ -1480,6 +1578,179 @@ public void render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec() thro verify(eventListener, times(2)).onVideoDecoderInitialized(any(), anyLong(), anyLong()); } + @Config(maxSdk = 29) + @Test + public void + render_withUnreliablePtsDuringIncompatibleFormatChange_stillFifoOrdersDrainingOldCodecOutput() + throws Exception { + // codecInputFormat (not inputFormat) must gate arrivalOrderPtsQueue, since inputFormat can + // advance to the new format before the old codec has finished draining its buffered output + // under the old format. Reuses render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec's + // 24fps->30fps trigger for a full codec reinit (REUSE_RESULT_NO) on SDK<=29, but the first + // format also has hasReliablePresentationTimestamps=false and out-of-order sample timestamps, + // and the codec adapter reorders dequeued output by ascending timestamp (simulating an + // untrustworthy decoder echo) so a wrong field choice is actually observable. + SystemClock.setCurrentTimeMillis(876_000_000); + Format unreliablePts24Fps = + VIDEO_H264_24FPS.buildUpon().setHasReliablePresentationTimestamps(false).build(); + Format reliablePts30Fps = + VIDEO_H264_30FPS.buildUpon().setHasReliablePresentationTimestamps(true).build(); + List processedTimestamps = new ArrayList<>(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePts24Fps, + ImmutableList.of( + // Deliberately out of order: with no ctts, these are decode-order values, not + // true presentation times, which is the whole point of hasReliablePresentationTimestamps. + oneByteSample(/* timeUs= */ 40_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 0))); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory( + new ForwardingSynchronousMediaCodecAdapterWithReordering.Factory()) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected void onProcessedOutputBuffer(long presentationTimeUs) { + super.onProcessedOutputBuffer(presentationTimeUs); + processedTimestamps.add(presentationTimeUs); + } + }; + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePts24Fps, reliablePts30Fps}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + // Render first sample to initialize codec. + renderer.render(/* positionUs= */ 0, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + shadowOf(testMainLooper).idle(); + + // Feed format change from 24->30 fps, which is not generally compatible and should reset + // codec, while the old codec may still be draining its format-A output. + fakeSampleStream.append( + ImmutableList.of( + format(reliablePts30Fps), + oneByteSample(/* timeUs= */ 80_000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 40_000); + renderer.setCurrentStreamFinal(); + int positionUs = 40_000; + while (!renderer.isEnded()) { + ShadowSystemClock.advanceBy(Duration.ofMillis(40)); + renderer.render(positionUs, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + positionUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener).onVideoDecoderReleased(any()); + verify(eventListener, times(2)).onVideoDecoderInitialized(any(), anyLong(), anyLong()); + // The reordering adapter would otherwise hand these back sorted ascending (0, 40_000). Seeing + // them in feed order proves codecInputFormat -- not the already-advanced inputFormat -- gated + // the FIFO replay while the old codec's format-A output was still draining. + assertThat(processedTimestamps.subList(0, 2)).containsExactly(40_000L, 0L).inOrder(); + } + + @Test + public void render_withUnreliablePtsAndVariableFrameSpacing_preservesPerSampleDurations() + throws Exception { + // Point 6: a VFR stream with real reordering and no ctts can't have its true per-frame + // presentation instants reconstructed exactly (there's no ctts to say what they are). What + // arrivalOrderPtsQueue can and must do is replay each sample's own queued timestamp exactly, + // not flatten irregular gaps to an assumed constant frame rate -- the bug this queue replaced. + Format unreliablePtsVfr = + VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); + List processedTimestamps = new ArrayList<>(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePtsVfr, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 40_000), + oneByteSample(/* timeUs= */ 80_000), + oneByteSample(/* timeUs= */ 280_000), // Large VFR jump, e.g. a scene cut. + oneByteSample(/* timeUs= */ 300_000), // Small gap right after. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory(codecAdapterFactory) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected void onProcessedOutputBuffer(long presentationTimeUs) { + super.onProcessedOutputBuffer(presentationTimeUs); + processedTimestamps.add(presentationTimeUs); + } + }; + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePtsVfr}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + renderer.setCurrentStreamFinal(); + int positionUs = 0; + while (!renderer.isEnded()) { + ShadowSystemClock.advanceBy(Duration.ofMillis(40)); + renderer.render(positionUs, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + positionUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + assertThat(processedTimestamps) + .containsExactly(0L, 40_000L, 80_000L, 280_000L, 300_000L) + .inOrder(); + } + @Config(minSdk = 30) @Test public void render_withIncompatibleFrameRateChangeFromSdk30_keepsCodec() throws Exception { diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java index fc741fe206c..d4aedc62168 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java @@ -498,6 +498,20 @@ public static TrackSampleTable parseStbl( GaplessInfoHolder gaplessInfoHolder, boolean omitTrackSampleTable) throws ParserException { + // Computed and applied to track up front, before any of the return paths below (including + // the sampleCount==0 and omitTrackSampleTable ones), so all of them carry the correct flag -- + // this doesn't depend on anything computed later in this method. See the field's javadoc for + // what NO_VALUE-vs-0 means here. + boolean hasReliablePresentationTimestamps = + !(track.type == C.TRACK_TYPE_VIDEO + && stblBox.getLeafBoxOfType(Mp4Box.TYPE_ctts) == null + && track.format.maxNumReorderSamples != 0); + if (!hasReliablePresentationTimestamps) { + track = + track.copyWithFormat( + track.format.buildUpon().setHasReliablePresentationTimestamps(false).build()); + } + SampleSizeBox sampleSizeBox; @Nullable LeafBox stszAtom = stblBox.getLeafBoxOfType(Mp4Box.TYPE_stsz); if (stszAtom != null) { From 7661dc83f465d9327a94714d4488eca927709555 Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Fri, 16 Oct 2026 15:37:07 -0500 Subject: [PATCH 2/3] Fix fragmented MP4 regression and relabeling desync, add requested tests Addresses the remaining points from the 2026-08-05 review on #3352. - BoxParser: only set hasReliablePresentationTimestamps=false for sample tables that have samples. FragmentedMp4Extractor also parses the moov box's empty sample table, which never has a ctts box because composition offsets come from trun boxes, so every fragmented video track with B-frames was being flagged and its correct timestamps scrambled. - MediaCodecRenderer: keep the relabeling queue in step when the codec discards samples without output. BUFFER_FLAG_DECODE_ONLY samples are no longer recorded, and each output's echoed timestamp is matched against the recorded ones. Pending samples queued more than maxNumReorderSamples before the match can't still be awaiting output (for example RASL pictures discarded after seeking to a CRA picture), so they're dropped. Outputs that don't match keep the codec's timestamp, which also makes the output side independent of codecInputFormat vs inputFormat during format changes. - Tests: extractor tests for no ctts with and without an edit list, with FLAG_OMIT_TRACK_SAMPLE_TABLE, non-reordered video, ctts present, and fragmented MP4 (three small generated H.264 assets); renderer tests for reordered output, codec-discarded samples, VFR with reordering, a seamless format change, EOS and flush. - Comments tightened; removed references to review point numbers. --- .../mediacodec/MediaCodecRenderer.java | 93 +++-- .../mediacodec/MediaCodecRendererTest.java | 71 +++- .../video/MediaCodecVideoRendererTest.java | 333 ++++++++++++++---- .../media3/extractor/mp4/BoxParser.java | 25 +- ...entedMp4ExtractorNonParameterizedTest.java | 18 + .../mp4/Mp4ExtractorNonParameterizedTest.java | 61 ++++ .../mp4/sample_no_ctts_not_reordered.mp4 | Bin 0 -> 5401 bytes .../media/mp4/sample_no_ctts_reordered.mp4 | Bin 0 -> 5255 bytes .../sample_no_ctts_reordered_edit_list.mp4 | Bin 0 -> 5291 bytes 9 files changed, 483 insertions(+), 118 deletions(-) create mode 100644 libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_not_reordered.mp4 create mode 100644 libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_reordered.mp4 create mode 100644 libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_reordered_edit_list.mp4 diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index 5f3bb589362..5ea0739f0e9 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -241,6 +241,15 @@ private static String buildCustomDiagnosticInfo(int errorCode) { */ private static final long MAX_CODEC_HOTSWAP_TIME_MS = 1000; + /** + * The reorder depth assumed when relabeling output timestamps for a format with an unknown + * {@link Format#maxNumReorderSamples}. Matches the maximum DPB size of H.264 and H.265. + */ + private static final int DEFAULT_MAX_NUM_REORDER_SAMPLES = 16; + + /** The maximum number of queued samples whose timestamps are retained for relabeling output. */ + private static final int MAX_ARRIVAL_ORDER_PTS_QUEUE_SIZE = 64; + @Documented @Retention(RetentionPolicy.SOURCE) @Target(TYPE_USE) @@ -399,11 +408,14 @@ private static String buildCustomDiagnosticInfo(int errorCode) { private CodecParameters activeCodecParameters; private CodecParameters lastDispatchedCodecParameters; private ImmutableSet subscribedCodecParameterKeys; - // Used only when Format#hasReliablePresentationTimestamps is false: replays each sample's own - // queued presentationTimeUs in output-arrival order instead of trusting MediaCodec's echoed - // (possibly non-monotonic) timestamp. Preserves true per-sample duration, unlike deriving from - // a single assumed frame rate. + // The timestamps of queued samples whose format has unreliable presentation timestamps, in queue + // order. Output buffers are relabeled with these in output order, instead of the timestamp the + // codec echoes back, which is out of order for reordered frames. private final ArrayDeque arrivalOrderPtsQueue = new ArrayDeque<>(); + // The timestamps passed to the codec for the samples in arrivalOrderPtsQueue, in queue order, + // removed as the codec echoes them back. Used to detect samples the codec discarded without + // producing output, so that arrivalOrderPtsQueue doesn't fall out of step. + private final ArrayDeque pendingCodecPtsQueue = new ArrayDeque<>(); @VisibleForTesting /* package */ int getArrivalOrderPtsQueueSizeForTesting() { @@ -1119,6 +1131,7 @@ protected void resetCodecStateForFlush() { skippedFlushOffsetUs = 0; largestStaleModifiedPresentationTimeUs = C.TIME_UNSET; arrivalOrderPtsQueue.clear(); + pendingCodecPtsQueue.clear(); } /** @@ -1667,22 +1680,18 @@ private boolean feedInputBuffer() throws ExoPlaybackException { handleInputBufferSupplementalData(buffer); } + onQueueInputBuffer(buffer); + int flags = getCodecBufferFlags(buffer); if (getTrackType() == C.TRACK_TYPE_VIDEO && codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps - && !getConfiguration().tunneling) { - // codecInputFormat, not inputFormat: this must describe the format actually governing the - // buffer being queued right now, which during a full codec reinit can briefly lag behind - // inputFormat (see drainOutputBuffer's matching check). Skipped entirely in tunneling mode, - // where drainOutputBuffer -- and so the corresponding pop -- never runs at all; queueing - // here without ever draining would just grow this unboundedly. - // - // Must run before the += skippedFlushOffsetUs below, to match the scale drainOutputBuffer - // converts dequeued timestamps back to. - arrivalOrderPtsQueue.addLast(presentationTimeUs); + && !getConfiguration().tunneling + && (flags & MediaCodec.BUFFER_FLAG_DECODE_ONLY) == 0) { + // codecInputFormat, not inputFormat: during a full codec reinit inputFormat already describes + // the next codec's samples. Skipped in tunneling mode, where output buffers are never + // dequeued, and for BUFFER_FLAG_DECODE_ONLY samples, which the codec doesn't output. + queueArrivalOrderPts(presentationTimeUs, presentationTimeUs + skippedFlushOffsetUs); } - onQueueInputBuffer(buffer); - int flags = getCodecBufferFlags(buffer); presentationTimeUs += skippedFlushOffsetUs; if (bufferEncrypted) { checkNotNull(codec) @@ -1719,6 +1728,43 @@ private long getLargestValidQueuedPresentationTimeUs() { : largestQueuedPresentationTimeUs; } + private void queueArrivalOrderPts(long presentationTimeUs, long codecPresentationTimeUs) { + if (arrivalOrderPtsQueue.size() >= MAX_ARRIVAL_ORDER_PTS_QUEUE_SIZE) { + // Backstop against a codec that never echoes back the timestamps it was given. + arrivalOrderPtsQueue.removeFirst(); + pendingCodecPtsQueue.removeFirst(); + } + arrivalOrderPtsQueue.addLast(presentationTimeUs); + pendingCodecPtsQueue.addLast(codecPresentationTimeUs); + } + + private long dequeueArrivalOrderPts(long codecPresentationTimeUs, long defaultPresentationTimeUs) { + int pendingIndex = 0; + for (long pendingCodecPts : pendingCodecPtsQueue) { + if (pendingCodecPts == codecPresentationTimeUs) { + break; + } + pendingIndex++; + } + if (pendingIndex == pendingCodecPtsQueue.size()) { + // Not a sample that was queued while relabeling was active. + return defaultPresentationTimeUs; + } + // At most maxNumReorderSamples samples queued before this one can still be awaiting output. + // Any older pending samples were discarded by the codec without being output (for example RASL + // pictures after a seek), so drop them to keep arrivalOrderPtsQueue in step. + int maxNumReorderSamples = + codecInputFormat != null && codecInputFormat.maxNumReorderSamples != Format.NO_VALUE + ? codecInputFormat.maxNumReorderSamples + : DEFAULT_MAX_NUM_REORDER_SAMPLES; + for (; pendingIndex > maxNumReorderSamples; pendingIndex--) { + pendingCodecPtsQueue.removeFirst(); + arrivalOrderPtsQueue.removeFirst(); + } + pendingCodecPtsQueue.removeFirstOccurrence(codecPresentationTimeUs); + return arrivalOrderPtsQueue.removeFirst(); + } + /** * Initializes the processing pipeline, if needed by the implementation. * @@ -2286,15 +2332,12 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) return false; } - if (getTrackType() == C.TRACK_TYPE_VIDEO - && codecInputFormat != null - && !codecInputFormat.hasReliablePresentationTimestamps - && !arrivalOrderPtsQueue.isEmpty()) { - // codecInputFormat, not inputFormat -- see the matching check in feedInputBuffer. No - // tunneling check needed here: in tunneling mode this method isn't called at all (see - // updateOutputFormatForTime's javadoc), so the push side already prevents anything from - // reaching this queue to begin with. - outputBufferInfo.presentationTimeUs = arrivalOrderPtsQueue.removeFirst(); + if (!arrivalOrderPtsQueue.isEmpty()) { + outputBufferInfo.presentationTimeUs = + dequeueArrivalOrderPts( + /* codecPresentationTimeUs= */ outputBufferInfo.presentationTimeUs + + skippedFlushOffsetUs, + /* defaultPresentationTimeUs= */ outputBufferInfo.presentationTimeUs); } this.outputIndex = outputIndex; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java index 7dc6c411584..c64062c2485 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -1030,9 +1030,7 @@ public void codecReinitialized_withSubscribedKeys_resubscribesToVendorParameters @Test public void feedInputBuffer_withUnreliablePtsAndTunneling_doesNotGrowArrivalOrderPtsQueue() throws Exception { - // In tunneling mode drainOutputBuffer -- and so the queue's pop side -- never runs (see its - // javadoc), so feedInputBuffer must skip pushing altogether or the queue grows unboundedly for - // the life of the session. Track type must be video: the guard is a no-op for other types. + // Output buffers are never dequeued in tunneling mode, so nothing would drain the queue. Format unreliablePtsVideo = new Format.Builder() .setSampleMimeType(MimeTypes.VIDEO_H264) @@ -1062,6 +1060,73 @@ public void feedInputBuffer_withUnreliablePtsAndTunneling_doesNotGrowArrivalOrde assertThat(renderer.getArrivalOrderPtsQueueSizeForTesting()).isEqualTo(0); } + @Test + public void render_withUnreliablePtsUntilEnded_drainsArrivalOrderPtsQueue() throws Exception { + Format unreliablePtsVideo = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_H264) + .setHasReliablePresentationTimestamps(false) + .build(); + FakeSampleStream fakeSampleStream = + createFakeSampleStream(unreliablePtsVideo, /* sampleTimesUs...= */ 0, 100, 200); + VideoTestRenderer renderer = new VideoTestRenderer(); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePtsVideo}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + renderer.setCurrentStreamFinal(); + long positionUs = 0; + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + assertThat(renderer.getArrivalOrderPtsQueueSizeForTesting()).isEqualTo(0); + } + + @Test + public void resetPosition_withUnreliablePts_clearsArrivalOrderPtsQueue() throws Exception { + Format unreliablePtsVideo = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_H264) + .setHasReliablePresentationTimestamps(false) + .build(); + FakeSampleStream fakeSampleStream = + createFakeSampleStream(unreliablePtsVideo, /* sampleTimesUs...= */ 0, 100, 200, 300); + VideoTestRenderer renderer = new VideoTestRenderer(); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePtsVideo}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + // Hold the position so that queued samples remain pending output. + for (int i = 0; i < 1000 && renderer.getArrivalOrderPtsQueueSizeForTesting() == 0; i++) { + renderer.render(/* positionUs= */ -1, SystemClock.elapsedRealtime()); + } + int queueSizeBeforeReset = renderer.getArrivalOrderPtsQueueSizeForTesting(); + + fakeSampleStream.seekToUs(/* positionUs= */ 0, /* allowTimeBeyondBuffer= */ true); + renderer.resetPosition(/* positionUs= */ 0, /* sampleStreamIsResetToKeyFrame= */ true); + + assertThat(queueSizeBeforeReset).isGreaterThan(0); + assertThat(renderer.getArrivalOrderPtsQueueSizeForTesting()).isEqualTo(0); + } + private TestRenderer setUpAndEnableRenderer(Format format) throws Exception { MediaCodecAdapter mockCodecAdapter = mock(MediaCodecAdapter.class); MediaCodecAdapter.Factory mockCodecAdapterFactory = configuration -> mockCodecAdapter; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index c6d2ea942e9..8cece5ff926 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -131,8 +131,10 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.PriorityQueue; +import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -400,9 +402,7 @@ public void render_withLateBuffer_dropsBuffer() throws Exception { @Test public void render_withUnreliablePtsAndModeratelyLateBuffer_doesNotDrop() throws Exception { - // Point 3: unreliable timestamps must not disable drop detection outright (see - // shouldDropOutputBuffer), but do need much more slack than normal since earlyUs isn't ground - // truth here. 100ms late clears the normal 30ms threshold but not the widened 500ms one. + // 100ms late is past the normal late threshold, but not the one for unreliable timestamps. Format unreliablePts = VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); FakeSampleStream fakeSampleStream = @@ -1583,13 +1583,9 @@ public void render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec() thro public void render_withUnreliablePtsDuringIncompatibleFormatChange_stillFifoOrdersDrainingOldCodecOutput() throws Exception { - // codecInputFormat (not inputFormat) must gate arrivalOrderPtsQueue, since inputFormat can - // advance to the new format before the old codec has finished draining its buffered output - // under the old format. Reuses render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec's - // 24fps->30fps trigger for a full codec reinit (REUSE_RESULT_NO) on SDK<=29, but the first - // format also has hasReliablePresentationTimestamps=false and out-of-order sample timestamps, - // and the codec adapter reorders dequeued output by ascending timestamp (simulating an - // untrustworthy decoder echo) so a wrong field choice is actually observable. + // Same 24fps -> 30fps codec reinitialization as + // render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec, but the first format has + // unreliable timestamps and the codec outputs in ascending timestamp order. SystemClock.setCurrentTimeMillis(876_000_000); Format unreliablePts24Fps = VIDEO_H264_24FPS.buildUpon().setHasReliablePresentationTimestamps(false).build(); @@ -1604,8 +1600,6 @@ public void render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec() thro new DrmSessionEventListener.EventDispatcher(), /* initialFormat= */ unreliablePts24Fps, ImmutableList.of( - // Deliberately out of order: with no ctts, these are decode-order values, not - // true presentation times, which is the whole point of hasReliablePresentationTimestamps. oneByteSample(/* timeUs= */ 40_000, C.BUFFER_FLAG_KEY_FRAME), oneByteSample(/* timeUs= */ 0))); fakeSampleStream.writeData(/* startPositionUs= */ 0); @@ -1670,81 +1664,131 @@ protected void onProcessedOutputBuffer(long presentationTimeUs) { verify(eventListener).onVideoDecoderReleased(any()); verify(eventListener, times(2)).onVideoDecoderInitialized(any(), anyLong(), anyLong()); - // The reordering adapter would otherwise hand these back sorted ascending (0, 40_000). Seeing - // them in feed order proves codecInputFormat -- not the already-advanced inputFormat -- gated - // the FIFO replay while the old codec's format-A output was still draining. assertThat(processedTimestamps.subList(0, 2)).containsExactly(40_000L, 0L).inOrder(); } @Test - public void render_withUnreliablePtsAndVariableFrameSpacing_preservesPerSampleDurations() + public void render_withUnreliablePtsAndReorderedOutput_relabelsOutputInArrivalOrder() throws Exception { - // Point 6: a VFR stream with real reordering and no ctts can't have its true per-frame - // presentation instants reconstructed exactly (there's no ctts to say what they are). What - // arrivalOrderPtsQueue can and must do is replay each sample's own queued timestamp exactly, - // not flatten irregular gaps to an assumed constant frame rate -- the bug this queue replaced. - Format unreliablePtsVfr = - VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); - List processedTimestamps = new ArrayList<>(); - FakeSampleStream fakeSampleStream = - new FakeSampleStream( - new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), - /* mediaSourceEventDispatcher= */ null, - DrmSessionManager.DRM_UNSUPPORTED, - new DrmSessionEventListener.EventDispatcher(), - /* initialFormat= */ unreliablePtsVfr, + // Decode order I0 P3 B1 B2 P6 B4 B5, with each sample's timestamp being its decode time. + Format format = + VIDEO_H264 + .buildUpon() + .setMaxNumReorderSamples(1) + .setHasReliablePresentationTimestamps(false) + .build(); + MediaCodecAdapter.Factory codecAdapterFactory = + new ForwardingSynchronousMediaCodecAdapterWithScriptedOutput.Factory( + /* outputTimestampsUs...= */ 0, 80_000, 120_000, 40_000, 200_000, 240_000, 160_000); + + List processedTimestamps = + renderToEndAndGetProcessedTimestamps( + format, + codecAdapterFactory, + /* sampleTimesUs...= */ 0, + 40_000, + 80_000, + 120_000, + 160_000, + 200_000, + 240_000); + + assertThat(processedTimestamps) + .containsExactly(0L, 40_000L, 80_000L, 120_000L, 160_000L, 200_000L, 240_000L) + .inOrder(); + } + + @Test + public void render_withUnreliablePtsAndSamplesDiscardedByCodec_resynchronizesRelabeling() + throws Exception { + // Decode order CRA, RASL, RASL, P3, B1, B2. The codec discards both RASL pictures without + // outputting them, as happens after seeking to a CRA picture. + Format format = + VIDEO_H264 + .buildUpon() + .setMaxNumReorderSamples(1) + .setHasReliablePresentationTimestamps(false) + .build(); + MediaCodecAdapter.Factory codecAdapterFactory = + new ForwardingSynchronousMediaCodecAdapterWithScriptedOutput.Factory( + /* outputTimestampsUs...= */ 0, 160_000, 200_000, 120_000); + + List processedTimestamps = + renderToEndAndGetProcessedTimestamps( + format, + codecAdapterFactory, + /* sampleTimesUs...= */ 0, + 40_000, + 80_000, + 120_000, + 160_000, + 200_000); + + // Without resynchronizing, B1, B2 and P3 would be relabeled 40_000, 80_000 and 120_000, two + // frames earlier than their decode-time slots. + assertThat(processedTimestamps).containsExactly(0L, 120_000L, 160_000L, 200_000L).inOrder(); + } + + @Test + public void render_withSeamlessChangeToReliablePts_onlyRelabelsSamplesOfUnreliableFormat() + throws Exception { + // The first format has decode order I0 P2 B1 with decode-time timestamps. The second format + // reuses the codec and has decode order I5 P7 B6 with correct presentation timestamps. + Format unreliablePts = + VIDEO_H264 + .buildUpon() + .setId("unreliable") + .setMaxNumReorderSamples(1) + .setHasReliablePresentationTimestamps(false) + .build(); + Format reliablePts = VIDEO_H264.buildUpon().setId("reliable").setMaxNumReorderSamples(1).build(); + MediaCodecAdapter.Factory codecAdapterFactory = + new ForwardingSynchronousMediaCodecAdapterWithScriptedOutput.Factory( + /* outputTimestampsUs...= */ 0, 80_000, 40_000, 200_000, 240_000, 280_000); + + List processedTimestamps = + renderToEndAndGetProcessedTimestamps( + new Format[] {unreliablePts, reliablePts}, + codecAdapterFactory, ImmutableList.of( oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), oneByteSample(/* timeUs= */ 40_000), oneByteSample(/* timeUs= */ 80_000), - oneByteSample(/* timeUs= */ 280_000), // Large VFR jump, e.g. a scene cut. - oneByteSample(/* timeUs= */ 300_000), // Small gap right after. + format(reliablePts), + oneByteSample(/* timeUs= */ 200_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 280_000), + oneByteSample(/* timeUs= */ 240_000), END_OF_STREAM_ITEM)); - fakeSampleStream.writeData(/* startPositionUs= */ 0); - MediaCodecVideoRenderer renderer = - new MediaCodecVideoRenderer( - new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) - .setCodecAdapterFactory(codecAdapterFactory) - .setMediaCodecSelector(mediaCodecSelector) - .setAllowedJoiningTimeMs(0) - .setEnableDecoderFallback(false) - .setEventHandler(new Handler(testMainLooper)) - .setEventListener(eventListener) - .setMaxDroppedFramesToNotify(1)) { - @Override - protected @Capabilities int supportsFormat( - MediaCodecSelector mediaCodecSelector, Format format) { - return RendererCapabilities.create(C.FORMAT_HANDLED); - } - @Override - protected void onProcessedOutputBuffer(long presentationTimeUs) { - super.onProcessedOutputBuffer(presentationTimeUs); - processedTimestamps.add(presentationTimeUs); - } - }; - renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); - renderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); - renderer.enable( - RendererConfiguration.DEFAULT, - new Format[] {unreliablePtsVfr}, - fakeSampleStream, - /* positionUs= */ 0, - /* joining= */ false, - /* mayRenderStartOfStream= */ true, - /* startPositionUs= */ 0, - /* offsetUs= */ 0, - new MediaSource.MediaPeriodId(new Object())); - renderer.start(); - renderer.setCurrentStreamFinal(); - int positionUs = 0; - while (!renderer.isEnded()) { - ShadowSystemClock.advanceBy(Duration.ofMillis(40)); - renderer.render(positionUs, msToUs(SystemClock.elapsedRealtime())); - codecAdapterFactory.idleQueueingAndCallbackThreads(); - positionUs += 40_000; - } - shadowOf(testMainLooper).idle(); + assertThat(processedTimestamps) + .containsExactly(0L, 40_000L, 80_000L, 200_000L, 240_000L, 280_000L) + .inOrder(); + } + + @Test + public void render_withUnreliablePtsAndVariableFrameSpacing_relabelsWithQueuedTimestamps() + throws Exception { + // Decode order I0 P2 B1 P4 B3 with irregular spacing. The true presentation times of reordered + // frames can't be recovered, so the queued timestamps are replayed in output order. + Format format = + VIDEO_H264 + .buildUpon() + .setMaxNumReorderSamples(1) + .setHasReliablePresentationTimestamps(false) + .build(); + MediaCodecAdapter.Factory codecAdapterFactory = + new ForwardingSynchronousMediaCodecAdapterWithScriptedOutput.Factory( + /* outputTimestampsUs...= */ 0, 80_000, 40_000, 300_000, 280_000); + + List processedTimestamps = + renderToEndAndGetProcessedTimestamps( + format, + codecAdapterFactory, + /* sampleTimesUs...= */ 0, + 40_000, + 80_000, + 280_000, + 300_000); assertThat(processedTimestamps) .containsExactly(0L, 40_000L, 80_000L, 280_000L, 300_000L) @@ -7621,6 +7665,143 @@ public void release() { } } + private List renderToEndAndGetProcessedTimestamps( + Format format, MediaCodecAdapter.Factory codecAdapterFactory, long... sampleTimesUs) + throws Exception { + ImmutableList.Builder items = ImmutableList.builder(); + for (int i = 0; i < sampleTimesUs.length; i++) { + items.add(oneByteSample(sampleTimesUs[i], i == 0 ? C.BUFFER_FLAG_KEY_FRAME : 0)); + } + items.add(END_OF_STREAM_ITEM); + return renderToEndAndGetProcessedTimestamps( + new Format[] {format}, codecAdapterFactory, items.build()); + } + + private List renderToEndAndGetProcessedTimestamps( + Format[] formats, + MediaCodecAdapter.Factory codecAdapterFactory, + ImmutableList items) + throws Exception { + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ formats[0], + items); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + List processedTimestamps = new ArrayList<>(); + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory(codecAdapterFactory) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected void onProcessedOutputBuffer(long presentationTimeUs) { + super.onProcessedOutputBuffer(presentationTimeUs); + processedTimestamps.add(presentationTimeUs); + } + }; + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + renderer.enable( + RendererConfiguration.DEFAULT, + formats, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + renderer.setCurrentStreamFinal(); + int positionUs = 0; + while (!renderer.isEnded()) { + ShadowSystemClock.advanceBy(Duration.ofMillis(40)); + renderer.render(positionUs, msToUs(SystemClock.elapsedRealtime())); + positionUs += 40_000; + } + shadowOf(testMainLooper).idle(); + return processedTimestamps; + } + + /** + * A {@link MediaCodecAdapter} that outputs buffers with a scripted sequence of timestamps, each + * of which must have been queued, and discards any queued buffers beyond the script. + */ + private static final class ForwardingSynchronousMediaCodecAdapterWithScriptedOutput + extends ForwardingMediaCodecAdapter { + /** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithScriptedOutput} instances. */ + public static final class Factory implements MediaCodecAdapter.Factory { + private final long[] outputTimestampsUs; + + Factory(long... outputTimestampsUs) { + this.outputTimestampsUs = outputTimestampsUs; + } + + @Override + public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException { + return new ForwardingSynchronousMediaCodecAdapterWithScriptedOutput( + outputTimestampsUs, + new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration)); + } + } + + private final long[] outputTimestampsUs; + private final Set queuedTimestampsUs; + private int outputCount; + + ForwardingSynchronousMediaCodecAdapterWithScriptedOutput( + long[] outputTimestampsUs, MediaCodecAdapter adapter) { + super(adapter); + this.outputTimestampsUs = outputTimestampsUs; + queuedTimestampsUs = new HashSet<>(); + } + + @Override + public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) { + while (true) { + if (outputCount < outputTimestampsUs.length + && !queuedTimestampsUs.contains(outputTimestampsUs[outputCount])) { + return INFO_TRY_AGAIN_LATER; + } + int outputBufferIndex = super.dequeueOutputBufferIndex(bufferInfo); + if (outputBufferIndex < 0 + || (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + return outputBufferIndex; + } + if (outputCount == outputTimestampsUs.length) { + super.releaseOutputBuffer(outputBufferIndex, /* render= */ false); + continue; + } + bufferInfo.presentationTimeUs = outputTimestampsUs[outputCount++]; + return outputBufferIndex; + } + } + + @Override + public void queueInputBuffer( + int index, int offset, int size, long presentationTimeUs, int flags) { + if ((flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) { + queuedTimestampsUs.add(presentationTimeUs); + } + super.queueInputBuffer(index, offset, size, presentationTimeUs, flags); + } + } + private static final class ForwardingSynchronousMediaCodecAdapterWithBufferLimit extends ForwardingMediaCodecAdapter { /** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithBufferLimit} instances. */ diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java index d4aedc62168..298ca84cf93 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java @@ -498,20 +498,6 @@ public static TrackSampleTable parseStbl( GaplessInfoHolder gaplessInfoHolder, boolean omitTrackSampleTable) throws ParserException { - // Computed and applied to track up front, before any of the return paths below (including - // the sampleCount==0 and omitTrackSampleTable ones), so all of them carry the correct flag -- - // this doesn't depend on anything computed later in this method. See the field's javadoc for - // what NO_VALUE-vs-0 means here. - boolean hasReliablePresentationTimestamps = - !(track.type == C.TRACK_TYPE_VIDEO - && stblBox.getLeafBoxOfType(Mp4Box.TYPE_ctts) == null - && track.format.maxNumReorderSamples != 0); - if (!hasReliablePresentationTimestamps) { - track = - track.copyWithFormat( - track.format.buildUpon().setHasReliablePresentationTimestamps(false).build()); - } - SampleSizeBox sampleSizeBox; @Nullable LeafBox stszAtom = stblBox.getLeafBoxOfType(Mp4Box.TYPE_stsz); if (stszAtom != null) { @@ -546,6 +532,17 @@ public static TrackSampleTable parseStbl( track = track.buildUpon().setFormat(format).build(); } + // Without a ctts box every sample's presentation time is its decode time, which is wrong for + // any track whose frames are reordered. Treat an unknown reorder depth (NO_VALUE) as possibly + // reordered. Applied after the sampleCount == 0 return: fragmented MP4 has an empty sample + // table here and carries composition offsets in trun boxes instead. + if (track.type == C.TRACK_TYPE_VIDEO + && stblBox.getLeafBoxOfType(Mp4Box.TYPE_ctts) == null + && track.format.maxNumReorderSamples != 0) { + Format format = track.format.buildUpon().setHasReliablePresentationTimestamps(false).build(); + track = track.buildUpon().setFormat(format).build(); + } + // Entries are byte offsets of chunks. boolean chunkOffsetsAreLongs = false; @Nullable LeafBox chunkOffsetsAtom = stblBox.getLeafBoxOfType(Mp4Box.TYPE_stco); diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNonParameterizedTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNonParameterizedTest.java index b4f76c601bb..35e7da88222 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNonParameterizedTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNonParameterizedTest.java @@ -15,9 +15,11 @@ */ package androidx.media3.extractor.mp4; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.US_ASCII; +import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.extractor.SniffFailure; import androidx.media3.extractor.text.SubtitleParser; @@ -106,6 +108,22 @@ public void read_withNegativeVersion1TfdtBaseMediaDecodeTime_outputsShiftedSampl assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(25_000L); } + @Test + public void extract_reorderedVideo_hasReliablePresentationTimestamps() throws Exception { + // The moov box's sample table has no ctts box, but composition offsets come from trun boxes. + FragmentedMp4Extractor extractor = + new FragmentedMp4Extractor(SubtitleParser.Factory.UNSUPPORTED); + + FakeExtractorOutput output = + TestUtil.extractAllSamplesFromFile( + extractor, ApplicationProvider.getApplicationContext(), "media/mp4/sample_fragmented.mp4"); + + Format videoFormat = checkNotNull(output.trackOutputs.get(0).lastFormat); + assertThat(videoFormat.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H264); + assertThat(videoFormat.maxNumReorderSamples).isGreaterThan(0); + assertThat(videoFormat.hasReliablePresentationTimestamps).isTrue(); + } + @Test public void extract_withoutMehdOrEditList_prefersMdhdTrackDuration() throws Exception { FragmentedMp4Extractor extractor = diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorNonParameterizedTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorNonParameterizedTest.java index 4e90e049e0f..494f551a11e 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorNonParameterizedTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorNonParameterizedTest.java @@ -15,6 +15,7 @@ */ package androidx.media3.extractor.mp4; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.truth.Truth.assertThat; import android.content.Context; @@ -406,6 +407,66 @@ public void extract_mebxHagcTrackWithFlagDisableHagcMetadata_ignoresMebxHagcTrac assertThat(output.trackOutputs.get(0).lastFormat.sampleMimeType).isEqualTo(MimeTypes.VIDEO_AV1); } + @Test + public void extract_reorderedVideoWithoutCttsOrEditList_hasUnreliablePresentationTimestamps() + throws Exception { + Format videoFormat = + extractFirstTrackFormat("media/mp4/sample_no_ctts_reordered.mp4", /* flags= */ 0); + + assertThat(videoFormat.maxNumReorderSamples).isGreaterThan(0); + assertThat(videoFormat.hasReliablePresentationTimestamps).isFalse(); + } + + @Test + public void extract_reorderedVideoWithoutCttsWithEditList_hasUnreliablePresentationTimestamps() + throws Exception { + Format videoFormat = + extractFirstTrackFormat( + "media/mp4/sample_no_ctts_reordered_edit_list.mp4", /* flags= */ 0); + + assertThat(videoFormat.maxNumReorderSamples).isGreaterThan(0); + assertThat(videoFormat.hasReliablePresentationTimestamps).isFalse(); + } + + @Test + public void + extract_reorderedVideoWithoutCttsAndOmitTrackSampleTable_hasUnreliablePresentationTimestamps() + throws Exception { + Format videoFormat = + extractFirstTrackFormat( + "media/mp4/sample_no_ctts_reordered.mp4", Mp4Extractor.FLAG_OMIT_TRACK_SAMPLE_TABLE); + + assertThat(videoFormat.hasReliablePresentationTimestamps).isFalse(); + } + + @Test + public void extract_nonReorderedVideoWithoutCtts_hasReliablePresentationTimestamps() + throws Exception { + Format videoFormat = + extractFirstTrackFormat("media/mp4/sample_no_ctts_not_reordered.mp4", /* flags= */ 0); + + assertThat(videoFormat.maxNumReorderSamples).isEqualTo(0); + assertThat(videoFormat.hasReliablePresentationTimestamps).isTrue(); + } + + @Test + public void extract_reorderedVideoWithCtts_hasReliablePresentationTimestamps() + throws Exception { + Format videoFormat = extractFirstTrackFormat("media/mp4/h265_bframes.mp4", /* flags= */ 0); + + assertThat(videoFormat.maxNumReorderSamples).isGreaterThan(0); + assertThat(videoFormat.hasReliablePresentationTimestamps).isTrue(); + } + + private static Format extractFirstTrackFormat(String inputFilePath, @Mp4Extractor.Flags int flags) + throws IOException { + Mp4Extractor mp4Extractor = new Mp4Extractor(SubtitleParser.Factory.UNSUPPORTED, flags); + FakeExtractorOutput output = + TestUtil.extractAllSamplesFromFile( + mp4Extractor, ApplicationProvider.getApplicationContext(), inputFilePath); + return checkNotNull(output.trackOutputs.get(0).lastFormat); + } + private static String getDumpFilePath(String inputFilePath, String suffix) { return inputFilePath.replaceFirst("media", "extractordumps") + suffix; } diff --git a/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_not_reordered.mp4 b/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_not_reordered.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2cf25c76378ae1fdf71936901806ea0a9a6ba1fc GIT binary patch literal 5401 zcmZ`+2|Scv_a8)(EhQx~won-}W0%REy@Wz&jG193GiJ<;T~b1pB9SdxXt5{Bl6^_C zFWD;lzJ#nao=PHuK+GTp1tuKj zfrb&u3NR3eMUhA%d4ND50?`A90q9?YWeNmhS^+Tu>tEV`7XZV*89^Y;zp?+#GXv*v zpixjRfXR;LvKy1-FW4V%K-~Wv{}Jc^jH?N3VoVUzHW7nI0SuUk`OmzdfcgmNQY`}q zlEq;NlwA&3Jn$IoKRkn824s=}$`wPvGB8XpiFj8hfH~wr+;#f*kt8s`Z9*6dmVsdm zhMm)$LID5fnVhLKM*_fCP-)bEip{{}G8zI<#`xRkZ;d99k^#UnCS&|J;&XAt-tk*{X&2a{TG&@1Ms`Pbpe~t z9~*-J`n`exT?EV=z?1^`Ent2HCU75^_WmUU7|;+fftY9#Kp6w@&BEv(${j;P0n~xgG^8!PD(I{G6V*d2Sj!! z3Kj@-pp*!J0Qvu|E@1mpUZ6y7Y^-!T@vGi+*+ic~;ltCt;6Cy13tn&~1+Xj_O`>4I zFo31V!=Nf)c^DR^>ZA%73?KkN&eTj#UseIEby){MVz6kyphF^i6R=J+FdPb%mxV*& z@_-UYqmk9*+pg+<7N zF<3_e3GIS_sX^7CV3aG0;7!FMpkDH7P%juv9!$g{a9A%e)!h-`l)+@GH?S30krWI9 z27v;bfCWs%dtot1hDR8%14%)-I%5$qB`_LCArVnXU@HtvqhJXHJQd(nyi_n~8o;34 zhzKZf1Qf=Hfc!@X15*ijG;o%G6i~1$Wj6*IPejoe=fJztuoMCc00DYOf;$D}jYN}(WE2gM z(LjlS_Y~e0Z~*`*CgtWM`)|(A4vLvk2(w-eD7aNovnd${S*4Ge$fWG;7TZ@k#A7@^QAUo0FH$ zJy6mA{+3NYk97Y=)VpAz_eL-MCg!jF%HTQUIU)7_r3$QO*chRc%AFGs-Ed>Mw5=;D z_H;4y+=EmV3A#l*)FQ`g%18>0(I55H7A=3&P>X|tD(JbS(0sDf>ckdR5^9OTIa1|P|Dwp`To)F33 zu$tPa87(UI6L9|kEz&uorf_dy&Spt9Y^ri~>OFI?Ki)6gC~r@0@3j4fBI4wfK~na!oj~bZo6`9PEd+^Bd4UvX@0D`Z+;>(P%ZiQ$XPsm2`7A(ne!h&c zc(h<=e8Z7#XJ|Cncae24W_9SxNF_FH-*BHJS0l@tX8Q|;RAHaik`1?U-#u%`I1yB~ zrsgllw^tw`?+mZWv$I^_aIiLr->~3|E@t7CV)wDv&Q0M}68Bg=h?r_D zA6*RX?~$_lE52T< z=Oa__R%;6JAI2I_ojHBnWmprGxHD_oXIpJ`e z9@3H`oW*{|H~E-)ydJ$8!Ylt2BRxAcdDHmv=VNoghkOMir<)o1Lc4C}!1^jwi&!b6 zSxn}A!5aDH@uk41qhdE8NBEP~uAba$xRT)wd9|!;Dqv1-Ltkj08xb0*+WvaO;AG0w zQx(Ne(|yk0X%4E0M@kE~9CA!PLAuOeH@Da{a|v5D%{>hrFEja+_~?@B#{k~Sn66{C zpqxCNU}^C&C(Lzg9=ia94rjr5ufaCtiFGnsa5E5*#cCE92j8P6_3!cC(ftuUtIr|Xj>Dl+tKiG%$aI*(`y2F-7>LhC0=01d_wWl3oC4_ zOj?%{VDQ9+DH`hVe%iGTdXP!^ei;#t<0s42k_EWS5oI|hM@mFa8=YoS=vGX*{pQ8J z`^*aaaXsbxOfEO%D~Ycm-Ls_bh2w&(2lx!|K{ z`gt*+u*s`Gp)EqPpj9noy71ert;hUL2C4iu>txJWZ&jIsMpHrUTeXtb8LA&5k-E!8$8wV)p;DILG=7oA0prK zNs49JPE)QQUNUAyus(GbEk3U*Ql00%^-DDLiuNRauuW)$P?d>yf?X2@Dj}|wNb%BX zh^Y*tFTp|@1BQwM$C8_R65(+{%nO6d=9WwN3ZfV9uO-KHOG_=*LUnp}Ur9mT-1NXX zRzp(mReO58*TX|;>wf!{O_KB%ukH8kH3)q&LJnI=wI!M&unQKxr+$u^#J3tmG(j)L z83k6+H7dt@gDc6&`V)ydT;Ik`P;{nkxvJyIJBls-tj~iDo#@rUGJ|+_zLZ0)pRbn7 zM8FFDHeR0W%-kC9uUS1s{FI-+Gq2UTW>VXB?F>u-R}d;F_t7fxs~XC~bpN?W>;-ch z;-kwQS6#bIC#|=h*PRnO<@` zZoMj1s;l&eSL5jCT}9x+fY!AVqyLoY{pXSwyl)KGQqwuxqdQw5 zYCTi4_ihT^1ogUlLgQ?t<%&1l%768-tk{oDcCyK&x`d~GJLqKd0nPh^O)A? zT7*br(bLBrd4AK8{z6C^a^8tx_lVVZS>L~ytJcdj-v`<=OZRtr6;PWVYcgJ<32FL? zR}#8fnGGs4Q>TpE$S!hp$L7_)o_>ZKpafjiRBwhvcleCIP+J)E2>He10o?+%C5RMu znlIX{h#6q*ADusDTgg)C}7|0w-?!U^yEUlyX5n;l;rel-kO28$383eA|H9JmmM$QwQ%5(lGoFl zk$g{|s4axkvh8Es0)-ywYx65Od+dAiV<@-oZSyx9cvz~F5PQt~c6CM5{2aZG>($D7 z)q_mjEZfuHHja)*+pF6??-LFs=ckk^3HQ4S7pH$S_2!Ii?EjupfTe*NKJ3FpoGGz_ zEg#H`m*U9RdSuZP^*+=RGq1NNmLv1yT_Cqbt;CK$xj6_uspg4a?20S<$#+f-AD{K& zh4Bc#bjQ!trl@VXt*@NMgX{FH_DN5dut7 zWj)QG^t)wtfGnwj`Eg#tq~16iGwZ1!E%O7m1_0AvDy#Fz89R|yhV3N;>oQD zR*lq9cucQu|3UG--?`=TV8~8IGVsLsX2^$YtzBz|9ZuYQk%tXN73PYD_cz2Lh>~rZ zIEyWTJmB9e6N^^jk3Tt=CARS@#RxFn`{sE`z8YsD(zdssKngRWY|2R8@rVkLis`p( z;8tNq5k{2lgiNXm7ajbCM|<|x)9K4=vw8!oZ8nj2vQCv|)*gVS+~U0QE5CR-(rZp+Q9lFEYyKYZs8DdS!1 zxv2#a{3%VipqV1H8byz1dO~6K*^rS=e09-~HN3ds=Xe^`g+3Lc_cr88-CSBWY`9f~ zK3o>-SP-Zvuaf>N&J#Cug1IGYVU2oxHi(zQA*t-9tyJ<@q=B!D1!h33?^Z-CocnXb zMTr5Ii|)HmbQ0fg8Sc?9e#x;`KOH|=&2GGBr3Mk}u8X~McyI|(ifjJfonTySt%49S z#I{e~woY${!Bts<`(`Ka*(1B(%czwKhXqxh-4L?s)E==fm#RLVmzTWqk?maK-hs3V zTF>>3RaY(l&8f3UA6xYBFkh0#MRoti550zwsb4t#AJ2aNp?-cpm2{!-Rsa5c)6^Kv z^b5^ub5||aP7Ig(*DQ@ed+3OE-rSuwp+U{rg+LTLY+uzZ7neOG)Xqa?V_Y`e0v)9s z90a49)eOGowLqDZdMP=e~ zybv(Z)(uWObf?GeyKGvr;iG~O_?WQZot9zcSNQapdo0%CuE9thqWKDt#{ds+8ZWRwaE!!BRnIZleNh2V3$h4I4tGClfBsVOjLuFy(EK&y8xI520Ka zdQ;S=oxtbQ0(>kCFA9RD_DGMV68%L|G3fyErOI zAzMkZWKEGR=KrQn_wU~O|KIbx-@9+$`_2M^K%fLx2%XHNQ6Ug+2nPeuKnG$GRJs-d z0^vbYX|zBH1VW((l5hb3c6cTs5U#HfE^z-Q{dWc^{!IyinEY=0-!wOeSzvE z%lAi4p5M&=i3ali=lPF3|7Ttk;KTt4H!l@OMgsy$#rOS9R4(FRe?xNs05 z_6S6p2yjss@4g0eFmwnn=V~ytPPXQP@RO!TpY3VbSXto&*iJ_?=>T(*a6kgXVz2-P zoY=)50D(Z6EEe;JjI&ff?6_tC-ujb;NPzZ!>3_wM1v2bEKEGtbKYbzQKzI5d9R~sV zmJ1-P2cW^8a5(}Dc*q=}H2{tSm@}tr01*J%0f3xX8o-8ib@ia2AT`cX(^H5!Ac&dH)iVo@o;hDK+RX?{!tC>D)DW043bSX`tg6o)bdLJhx##onSjBq1~KKl%#7lZhl22Iy#X zyq`CbMhDuTLOKZJiw^;@ks8|Ye@JieO@q|Xf-Tr{(;fKNF9A3V6yOZq!yG+ z2SWl20cHqdq5Z-9In#uMLm<5Q_~?s}_<_+IWEtCHRZnwPS?Pl1%jiO%;%LIyfwP(~ zZBLzrH|sA-6or%ylpiM>wsenOKyAJ4w1G(#Aub2yjPOYZ$PoPVY07i5$>o(o?L4Q% zSA3u$?RLeUrG+7hImy+dMB|bV5n&>kj_q!l?DI}C%*qyBw^^=*w<2RJclZig}*`d!KBhFYo zUqc?rcrI$EFc%#?!ab^mHn(^kSna+~Tkhk%2C~6cH+)`2rv2k)ScIcAMTLHEiY$Ie`V-3;rF z8C&5uWm1jId!N2wJ}--li+E7d*^d+rE1jInE-rcIwP5l z(lwd>{Bhi{vEU&7VQF{quHomYk*;0Sdz&CaVL6hzmgeWLQs4J%ypg~5Ep+Fx!UYq_ z=)4to_UTtbLROiBR*B#BTQ0aRo7^}&@K8Nsyyx8LqRm`RfyV>O8}Gz?`{ce=Hct7- zTYM%&SWO_)JCM1z{e^0st!XG0{^f+Ct3&x)b@ZW}%_A`xNmcW4mg+?cQ zTB~gcLP_y0KHage)5}K6Y1n=jTek~k%-96tz@&1O`_SNV&+q(#MszW)S6}8UVaYa9 zr(4Q%%KL?S^F1F#+a1ZOSbaSg`_=H|oH%K&OEclDwu|}2zV8^x2G9Otr@9@{PtGT2 zUp9Q9$YuAXBAueVjUPJjad6zAMD0Ze_x1q=r8o6EkLXNwe-I-d=;`UL9bIl*?xOSy z?927<51Zu?9Tnd-cWM|z+~+=J`Y@kTe~IUM#5=C`1I2d|$jIu7`h8~%znJZNn`W)q z@9^+%y6)YR^++GxkUYw_lTF2Adv1(zUnGczJg9bxQGW@GLSabCm_J`^bSvlu9!+O3g$5L!O$^$y#sI~(s$AN!nrMpG~?kWJa7eDlN%neu<>tNP2Q+UY#ki}_RMF8h=jug857dv|63 zQU1L>uIX*ObUm-KnzWV(k;bD+RZ=WAMQpE-?RMeYCBco=XG?``DuXhOuMaDa3k|DG zj!`5{uImju_b2t9jX0M!bynWe``y#Y<~dBx4(aeWMfgcHnIe4eNi_T9VQuCeZn@T* z<;|fg7cZ?k=g~zj9>4ZhSAlqDkv@6GK((S#@!}t(U0Zj1SpDIfe*3;zl0I=lQwtNS z`3*7Yq@2eV<%W&!bZ-oOpP|9}v;U@q4q++v*{CtN`Wm&AH{~{hpMDoqx?*1}yLyYgo@nySy~J#N zkEep(Ns@ngZdXbj`W}n_?4@_vLUgR9b*!j=1?HGlcV)Q7Wi8Sp5N?r8Q_d+toay;! zXYCn`(%LH26pctvAxRCr`1>oATdG@FP9UT&zAl4%&fO*zQy-Xj|J<^_%s_zi<(NtT zoZkFd%+n=9VtAUHWFF)gEg8MrtDUDRf_M8CE5YFRG1&E`zgmL}YiFKK;(6R$g6y4T zUEU7gteiDur|;L1J|DxcAnbthyuV#f_>ISHQQfd%GmP!T6Jz+D0aWEHszR z$*qbhx#woIqTNK^16NOGOVhOUY@@1MtJ|@OjXrK8@4$UQT#&zglB``s zlmrTSlT-mG&F(l;4(!&r3W;Y%dY#aP4GtkLZ?w_}R8~7K)2kGvwtadD=6XYpavRr|0N1-}zXaIbVl<%TS!!!1| zWu{qZ>Mp+AixL{x>z%LX$xZ<}`RhydzBx^V>>UnC_gAlNmy+#Dw0I`Jth5zN-m_D} z?~wt&-!so+X|mE@2k%;1hR7JpugVF-)ssL8Q>gS)Nlm-$@_1cMCO%dz@M~E~dd;)t z1sCIs)8xjE1Ysj^XScWJeSm#Ep*8pTzG&pFyO#wAxk}5r8kbb=Bz~BW8aegA`g*+S z)!obQ#F9_G{;K!7u9{Rdyb@&|^fX4JT41?av{fzg?ZdKmopTE!Cv{tU++Sw|I*sqa zr@Bl*ElQ1-NHrQVhhm%DP3#r@?QHoIs(aPi1v7MOWj45BEkr_|)=svia`=o#_Ecko zM@`7+u*`R4%(g=rD~Z!*X0o-vTh70CNlQ^OZt9~p$gLl75oO3+GOm~Sw&d~Y@_Y4> z!(tvH9fu>p4y&Qk<9>EatRMJP24%+?6$+8Ou*-NyHNNRG^*8~q$NkTtv3^+_v2(N? z*u~84eFB1ak1MQ4-8Al=De2i5tvP3eZH_m37C`#i;hn0W9cSQYe3+G^t5pko`i0(x zTU8Bt)Y&CH808nEJ}zSUByrof zUi`J%Mkyxs5WMmbY;g8-mhXK?pdSF#xt1;+N-$U2%>V(Iep`}P2<*9p3RAd z8nXqD@|ia1$B@aR$}3ZC@LVmHxuFbcG3+Yeh7S!FFFeipAwi?k%YV*KiT#3jCeFHh zbTYP~j?KVKt)gbRE;Zu&1hbB5HEawub#;<5Qp*08%f04vv`y`*+yl|TW2=rR+q*mA z0|n;u?)rV*@iUo?Vf>gb_s#%u3*Uwr;klHs+xOK<s-yEK4nMrzav=dVo>W`XEAzeMT~qd!oyq#m2D_c5_DhbM?w`RG-Tf-F zx>ZEep+BfpjXRs7ChAsWy#H0um|bj}VE4=HQpM*9)Cf=ce5A91k&Njg(=H3c=T4j0 z>+Iq1%qH1723Z`r7TM~VU(S=h*IROR*~-^26pZ{PSJq1~@)fK2;i4IF+6Nua%|PjS z-Mz`JH1_7BAQGpYpX|ks|Kyh%(NvKTQmG+||%k z*g5VJqasiEK9hV=1yMo~?t670fERO&d?gO^2xhtj-EJRiD!Z)!ReySLc&V)}EUAp` zA15Y!c^t1My>vlmhPMs~57uU7ERF ziGCWX019}5O7H!-gfKRye57xoXvdsD5lu%iWOHc?_lpX&&Wv$*NWHH0ftd#^5C3oG zeYwsgy_2*04mMo6B$^-E(YA5NN8W4cOBRv7pO)Ze0&PQFj$P(=gPQB0`OgzRu&3+a z=7%%Zq7~F~l3&-Rczd|i=KWpZJfA&r@HnBB|AklPcz2=kklp))OYVl-mIo5kxhoX? z$3m>|`5)qRW>l1RrS4OZ_9z}hNs*cto*{NVpFJh`{ZvAV?(n;f2h&^k(?8q)52lBH AHUIzs literal 0 HcmV?d00001 diff --git a/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_reordered_edit_list.mp4 b/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_reordered_edit_list.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ee47e09424ca15170c29bf0e2f5a6f310829d904 GIT binary patch literal 5291 zcmZ`b2{;t$+A~N9DZ&vMMW}`uI~j~!RLbroW}nH788c%Gq3jh|i!3Ek5!nx?>~XY7 zAxn{D$(kZm%>NDN+`oJ8|G&@keeb?~?>h?w0)Y|PK@19uPJ=*rAY2TN2F4$Uq%pLR z5C|`ZMyLBjAP_3epNt3i7vP zZ10VnyuXcW&tcqpt9H-l8s8CPXLh=7prr)>oN zwQNQFrcM^mByerGz|{-;F{!Xk8cJueu~a}eu-L4B2Ew%|N7X9OZm0=T$~Z-1jXSPTds_i8k>PPK+Y_{r0wPxm&i zt*meZh%qoM2Eg1T+#`WyGdJHDx-$~OJrD?t#ol;-K!U~FumkT7fo$7kArhcH?tHlL zPaIhw!)=QGB@_NzZw_>4|JHF4kZ-*JT-^Z;b_wbLFyJ9`fYtyw0buT&a>0oJrvn_2 z6GsP_3m_+N62KTgJR1YB8;!7mxjKbk=M8p*iD59fBl#B`qHHde;$1eK8&DRH*>L7I z0Ct$J#;W!deX&s2{bru!3vB(9UMS5>LPZ zg&Ca@L?sZ}FbxDk6Rv^K&;&v{MqcchB+$?HjHjN5w=nOW6 z?!z*G;V@VX4uyn)#YJht@B}QCj`Kz#4G;zh7{&)f4Pp^ch(JvPL?9BW38N8EWI`Z} z<%b27E{wqn0dlZo*m5l~1S7>>-O(=eXE6$xWA z2~;YD1t`5hJv@#L2%Iksg#aaB@Mq~h1e6By5E2O^VpwcX2Fshm;96`3@MU!La6Bg zlEDqUfiZz79ep5Ru?Y;67L39GLjnr{W(Z{*hom?!00uKjBSak zhdH~Ve8KW%Y>`(k|x7~DH!={UnmILxe zwn+QQ5Pb{j%5$=*mDNHWyl2E$ykJ2cb|oI=MM24VskNgdQ-IXS?Iz$m7XT^!`B`2mC_br_kMVZx;!bXJTDQildOFIddcTbqr)ib{cX=vliV*z z{{FE3HL)8pLCr5mIovdQ>?;=?5E&71*BUSH6Kxf1-f}D=MB!ve2A{Hi@|JC@_cTM_ z_gyrX4(HuEne26!6Y}&SGT7?bchu3WXQFlrbFr}_Jfm6|bBovhwQl>hT{3y=c%YZ6eK4<56vRZ_I;d z_Fm*s9XQh^;7-NE_?b)a>!ICoV=MfoEUV82b>h7zT;GdlGf|&(A$kpIXAa*(uZ>o~ zC3suj-TK^Hts4Dx#*a{)$xP*-bxo!}e;hY#x_5~1puD?e_wcjyD3`A3ea#S|&^$?9 zOY`%WY43a1-pF717P9Mj(SnI&Y{7~f=j$i z){n+zrPRzPU?Y^}t3?L46&d~R)mCdm6iP{K_3Dm)omnwbNyqg&+qzz;V8thq1}2qj z+};kJ@c79uXvEm5_3F!fH9XZu>TGLeUS+>fZ=uKiSi7S+RjaS(;=dZ6niD6_b!jG@ z({?tG?E8t8Z1m_aajf4F`}lln?nT2Ficq^RRhd-f?fkHTkAvd|rD`v-c*F)2l-@M# zI;u0({b48NU{6o)qtWH2d<5NoGqn|IAp^hpz%TG$ZHZ^%hUcZ~ueWd(`((+G6rSNKSpYdsPsQvx-N9QE!6&xWN-*$! z9PY=`pKXCfk7k}u5_nym1MHn-o!<>#ubwsJWFF9wJ|D-gAbb+-aaT-F_>KEbQQgpC zU39H$^8QNF;{`0REm9zfvR)ocbM{G~do0^2Ez3OFy+wY#+bMCf&=krEK8Cl=U z)qRzNBuC2*l|s?=p%Ou--axU4kT=Pt^80ZSHnZylXU(nx*@_sRiT9b#F4}TwAGTxe z4;^ZK>xUNKzHsZuq1_*%Xg{XAn`KRV3r$UUZ;hhs*AEG}7IOdmkf1rHUnDsz0k9{N zTR$&#-mVSKoBqBK7`|-&eXZwRMYYzely#RsIqRB}=Nh&Ike{DO$N!e=7@c>&^RY+| zbShh^Pl$FDf6a0(uTJOSqF_?j_kxc_CI-}Ur=JdVz^4m8{E zAW*u1O#0-Icyr5#A6tvlU2KZhVcMsC^go)tupoMB1vy%}mjxWGd!){z3&S1o)}|hp z67R_9)pU?y{R#CTWviZVsfKuO758%y?76a@8IQAIg^$I`Q^a`n7Sa@71yNd z+U9zEVy%zI$kTs+02K0h?2m)>9vsgQ9hnzlw(jm+kSoGbL??H1~ivlwG#z- zy!0UGB&(YM53~(@mF@57x!Pcm-FY%4yC6n+|H)2)gEq3Q&8iSXn9;&syd+yf@Pl1bb$+ZrrYyMr5tZQwI z0hQI~7xE1s81(!f}mKSm)nsqlj0SfYT{rH`M8zk4r` z-s{+2m9u{-bt53@+~M5?vjHYxdSTEUVs})-g1|TXV{R)p_HX!H9+;?!?-Y}TE1%7+ zl&nOOex!s3?rP`jd5WW-PT`NG2JgIPV(yNU zDR)<|h)KzIC0jg|Usl?NqwL)!;ZtV7@AK5-c!sR>=|i_IErVo?NU{;oCm@UCdojoTLm z2chK^T}?|Ww~{~1M~|GjZ+$h<^zxqNp`EFxUVqhlU0+Ks9$twy4|o!%Q7f=qE831H;F(^Juzz(aSGZQwuB{d9us(^74jEaQFr*X>! z2emEJ73v8Bryuq|gT?#gti{jKci35q;gbd#1E!ZL}`T2-lKm^wf|1 z^|@!df_8#|kMR+9p03s-_>(V;cKoVpP+4c|=Nn}zUv7`^N=%Mj|61a*`#Lf)#=d8r z^hg6+d-woaFFD#LPJLX&@^SL^Z@q*okD8=dw8Mz%!|=h`&pF;ZzSjth{-!IB!Xc4$ zS+Wx&H=@vAGSGzTo1?McmqJfE1(SB>P0RU+MmU=P&{uaWr6o#Aj&arxe8kCS>VJBk zrCLVgf9{dUI#z&IlJNgC`F>51@#;`wKxD0*ZBoaPuq&@jGX!3Yuwzd6mut5jMd`fN zeZGViscld8=n0+{P;z9+e;a>*2Kx~Y%_#;e`lBga%ZN4}*129`ezj~o`ye)Z{i?Oj zKIg_>{;$fD!rj`oOF0*p-@PSeLLIF<)E&qC&*;ss+=7eKo|yYrQL@@crn_g2)|-N} z6|`6Jy%8kkfG~aIgmvS#Hs1A##yYbF_sW@e*vFvBW6CR2?TCCWwz;7Uc`@|zmNhRr zK2dm@`$K|3XI5^`Pl@w_6r5n)JvtfRSkGZ%r&iIk(3mDdpJ2{$t;V&t&0U@3th9>1 z&Lu}RVdfmvSW8rg7nQ^4B;Hs`0M+%x1Sa6xg zE_E1rgZh~f`4ewc2V~ij~nHI`P3%WIu2D5^*yT1qp*@UbDyW=>avx$VF(!c zb!g5@F!B|v#Npx@ary@xk9B|PdEI@fZFJ83yi6Z7Zf&p6@Xr@lwY#4_=XtboVZ&KcxWn$ouA5bmaK2HNB9i#WRNGfY-D2H&&1OFgjCZ@Pw#{TF$IQ%)Wc-4pY2a-VT1av{Y0Y zOIPs;dh4_({!o=~vT9G8JUV9PW;NzXlmaN=F*>t%a|z)bY~@JbLh+6{fnvIjV$k~1 zR-PAC7@Zm8@Sp}=>w`1*+3vpI%=_}4$a<${^-tPBb;)!ejDu~{j*om-GM6l(ygw}= z%mmusLLIuyZwIv0!wR1zec((tyekZ6eveg9%S(OzD9zK|`BA}N_nhW)Ck~w;w(-9> zoju-NWc=3deNv2@;r8W$C_Mc=U?D?;Ii1f3ZbrQPZK6{Ouu#?VsamW8LtUC(CE W2>v{il%_j8w03`b+X2RB`~Lw<0fV#v literal 0 HcmV?d00001 From 2a6eaa97753e8eb9de5399bd6a9e273b2b02906e Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Fri, 16 Oct 2026 18:07:32 -0500 Subject: [PATCH 3/3] Keep MediaCodecVideoRenderer's drop policy unchanged With the relabeling in MediaCodecRenderer, output buffers of a track with unreliable timestamps carry their decode-time slots in display order, which for constant frame rate content are their exact presentation times. earlyUs is therefore a real lateness signal again, so the widened threshold in shouldDropOutputBuffer and the disabled shouldDropBuffersToKeyframe aren't needed, and keeping them would weaken recovery from genuine decoder slowness (point 3 of the 2026-08-05 review on #3352). Verified on a Google TV Streamer 4K (c2.mtk.hevc.decoder) with the stock drop policy: a 95s B-frame HEVC file without ctts plays with 0 dropped frames and monotonic relabeled timestamps. --- .../video/MediaCodecVideoRenderer.java | 17 ---- .../video/MediaCodecVideoRendererTest.java | 96 ------------------- 2 files changed, 113 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index f4501edf0d0..30d77966963 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -2221,16 +2221,6 @@ protected void onProcessedStreamChange() { */ protected boolean shouldDropOutputBuffer( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { - @Nullable Format codecInputFormat = getCodecInputFormat(); - if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { - // earlyUs is derived from a reconstructed timestamp here, not ground truth, so the normal - // threshold is too trigger-happy: pipeline depth from the decoder's own reorder buffering - // can plausibly look this late even with no real backlog. But it's still a genuine - // pacing signal (see arrivalOrderPtsQueue's declaration), so don't disable dropping - // outright either -- that would leave no recovery path if the decoder is genuinely falling - // behind. Require VERY_LATE-level lateness instead of the normal threshold as a backstop. - return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer; - } return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && !isLastBuffer; } @@ -2246,13 +2236,6 @@ protected boolean shouldDropOutputBuffer( */ protected boolean shouldDropBuffersToKeyframe( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { - @Nullable Format codecInputFormat = getCodecInputFormat(); - if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { - // Unlike shouldDropOutputBuffer, stays disabled here: this skips to a keyframe based on - // *how many* buffers to discard, which needs real per-sample identity we don't have in this - // case -- getting that wrong risks skipping past buffers that were never actually late. - return false; - } return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer; } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index 8cece5ff926..baaa5982bb4 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -44,7 +44,6 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assume.assumeTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.clearInvocations; @@ -400,101 +399,6 @@ public void render_withLateBuffer_dropsBuffer() throws Exception { verify(eventListener).onDroppedFrames(eq(1), anyLong()); } - @Test - public void render_withUnreliablePtsAndModeratelyLateBuffer_doesNotDrop() throws Exception { - // 100ms late is past the normal late threshold, but not the one for unreliable timestamps. - Format unreliablePts = - VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); - FakeSampleStream fakeSampleStream = - new FakeSampleStream( - new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), - /* mediaSourceEventDispatcher= */ null, - DrmSessionManager.DRM_UNSUPPORTED, - new DrmSessionEventListener.EventDispatcher(), - /* initialFormat= */ unreliablePts, - ImmutableList.of( - oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. - oneByteSample(/* timeUs= */ 50_000), // Moderately late buffer. - oneByteSample(/* timeUs= */ 100_000), // Last buffer. - END_OF_STREAM_ITEM)); - fakeSampleStream.writeData(/* startPositionUs= */ 0); - mediaCodecVideoRenderer.enable( - RendererConfiguration.DEFAULT, - new Format[] {unreliablePts}, - fakeSampleStream, - /* positionUs= */ 0, - /* joining= */ false, - /* mayRenderStartOfStream= */ true, - /* startPositionUs= */ 0, - /* offsetUs= */ 0, - /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); - - mediaCodecVideoRenderer.start(); - mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); - for (int i = 0; i < 5; i++) { - mediaCodecVideoRenderer.render(40_000, SystemClock.elapsedRealtime() * 1000); - codecAdapterFactory.idleQueueingAndCallbackThreads(); - } - mediaCodecVideoRenderer.setCurrentStreamFinal(); - int posUs = 150_001; // ~100ms late relative to the 50_000 sample. - while (!mediaCodecVideoRenderer.isEnded()) { - mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); - codecAdapterFactory.idleQueueingAndCallbackThreads(); - posUs += 40_000; - } - shadowOf(testMainLooper).idle(); - - verify(eventListener, never()).onDroppedFrames(anyInt(), anyLong()); - } - - @Test - public void render_withUnreliablePtsAndVeryLateBuffer_stillDrops() throws Exception { - // Same as render_withUnreliablePtsAndModeratelyLateBuffer_doesNotDrop, but ~600ms late -- - // past the widened threshold, proving the backstop actually engages. - Format unreliablePts = - VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); - FakeSampleStream fakeSampleStream = - new FakeSampleStream( - new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), - /* mediaSourceEventDispatcher= */ null, - DrmSessionManager.DRM_UNSUPPORTED, - new DrmSessionEventListener.EventDispatcher(), - /* initialFormat= */ unreliablePts, - ImmutableList.of( - oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. - oneByteSample(/* timeUs= */ 50_000), // Very late buffer. - oneByteSample(/* timeUs= */ 100_000), // Last buffer. - END_OF_STREAM_ITEM)); - fakeSampleStream.writeData(/* startPositionUs= */ 0); - mediaCodecVideoRenderer.enable( - RendererConfiguration.DEFAULT, - new Format[] {unreliablePts}, - fakeSampleStream, - /* positionUs= */ 0, - /* joining= */ false, - /* mayRenderStartOfStream= */ true, - /* startPositionUs= */ 0, - /* offsetUs= */ 0, - /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); - - mediaCodecVideoRenderer.start(); - mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); - for (int i = 0; i < 5; i++) { - mediaCodecVideoRenderer.render(40_000, SystemClock.elapsedRealtime() * 1000); - codecAdapterFactory.idleQueueingAndCallbackThreads(); - } - mediaCodecVideoRenderer.setCurrentStreamFinal(); - int posUs = 650_001; // ~600ms late relative to the 50_000 sample. - while (!mediaCodecVideoRenderer.isEnded()) { - mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); - codecAdapterFactory.idleQueueingAndCallbackThreads(); - posUs += 40_000; - } - shadowOf(testMainLooper).idle(); - - verify(eventListener).onDroppedFrames(eq(1), anyLong()); - } - @Test public void render_withVeryLateBuffer_dropsBuffersUpstream() throws Exception { FakeSampleStream fakeSampleStream =