diff --git a/libraries/common/src/main/java/androidx/media3/common/Format.java b/libraries/common/src/main/java/androidx/media3/common/Format.java index e1f58007c1a..3ba05ce012d 100644 --- a/libraries/common/src/main/java/androidx/media3/common/Format.java +++ b/libraries/common/src/main/java/androidx/media3/common/Format.java @@ -177,6 +177,7 @@ public static final class Builder { @Nullable private List initializationData; @Nullable private DrmInitData drmInitData; private long subsampleOffsetUs; + private boolean hasReliablePresentationTimestamps; // Video specific. @@ -227,6 +228,7 @@ public Builder() { maxInputSize = NO_VALUE; maxNumReorderSamples = NO_VALUE; subsampleOffsetUs = OFFSET_SAMPLE_RELATIVE; + hasReliablePresentationTimestamps = true; // most formats have accurate per-sample timestamps // Video specific. width = NO_VALUE; height = NO_VALUE; @@ -281,6 +283,7 @@ private Builder(Format format) { this.initializationData = format.initializationData; this.drmInitData = format.drmInitData; this.subsampleOffsetUs = format.subsampleOffsetUs; + this.hasReliablePresentationTimestamps = format.hasReliablePresentationTimestamps; // Video specific. this.width = format.width; this.height = format.height; @@ -604,6 +607,20 @@ public Builder setSubsampleOffsetUs(long subsampleOffsetUs) { return this; } + /** + * Sets {@link Format#hasReliablePresentationTimestamps}. The default value is {@code true}. + * + * @param hasReliablePresentationTimestamps The {@link + * Format#hasReliablePresentationTimestamps}. + * @return The builder. + */ + @CanIgnoreReturnValue + public Builder setHasReliablePresentationTimestamps( + boolean hasReliablePresentationTimestamps) { + this.hasReliablePresentationTimestamps = hasReliablePresentationTimestamps; + return this; + } + // Video specific. /** @@ -1117,6 +1134,15 @@ public Format build() { */ @UnstableApi public final long subsampleOffsetUs; + /** + * Indicates whether per-sample presentation timestamps for this track are known to be accurate. + * + *

When {@code false}, the source couldn't determine each sample's true presentation time + * (for example an MP4 track with no {@code ctts} box). Consumers shouldn't treat the + * timestamps as ground truth for scheduling decisions. Defaults to {@code true}. + */ + @UnstableApi public final boolean hasReliablePresentationTimestamps; + // Video specific. /** The width of the video in pixels, or {@link #NO_VALUE} if unknown or not applicable. */ @@ -1317,6 +1343,7 @@ private Format(Builder builder) { builder.initializationData == null ? Collections.emptyList() : builder.initializationData; drmInitData = builder.drmInitData; subsampleOffsetUs = builder.subsampleOffsetUs; + hasReliablePresentationTimestamps = builder.hasReliablePresentationTimestamps; // Video specific. width = builder.width; height = builder.height; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index c0a22e0d301..5ea0739f0e9 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -46,6 +46,7 @@ import androidx.annotation.IntDef; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; +import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.Flags; import androidx.media3.common.Format; @@ -240,6 +241,15 @@ private static String buildCustomDiagnosticInfo(int errorCode) { */ private static final long MAX_CODEC_HOTSWAP_TIME_MS = 1000; + /** + * The reorder depth assumed when relabeling output timestamps for a format with an unknown + * {@link Format#maxNumReorderSamples}. Matches the maximum DPB size of H.264 and H.265. + */ + private static final int DEFAULT_MAX_NUM_REORDER_SAMPLES = 16; + + /** The maximum number of queued samples whose timestamps are retained for relabeling output. */ + private static final int MAX_ARRIVAL_ORDER_PTS_QUEUE_SIZE = 64; + @Documented @Retention(RetentionPolicy.SOURCE) @Target(TYPE_USE) @@ -398,6 +408,19 @@ private static String buildCustomDiagnosticInfo(int errorCode) { private CodecParameters activeCodecParameters; private CodecParameters lastDispatchedCodecParameters; private ImmutableSet subscribedCodecParameterKeys; + // The timestamps of queued samples whose format has unreliable presentation timestamps, in queue + // order. Output buffers are relabeled with these in output order, instead of the timestamp the + // codec echoes back, which is out of order for reordered frames. + private final ArrayDeque arrivalOrderPtsQueue = new ArrayDeque<>(); + // The timestamps passed to the codec for the samples in arrivalOrderPtsQueue, in queue order, + // removed as the codec echoes them back. Used to detect samples the codec discarded without + // producing output, so that arrivalOrderPtsQueue doesn't fall out of step. + private final ArrayDeque pendingCodecPtsQueue = new ArrayDeque<>(); + + @VisibleForTesting + /* package */ int getArrivalOrderPtsQueueSizeForTesting() { + return arrivalOrderPtsQueue.size(); + } /** * @param context A context. @@ -1107,6 +1130,8 @@ protected void resetCodecStateForFlush() { hasSkippedFlushAndWaitingForQueueInputBuffer = false; skippedFlushOffsetUs = 0; largestStaleModifiedPresentationTimeUs = C.TIME_UNSET; + arrivalOrderPtsQueue.clear(); + pendingCodecPtsQueue.clear(); } /** @@ -1657,6 +1682,16 @@ private boolean feedInputBuffer() throws ExoPlaybackException { onQueueInputBuffer(buffer); int flags = getCodecBufferFlags(buffer); + if (getTrackType() == C.TRACK_TYPE_VIDEO + && codecInputFormat != null + && !codecInputFormat.hasReliablePresentationTimestamps + && !getConfiguration().tunneling + && (flags & MediaCodec.BUFFER_FLAG_DECODE_ONLY) == 0) { + // codecInputFormat, not inputFormat: during a full codec reinit inputFormat already describes + // the next codec's samples. Skipped in tunneling mode, where output buffers are never + // dequeued, and for BUFFER_FLAG_DECODE_ONLY samples, which the codec doesn't output. + queueArrivalOrderPts(presentationTimeUs, presentationTimeUs + skippedFlushOffsetUs); + } presentationTimeUs += skippedFlushOffsetUs; if (bufferEncrypted) { checkNotNull(codec) @@ -1693,6 +1728,43 @@ private long getLargestValidQueuedPresentationTimeUs() { : largestQueuedPresentationTimeUs; } + private void queueArrivalOrderPts(long presentationTimeUs, long codecPresentationTimeUs) { + if (arrivalOrderPtsQueue.size() >= MAX_ARRIVAL_ORDER_PTS_QUEUE_SIZE) { + // Backstop against a codec that never echoes back the timestamps it was given. + arrivalOrderPtsQueue.removeFirst(); + pendingCodecPtsQueue.removeFirst(); + } + arrivalOrderPtsQueue.addLast(presentationTimeUs); + pendingCodecPtsQueue.addLast(codecPresentationTimeUs); + } + + private long dequeueArrivalOrderPts(long codecPresentationTimeUs, long defaultPresentationTimeUs) { + int pendingIndex = 0; + for (long pendingCodecPts : pendingCodecPtsQueue) { + if (pendingCodecPts == codecPresentationTimeUs) { + break; + } + pendingIndex++; + } + if (pendingIndex == pendingCodecPtsQueue.size()) { + // Not a sample that was queued while relabeling was active. + return defaultPresentationTimeUs; + } + // At most maxNumReorderSamples samples queued before this one can still be awaiting output. + // Any older pending samples were discarded by the codec without being output (for example RASL + // pictures after a seek), so drop them to keep arrivalOrderPtsQueue in step. + int maxNumReorderSamples = + codecInputFormat != null && codecInputFormat.maxNumReorderSamples != Format.NO_VALUE + ? codecInputFormat.maxNumReorderSamples + : DEFAULT_MAX_NUM_REORDER_SAMPLES; + for (; pendingIndex > maxNumReorderSamples; pendingIndex--) { + pendingCodecPtsQueue.removeFirst(); + arrivalOrderPtsQueue.removeFirst(); + } + pendingCodecPtsQueue.removeFirstOccurrence(codecPresentationTimeUs); + return arrivalOrderPtsQueue.removeFirst(); + } + /** * Initializes the processing pipeline, if needed by the implementation. * @@ -2260,6 +2332,14 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) return false; } + if (!arrivalOrderPtsQueue.isEmpty()) { + outputBufferInfo.presentationTimeUs = + dequeueArrivalOrderPts( + /* codecPresentationTimeUs= */ outputBufferInfo.presentationTimeUs + + skippedFlushOffsetUs, + /* defaultPresentationTimeUs= */ outputBufferInfo.presentationTimeUs); + } + this.outputIndex = outputIndex; outputBuffer = codec.getOutputBuffer(outputIndex); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java index 9d0aeb98920..c64062c2485 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -1027,6 +1027,106 @@ public void codecReinitialized_withSubscribedKeys_resubscribesToVendorParameters assertThat(stringListCaptor.getValue()).containsExactly("key1", "key2"); } + @Test + public void feedInputBuffer_withUnreliablePtsAndTunneling_doesNotGrowArrivalOrderPtsQueue() + throws Exception { + // Output buffers are never dequeued in tunneling mode, so nothing would drain the queue. + Format unreliablePtsVideo = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_H264) + .setHasReliablePresentationTimestamps(false) + .build(); + FakeSampleStream fakeSampleStream = + createFakeSampleStream(unreliablePtsVideo, /* sampleTimesUs...= */ 0, 100, 200); + VideoTestRenderer renderer = new VideoTestRenderer(); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.enable( + new RendererConfiguration(/* tunneling= */ true), + new Format[] {unreliablePtsVideo}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + assertThat(renderer.getArrivalOrderPtsQueueSizeForTesting()).isEqualTo(0); + } + + @Test + public void render_withUnreliablePtsUntilEnded_drainsArrivalOrderPtsQueue() throws Exception { + Format unreliablePtsVideo = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_H264) + .setHasReliablePresentationTimestamps(false) + .build(); + FakeSampleStream fakeSampleStream = + createFakeSampleStream(unreliablePtsVideo, /* sampleTimesUs...= */ 0, 100, 200); + VideoTestRenderer renderer = new VideoTestRenderer(); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePtsVideo}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + renderer.setCurrentStreamFinal(); + long positionUs = 0; + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + assertThat(renderer.getArrivalOrderPtsQueueSizeForTesting()).isEqualTo(0); + } + + @Test + public void resetPosition_withUnreliablePts_clearsArrivalOrderPtsQueue() throws Exception { + Format unreliablePtsVideo = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_H264) + .setHasReliablePresentationTimestamps(false) + .build(); + FakeSampleStream fakeSampleStream = + createFakeSampleStream(unreliablePtsVideo, /* sampleTimesUs...= */ 0, 100, 200, 300); + VideoTestRenderer renderer = new VideoTestRenderer(); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePtsVideo}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + // Hold the position so that queued samples remain pending output. + for (int i = 0; i < 1000 && renderer.getArrivalOrderPtsQueueSizeForTesting() == 0; i++) { + renderer.render(/* positionUs= */ -1, SystemClock.elapsedRealtime()); + } + int queueSizeBeforeReset = renderer.getArrivalOrderPtsQueueSizeForTesting(); + + fakeSampleStream.seekToUs(/* positionUs= */ 0, /* allowTimeBeyondBuffer= */ true); + renderer.resetPosition(/* positionUs= */ 0, /* sampleStreamIsResetToKeyFrame= */ true); + + assertThat(queueSizeBeforeReset).isGreaterThan(0); + assertThat(renderer.getArrivalOrderPtsQueueSizeForTesting()).isEqualTo(0); + } + private TestRenderer setUpAndEnableRenderer(Format format) throws Exception { MediaCodecAdapter mockCodecAdapter = mock(MediaCodecAdapter.class); MediaCodecAdapter.Factory mockCodecAdapterFactory = configuration -> mockCodecAdapter; @@ -1191,6 +1291,106 @@ protected DecoderReuseEvaluation canReuseCodec( } } + /** Same as {@link TestRenderer}, but with {@link C#TRACK_TYPE_VIDEO} for video-only guards. */ + private static class VideoTestRenderer extends MediaCodecRenderer { + + VideoTestRenderer() { + this(MediaCodecAdapter.Factory.getDefault(ApplicationProvider.getApplicationContext())); + } + + VideoTestRenderer(MediaCodecAdapter.Factory mediaCodecAdapterFactory) { + super( + ApplicationProvider.getApplicationContext(), + C.TRACK_TYPE_VIDEO, + mediaCodecAdapterFactory, + /* mediaCodecSelector= */ (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> + Collections.singletonList( + MediaCodecInfo.newInstance( + /* name= */ "name", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ null, + /* hardwareAccelerated= */ false, + /* softwareOnly= */ true, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)), + /* enableDecoderFallback= */ false, + /* assumedMinimumCodecOperatingRate= */ 44100); + } + + @Override + public String getName() { + return "videoTest"; + } + + @Override + protected @Capabilities int supportsFormat(MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected List getDecoderInfos( + MediaCodecSelector mediaCodecSelector, Format format, boolean requiresSecureDecoder) + throws MediaCodecUtil.DecoderQueryException { + return mediaCodecSelector.getDecoderInfos( + format.sampleMimeType, + /* requiresSecureDecoder= */ false, + /* requiresTunnelingDecoder= */ false); + } + + @Override + protected MediaCodecAdapter.Configuration getMediaCodecConfiguration( + MediaCodecInfo codecInfo, + Format format, + @Nullable MediaCrypto crypto, + float codecOperatingRate) { + MediaFormat mediaFormat = new MediaFormat(); + applyCodecParametersToMediaFormat(mediaFormat); + return MediaCodecAdapter.Configuration.createForVideoDecoding( + codecInfo, mediaFormat, format, /* surface= */ null, crypto); + } + + @Override + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + @Nullable MediaCodecAdapter codec, + @Nullable ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + int sampleCount, + long bufferPresentationTimeUs, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, + Format format) + throws ExoPlaybackException { + if (bufferPresentationTimeUs <= positionUs) { + // Only release buffers when the position advances far enough for realistic behavior where + // input of buffers to the codec is faster than output. + if (codec != null) { + codec.releaseOutputBuffer(bufferIndex, /* render= */ true); + } + return true; + } + return false; + } + + @Override + protected void onCodecParametersChanged(CodecParameters codecParameters) { + // No-op for this test renderer. + } + + @Override + protected DecoderReuseEvaluation canReuseCodec( + MediaCodecInfo codecInfo, + Format oldFormat, + Format newFormat, + boolean isAdaptiveFormatChange) { + return codecInfo.canReuseCodec(oldFormat, newFormat); + } + } + /** * A {@link MediaCodecAdapter} that throws a pre-specified exception from every decoding-related * interaction. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index b45730f25ef..baaa5982bb4 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -130,8 +130,10 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.PriorityQueue; +import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -1480,6 +1482,223 @@ public void render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec() thro verify(eventListener, times(2)).onVideoDecoderInitialized(any(), anyLong(), anyLong()); } + @Config(maxSdk = 29) + @Test + public void + render_withUnreliablePtsDuringIncompatibleFormatChange_stillFifoOrdersDrainingOldCodecOutput() + throws Exception { + // Same 24fps -> 30fps codec reinitialization as + // render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec, but the first format has + // unreliable timestamps and the codec outputs in ascending timestamp order. + SystemClock.setCurrentTimeMillis(876_000_000); + Format unreliablePts24Fps = + VIDEO_H264_24FPS.buildUpon().setHasReliablePresentationTimestamps(false).build(); + Format reliablePts30Fps = + VIDEO_H264_30FPS.buildUpon().setHasReliablePresentationTimestamps(true).build(); + List processedTimestamps = new ArrayList<>(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePts24Fps, + ImmutableList.of( + oneByteSample(/* timeUs= */ 40_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 0))); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory( + new ForwardingSynchronousMediaCodecAdapterWithReordering.Factory()) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected void onProcessedOutputBuffer(long presentationTimeUs) { + super.onProcessedOutputBuffer(presentationTimeUs); + processedTimestamps.add(presentationTimeUs); + } + }; + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePts24Fps, reliablePts30Fps}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + // Render first sample to initialize codec. + renderer.render(/* positionUs= */ 0, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + shadowOf(testMainLooper).idle(); + + // Feed format change from 24->30 fps, which is not generally compatible and should reset + // codec, while the old codec may still be draining its format-A output. + fakeSampleStream.append( + ImmutableList.of( + format(reliablePts30Fps), + oneByteSample(/* timeUs= */ 80_000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 40_000); + renderer.setCurrentStreamFinal(); + int positionUs = 40_000; + while (!renderer.isEnded()) { + ShadowSystemClock.advanceBy(Duration.ofMillis(40)); + renderer.render(positionUs, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + positionUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener).onVideoDecoderReleased(any()); + verify(eventListener, times(2)).onVideoDecoderInitialized(any(), anyLong(), anyLong()); + assertThat(processedTimestamps.subList(0, 2)).containsExactly(40_000L, 0L).inOrder(); + } + + @Test + public void render_withUnreliablePtsAndReorderedOutput_relabelsOutputInArrivalOrder() + throws Exception { + // Decode order I0 P3 B1 B2 P6 B4 B5, with each sample's timestamp being its decode time. + Format format = + VIDEO_H264 + .buildUpon() + .setMaxNumReorderSamples(1) + .setHasReliablePresentationTimestamps(false) + .build(); + MediaCodecAdapter.Factory codecAdapterFactory = + new ForwardingSynchronousMediaCodecAdapterWithScriptedOutput.Factory( + /* outputTimestampsUs...= */ 0, 80_000, 120_000, 40_000, 200_000, 240_000, 160_000); + + List processedTimestamps = + renderToEndAndGetProcessedTimestamps( + format, + codecAdapterFactory, + /* sampleTimesUs...= */ 0, + 40_000, + 80_000, + 120_000, + 160_000, + 200_000, + 240_000); + + assertThat(processedTimestamps) + .containsExactly(0L, 40_000L, 80_000L, 120_000L, 160_000L, 200_000L, 240_000L) + .inOrder(); + } + + @Test + public void render_withUnreliablePtsAndSamplesDiscardedByCodec_resynchronizesRelabeling() + throws Exception { + // Decode order CRA, RASL, RASL, P3, B1, B2. The codec discards both RASL pictures without + // outputting them, as happens after seeking to a CRA picture. + Format format = + VIDEO_H264 + .buildUpon() + .setMaxNumReorderSamples(1) + .setHasReliablePresentationTimestamps(false) + .build(); + MediaCodecAdapter.Factory codecAdapterFactory = + new ForwardingSynchronousMediaCodecAdapterWithScriptedOutput.Factory( + /* outputTimestampsUs...= */ 0, 160_000, 200_000, 120_000); + + List processedTimestamps = + renderToEndAndGetProcessedTimestamps( + format, + codecAdapterFactory, + /* sampleTimesUs...= */ 0, + 40_000, + 80_000, + 120_000, + 160_000, + 200_000); + + // Without resynchronizing, B1, B2 and P3 would be relabeled 40_000, 80_000 and 120_000, two + // frames earlier than their decode-time slots. + assertThat(processedTimestamps).containsExactly(0L, 120_000L, 160_000L, 200_000L).inOrder(); + } + + @Test + public void render_withSeamlessChangeToReliablePts_onlyRelabelsSamplesOfUnreliableFormat() + throws Exception { + // The first format has decode order I0 P2 B1 with decode-time timestamps. The second format + // reuses the codec and has decode order I5 P7 B6 with correct presentation timestamps. + Format unreliablePts = + VIDEO_H264 + .buildUpon() + .setId("unreliable") + .setMaxNumReorderSamples(1) + .setHasReliablePresentationTimestamps(false) + .build(); + Format reliablePts = VIDEO_H264.buildUpon().setId("reliable").setMaxNumReorderSamples(1).build(); + MediaCodecAdapter.Factory codecAdapterFactory = + new ForwardingSynchronousMediaCodecAdapterWithScriptedOutput.Factory( + /* outputTimestampsUs...= */ 0, 80_000, 40_000, 200_000, 240_000, 280_000); + + List processedTimestamps = + renderToEndAndGetProcessedTimestamps( + new Format[] {unreliablePts, reliablePts}, + codecAdapterFactory, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 40_000), + oneByteSample(/* timeUs= */ 80_000), + format(reliablePts), + oneByteSample(/* timeUs= */ 200_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 280_000), + oneByteSample(/* timeUs= */ 240_000), + END_OF_STREAM_ITEM)); + + assertThat(processedTimestamps) + .containsExactly(0L, 40_000L, 80_000L, 200_000L, 240_000L, 280_000L) + .inOrder(); + } + + @Test + public void render_withUnreliablePtsAndVariableFrameSpacing_relabelsWithQueuedTimestamps() + throws Exception { + // Decode order I0 P2 B1 P4 B3 with irregular spacing. The true presentation times of reordered + // frames can't be recovered, so the queued timestamps are replayed in output order. + Format format = + VIDEO_H264 + .buildUpon() + .setMaxNumReorderSamples(1) + .setHasReliablePresentationTimestamps(false) + .build(); + MediaCodecAdapter.Factory codecAdapterFactory = + new ForwardingSynchronousMediaCodecAdapterWithScriptedOutput.Factory( + /* outputTimestampsUs...= */ 0, 80_000, 40_000, 300_000, 280_000); + + List processedTimestamps = + renderToEndAndGetProcessedTimestamps( + format, + codecAdapterFactory, + /* sampleTimesUs...= */ 0, + 40_000, + 80_000, + 280_000, + 300_000); + + assertThat(processedTimestamps) + .containsExactly(0L, 40_000L, 80_000L, 280_000L, 300_000L) + .inOrder(); + } + @Config(minSdk = 30) @Test public void render_withIncompatibleFrameRateChangeFromSdk30_keepsCodec() throws Exception { @@ -7350,6 +7569,143 @@ public void release() { } } + private List renderToEndAndGetProcessedTimestamps( + Format format, MediaCodecAdapter.Factory codecAdapterFactory, long... sampleTimesUs) + throws Exception { + ImmutableList.Builder items = ImmutableList.builder(); + for (int i = 0; i < sampleTimesUs.length; i++) { + items.add(oneByteSample(sampleTimesUs[i], i == 0 ? C.BUFFER_FLAG_KEY_FRAME : 0)); + } + items.add(END_OF_STREAM_ITEM); + return renderToEndAndGetProcessedTimestamps( + new Format[] {format}, codecAdapterFactory, items.build()); + } + + private List renderToEndAndGetProcessedTimestamps( + Format[] formats, + MediaCodecAdapter.Factory codecAdapterFactory, + ImmutableList items) + throws Exception { + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ formats[0], + items); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + List processedTimestamps = new ArrayList<>(); + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory(codecAdapterFactory) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected void onProcessedOutputBuffer(long presentationTimeUs) { + super.onProcessedOutputBuffer(presentationTimeUs); + processedTimestamps.add(presentationTimeUs); + } + }; + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + renderer.enable( + RendererConfiguration.DEFAULT, + formats, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + renderer.setCurrentStreamFinal(); + int positionUs = 0; + while (!renderer.isEnded()) { + ShadowSystemClock.advanceBy(Duration.ofMillis(40)); + renderer.render(positionUs, msToUs(SystemClock.elapsedRealtime())); + positionUs += 40_000; + } + shadowOf(testMainLooper).idle(); + return processedTimestamps; + } + + /** + * A {@link MediaCodecAdapter} that outputs buffers with a scripted sequence of timestamps, each + * of which must have been queued, and discards any queued buffers beyond the script. + */ + private static final class ForwardingSynchronousMediaCodecAdapterWithScriptedOutput + extends ForwardingMediaCodecAdapter { + /** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithScriptedOutput} instances. */ + public static final class Factory implements MediaCodecAdapter.Factory { + private final long[] outputTimestampsUs; + + Factory(long... outputTimestampsUs) { + this.outputTimestampsUs = outputTimestampsUs; + } + + @Override + public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException { + return new ForwardingSynchronousMediaCodecAdapterWithScriptedOutput( + outputTimestampsUs, + new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration)); + } + } + + private final long[] outputTimestampsUs; + private final Set queuedTimestampsUs; + private int outputCount; + + ForwardingSynchronousMediaCodecAdapterWithScriptedOutput( + long[] outputTimestampsUs, MediaCodecAdapter adapter) { + super(adapter); + this.outputTimestampsUs = outputTimestampsUs; + queuedTimestampsUs = new HashSet<>(); + } + + @Override + public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) { + while (true) { + if (outputCount < outputTimestampsUs.length + && !queuedTimestampsUs.contains(outputTimestampsUs[outputCount])) { + return INFO_TRY_AGAIN_LATER; + } + int outputBufferIndex = super.dequeueOutputBufferIndex(bufferInfo); + if (outputBufferIndex < 0 + || (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + return outputBufferIndex; + } + if (outputCount == outputTimestampsUs.length) { + super.releaseOutputBuffer(outputBufferIndex, /* render= */ false); + continue; + } + bufferInfo.presentationTimeUs = outputTimestampsUs[outputCount++]; + return outputBufferIndex; + } + } + + @Override + public void queueInputBuffer( + int index, int offset, int size, long presentationTimeUs, int flags) { + if ((flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) { + queuedTimestampsUs.add(presentationTimeUs); + } + super.queueInputBuffer(index, offset, size, presentationTimeUs, flags); + } + } + private static final class ForwardingSynchronousMediaCodecAdapterWithBufferLimit extends ForwardingMediaCodecAdapter { /** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithBufferLimit} instances. */ diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java index fc741fe206c..298ca84cf93 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java @@ -532,6 +532,17 @@ public static TrackSampleTable parseStbl( track = track.buildUpon().setFormat(format).build(); } + // Without a ctts box every sample's presentation time is its decode time, which is wrong for + // any track whose frames are reordered. Treat an unknown reorder depth (NO_VALUE) as possibly + // reordered. Applied after the sampleCount == 0 return: fragmented MP4 has an empty sample + // table here and carries composition offsets in trun boxes instead. + if (track.type == C.TRACK_TYPE_VIDEO + && stblBox.getLeafBoxOfType(Mp4Box.TYPE_ctts) == null + && track.format.maxNumReorderSamples != 0) { + Format format = track.format.buildUpon().setHasReliablePresentationTimestamps(false).build(); + track = track.buildUpon().setFormat(format).build(); + } + // Entries are byte offsets of chunks. boolean chunkOffsetsAreLongs = false; @Nullable LeafBox chunkOffsetsAtom = stblBox.getLeafBoxOfType(Mp4Box.TYPE_stco); diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNonParameterizedTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNonParameterizedTest.java index b4f76c601bb..35e7da88222 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNonParameterizedTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNonParameterizedTest.java @@ -15,9 +15,11 @@ */ package androidx.media3.extractor.mp4; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.US_ASCII; +import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.extractor.SniffFailure; import androidx.media3.extractor.text.SubtitleParser; @@ -106,6 +108,22 @@ public void read_withNegativeVersion1TfdtBaseMediaDecodeTime_outputsShiftedSampl assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(25_000L); } + @Test + public void extract_reorderedVideo_hasReliablePresentationTimestamps() throws Exception { + // The moov box's sample table has no ctts box, but composition offsets come from trun boxes. + FragmentedMp4Extractor extractor = + new FragmentedMp4Extractor(SubtitleParser.Factory.UNSUPPORTED); + + FakeExtractorOutput output = + TestUtil.extractAllSamplesFromFile( + extractor, ApplicationProvider.getApplicationContext(), "media/mp4/sample_fragmented.mp4"); + + Format videoFormat = checkNotNull(output.trackOutputs.get(0).lastFormat); + assertThat(videoFormat.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H264); + assertThat(videoFormat.maxNumReorderSamples).isGreaterThan(0); + assertThat(videoFormat.hasReliablePresentationTimestamps).isTrue(); + } + @Test public void extract_withoutMehdOrEditList_prefersMdhdTrackDuration() throws Exception { FragmentedMp4Extractor extractor = diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorNonParameterizedTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorNonParameterizedTest.java index 4e90e049e0f..494f551a11e 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorNonParameterizedTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorNonParameterizedTest.java @@ -15,6 +15,7 @@ */ package androidx.media3.extractor.mp4; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.truth.Truth.assertThat; import android.content.Context; @@ -406,6 +407,66 @@ public void extract_mebxHagcTrackWithFlagDisableHagcMetadata_ignoresMebxHagcTrac assertThat(output.trackOutputs.get(0).lastFormat.sampleMimeType).isEqualTo(MimeTypes.VIDEO_AV1); } + @Test + public void extract_reorderedVideoWithoutCttsOrEditList_hasUnreliablePresentationTimestamps() + throws Exception { + Format videoFormat = + extractFirstTrackFormat("media/mp4/sample_no_ctts_reordered.mp4", /* flags= */ 0); + + assertThat(videoFormat.maxNumReorderSamples).isGreaterThan(0); + assertThat(videoFormat.hasReliablePresentationTimestamps).isFalse(); + } + + @Test + public void extract_reorderedVideoWithoutCttsWithEditList_hasUnreliablePresentationTimestamps() + throws Exception { + Format videoFormat = + extractFirstTrackFormat( + "media/mp4/sample_no_ctts_reordered_edit_list.mp4", /* flags= */ 0); + + assertThat(videoFormat.maxNumReorderSamples).isGreaterThan(0); + assertThat(videoFormat.hasReliablePresentationTimestamps).isFalse(); + } + + @Test + public void + extract_reorderedVideoWithoutCttsAndOmitTrackSampleTable_hasUnreliablePresentationTimestamps() + throws Exception { + Format videoFormat = + extractFirstTrackFormat( + "media/mp4/sample_no_ctts_reordered.mp4", Mp4Extractor.FLAG_OMIT_TRACK_SAMPLE_TABLE); + + assertThat(videoFormat.hasReliablePresentationTimestamps).isFalse(); + } + + @Test + public void extract_nonReorderedVideoWithoutCtts_hasReliablePresentationTimestamps() + throws Exception { + Format videoFormat = + extractFirstTrackFormat("media/mp4/sample_no_ctts_not_reordered.mp4", /* flags= */ 0); + + assertThat(videoFormat.maxNumReorderSamples).isEqualTo(0); + assertThat(videoFormat.hasReliablePresentationTimestamps).isTrue(); + } + + @Test + public void extract_reorderedVideoWithCtts_hasReliablePresentationTimestamps() + throws Exception { + Format videoFormat = extractFirstTrackFormat("media/mp4/h265_bframes.mp4", /* flags= */ 0); + + assertThat(videoFormat.maxNumReorderSamples).isGreaterThan(0); + assertThat(videoFormat.hasReliablePresentationTimestamps).isTrue(); + } + + private static Format extractFirstTrackFormat(String inputFilePath, @Mp4Extractor.Flags int flags) + throws IOException { + Mp4Extractor mp4Extractor = new Mp4Extractor(SubtitleParser.Factory.UNSUPPORTED, flags); + FakeExtractorOutput output = + TestUtil.extractAllSamplesFromFile( + mp4Extractor, ApplicationProvider.getApplicationContext(), inputFilePath); + return checkNotNull(output.trackOutputs.get(0).lastFormat); + } + private static String getDumpFilePath(String inputFilePath, String suffix) { return inputFilePath.replaceFirst("media", "extractordumps") + suffix; } diff --git a/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_not_reordered.mp4 b/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_not_reordered.mp4 new file mode 100644 index 00000000000..2cf25c76378 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_not_reordered.mp4 differ diff --git a/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_reordered.mp4 b/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_reordered.mp4 new file mode 100644 index 00000000000..812aeb720ca Binary files /dev/null and b/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_reordered.mp4 differ diff --git a/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_reordered_edit_list.mp4 b/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_reordered_edit_list.mp4 new file mode 100644 index 00000000000..ee47e09424c Binary files /dev/null and b/libraries/test_data/src/test/assets/media/mp4/sample_no_ctts_reordered_edit_list.mp4 differ