Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions libraries/common/src/main/java/androidx/media3/common/Format.java
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ public static final class Builder {
@Nullable private List<byte[]> initializationData;
@Nullable private DrmInitData drmInitData;
private long subsampleOffsetUs;
private boolean hasReliablePresentationTimestamps;

// Video specific.

Expand Down Expand Up @@ -227,6 +228,7 @@ public Builder() {
maxInputSize = NO_VALUE;
maxNumReorderSamples = NO_VALUE;
subsampleOffsetUs = OFFSET_SAMPLE_RELATIVE;
hasReliablePresentationTimestamps = true; // most formats have accurate per-sample timestamps
// Video specific.
width = NO_VALUE;
height = NO_VALUE;
Expand Down Expand Up @@ -281,6 +283,7 @@ private Builder(Format format) {
this.initializationData = format.initializationData;
this.drmInitData = format.drmInitData;
this.subsampleOffsetUs = format.subsampleOffsetUs;
this.hasReliablePresentationTimestamps = format.hasReliablePresentationTimestamps;
// Video specific.
this.width = format.width;
this.height = format.height;
Expand Down Expand Up @@ -604,6 +607,20 @@ public Builder setSubsampleOffsetUs(long subsampleOffsetUs) {
return this;
}

/**
* Sets {@link Format#hasReliablePresentationTimestamps}. The default value is {@code true}.
*
* @param hasReliablePresentationTimestamps The {@link
* Format#hasReliablePresentationTimestamps}.
* @return The builder.
*/
@CanIgnoreReturnValue
public Builder setHasReliablePresentationTimestamps(
boolean hasReliablePresentationTimestamps) {
this.hasReliablePresentationTimestamps = hasReliablePresentationTimestamps;
return this;
}

// Video specific.

/**
Expand Down Expand Up @@ -1117,6 +1134,15 @@ public Format build() {
*/
@UnstableApi public final long subsampleOffsetUs;

/**
* Indicates whether per-sample presentation timestamps for this track are known to be accurate.
*
* <p>When {@code false}, the source couldn't determine each sample's true presentation time
* (for example an MP4 track with no {@code ctts} box). Consumers shouldn't treat the
* timestamps as ground truth for scheduling decisions. Defaults to {@code true}.
*/
@UnstableApi public final boolean hasReliablePresentationTimestamps;

// Video specific.

/** The width of the video in pixels, or {@link #NO_VALUE} if unknown or not applicable. */
Expand Down Expand Up @@ -1317,6 +1343,7 @@ private Format(Builder builder) {
builder.initializationData == null ? Collections.emptyList() : builder.initializationData;
drmInitData = builder.drmInitData;
subsampleOffsetUs = builder.subsampleOffsetUs;
hasReliablePresentationTimestamps = builder.hasReliablePresentationTimestamps;
// Video specific.
width = builder.width;
height = builder.height;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.Flags;
import androidx.media3.common.Format;
Expand Down Expand Up @@ -240,6 +241,15 @@ private static String buildCustomDiagnosticInfo(int errorCode) {
*/
private static final long MAX_CODEC_HOTSWAP_TIME_MS = 1000;

/**
* The reorder depth assumed when relabeling output timestamps for a format with an unknown
* {@link Format#maxNumReorderSamples}. Matches the maximum DPB size of H.264 and H.265.
*/
private static final int DEFAULT_MAX_NUM_REORDER_SAMPLES = 16;

/** The maximum number of queued samples whose timestamps are retained for relabeling output. */
private static final int MAX_ARRIVAL_ORDER_PTS_QUEUE_SIZE = 64;

@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
Expand Down Expand Up @@ -398,6 +408,19 @@ private static String buildCustomDiagnosticInfo(int errorCode) {
private CodecParameters activeCodecParameters;
private CodecParameters lastDispatchedCodecParameters;
private ImmutableSet<String> subscribedCodecParameterKeys;
// The timestamps of queued samples whose format has unreliable presentation timestamps, in queue
// order. Output buffers are relabeled with these in output order, instead of the timestamp the
// codec echoes back, which is out of order for reordered frames.
private final ArrayDeque<Long> arrivalOrderPtsQueue = new ArrayDeque<>();
// The timestamps passed to the codec for the samples in arrivalOrderPtsQueue, in queue order,
// removed as the codec echoes them back. Used to detect samples the codec discarded without
// producing output, so that arrivalOrderPtsQueue doesn't fall out of step.
private final ArrayDeque<Long> pendingCodecPtsQueue = new ArrayDeque<>();

@VisibleForTesting
/* package */ int getArrivalOrderPtsQueueSizeForTesting() {
return arrivalOrderPtsQueue.size();
}

/**
* @param context A context.
Expand Down Expand Up @@ -1107,6 +1130,8 @@ protected void resetCodecStateForFlush() {
hasSkippedFlushAndWaitingForQueueInputBuffer = false;
skippedFlushOffsetUs = 0;
largestStaleModifiedPresentationTimeUs = C.TIME_UNSET;
arrivalOrderPtsQueue.clear();
pendingCodecPtsQueue.clear();
}

/**
Expand Down Expand Up @@ -1657,6 +1682,16 @@ private boolean feedInputBuffer() throws ExoPlaybackException {

onQueueInputBuffer(buffer);
int flags = getCodecBufferFlags(buffer);
if (getTrackType() == C.TRACK_TYPE_VIDEO
&& codecInputFormat != null
&& !codecInputFormat.hasReliablePresentationTimestamps
&& !getConfiguration().tunneling
&& (flags & MediaCodec.BUFFER_FLAG_DECODE_ONLY) == 0) {
// codecInputFormat, not inputFormat: during a full codec reinit inputFormat already describes
// the next codec's samples. Skipped in tunneling mode, where output buffers are never
// dequeued, and for BUFFER_FLAG_DECODE_ONLY samples, which the codec doesn't output.
queueArrivalOrderPts(presentationTimeUs, presentationTimeUs + skippedFlushOffsetUs);
}
presentationTimeUs += skippedFlushOffsetUs;
if (bufferEncrypted) {
checkNotNull(codec)
Expand Down Expand Up @@ -1693,6 +1728,43 @@ private long getLargestValidQueuedPresentationTimeUs() {
: largestQueuedPresentationTimeUs;
}

private void queueArrivalOrderPts(long presentationTimeUs, long codecPresentationTimeUs) {
if (arrivalOrderPtsQueue.size() >= MAX_ARRIVAL_ORDER_PTS_QUEUE_SIZE) {
// Backstop against a codec that never echoes back the timestamps it was given.
arrivalOrderPtsQueue.removeFirst();
pendingCodecPtsQueue.removeFirst();
}
arrivalOrderPtsQueue.addLast(presentationTimeUs);
pendingCodecPtsQueue.addLast(codecPresentationTimeUs);
}

private long dequeueArrivalOrderPts(long codecPresentationTimeUs, long defaultPresentationTimeUs) {
int pendingIndex = 0;
for (long pendingCodecPts : pendingCodecPtsQueue) {
if (pendingCodecPts == codecPresentationTimeUs) {
break;
}
pendingIndex++;
}
if (pendingIndex == pendingCodecPtsQueue.size()) {
// Not a sample that was queued while relabeling was active.
return defaultPresentationTimeUs;
}
// At most maxNumReorderSamples samples queued before this one can still be awaiting output.
// Any older pending samples were discarded by the codec without being output (for example RASL
// pictures after a seek), so drop them to keep arrivalOrderPtsQueue in step.
int maxNumReorderSamples =
codecInputFormat != null && codecInputFormat.maxNumReorderSamples != Format.NO_VALUE
? codecInputFormat.maxNumReorderSamples
: DEFAULT_MAX_NUM_REORDER_SAMPLES;
for (; pendingIndex > maxNumReorderSamples; pendingIndex--) {
pendingCodecPtsQueue.removeFirst();
arrivalOrderPtsQueue.removeFirst();
}
pendingCodecPtsQueue.removeFirstOccurrence(codecPresentationTimeUs);
return arrivalOrderPtsQueue.removeFirst();
}

/**
* Initializes the processing pipeline, if needed by the implementation.
*
Expand Down Expand Up @@ -2260,6 +2332,14 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
return false;
}

if (!arrivalOrderPtsQueue.isEmpty()) {
outputBufferInfo.presentationTimeUs =
dequeueArrivalOrderPts(
/* codecPresentationTimeUs= */ outputBufferInfo.presentationTimeUs
+ skippedFlushOffsetUs,
/* defaultPresentationTimeUs= */ outputBufferInfo.presentationTimeUs);
}

this.outputIndex = outputIndex;
outputBuffer = codec.getOutputBuffer(outputIndex);

Expand Down
Loading