diff --git a/README.md b/README.md
index cb3c2bc8..df41e509 100644
--- a/README.md
+++ b/README.md
@@ -92,7 +92,6 @@ Shows all configured slots in a 2×2 grid. Each cell has its own connection and
> These are known gaps that we plan to address. Each one has an issue — comments, test results and pull requests are welcome.
>
> - **RTSP over TCP only.** No UDP or multicast — [#7](https://github.com/OpenIPC/decoder/issues/7)
-> - **Simplified RTP handling for video:** no STAP-A/AP aggregation, packets with CSRC or header extensions are dropped, no packet-loss recovery — [#10](https://github.com/OpenIPC/decoder/issues/10)
> - **No audio in Quad mode** — [#12](https://github.com/OpenIPC/decoder/issues/12)
> - **Only 4 camera slots** — [#13](https://github.com/OpenIPC/decoder/issues/13)
> - **Audio formats not supported:** MP4A-LATM, Opus, G.726, and AAC frames split across RTP packets — [#14](https://github.com/OpenIPC/decoder/issues/14)
diff --git a/README.ru.md b/README.ru.md
index 1a05c58a..50208585 100644
--- a/README.ru.md
+++ b/README.ru.md
@@ -92,7 +92,6 @@
> Это известные пробелы, которые мы планируем закрыть. На каждый заведён issue — комментарии, результаты проверок и pull request приветствуются.
>
> - **Только RTSP поверх TCP.** UDP и multicast не поддерживаются — [#7](https://github.com/OpenIPC/decoder/issues/7)
-> - **Упрощённая обработка RTP для видео:** нет агрегации STAP-A/AP, пакеты с CSRC или расширением заголовка отбрасываются, потери пакетов не обрабатываются — [#10](https://github.com/OpenIPC/decoder/issues/10)
> - **В режиме Quad нет звука** — [#12](https://github.com/OpenIPC/decoder/issues/12)
> - **Только 4 слота для камер** — [#13](https://github.com/OpenIPC/decoder/issues/13)
> - **Не поддерживаются форматы звука** MP4A-LATM, Opus, G.726, а также AAC-кадры, разбитые на несколько RTP-пакетов — [#14](https://github.com/OpenIPC/decoder/issues/14)
diff --git a/README.zh-CN.md b/README.zh-CN.md
index a6f08c21..e1f6bfbd 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -92,7 +92,6 @@
> 以下是我们计划解决的已知不足，每一项都有对应的 issue。欢迎留言、提交测试结果和 pull request。
>
> - **仅支持 RTSP over TCP**，不支持 UDP 和组播 — [#7](https://github.com/OpenIPC/decoder/issues/7)
-> - **视频的 RTP 处理较为简单：** 不支持 STAP-A/AP 聚合包，带 CSRC 或扩展头的包会被丢弃，也不处理丢包 — [#10](https://github.com/OpenIPC/decoder/issues/10)
> - **Quad 模式下没有声音** — [#12](https://github.com/OpenIPC/decoder/issues/12)
> - **仅有 4 个摄像机槽位** — [#13](https://github.com/OpenIPC/decoder/issues/13)
> - **不支持的音频格式：** MP4A-LATM、Opus、G.726，以及跨多个 RTP 包分片的 AAC 帧 — [#14](https://github.com/OpenIPC/decoder/issues/14)
diff --git a/android/app/src/main/java/com/openipc/decoder/NalAssembler.java b/android/app/src/main/java/com/openipc/decoder/NalAssembler.java
index c1b4788c..652905ba 100644
--- a/android/app/src/main/java/com/openipc/decoder/NalAssembler.java
+++ b/android/app/src/main/java/com/openipc/decoder/NalAssembler.java
@@ -1,7 +1,7 @@
/*
* Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0
*
- * NalAssembler.java — reusable NAL unit reassembler from RTP fragmentation units
+ * NalAssembler.java — H.264/H.265 NAL units from RTP packets (RFC 6184 / RFC 7798)
*
*/
@@ -10,109 +10,194 @@
import android.util.Log;
/**
- * Reassembles NAL units from RTP fragmentation units (FU-A for H.264, FU for H.265).
- * Shared by the main decoder and QuadCell streams. Replaces duplicated buildFrame()/assembleNal().
+ * Turns RTP video packets into Annex B NAL units (with a 4-byte start code). Handles single
+ * NAL unit packets, fragmentation units (FU-A, FU) and aggregation packets (STAP-A, AP),
+ * any RTP header length (CSRC, extension, padding) and packet loss: a fragmented NAL unit
+ * with a missing piece is dropped instead of being passed on corrupted.
*/
class NalAssembler {
private static final String TAG = "OpenIPCDecoder";
- private static final int RTP_FU_H264 = 28;
- private static final int RTP_FU_H265 = 49;
+ private static final int H264_STAP_A = 24;
+ private static final int H264_FU_A = 28;
+ private static final int H265_AP = 48;
+ private static final int H265_FU = 49;
+ private static final int H265_PACI = 50;
+
+ interface Sink {
+ /** A complete NAL unit; the sink owns the frame (recycle it when done). */
+ void onNal(Frame nal);
+ }
private final byte[] nalBuffer;
- private int nalSize;
- private boolean lastCodec;
private final Runnable onCodecSwitch;
private final FramePool framePool;
+ private int nalSize;
+ /** A fragmented NAL unit is being collected in nalBuffer. */
+ private boolean inFragment;
+ /** Skipping the rest of a fragmented NAL unit whose start was not seen (counted once). */
+ private boolean orphan;
+ private int lastSeq = -1;
+ private boolean lastCodec;
+ private int dropped;
+
NalAssembler(int bufferSize, Runnable onCodecSwitch, FramePool framePool) {
this.nalBuffer = new byte[bufferSize];
this.onCodecSwitch = onCodecSwitch;
this.framePool = framePool;
}
- void reset() { nalSize = 0; }
+ void reset() {
+ nalSize = 0;
+ inFragment = false;
+ orphan = false;
+ lastSeq = -1;
+ }
- /** Reassemble one RTP packet into (possibly null) NAL frame. */
- Frame assemble(Frame frame, boolean codecH265) {
- byte[] rx = frame.data();
- int rxSize = frame.length();
- int cp = 12;
- rxSize -= cp;
- if (rxSize <= 0) return null;
+ /** NAL units dropped so far because of packet loss or size (for diagnostics). */
+ int droppedCount() { return dropped; }
+
+ /**
+ * Process one RTP packet ({@code length} bytes at {@code offset}, header included).
+ * Returns false if the RTP header is malformed.
+ */
+ boolean assemble(byte[] buf, int offset, int length, boolean codecH265, Sink sink) {
+ int header = RtspSession.rtpHeaderLength(buf, offset, length);
+ if (header < 0) return false;
+ int end = RtspSession.rtpPayloadEnd(buf, offset, length);
+ if (end < header) return false;
+ int start = offset + header;
+ end += offset;
if (lastCodec != codecH265) {
lastCodec = codecH265;
- nalSize = 0;
+ reset();
onCodecSwitch.run();
}
- int frag = fragment(rx[cp], codecH265);
- if (frag == RTP_FU_H264 || frag == RTP_FU_H265) {
- int minPayload = codecH265 ? 3 : 2;
- if (rxSize < minPayload) return null;
+ int seq = ((buf[offset + 2] & 0xFF) << 8) | (buf[offset + 3] & 0xFF);
+ boolean gap = lastSeq >= 0 && seq != ((lastSeq + 1) & 0xFFFF);
+ lastSeq = seq;
+ if (gap && inFragment) {
+ drop("packet loss inside a fragmented NAL unit");
+ orphan = true; // its remaining pieces belong to the unit just counted
+ }
- int staBit, endBit;
- if (codecH265) {
- staBit = rx[cp + 2] & 0x80;
- endBit = rx[cp + 2] & 0x40;
- } else {
- staBit = rx[cp + 1] & 0x80;
- endBit = rx[cp + 1] & 0x40;
- }
+ int nalHeader = codecH265 ? 2 : 1;
+ if (end - start < nalHeader) return true;
+ int type = fragment(buf[start], codecH265);
+
+ if (codecH265 ? type == H265_FU : type == H264_FU_A) {
+ fragmentUnit(buf, start, end, codecH265, sink);
+ return true;
+ }
+ // Any other packet ends a fragmented NAL unit that did not finish; its remaining
+ // pieces, if any arrive, belong to the unit counted here
+ if (inFragment) {
+ drop("fragmented NAL unit interrupted by another packet");
+ orphan = true;
+ } else {
+ orphan = false;
+ }
+ if (codecH265 ? type == H265_AP : type == H264_STAP_A) {
+ aggregation(buf, start + nalHeader, end, sink);
+ } else if (codecH265 ? type == H265_PACI : (type >= 25 && type <= 29)) {
+ // PACI, STAP-B, MTAP and FU-B are not used by cameras in non-interleaved mode
+ return true;
+ } else {
+ emit(buf, start, end - start, sink);
+ }
+ return true;
+ }
+
+ private void fragmentUnit(byte[] buf, int start, int end, boolean h265, Sink sink) {
+ int payloadHeader = h265 ? 2 : 1;
+ if (end - start <= payloadHeader) return;
+ int fu = buf[start + payloadHeader] & 0xFF;
+ boolean first = (fu & 0x80) != 0;
+ boolean last = (fu & 0x40) != 0;
+ int pos = start + payloadHeader + 1;
- if (staBit != 0) {
- nalSize = 0;
- if (codecH265) {
- nalBuffer[nalSize++] = 0;
- nalBuffer[nalSize++] = 0;
- nalBuffer[nalSize++] = 0;
- nalBuffer[nalSize++] = 1;
- // 2-byte NAL header: F + FU type + layer-id MSB, then layer-id/TID as sent
- nalBuffer[nalSize++] = (byte)((rx[cp] & 0x81) | ((rx[cp + 2] & 0x3F) << 1));
- nalBuffer[nalSize++] = rx[cp + 1];
- cp += 3; // payload header + FU header
- } else {
- nalBuffer[nalSize++] = 0;
- nalBuffer[nalSize++] = 0;
- nalBuffer[nalSize++] = 0;
- nalBuffer[nalSize++] = 1;
- nalBuffer[nalSize++] = (byte)((rx[cp] & 0xE0) | (rx[cp + 1] & 0x1F));
- cp += 2;
- }
- rxSize = frame.length() - cp;
+ if (first) {
+ if (inFragment) drop("fragmented NAL unit without its end");
+ orphan = false;
+ nalSize = 0;
+ inFragment = true;
+ nalBuffer[nalSize++] = 0;
+ nalBuffer[nalSize++] = 0;
+ nalBuffer[nalSize++] = 0;
+ nalBuffer[nalSize++] = 1;
+ if (h265) {
+ // 2-byte NAL header: F + FU type + layer-id MSB, then layer-id/TID as sent
+ nalBuffer[nalSize++] = (byte) ((buf[start] & 0x81) | ((fu & 0x3F) << 1));
+ nalBuffer[nalSize++] = buf[start + 1];
} else {
- cp += codecH265 ? 3 : 2;
- rxSize = frame.length() - cp;
+ nalBuffer[nalSize++] = (byte) ((buf[start] & 0xE0) | (fu & 0x1F));
}
-
- int copyLen = Math.min(rxSize, nalBuffer.length - nalSize);
- if (copyLen > 0) {
- System.arraycopy(rx, cp, nalBuffer, nalSize, copyLen);
- nalSize += copyLen;
+ } else if (!inFragment) {
+ // The start was lost (or we joined mid-unit): nothing to attach this piece to.
+ // Count the lost unit once, not once per remaining piece.
+ if (!orphan) {
+ orphan = true;
+ dropped++;
+ logDrop("fragmented NAL unit without its start");
}
+ if (last) orphan = false;
+ return;
+ }
- if (endBit != 0) {
- Frame out = framePool.obtain(nalSize);
- System.arraycopy(nalBuffer, 0, out.data(), 0, nalSize);
- out.setLength(nalSize);
- nalSize = 0;
- return out;
- }
- return null;
- } else {
+ int len = end - pos;
+ if (nalSize + len > nalBuffer.length) {
+ drop("NAL unit larger than " + nalBuffer.length + " bytes");
+ return;
+ }
+ System.arraycopy(buf, pos, nalBuffer, nalSize, len);
+ nalSize += len;
+
+ if (last) {
+ Frame out = framePool.obtain(nalSize);
+ System.arraycopy(nalBuffer, 0, out.data(), 0, nalSize);
+ out.setLength(nalSize);
nalSize = 0;
- int copyLen = rxSize + 4;
- if (copyLen > nalBuffer.length || copyLen < 5) return null;
- nalBuffer[0] = 0;
- nalBuffer[1] = 0;
- nalBuffer[2] = 0;
- nalBuffer[3] = 1;
- System.arraycopy(rx, cp, nalBuffer, 4, rxSize);
- Frame out = framePool.obtain(copyLen);
- System.arraycopy(nalBuffer, 0, out.data(), 0, copyLen);
- out.setLength(copyLen);
- return out;
+ inFragment = false;
+ sink.onNal(out);
+ }
+ }
+
+ /** STAP-A / AP: a sequence of (16-bit size, NAL unit). */
+ private void aggregation(byte[] buf, int pos, int end, Sink sink) {
+ while (end - pos >= 2) {
+ int size = ((buf[pos] & 0xFF) << 8) | (buf[pos + 1] & 0xFF);
+ pos += 2;
+ if (size == 0 || pos + size > end) break;
+ emit(buf, pos, size, sink);
+ pos += size;
+ }
+ }
+
+ private void emit(byte[] buf, int pos, int size, Sink sink) {
+ Frame out = framePool.obtain(size + 4);
+ byte[] d = out.data();
+ d[0] = 0;
+ d[1] = 0;
+ d[2] = 0;
+ d[3] = 1;
+ System.arraycopy(buf, pos, d, 4, size);
+ out.setLength(size + 4);
+ sink.onNal(out);
+ }
+
+ private void drop(String reason) {
+ nalSize = 0;
+ inFragment = false;
+ dropped++;
+ logDrop(reason);
+ }
+
+ private void logDrop(String reason) {
+ if (dropped <= 3 || dropped % 100 == 0) {
+ Log.w(TAG, "NAL unit dropped (" + dropped + "): " + reason);
}
}
diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java
index 4a34b549..031e020e 100644
--- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java
+++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java
@@ -42,6 +42,8 @@ class QuadCell implements RtspSession.Callback {
private final BlockingQueue nalQueue = new ArrayBlockingQueue<>(30);
private final NalAssembler nalAssembler;
+ private final NalAssembler.Sink nalSink;
+ private int queueFullDrops;
private int lastUnknownPayload = -1;
private ExecutorService executor;
@@ -61,6 +63,15 @@ interface Listener {
this.userAgent = userAgent;
this.listener = listener;
this.session = new RtspSession(tag, this);
+ this.nalSink = nal -> {
+ if (!nalQueue.offer(nal)) {
+ // Rate-limited: one aggregation packet can carry many NAL units
+ if (queueFullDrops++ % 100 == 0) {
+ Log.w(TAG, tag + " queue full, NAL dropped (" + queueFullDrops + ")");
+ }
+ pool.recycle(nal);
+ }
+ };
this.nalAssembler = new NalAssembler(512 * 1024, () -> {
nalQueue.clear();
synchronized (decoderLock) {
@@ -129,7 +140,14 @@ private void startThreads() {
while (running) {
try {
Frame f = nalQueue.poll(5, TimeUnit.MILLISECONDS);
- if (f != null) decode(f);
+ if (f != null) {
+ try {
+ decode(f);
+ } finally {
+ // The decoder copied the data: give the buffer back to the pool
+ cellFramePool.recycle(f);
+ }
+ }
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
@@ -189,8 +207,6 @@ public void onSdp(Sdp sdp) {
@Override
public boolean onPacket(int channel, byte[] buf, int offset, int length) {
if (channel != RtspSession.CHANNEL_VIDEO || length < 12) return false;
- // CSRC and header extensions are not supported for video yet (#10)
- if ((buf[offset] & 0x0F) != 0 || (buf[offset] & 0x10) != 0) return false;
int pt = buf[offset + 1] & 0x7F;
if (pt != videoPt) {
if (pt != lastUnknownPayload) {
@@ -200,18 +216,10 @@ public boolean onPacket(int channel, byte[] buf, int offset, int length) {
return false;
}
- Frame frame = cellFramePool.obtain(length);
- System.arraycopy(buf, offset, frame.data(), 0, length);
- frame.setLength(length);
- Frame output = nalAssembler.assemble(frame, codecH265);
- cellFramePool.recycle(frame);
- if (output != null && !nalQueue.offer(output)) {
- Log.w(TAG, tag + " queue full, dropping frame");
- cellFramePool.recycle(output);
- }
- return true;
+ return nalAssembler.assemble(buf, offset, length, codecH265, nalSink);
}
+
private void decode(Frame buffer) {
if (buffer.length() < 5) return;
diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java
index 2a7ed1e5..f4086bb1 100644
--- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java
+++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java
@@ -47,6 +47,15 @@ interface Listener {
this.framePool = framePool;
this.nalAssembler = nalAssembler;
this.nalQueue = nalQueue;
+ this.nalSink = nal -> {
+ if (!nalQueue.offer(nal)) {
+ // Rate-limited: one aggregation packet can carry many NAL units
+ if (queueFullDrops++ % 100 == 0) {
+ Log.w(TAG, "Video queue full, NAL dropped (" + queueFullDrops + ")");
+ }
+ framePool.recycle(nal);
+ }
+ };
}
void setListener(Listener cb) { this.listenerCb = cb; }
@@ -93,30 +102,28 @@ public void onSdp(Sdp sdp) {
if (l != null) l.onCodecChanged(codecH265, sdp.audio);
}
+ private final NalAssembler.Sink nalSink;
+ private int queueFullDrops;
+
@Override
public boolean onPacket(int channel, byte[] buf, int offset, int length) {
if (length < 12) return false;
- Frame frame = framePool.obtain(length);
- System.arraycopy(buf, offset, frame.data(), 0, length);
- frame.setLength(length);
-
- boolean usable = false;
if (channel == RtspSession.CHANNEL_VIDEO) {
- usable = processVideo(frame);
- } else if (channel == RtspSession.CHANNEL_AUDIO && audioManager != null) {
+ return processVideo(buf, offset, length);
+ }
+ if (channel == RtspSession.CHANNEL_AUDIO && audioManager != null) {
+ Frame frame = framePool.obtain(length);
+ System.arraycopy(buf, offset, frame.data(), 0, length);
+ frame.setLength(length);
audioManager.onAudioFrame(frame);
+ framePool.recycle(frame);
}
- framePool.recycle(frame);
- return usable;
+ return false;
}
- /** Returns false for packets of the wrong payload type or with unsupported headers. */
- private boolean processVideo(Frame frame) {
- byte[] data = frame.data();
- // CSRC and header extensions are not supported for video yet (#10)
- if ((data[0] & 0x0F) != 0 || (data[0] & 0x10) != 0) return false;
-
- int payload = data[1] & 0x7F;
+ /** Returns false for packets of the wrong payload type or with a malformed header. */
+ private boolean processVideo(byte[] data, int offset, int length) {
+ int payload = data[offset + 1] & 0x7F;
if (payload != videoPt) {
if (payload != lastUnknownPayload) {
lastUnknownPayload = payload;
@@ -125,17 +132,12 @@ private boolean processVideo(Frame frame) {
return false;
}
- updateJitter(data);
-
- Frame output = nalAssembler.assemble(frame, codecH265);
- if (output != null && !nalQueue.offer(output)) {
- Log.w(TAG, "Video queue full, frame dropped");
- framePool.recycle(output);
- }
+ if (!nalAssembler.assemble(data, offset, length, codecH265, nalSink)) return false;
+ updateJitter(data, offset);
return true;
}
- private void updateJitter(byte[] data) {
+ private void updateJitter(byte[] data, int offset) {
long now = SystemClock.elapsedRealtime();
if (now - lastQualityUpdateTime > 1000) {
lastQualityUpdateTime = now;
@@ -146,10 +148,10 @@ private void updateJitter(byte[] data) {
jitterAccumulator = 0;
jitterSampleCount = 0;
}
- long rtpTs = Integer.toUnsignedLong(((data[4] & 0xFF) << 24)
- | ((data[5] & 0xFF) << 16)
- | ((data[6] & 0xFF) << 8)
- | (data[7] & 0xFF));
+ long rtpTs = Integer.toUnsignedLong(((data[offset + 4] & 0xFF) << 24)
+ | ((data[offset + 5] & 0xFF) << 16)
+ | ((data[offset + 6] & 0xFF) << 8)
+ | (data[offset + 7] & 0xFF));
long arrivalNs = System.nanoTime();
if (lastRtpTimestamp >= 0 && lastRtpArrivalNs >= 0) {
long rtpDeltaMs = (rtpTs - lastRtpTimestamp) / 90L;
diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java
index 030f6cd9..2c5da084 100644
--- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java
+++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java
@@ -580,8 +580,14 @@ static int rtpHeaderLength(byte[] data, int offset, int length) {
/** Payload end: packet length minus RTP padding, or -1 if malformed. */
static int rtpPayloadEnd(byte[] data, int length) {
- if ((data[0] & 0x20) == 0) return length;
- int pad = data[length - 1] & 0xFF;
+ return rtpPayloadEnd(data, 0, length);
+ }
+
+ /** Payload end relative to {@code offset}: length minus RTP padding, or -1 if malformed. */
+ static int rtpPayloadEnd(byte[] data, int offset, int length) {
+ if (length < 1) return -1;
+ if ((data[offset] & 0x20) == 0) return length;
+ int pad = data[offset + length - 1] & 0xFF;
return pad <= length ? length - pad : -1;
}
diff --git a/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java b/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java
index 0e888747..59777581 100644
--- a/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java
+++ b/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java
@@ -1,7 +1,7 @@
/*
* Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0
*
- * NalAssemblerTest.java — NAL unit reassembly from RTP single NAL and FU packets
+ * NalAssemblerTest.java — NAL units from RTP: single, FU, STAP-A/AP, headers and loss
*
*/
@@ -9,28 +9,21 @@
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import org.junit.Test;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
public class NalAssemblerTest {
private int codecSwitches;
- private final NalAssembler nal = new NalAssembler(1024 * 1024, () -> codecSwitches++,
- new FramePool(4));
-
- /** RTP packet: 12-byte header followed by the given payload bytes. */
- private static Frame rtp(int... payload) {
- byte[] d = new byte[12 + payload.length];
- d[0] = (byte) 0x80;
- for (int i = 0; i < payload.length; i++) d[12 + i] = (byte) payload[i];
- return new Frame(d, d.length);
- }
-
- private static byte[] bytes(Frame f) {
- return Arrays.copyOf(f.data(), f.length());
- }
+ private final List out = new ArrayList<>();
+ private final NalAssembler.Sink sink = nal -> out.add(Arrays.copyOf(nal.data(), nal.length()));
+ private NalAssembler nal = new NalAssembler(1024 * 1024, () -> codecSwitches++, new FramePool(4));
+ private int seq = 100;
private static byte[] b(int... v) {
byte[] r = new byte[v.length];
@@ -38,47 +31,192 @@ private static byte[] b(int... v) {
return r;
}
+ /** RTP packet with the next sequence number: 12-byte header + payload. */
+ private byte[] rtp(int... payload) {
+ return rtpWithHeader(b(0x80, 96, (seq >> 8) & 0xFF, seq++ & 0xFF, 0, 0, 0, 0, 0, 0, 0, 0),
+ payload);
+ }
+
+ private static byte[] rtpWithHeader(byte[] header, int... payload) {
+ byte[] p = Arrays.copyOf(header, header.length + payload.length);
+ for (int i = 0; i < payload.length; i++) p[header.length + i] = (byte) payload[i];
+ return p;
+ }
+
+ private boolean feed(byte[] packet, boolean h265) {
+ return nal.assemble(packet, 0, packet.length, h265, sink);
+ }
+
@Test
public void h264SingleNal() {
- Frame out = nal.assemble(rtp(0x67, 0x42, 0x00, 0x1F), false);
- assertArrayEquals(b(0, 0, 0, 1, 0x67, 0x42, 0x00, 0x1F), bytes(out));
+ feed(rtp(0x67, 0x42, 0x00, 0x1F), false);
+ assertEquals(1, out.size());
+ assertArrayEquals(b(0, 0, 0, 1, 0x67, 0x42, 0x00, 0x1F), out.get(0));
}
@Test
public void h264FuA() {
// FU indicator: NRI 3, type 28; FU header: S/E + type 5 (IDR)
- assertNull(nal.assemble(rtp(0x7C, 0x85, 0xA1, 0xA2), false));
- assertNull(nal.assemble(rtp(0x7C, 0x05, 0xB1), false));
- Frame out = nal.assemble(rtp(0x7C, 0x45, 0xC1, 0xC2), false);
- assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xA1, 0xA2, 0xB1, 0xC1, 0xC2), bytes(out));
+ feed(rtp(0x7C, 0x85, 0xA1, 0xA2), false);
+ feed(rtp(0x7C, 0x05, 0xB1), false);
+ assertEquals(0, out.size());
+ feed(rtp(0x7C, 0x45, 0xC1, 0xC2), false);
+ assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xA1, 0xA2, 0xB1, 0xC1, 0xC2), out.get(0));
}
@Test
public void h265SingleNal() {
- Frame out = nal.assemble(rtp(0x40, 0x01, 0x0C, 0x01), true);
- assertArrayEquals(b(0, 0, 0, 1, 0x40, 0x01, 0x0C, 0x01), bytes(out));
+ feed(rtp(0x40, 0x01, 0x0C, 0x01), true);
+ assertArrayEquals(b(0, 0, 0, 1, 0x40, 0x01, 0x0C, 0x01), out.get(0));
}
@Test
public void h265Fu() {
- // Payload header: type 49, layer 0, TID 1 (0x62 0x01); FU header: S/E + type 19 (IDR_W_RADL)
- assertNull(nal.assemble(rtp(0x62, 0x01, 0x93, 0xA1, 0xA2), true));
- assertNull(nal.assemble(rtp(0x62, 0x01, 0x13, 0xB1), true));
- Frame out = nal.assemble(rtp(0x62, 0x01, 0x53, 0xC1), true);
+ // Payload header: type 49, layer 0, TID 1 (0x62 0x01); FU header: S/E + type 19
+ feed(rtp(0x62, 0x01, 0x93, 0xA1, 0xA2), true);
+ feed(rtp(0x62, 0x01, 0x13, 0xB1), true);
+ feed(rtp(0x62, 0x01, 0x53, 0xC1), true);
+ assertEquals(1, out.size());
// NAL header must be 0x26 0x01 (type 19, layer 0, TID 1)
- assertArrayEquals(b(0, 0, 0, 1, 0x26, 0x01, 0xA1, 0xA2, 0xB1, 0xC1), bytes(out));
+ assertArrayEquals(b(0, 0, 0, 1, 0x26, 0x01, 0xA1, 0xA2, 0xB1, 0xC1), out.get(0));
+ }
+
+ @Test
+ public void h264StapA() {
+ // STAP-A (NRI 3, type 24): SPS (3 bytes) + PPS (2 bytes)
+ feed(rtp(0x78, 0x00, 0x03, 0x67, 0x42, 0x1F, 0x00, 0x02, 0x68, 0xCE), false);
+ assertEquals(2, out.size());
+ assertArrayEquals(b(0, 0, 0, 1, 0x67, 0x42, 0x1F), out.get(0));
+ assertArrayEquals(b(0, 0, 0, 1, 0x68, 0xCE), out.get(1));
+ }
+
+ @Test
+ public void h265AggregationPacket() {
+ // AP (type 48 -> 0x60 0x01): VPS (3 bytes) + SPS (2 bytes)
+ feed(rtp(0x60, 0x01, 0x00, 0x03, 0x40, 0x01, 0x0C, 0x00, 0x02, 0x42, 0x01), true);
+ assertEquals(2, out.size());
+ assertArrayEquals(b(0, 0, 0, 1, 0x40, 0x01, 0x0C), out.get(0));
+ assertArrayEquals(b(0, 0, 0, 1, 0x42, 0x01), out.get(1));
+ }
+
+ @Test
+ public void truncatedAggregationStopsAtTheBadUnit() {
+ feed(rtp(0x78, 0x00, 0x02, 0x67, 0x42, 0x00, 0x09, 0x68), false);
+ assertEquals(1, out.size());
+ }
+
+ @Test
+ public void csrcExtensionAndPaddingAreSkipped() {
+ // V=2, P=1, X=1, CC=1: 12 + 4 (CSRC) + 4 + 4 (extension, 1 word) header, 2 bytes padding
+ byte[] header = b(0xB1, 96, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 2, 3, 4,
+ 0xBE, 0xDE, 0x00, 0x01, 9, 9, 9, 9);
+ byte[] p = rtpWithHeader(header, 0x67, 0x42, 0x1F, 0x00, 0x02);
+ assertTrue(feed(p, false));
+ assertArrayEquals(b(0, 0, 0, 1, 0x67, 0x42, 0x1F), out.get(0));
+ }
+
+ @Test
+ public void malformedHeaderIsRejected() {
+ assertFalse(feed(b(0x40, 96, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0x67), false));
+ // CSRC count larger than the packet
+ assertFalse(feed(b(0x8F, 96, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0x67), false));
+ assertEquals(0, out.size());
+ }
+
+ @Test
+ public void lostMiddleFragmentDropsTheNal() {
+ feed(rtp(0x7C, 0x85, 0xA1), false);
+ seq++; // one packet lost
+ feed(rtp(0x7C, 0x05, 0xB1), false);
+ feed(rtp(0x7C, 0x45, 0xC1), false);
+ assertEquals(0, out.size());
+ assertEquals(1, nal.droppedCount());
+ // The next complete unit is assembled normally
+ feed(rtp(0x7C, 0x85, 0xD1), false);
+ feed(rtp(0x7C, 0x45, 0xD2), false);
+ assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xD1, 0xD2), out.get(0));
+ }
+
+ @Test
+ public void continuationWithoutStartIsIgnoredAndCountedOnce() {
+ feed(rtp(0x7C, 0x05, 0xB1), false);
+ feed(rtp(0x7C, 0x05, 0xB2), false);
+ feed(rtp(0x7C, 0x45, 0xC1), false);
+ assertEquals(0, out.size());
+ assertEquals("one lost unit, not one per piece", 1, nal.droppedCount());
+ // The next orphan unit is a new loss
+ feed(rtp(0x7C, 0x05, 0xD1), false);
+ assertEquals(2, nal.droppedCount());
+ }
+
+ @Test
+ public void otherPacketInterruptsAFragment() {
+ feed(rtp(0x7C, 0x85, 0xA1), false); // FU start
+ feed(rtp(0x41, 0x01), false); // single NAL in between
+ feed(rtp(0x7C, 0x45, 0xA2), false); // continuation must not reuse stale bytes
+ assertEquals(1, out.size());
+ assertArrayEquals(b(0, 0, 0, 1, 0x41, 0x01), out.get(0));
+ assertEquals(1, nal.droppedCount()); // the interrupted unit, counted once
+ }
+
+ @Test
+ public void startWithoutEndIsDroppedOnNextStart() {
+ feed(rtp(0x7C, 0x85, 0xA1), false);
+ feed(rtp(0x7C, 0x85, 0xB1), false); // new start, previous never ended
+ feed(rtp(0x7C, 0x45, 0xB2), false);
+ assertEquals(1, out.size());
+ assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xB1, 0xB2), out.get(0));
+ assertEquals(1, nal.droppedCount());
+ }
+
+ @Test
+ public void lossBetweenSingleNalsIsHarmless() {
+ feed(rtp(0x41, 0x01), false);
+ seq += 5;
+ feed(rtp(0x41, 0x02), false);
+ assertEquals(2, out.size());
+ assertEquals(0, nal.droppedCount());
+ }
+
+ @Test
+ public void sequenceNumberWrapIsNotALoss() {
+ seq = 65535;
+ feed(rtp(0x7C, 0x85, 0xA1), false);
+ feed(rtp(0x7C, 0x45, 0xA2), false); // seq 0
+ assertEquals(1, out.size());
+ assertEquals(0, nal.droppedCount());
+ }
+
+ @Test
+ public void oversizedNalIsDroppedNotTruncated() {
+ nal = new NalAssembler(16, () -> codecSwitches++, new FramePool(4));
+ feed(rtp(0x7C, 0x85, 1, 2, 3, 4, 5, 6, 7, 8), false);
+ feed(rtp(0x7C, 0x45, 1, 2, 3, 4, 5, 6, 7, 8), false);
+ assertEquals(0, out.size());
+ assertEquals(1, nal.droppedCount());
}
@Test
public void codecSwitchResetsAndNotifies() {
- nal.assemble(rtp(0x67, 0x42), false);
+ feed(rtp(0x67, 0x42), false);
assertEquals(0, codecSwitches);
- nal.assemble(rtp(0x40, 0x01, 0x0C), true);
+ feed(rtp(0x40, 0x01, 0x0C), true);
assertEquals(1, codecSwitches);
}
@Test
public void emptyPayloadIsIgnored() {
- assertNull(nal.assemble(rtp(), false));
+ assertTrue(feed(rtp(), false));
+ assertEquals(0, out.size());
+ }
+
+ @Test
+ public void packetAtAnOffset() {
+ byte[] packet = rtp(0x67, 0x42);
+ byte[] buf = new byte[packet.length + 4];
+ System.arraycopy(packet, 0, buf, 4, packet.length);
+ assertTrue(nal.assemble(buf, 4, packet.length, false, sink));
+ assertArrayEquals(b(0, 0, 0, 1, 0x67, 0x42), out.get(0));
}
}