From dad839207d0fafbe4c7b14460487fd667b37cbd9 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Mon, 12 Oct 2026 13:01:44 +0000 Subject: [PATCH 1/3] feat: Opus, MP4A-LATM and fragmented AAC audio Part of #14 (G.726 remains): - Opus (RFC 7587): one Opus packet per RTP packet, decoded by MediaCodec "audio/opus" (Android 5.0+) with an OpusHead csd-0 and codec delay / seek pre-roll csd-1/csd-2; output is 48 kHz. - MP4A-LATM (RFC 6416) with out-of-band config: the fmtp StreamMuxConfig is turned into an AudioSpecificConfig (audioMuxVersion 0, one program and layer), sample rate and channels come from it, and each AudioMuxElement is split by its PayloadLengthInfo. cpresent=1 and other layouts stay unsupported. - RFC 3640 AUs spanning several RTP packets are reassembled (AU-size in each fragment, RTP marker on the last); a fragment with missing pieces is dropped. - The AAC MediaCodec path is shared by AAC, LATM and Opus. - README (EN/RU/ZH): formats listed; only G.726 remains in the limitations. Tests: Opus SDP and header, StreamMuxConfig to ASC, LATM SDP, unsupported LATM layouts, PayloadLengthInfo splitting, fragmented AU reassembly, dropped incomplete fragment, whole AUs passing through. --- README.md | 4 +- README.ru.md | 4 +- README.zh-CN.md | 4 +- .../java/com/openipc/decoder/AudioConfig.java | 88 +++++++++- .../com/openipc/decoder/AudioManager.java | 136 +++++++++++++-- .../com/openipc/decoder/AudioFormatsTest.java | 156 ++++++++++++++++++ .../com/openipc/decoder/SdpParserTest.java | 2 +- 7 files changed, 375 insertions(+), 19 deletions(-) create mode 100644 android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java diff --git a/README.md b/README.md index 7ff94bcb..6b2beb6c 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ A beta channel is available on [Google Play](https://play.google.com/apps/intern ## Features - **Video:** RTSP playback, hardware H.264/H.265 decoding via MediaCodec -- **Audio:** AAC (RFC 3640 `mpeg4-generic`, ADTS), G.711 μ-law/A-law, L16 PCM (mono/stereo) +- **Audio:** AAC (RFC 3640 `mpeg4-generic`, ADTS, MP4A-LATM), Opus, G.711 μ-law/A-law, L16 PCM (mono/stereo) - **Display:** single camera, or Quad mode with 4 cameras at once - **Controls:** pinch-to-zoom, pan, double-tap to reset, Android TV remote (Leanback launcher) - **Network:** Basic and Digest authentication, RTSP keepalive, automatic reconnection with exponential back-off, connection-quality indicator @@ -93,7 +93,7 @@ Shows all configured slots in a 2×2 grid. Each cell has its own connection and > > - **RTSP over TCP only.** No UDP or multicast — [#7](https://github.com/OpenIPC/decoder/issues/7) > - **Only 4 camera slots** — [#13](https://github.com/OpenIPC/decoder/issues/13) -> - **Audio formats not supported:** MP4A-LATM, Opus, G.726, and AAC frames split across RTP packets — [#14](https://github.com/OpenIPC/decoder/issues/14) +> - **G.726 audio is not supported** — [#14](https://github.com/OpenIPC/decoder/issues/14) > - **Possible dependency on Google services (GAPPS)** on some devices; under investigation, so disable GAPPS with care — [#15](https://github.com/OpenIPC/decoder/issues/15) > - **Credentials are stored and sent in clear text** (plain RTSP/HTTP; Basic auth is sent unless the camera asks for Digest) — [#16](https://github.com/OpenIPC/decoder/issues/16) > diff --git a/README.ru.md b/README.ru.md index 0fda88f6..3c772451 100644 --- a/README.ru.md +++ b/README.ru.md @@ -34,7 +34,7 @@ ## Возможности - **Видео:** воспроизведение RTSP, аппаратное декодирование H.264/H.265 через MediaCodec -- **Звук:** AAC (RFC 3640 `mpeg4-generic`, ADTS), G.711 μ-law/A-law, L16 PCM (моно и стерео) +- **Звук:** AAC (RFC 3640 `mpeg4-generic`, ADTS, MP4A-LATM), Opus, G.711 μ-law/A-law, L16 PCM (моно и стерео) - **Режимы:** одна камера или Quad — 4 камеры на одном экране - **Управление:** масштаб двумя пальцами, перемещение, сброс двойным касанием, пульт Android TV (Leanback) - **Сеть:** авторизация Basic и Digest, RTSP keepalive, автоматическое переподключение с растущей задержкой, индикатор качества связи @@ -93,7 +93,7 @@ > > - **Только RTSP поверх TCP.** UDP и multicast не поддерживаются — [#7](https://github.com/OpenIPC/decoder/issues/7) > - **Только 4 слота для камер** — [#13](https://github.com/OpenIPC/decoder/issues/13) -> - **Не поддерживаются форматы звука** MP4A-LATM, Opus, G.726, а также AAC-кадры, разбитые на несколько RTP-пакетов — [#14](https://github.com/OpenIPC/decoder/issues/14) +> - **Не поддерживается звук G.726** — [#14](https://github.com/OpenIPC/decoder/issues/14) > - **Возможная зависимость от сервисов Google (GAPPS)** на некоторых устройствах; изучается, поэтому отключайте GAPPS осознанно — [#15](https://github.com/OpenIPC/decoder/issues/15) > - **Логин и пароль хранятся и передаются в открытом виде** (RTSP/HTTP без шифрования; Basic отправляется, если камера не запросила Digest) — [#16](https://github.com/OpenIPC/decoder/issues/16) > diff --git a/README.zh-CN.md b/README.zh-CN.md index 2e79fcaa..17859bb5 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -34,7 +34,7 @@ ## 功能 - **视频：** RTSP 播放，通过 MediaCodec 进行 H.264/H.265 硬件解码 -- **音频：** AAC（RFC 3640 `mpeg4-generic`、ADTS）、G.711 μ-law/A-law、L16 PCM（单声道/立体声） +- **音频：** AAC（RFC 3640 `mpeg4-generic`、ADTS、MP4A-LATM）、Opus、G.711 μ-law/A-law、L16 PCM（单声道/立体声） - **显示：** 单路画面，或 Quad 四分屏（同时显示 4 路摄像机） - **操作：** 双指缩放、拖动平移、双击复位，支持 Android TV 遥控器（Leanback 启动器） - **网络：** 支持 Basic 和 Digest 认证、RTSP 保活，断线后自动重连（指数退避），连接质量指示 @@ -93,7 +93,7 @@ > > - **仅支持 RTSP over TCP**，不支持 UDP 和组播 — [#7](https://github.com/OpenIPC/decoder/issues/7) > - **仅有 4 个摄像机槽位** — [#13](https://github.com/OpenIPC/decoder/issues/13) -> - **不支持的音频格式：** MP4A-LATM、Opus、G.726，以及跨多个 RTP 包分片的 AAC 帧 — [#14](https://github.com/OpenIPC/decoder/issues/14) +> - **不支持 G.726 音频** — [#14](https://github.com/OpenIPC/decoder/issues/14) > - **部分设备上可能依赖 Google 服务（GAPPS）**，仍在调查中，请谨慎禁用 GAPPS — [#15](https://github.com/OpenIPC/decoder/issues/15) > - **用户名和密码以明文存储和传输**（未加密的 RTSP/HTTP；摄像机未要求 Digest 时发送 Basic） — [#16](https://github.com/OpenIPC/decoder/issues/16) > diff --git a/android/app/src/main/java/com/openipc/decoder/AudioConfig.java b/android/app/src/main/java/com/openipc/decoder/AudioConfig.java index 51a24ec3..f5a49970 100644 --- a/android/app/src/main/java/com/openipc/decoder/AudioConfig.java +++ b/android/app/src/main/java/com/openipc/decoder/AudioConfig.java @@ -13,6 +13,8 @@ class AudioConfig { static final String PCMU = "PCMU"; // G.711 mu-law static final String PCMA = "PCMA"; // G.711 A-law static final String AAC = "AAC"; + static final String LATM = "LATM"; // MP4A-LATM (RFC 6416): AAC in LATM framing + static final String OPUS = "OPUS"; String codec = NONE; int pt = -1; @@ -26,6 +28,8 @@ class AudioConfig { int sizeLength; int indexLength; int indexDeltaLength; + // MP4A-LATM: "cpresent=1" puts the configuration in-band, which is not supported + boolean latmConfigInBand; /** Fill defaults for static RTP payload types (RFC 3551), which may come without rtpmap. */ void applyStaticPayloadType() { @@ -46,6 +50,10 @@ void applyRtpmap(String encoding) { bigEndian = false; if (name.equals("MPEG4-GENERIC") || name.equals("AAC") || name.equals("MP4A")) { codec = AAC; + } else if (name.equals("MP4A-LATM")) { + codec = LATM; + } else if (name.equals("OPUS")) { + codec = OPUS; } else if (name.equals("PCMU")) { codec = PCMU; } else if (name.equals("PCMA")) { @@ -54,7 +62,7 @@ void applyRtpmap(String encoding) { codec = PCM; bigEndian = true; } else { - // MP4A-LATM, OPUS, G726, ... are not supported + // G.726 and others are not supported codec = NONE; } @@ -86,6 +94,7 @@ void applyFmtp(String params) { case "sizelength": sizeLength = Integer.parseInt(value); break; case "indexlength": indexLength = Integer.parseInt(value); break; case "indexdeltalength": indexDeltaLength = Integer.parseInt(value); break; + case "cpresent": latmConfigInBand = !"0".equals(value); break; default: break; } } catch (NumberFormatException ignored) {} @@ -102,6 +111,83 @@ void finish() { || indexDeltaLength < 0 || indexDeltaLength > 16)) { codec = NONE; } + if (OPUS.equals(codec)) { + // RFC 7587: always 48 kHz and "/2" in rtpmap; the decoder handles mono streams too + sampleRate = 48000; + channels = 2; + } + if (LATM.equals(codec)) { + // fmtp config= is a StreamMuxConfig here; turn it into an AudioSpecificConfig + byte[] asc = latmConfigInBand || aacConfig == null ? null : latmToAsc(aacConfig); + if (asc == null) { + codec = NONE; + } else { + aacConfig = asc; + int[] info = ascInfo(asc); + if (info != null) { + sampleRate = info[0]; + channels = info[1]; + } + } + } + } + + private static final int[] AAC_RATES = { + 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, + 16000, 12000, 11025, 8000, 7350 + }; + + /** + * StreamMuxConfig (audioMuxVersion 0, one program, one layer) to a 2-byte + * AudioSpecificConfig, or null if the layout is not supported. + */ + static byte[] latmToAsc(byte[] smc) { + BitReader r = new BitReader(smc); + if (r.bits(1) != 0) return null; // audioMuxVersion 1 is not supported + r.bits(1); // allStreamsSameTimeFraming + r.bits(6); // numSubFrames + if (r.bits(4) != 0) return null; // numProgram + 1 must be 1 + if (r.bits(3) != 0) return null; // numLayer + 1 must be 1 + int objectType = r.bits(5); + if (objectType == 31) objectType = 32 + r.bits(6); + int freqIndex = r.bits(4); + if (freqIndex == 15) return null; // explicit frequency: not supported + int channelConfig = r.bits(4); + if (r.overrun() || objectType < 1 || objectType > 4 || channelConfig < 1 + || channelConfig > 7) { + return null; + } + return new byte[] { + (byte) ((objectType << 3) | (freqIndex >> 1)), + (byte) (((freqIndex & 1) << 7) | (channelConfig << 3)) + }; + } + + /** {sample rate, channels} of a 2-byte AudioSpecificConfig, or null. */ + static int[] ascInfo(byte[] asc) { + if (asc == null || asc.length < 2) return null; + int freqIndex = ((asc[0] & 0x07) << 1) | ((asc[1] & 0x80) >> 7); + int channelConfig = (asc[1] & 0x78) >> 3; + if (freqIndex >= AAC_RATES.length || channelConfig < 1) return null; + return new int[] {AAC_RATES[freqIndex], Math.min(channelConfig, 2)}; + } + + private static final class BitReader { + private final byte[] data; + private int pos; + + BitReader(byte[] data) { this.data = data; } + + int bits(int n) { + int v = 0; + for (int i = 0; i < n; i++, pos++) { + int bit = pos >> 3 < data.length ? (data[pos >> 3] >> (7 - (pos & 7))) & 1 : 0; + v = (v << 1) | bit; + } + return v; + } + + boolean overrun() { return pos > data.length * 8; } } private static byte[] parseHex(String hex) { diff --git a/android/app/src/main/java/com/openipc/decoder/AudioManager.java b/android/app/src/main/java/com/openipc/decoder/AudioManager.java index 4a0739ad..c476a01c 100644 --- a/android/app/src/main/java/com/openipc/decoder/AudioManager.java +++ b/android/app/src/main/java/com/openipc/decoder/AudioManager.java @@ -129,6 +129,7 @@ private void configure(AudioConfig audio) { // Parameters may differ after a reconnect: the AAC decoder is recreated on the next packet releaseAacDecoder(); aacQueue.clear(); + auAssembler.reset(); if (!AudioConfig.AAC.equals(audio.codec)) { pcmSampleRate = audio.sampleRate; pcmChannels = audio.channels; @@ -147,7 +148,18 @@ void onAudioFrame(Frame frame, int r) { AudioConfig c = config; switch (c.codec) { case AudioConfig.AAC: - processAac(c, data, start, end, r); + processAac(c, data, start, end, (data[1] & 0x80) != 0, r); + break; + case AudioConfig.LATM: + ensureAacDecoder(c, false); + for (int[] u : splitLatm(data, start, end)) { + offerAac(Arrays.copyOfRange(data, u[0], u[0] + u[1]), r); + } + break; + case AudioConfig.OPUS: + // RFC 7587: one Opus packet per RTP packet + ensureAacDecoder(c, false); + offerAac(Arrays.copyOfRange(data, start, end), r); break; case AudioConfig.PCMU: offerPcm(decodeG711(data, start, end, ULAW_TABLE), r); @@ -357,12 +369,15 @@ private void releaseAacDecoder() { } } - private void processAac(AudioConfig c, byte[] data, int start, int end, int r) { + private final AuAssembler auAssembler = new AuAssembler(); + + private void processAac(AudioConfig c, byte[] data, int start, int end, boolean marker, + int r) { List units; if (c.sizeLength > 0) { ensureAacDecoder(c, false); - units = splitAuHeaders(data, start, end, - c.sizeLength, c.indexLength, c.indexDeltaLength); + for (byte[] au : auAssembler.push(data, start, end, c, marker)) offerAac(au, r); + return; } else if (end - start >= 7 && isAdtsSync(data, start)) { ensureAacDecoder(c, true); units = splitAdts(data, start, end); @@ -404,6 +419,79 @@ static List splitAuHeaders(byte[] data, int start, int end, return units; } + /** + * RFC 3640 with AUs that may span several RTP packets: complete AUs are returned as + * soon as they are whole. A fragment whose pieces do not add up is dropped. + */ + static final class AuAssembler { + private byte[] fragment; + private int have; + + void reset() { fragment = null; } + + List push(byte[] data, int start, int end, AudioConfig c, boolean marker) { + List out = new ArrayList<>(); + if (end - start < 2) return out; + int headerBits = ((data[start] & 0xFF) << 8) | (data[start + 1] & 0xFF); + int pos = start + 2 + (headerBits + 7) / 8; + if (pos > end || headerBits < c.sizeLength + c.indexLength) return out; + int size = readBits(data, (start + 2) * 8, c.sizeLength); + + if (fragment != null) { + // Continuation: one AU header with the full AU size, then the next piece + int len = end - pos; + if (size != fragment.length || have + len > fragment.length) { + fragment = null; // does not belong to the AU being collected + } else { + System.arraycopy(data, pos, fragment, have, len); + have += len; + if (have == fragment.length) { + out.add(fragment); + fragment = null; + } else if (marker) { + fragment = null; // last packet of the AU, but pieces are missing + } + return out; + } + } + + if (size > end - pos && !marker) { + // First piece of an AU larger than this packet + fragment = new byte[size]; + have = end - pos; + System.arraycopy(data, pos, fragment, 0, have); + return out; + } + for (int[] u : splitAuHeaders(data, start, end, + c.sizeLength, c.indexLength, c.indexDeltaLength)) { + out.add(Arrays.copyOfRange(data, u[0], u[0] + u[1])); + } + return out; + } + } + + /** + * RFC 6416 MP4A-LATM with out-of-band config: each AudioMuxElement is a + * PayloadLengthInfo (bytes of 255 plus a final byte, summed) and the AU. + */ + static List splitLatm(byte[] data, int start, int end) { + List units = new ArrayList<>(); + int pos = start; + while (pos < end) { + int len = 0; + int b; + do { + if (pos >= end) return units; + b = data[pos++] & 0xFF; + len += b; + } while (b == 0xFF); + if (len == 0 || pos + len > end) break; + units.add(new int[] {pos, len}); + pos += len; + } + return units; + } + /** ADTS frames: {offset, length} of each complete frame, stopping at bad sync or length. */ static List splitAdts(byte[] data, int start, int end) { List frames = new ArrayList<>(); @@ -441,6 +529,26 @@ private void offerAac(byte[] au, int r) { } } + /** Opus identification header (RFC 7845) used as csd-0. */ + static byte[] opusHead(int channels) { + ByteBuffer b = ByteBuffer.allocate(19).order(java.nio.ByteOrder.LITTLE_ENDIAN); + b.put("OpusHead".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + b.put((byte) 1); // version + b.put((byte) channels); + b.putShort((short) 0); // pre-skip + b.putInt(48000); // input sample rate + b.putShort((short) 0); // output gain + b.put((byte) 0); // mapping family 0: mono or stereo + return b.array(); + } + + private static ByteBuffer nanos(long ns) { + ByteBuffer b = ByteBuffer.allocate(8).order(java.nio.ByteOrder.nativeOrder()); + b.putLong(ns); + b.flip(); + return b; + } + /** AAC-LC AudioSpecificConfig built from the sample rate and channel count. */ static byte[] buildAacConfig(int sampleRate, int channels) { int index = 4; // 44100 @@ -459,11 +567,17 @@ private void ensureAacDecoder(AudioConfig c, boolean adts) { synchronized (aacDecoderLock) { if (aacDecoder != null || audioFailed || !enabled) return; - Log.d(TAG, "Creating AAC decoder (" + c.sampleRate + "hz, " + c.channels + "ch, " - + (adts ? "ADTS" : "raw") + ")"); - MediaFormat fmt = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, - c.sampleRate, c.channels); - if (adts) { + boolean opus = AudioConfig.OPUS.equals(c.codec); + String mime = opus ? MediaFormat.MIMETYPE_AUDIO_OPUS : MediaFormat.MIMETYPE_AUDIO_AAC; + Log.d(TAG, "Creating " + c.codec + " decoder (" + c.sampleRate + "hz, " + c.channels + + "ch" + (opus ? "" : adts ? ", ADTS" : ", raw") + ")"); + MediaFormat fmt = MediaFormat.createAudioFormat(mime, c.sampleRate, c.channels); + if (opus) { + fmt.setByteBuffer("csd-0", ByteBuffer.wrap(opusHead(c.channels))); + // Codec delay and seek pre-roll in ns; RTP does not carry the pre-skip + fmt.setByteBuffer("csd-1", nanos(0)); + fmt.setByteBuffer("csd-2", nanos(80_000_000L)); + } else if (adts) { fmt.setInteger(MediaFormat.KEY_IS_ADTS, 1); } else { byte[] asc = c.aacConfig != null ? c.aacConfig @@ -475,7 +589,7 @@ private void ensureAacDecoder(AudioConfig c, boolean adts) { MediaCodec codec; try { - codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_AUDIO_AAC); + codec = MediaCodec.createDecoderByType(mime); try { codec.configure(fmt, null, null, 0); codec.start(); @@ -484,7 +598,7 @@ private void ensureAacDecoder(AudioConfig c, boolean adts) { throw e; } } catch (Exception e) { - Log.e(TAG, "Cannot create AAC decoder, audio disabled", e); + Log.e(TAG, "Cannot create " + c.codec + " decoder, audio disabled", e); audioFailed = true; return; } diff --git a/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java b/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java new file mode 100644 index 00000000..ea8a84c7 --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * AudioFormatsTest.java — Opus, MP4A-LATM and fragmented AAC (#14) + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +import java.util.List; + +public class AudioFormatsTest { + /** Builds a byte array from a string of '0'/'1' bits (other characters ignored). */ + private static byte[] bits(String s) { + String b = s.replaceAll("[^01]", ""); + byte[] out = new byte[(b.length() + 7) / 8]; + for (int i = 0; i < b.length(); i++) { + if (b.charAt(i) == '1') out[i / 8] |= (byte) (0x80 >> (i % 8)); + } + return out; + } + + // StreamMuxConfig: version 0, sameTimeFraming 1, numSubFrames 0, numProgram 0, + // numLayer 0, then ASC: AAC LC (2), 16 kHz (index 8), mono (1), GASpecific 000 + private static final String SMC_16K_MONO = + "0 1 000000 0000 000 00010 1000 0001 000 000 00000000"; + + private static AudioConfig parse(String sdp) { + return Sdp.parse(sdp, "rtsp://x").audio; + } + + @Test + public void opusFromSdp() { + AudioConfig a = parse("m=audio 0 RTP/AVP 111\r\na=rtpmap:111 opus/48000/2\r\n"); + assertEquals(AudioConfig.OPUS, a.codec); + assertEquals(48000, a.sampleRate); + assertEquals(2, a.channels); + } + + @Test + public void opusHeaderLayout() { + byte[] h = AudioManager.opusHead(2); + assertEquals(19, h.length); + assertEquals("OpusHead", new String(h, 0, 8, java.nio.charset.StandardCharsets.US_ASCII)); + assertEquals(1, h[8]); + assertEquals(2, h[9]); + // 48000 little-endian at offset 12 + assertArrayEquals(new byte[] {(byte) 0x80, (byte) 0xBB, 0, 0}, + java.util.Arrays.copyOfRange(h, 12, 16)); + assertEquals(0, h[18]); + } + + @Test + public void latmStreamMuxConfigToAsc() { + byte[] asc = AudioConfig.latmToAsc(bits(SMC_16K_MONO)); + assertArrayEquals(AudioManager.buildAacConfig(16000, 1), asc); + assertArrayEquals(new int[] {16000, 1}, AudioConfig.ascInfo(asc)); + } + + @Test + public void latmFromSdp() { + StringBuilder hex = new StringBuilder(); + for (byte b : bits(SMC_16K_MONO)) hex.append(String.format("%02x", b & 0xFF)); + AudioConfig a = parse("m=audio 0 RTP/AVP 96\r\na=rtpmap:96 MP4A-LATM/16000/1\r\n" + + "a=fmtp:96 profile-level-id=15; object=2; cpresent=0; config=" + hex + "\r\n"); + assertEquals(AudioConfig.LATM, a.codec); + assertEquals(16000, a.sampleRate); + assertEquals(1, a.channels); + assertArrayEquals(AudioManager.buildAacConfig(16000, 1), a.aacConfig); + } + + @Test + public void unsupportedLatmLayouts() { + assertEquals(AudioConfig.NONE, parse("m=audio 0 RTP/AVP 96\r\n" + + "a=rtpmap:96 MP4A-LATM/16000\r\na=fmtp:96 cpresent=1\r\n").codec); + assertEquals(AudioConfig.NONE, parse("m=audio 0 RTP/AVP 96\r\n" + + "a=rtpmap:96 MP4A-LATM/16000\r\n").codec); + // audioMuxVersion 1 + assertNull(AudioConfig.latmToAsc(bits("1" + SMC_16K_MONO.substring(1)))); + // two programs + assertNull(AudioConfig.latmToAsc(bits("0 1 000000 0001 000 00010 1000 0001 000"))); + } + + @Test + public void latmPayloadLengths() { + byte[] p = new byte[2 + 1 + 3 + 2 + 257]; + p[2] = 3; // first AU: 3 bytes + p[6] = (byte) 0xFF; // second AU: 255 + 2 = 257 bytes + p[7] = 2; + List u = AudioManager.splitLatm(p, 2, p.length); + assertEquals(2, u.size()); + assertArrayEquals(new int[] {3, 3}, u.get(0)); + assertArrayEquals(new int[] {8, 257}, u.get(1)); + // Truncated: the length points past the end + assertEquals(1, AudioManager.splitLatm(p, 2, p.length - 1).size()); + } + + private static AudioConfig aacHbr() { + AudioConfig c = new AudioConfig(); + c.codec = AudioConfig.AAC; + c.sizeLength = 13; + c.indexLength = 3; + c.indexDeltaLength = 3; + return c; + } + + /** RFC 3640 packet: one 16-bit AU header (size << 3) and the given piece. */ + private static byte[] auPacket(int auSize, int pieceLen, int fill) { + byte[] p = new byte[4 + pieceLen]; + p[1] = 16; + p[2] = (byte) ((auSize << 3) >> 8); + p[3] = (byte) (auSize << 3); + for (int i = 0; i < pieceLen; i++) p[4 + i] = (byte) fill; + return p; + } + + @Test + public void fragmentedAuIsReassembled() { + AudioManager.AuAssembler a = new AudioManager.AuAssembler(); + AudioConfig c = aacHbr(); + assertEquals(0, a.push(auPacket(10, 4, 1), 0, 8, c, false).size()); + assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, false).size()); + List out = a.push(auPacket(10, 2, 3), 0, 6, c, true); + assertEquals(1, out.size()); + assertArrayEquals(new byte[] {1, 1, 1, 1, 2, 2, 2, 2, 3, 3}, out.get(0)); + } + + @Test + public void incompleteFragmentIsDropped() { + AudioManager.AuAssembler a = new AudioManager.AuAssembler(); + AudioConfig c = aacHbr(); + a.push(auPacket(10, 4, 1), 0, 8, c, false); + // Marker on the next piece, but 2 bytes are still missing + assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, true).size()); + // A normal, complete AU afterwards is delivered + List out = a.push(auPacket(3, 3, 7), 0, 7, c, true); + assertEquals(1, out.size()); + assertArrayEquals(new byte[] {7, 7, 7}, out.get(0)); + } + + @Test + public void wholeAusPassThrough() { + AudioManager.AuAssembler a = new AudioManager.AuAssembler(); + // Two AUs (3 and 2 bytes) in one packet + byte[] p = {0x00, 0x20, 0x00, 0x18, 0x00, 0x10, 1, 1, 1, 2, 2}; + List out = a.push(p, 0, p.length, aacHbr(), true); + assertEquals(2, out.size()); + assertArrayEquals(new byte[] {2, 2}, out.get(1)); + } +} diff --git a/android/app/src/test/java/com/openipc/decoder/SdpParserTest.java b/android/app/src/test/java/com/openipc/decoder/SdpParserTest.java index a387ec58..1594abb2 100644 --- a/android/app/src/test/java/com/openipc/decoder/SdpParserTest.java +++ b/android/app/src/test/java/com/openipc/decoder/SdpParserTest.java @@ -95,7 +95,7 @@ public void l16BigEndianAndRtpmapOfVideoIsIgnored() { @Test public void unsupportedAndMissingAudio() { - AudioConfig a = parse("m=audio 0 RTP/AVP 111\r\na=rtpmap:111 opus/48000/2\r\n", "rtsp://x").audio; + AudioConfig a = parse("m=audio 0 RTP/AVP 111\r\na=rtpmap:111 G726-32/8000\r\n", "rtsp://x").audio; assertEquals(AudioConfig.NONE, a.codec); AudioConfig b = parse("m=video 0 RTP/AVP 96\r\n", "rtsp://x").audio; From 6f43336d2fee8fabcdc9637c6ea438bf69bfaff3 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Mon, 12 Oct 2026 13:09:43 +0000 Subject: [PATCH 2/3] fix: address review findings for LATM and fragmented AAC - latmToAsc() parses the whole StreamMuxConfig and accepts only the layout the RTP path handles: same time framing, one subframe, one program and layer, AAC-LC, a standard sampling rate, mono or stereo, frameLengthType 0, no other data, no CRC. Anything else disables LATM instead of reaching the splitter or MediaCodec. - ascInfo() rejects more than two channels; finish() disables LATM when the ASC gives no valid rate/channels. - LatmAssembler joins an AudioMuxElement that spans several RTP packets (marker on the last) and drops an element with a missing piece. - AuAssembler tracks RTP sequence numbers: after a loss the pending AAC fragment is dropped, so bytes of two AUs are never joined. --- .../java/com/openipc/decoder/AudioConfig.java | 46 +++++++---- .../com/openipc/decoder/AudioManager.java | 59 ++++++++++++-- .../com/openipc/decoder/AudioFormatsTest.java | 77 +++++++++++++++++-- 3 files changed, 150 insertions(+), 32 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/AudioConfig.java b/android/app/src/main/java/com/openipc/decoder/AudioConfig.java index f5a49970..a83f5660 100644 --- a/android/app/src/main/java/com/openipc/decoder/AudioConfig.java +++ b/android/app/src/main/java/com/openipc/decoder/AudioConfig.java @@ -122,9 +122,11 @@ void finish() { if (asc == null) { codec = NONE; } else { - aacConfig = asc; int[] info = ascInfo(asc); - if (info != null) { + if (info == null) { + codec = NONE; + } else { + aacConfig = asc; sampleRate = info[0]; channels = info[1]; } @@ -138,25 +140,35 @@ void finish() { }; /** - * StreamMuxConfig (audioMuxVersion 0, one program, one layer) to a 2-byte - * AudioSpecificConfig, or null if the layout is not supported. + * StreamMuxConfig to a 2-byte AudioSpecificConfig, or null unless it is the one layout + * the RTP path handles: audioMuxVersion 0, all streams same time framing, one subframe, + * one program, one layer, AAC-LC, a standard sampling rate, mono or stereo, + * frameLengthType 0 (byte-aligned PayloadLengthInfo), no other data and no CRC. */ static byte[] latmToAsc(byte[] smc) { BitReader r = new BitReader(smc); - if (r.bits(1) != 0) return null; // audioMuxVersion 1 is not supported - r.bits(1); // allStreamsSameTimeFraming - r.bits(6); // numSubFrames - if (r.bits(4) != 0) return null; // numProgram + 1 must be 1 - if (r.bits(3) != 0) return null; // numLayer + 1 must be 1 + if (r.bits(1) != 0) return null; // audioMuxVersion 1 + if (r.bits(1) != 1) return null; // allStreamsSameTimeFraming + if (r.bits(6) != 0) return null; // numSubFrames: exactly one + if (r.bits(4) != 0) return null; // numProgram: exactly one + if (r.bits(3) != 0) return null; // numLayer: exactly one + // AudioSpecificConfig int objectType = r.bits(5); - if (objectType == 31) objectType = 32 + r.bits(6); int freqIndex = r.bits(4); - if (freqIndex == 15) return null; // explicit frequency: not supported int channelConfig = r.bits(4); - if (r.overrun() || objectType < 1 || objectType > 4 || channelConfig < 1 - || channelConfig > 7) { - return null; - } + if (objectType != 2) return null; // AAC-LC only (decoder profile) + if (freqIndex >= AAC_RATES.length) return null; // reserved / explicit rate + if (channelConfig < 1 || channelConfig > 2) return null; // playback is mono/stereo + // GASpecificConfig + r.bits(1); // frameLengthFlag + if (r.bits(1) == 1) r.bits(14); // dependsOnCoreCoder + if (r.bits(1) != 0) return null; // extensionFlag + // Back in StreamMuxConfig + if (r.bits(3) != 0) return null; // frameLengthType 0: PayloadLengthInfo in bytes + r.bits(8); // latmBufferFullness + if (r.bits(1) != 0) return null; // otherDataPresent + if (r.bits(1) != 0) return null; // crcCheckPresent + if (r.overrun()) return null; return new byte[] { (byte) ((objectType << 3) | (freqIndex >> 1)), (byte) (((freqIndex & 1) << 7) | (channelConfig << 3)) @@ -168,8 +180,8 @@ static int[] ascInfo(byte[] asc) { if (asc == null || asc.length < 2) return null; int freqIndex = ((asc[0] & 0x07) << 1) | ((asc[1] & 0x80) >> 7); int channelConfig = (asc[1] & 0x78) >> 3; - if (freqIndex >= AAC_RATES.length || channelConfig < 1) return null; - return new int[] {AAC_RATES[freqIndex], Math.min(channelConfig, 2)}; + if (freqIndex >= AAC_RATES.length || channelConfig < 1 || channelConfig > 2) return null; + return new int[] {AAC_RATES[freqIndex], channelConfig}; } private static final class BitReader { diff --git a/android/app/src/main/java/com/openipc/decoder/AudioManager.java b/android/app/src/main/java/com/openipc/decoder/AudioManager.java index c476a01c..591921d6 100644 --- a/android/app/src/main/java/com/openipc/decoder/AudioManager.java +++ b/android/app/src/main/java/com/openipc/decoder/AudioManager.java @@ -130,6 +130,7 @@ private void configure(AudioConfig audio) { releaseAacDecoder(); aacQueue.clear(); auAssembler.reset(); + latmAssembler.reset(); if (!AudioConfig.AAC.equals(audio.codec)) { pcmSampleRate = audio.sampleRate; pcmChannels = audio.channels; @@ -145,15 +146,20 @@ void onAudioFrame(Frame frame, int r) { int end = RtspSession.rtpPayloadEnd(data, frame.length()); if (end <= start) return; + boolean marker = (data[1] & 0x80) != 0; + int seq = ((data[2] & 0xFF) << 8) | (data[3] & 0xFF); AudioConfig c = config; switch (c.codec) { case AudioConfig.AAC: - processAac(c, data, start, end, (data[1] & 0x80) != 0, r); + processAac(c, data, start, end, marker, seq, r); break; case AudioConfig.LATM: ensureAacDecoder(c, false); - for (int[] u : splitLatm(data, start, end)) { - offerAac(Arrays.copyOfRange(data, u[0], u[0] + u[1]), r); + byte[] element = latmAssembler.push(data, start, end, marker, seq); + if (element != null) { + for (int[] u : splitLatm(element, 0, element.length)) { + offerAac(Arrays.copyOfRange(element, u[0], u[0] + u[1]), r); + } } break; case AudioConfig.OPUS: @@ -370,13 +376,14 @@ private void releaseAacDecoder() { } private final AuAssembler auAssembler = new AuAssembler(); + private final LatmAssembler latmAssembler = new LatmAssembler(); private void processAac(AudioConfig c, byte[] data, int start, int end, boolean marker, - int r) { + int seq, int r) { List units; if (c.sizeLength > 0) { ensureAacDecoder(c, false); - for (byte[] au : auAssembler.push(data, start, end, c, marker)) offerAac(au, r); + for (byte[] au : auAssembler.push(data, start, end, c, marker, seq)) offerAac(au, r); return; } else if (end - start >= 7 && isAdtsSync(data, start)) { ensureAacDecoder(c, true); @@ -426,11 +433,19 @@ static List splitAuHeaders(byte[] data, int start, int end, static final class AuAssembler { private byte[] fragment; private int have; + private int lastSeq = -1; - void reset() { fragment = null; } + void reset() { + fragment = null; + lastSeq = -1; + } - List push(byte[] data, int start, int end, AudioConfig c, boolean marker) { + List push(byte[] data, int start, int end, AudioConfig c, boolean marker, + int seq) { List out = new ArrayList<>(); + // A lost packet in the middle of an AU: the pending fragment is not trustworthy + if (lastSeq >= 0 && seq != ((lastSeq + 1) & 0xFFFF)) fragment = null; + lastSeq = seq; if (end - start < 2) return out; int headerBits = ((data[start] & 0xFF) << 8) | (data[start + 1] & 0xFF); int pos = start + 2 + (headerBits + 7) / 8; @@ -470,6 +485,36 @@ List push(byte[] data, int start, int end, AudioConfig c, boolean marker } } + /** + * RFC 6416: an AudioMuxElement may span several RTP packets, the marker bit is set on + * the last one. Returns the complete element, or null while it is incomplete or after + * a loss (then the rest of that element is skipped). + */ + static final class LatmAssembler { + private final java.io.ByteArrayOutputStream pending = new java.io.ByteArrayOutputStream(); + private int lastSeq = -1; + private boolean broken; + + void reset() { + pending.reset(); + lastSeq = -1; + broken = false; + } + + byte[] push(byte[] data, int start, int end, boolean marker, int seq) { + if (lastSeq >= 0 && seq != ((lastSeq + 1) & 0xFFFF) && pending.size() > 0) { + broken = true; // a piece of the current element is missing + } + lastSeq = seq; + if (!broken) pending.write(data, start, end - start); + if (!marker) return null; + byte[] element = broken ? null : pending.toByteArray(); + pending.reset(); + broken = false; + return element; + } + } + /** * RFC 6416 MP4A-LATM with out-of-band config: each AudioMuxElement is a * PayloadLengthInfo (bytes of 255 plus a final byte, summed) and the AU. diff --git a/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java b/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java index ea8a84c7..47bbce03 100644 --- a/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java +++ b/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java @@ -9,6 +9,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import org.junit.Test; @@ -28,8 +29,16 @@ private static byte[] bits(String s) { // StreamMuxConfig: version 0, sameTimeFraming 1, numSubFrames 0, numProgram 0, // numLayer 0, then ASC: AAC LC (2), 16 kHz (index 8), mono (1), GASpecific 000 + // ... GASpecificConfig 000, frameLengthType 000, latmBufferFullness 0xFF, + // otherDataPresent 0, crcCheckPresent 0 private static final String SMC_16K_MONO = - "0 1 000000 0000 000 00010 1000 0001 000 000 00000000"; + "0 1 000000 0000 000 00010 1000 0001 000 000 11111111 0 0"; + + /** SMC_16K_MONO with the bits from {@code from} replaced by {@code bits}. */ + private static String smcWith(int from, String bits) { + String b = SMC_16K_MONO.replaceAll("[^01]", ""); + return b.substring(0, from) + bits + b.substring(from + bits.length()); + } private static AudioConfig parse(String sdp) { return Sdp.parse(sdp, "rtsp://x").audio; @@ -124,9 +133,9 @@ private static byte[] auPacket(int auSize, int pieceLen, int fill) { public void fragmentedAuIsReassembled() { AudioManager.AuAssembler a = new AudioManager.AuAssembler(); AudioConfig c = aacHbr(); - assertEquals(0, a.push(auPacket(10, 4, 1), 0, 8, c, false).size()); - assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, false).size()); - List out = a.push(auPacket(10, 2, 3), 0, 6, c, true); + assertEquals(0, a.push(auPacket(10, 4, 1), 0, 8, c, false, 1).size()); + assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, false, 2).size()); + List out = a.push(auPacket(10, 2, 3), 0, 6, c, true, 3); assertEquals(1, out.size()); assertArrayEquals(new byte[] {1, 1, 1, 1, 2, 2, 2, 2, 3, 3}, out.get(0)); } @@ -135,11 +144,11 @@ public void fragmentedAuIsReassembled() { public void incompleteFragmentIsDropped() { AudioManager.AuAssembler a = new AudioManager.AuAssembler(); AudioConfig c = aacHbr(); - a.push(auPacket(10, 4, 1), 0, 8, c, false); + a.push(auPacket(10, 4, 1), 0, 8, c, false, 4); // Marker on the next piece, but 2 bytes are still missing - assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, true).size()); + assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, true, 5).size()); // A normal, complete AU afterwards is delivered - List out = a.push(auPacket(3, 3, 7), 0, 7, c, true); + List out = a.push(auPacket(3, 3, 7), 0, 7, c, true, 6); assertEquals(1, out.size()); assertArrayEquals(new byte[] {7, 7, 7}, out.get(0)); } @@ -149,8 +158,60 @@ public void wholeAusPassThrough() { AudioManager.AuAssembler a = new AudioManager.AuAssembler(); // Two AUs (3 and 2 bytes) in one packet byte[] p = {0x00, 0x20, 0x00, 0x18, 0x00, 0x10, 1, 1, 1, 2, 2}; - List out = a.push(p, 0, p.length, aacHbr(), true); + List out = a.push(p, 0, p.length, aacHbr(), true, 7); assertEquals(2, out.size()); assertArrayEquals(new byte[] {2, 2}, out.get(1)); } + + @Test + public void lossBetweenFragmentsDropsTheStaleAu() { + AudioManager.AuAssembler a = new AudioManager.AuAssembler(); + AudioConfig c = aacHbr(); + a.push(auPacket(10, 4, 1), 0, 8, c, false, 100); + // Packet 101 lost; 102 is the start of a new AU of the same size + assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, false, 102).size()); + assertEquals(0, a.push(auPacket(10, 4, 3), 0, 8, c, false, 103).size()); + List out = a.push(auPacket(10, 2, 4), 0, 6, c, true, 104); + assertEquals(1, out.size()); + assertArrayEquals("only bytes of the new AU", + new byte[] {2, 2, 2, 2, 3, 3, 3, 3, 4, 4}, out.get(0)); + } + + @Test + public void latmElementAcrossPackets() { + AudioManager.LatmAssembler a = new AudioManager.LatmAssembler(); + byte[] part1 = {5, 1, 1}; // length 5, first 2 bytes + byte[] part2 = {1, 1, 1}; // remaining 3 bytes, marker set + assertNull(a.push(part1, 0, 3, false, 10)); + byte[] element = a.push(part2, 0, 3, true, 11); + assertArrayEquals(new byte[] {5, 1, 1, 1, 1, 1}, element); + List u = AudioManager.splitLatm(element, 0, element.length); + assertArrayEquals(new int[] {1, 5}, u.get(0)); + } + + @Test + public void latmElementWithALostPieceIsDropped() { + AudioManager.LatmAssembler a = new AudioManager.LatmAssembler(); + assertNull(a.push(new byte[] {5, 1}, 0, 2, false, 10)); + assertNull("piece 11 lost", a.push(new byte[] {1, 1}, 0, 2, true, 12)); + // The next element is fine + assertArrayEquals(new byte[] {1, 9}, a.push(new byte[] {1, 9}, 0, 2, true, 13)); + } + + @Test + public void latmRejectsLayoutsTheRtpPathCannotHandle() { + // bit offsets in SMC_16K_MONO: 1 sameTimeFraming, 2..7 numSubFrames, + // 15..19 objectType, 20..23 freqIndex, 24..27 channels, 31..33 frameLengthType, + // 42 otherDataPresent, 43 crcCheckPresent + assertNotNull(AudioConfig.latmToAsc(bits(SMC_16K_MONO))); + assertNull("not same time framing", AudioConfig.latmToAsc(bits(smcWith(1, "0")))); + assertNull("two subframes", AudioConfig.latmToAsc(bits(smcWith(2, "000001")))); + assertNull("AAC Main", AudioConfig.latmToAsc(bits(smcWith(15, "00001")))); + assertNull("reserved rate", AudioConfig.latmToAsc(bits(smcWith(20, "1101")))); + assertNull("5.1", AudioConfig.latmToAsc(bits(smcWith(24, "0110")))); + assertNull("frameLengthType 1", AudioConfig.latmToAsc(bits(smcWith(31, "001")))); + assertNull("other data", AudioConfig.latmToAsc(bits(smcWith(42, "1")))); + assertNull("CRC", AudioConfig.latmToAsc(bits(smcWith(43, "1")))); + assertNull("multichannel ASC info", AudioConfig.ascInfo(new byte[] {0x14, 0x30})); + } } From 68c8c25ad6bd5f7555c0813c9b25e5ed4f6f1bcf Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Mon, 12 Oct 2026 14:59:27 +0000 Subject: [PATCH 3/3] LATM: drop elements after any loss, cap the buffer, refuse 960-sample and core-coder configs - A sequence gap right after a completed element may have lost the next element's start, so its continuation is no longer taken for a new element - Without markers the element buffer grew forever; it is capped at 64 KiB - frameLengthFlag and dependsOnCoreCoder are not carried by the 2-byte ASC, so such StreamMuxConfigs are refused instead of reaching the decoder with a wrong csd-0 --- .../java/com/openipc/decoder/AudioConfig.java | 5 ++-- .../com/openipc/decoder/AudioManager.java | 18 ++++++++++--- .../com/openipc/decoder/AudioFormatsTest.java | 25 ++++++++++++++++++- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/AudioConfig.java b/android/app/src/main/java/com/openipc/decoder/AudioConfig.java index a83f5660..0cf130db 100644 --- a/android/app/src/main/java/com/openipc/decoder/AudioConfig.java +++ b/android/app/src/main/java/com/openipc/decoder/AudioConfig.java @@ -160,8 +160,9 @@ static byte[] latmToAsc(byte[] smc) { if (freqIndex >= AAC_RATES.length) return null; // reserved / explicit rate if (channelConfig < 1 || channelConfig > 2) return null; // playback is mono/stereo // GASpecificConfig - r.bits(1); // frameLengthFlag - if (r.bits(1) == 1) r.bits(14); // dependsOnCoreCoder + // The 2-byte ASC built below carries neither setting, so streams using them are refused + if (r.bits(1) != 0) return null; // frameLengthFlag: 960 samples + if (r.bits(1) != 0) return null; // dependsOnCoreCoder if (r.bits(1) != 0) return null; // extensionFlag // Back in StreamMuxConfig if (r.bits(3) != 0) return null; // frameLengthType 0: PayloadLengthInfo in bytes diff --git a/android/app/src/main/java/com/openipc/decoder/AudioManager.java b/android/app/src/main/java/com/openipc/decoder/AudioManager.java index 591921d6..31b69a44 100644 --- a/android/app/src/main/java/com/openipc/decoder/AudioManager.java +++ b/android/app/src/main/java/com/openipc/decoder/AudioManager.java @@ -491,6 +491,8 @@ List push(byte[] data, int start, int end, AudioConfig c, boolean marker * a loss (then the rest of that element is skipped). */ static final class LatmAssembler { + /** Far above any AAC frame: without markers the buffer would otherwise grow forever. */ + static final int MAX_ELEMENT = 64 * 1024; private final java.io.ByteArrayOutputStream pending = new java.io.ByteArrayOutputStream(); private int lastSeq = -1; private boolean broken; @@ -502,11 +504,21 @@ void reset() { } byte[] push(byte[] data, int start, int end, boolean marker, int seq) { - if (lastSeq >= 0 && seq != ((lastSeq + 1) & 0xFFFF) && pending.size() > 0) { - broken = true; // a piece of the current element is missing + if (lastSeq >= 0 && seq != ((lastSeq + 1) & 0xFFFF)) { + // A piece is missing: possibly the start of this element, even when nothing is + // pending yet, so skip everything up to the next marker + broken = true; } lastSeq = seq; - if (!broken) pending.write(data, start, end - start); + if (!broken && pending.size() + (end - start) > MAX_ELEMENT) { + Log.w(TAG, "LATM element larger than " + MAX_ELEMENT + " bytes, dropped"); + broken = true; + } + if (broken) { + pending.reset(); + } else { + pending.write(data, start, end - start); + } if (!marker) return null; byte[] element = broken ? null : pending.toByteArray(); pending.reset(); diff --git a/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java b/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java index 47bbce03..e3d294bb 100644 --- a/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java +++ b/android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java @@ -198,10 +198,31 @@ public void latmElementWithALostPieceIsDropped() { assertArrayEquals(new byte[] {1, 9}, a.push(new byte[] {1, 9}, 0, 2, true, 13)); } + @Test + public void latmLostFirstPieceDropsTheElement() { + AudioManager.LatmAssembler a = new AudioManager.LatmAssembler(); + assertArrayEquals(new byte[] {1, 7}, a.push(new byte[] {1, 7}, 0, 2, true, 10)); + // 11 (the start of the next element) is lost: its continuation is not a new element + assertNull(a.push(new byte[] {9, 9}, 0, 2, true, 12)); + assertArrayEquals(new byte[] {1, 8}, a.push(new byte[] {1, 8}, 0, 2, true, 13)); + } + + @Test + public void latmWithoutMarkersStaysBounded() { + AudioManager.LatmAssembler a = new AudioManager.LatmAssembler(); + byte[] chunk = new byte[1000]; + int seq = 0; + for (int i = 0; i < 100; i++) assertNull(a.push(chunk, 0, chunk.length, false, seq++)); + // The oversized element is dropped; the next one assembles normally + assertNull(a.push(chunk, 0, 10, true, seq++)); + assertArrayEquals(new byte[] {1, 5}, a.push(new byte[] {1, 5}, 0, 2, true, seq)); + } + @Test public void latmRejectsLayoutsTheRtpPathCannotHandle() { // bit offsets in SMC_16K_MONO: 1 sameTimeFraming, 2..7 numSubFrames, - // 15..19 objectType, 20..23 freqIndex, 24..27 channels, 31..33 frameLengthType, + // 15..19 objectType, 20..23 freqIndex, 24..27 channels, 28 frameLengthFlag, + // 29 dependsOnCoreCoder, 31..33 frameLengthType, // 42 otherDataPresent, 43 crcCheckPresent assertNotNull(AudioConfig.latmToAsc(bits(SMC_16K_MONO))); assertNull("not same time framing", AudioConfig.latmToAsc(bits(smcWith(1, "0")))); @@ -209,6 +230,8 @@ public void latmRejectsLayoutsTheRtpPathCannotHandle() { assertNull("AAC Main", AudioConfig.latmToAsc(bits(smcWith(15, "00001")))); assertNull("reserved rate", AudioConfig.latmToAsc(bits(smcWith(20, "1101")))); assertNull("5.1", AudioConfig.latmToAsc(bits(smcWith(24, "0110")))); + assertNull("960-sample frames", AudioConfig.latmToAsc(bits(smcWith(28, "1")))); + assertNull("depends on core coder", AudioConfig.latmToAsc(bits(smcWith(29, "1")))); assertNull("frameLengthType 1", AudioConfig.latmToAsc(bits(smcWith(31, "001")))); assertNull("other data", AudioConfig.latmToAsc(bits(smcWith(42, "1")))); assertNull("CRC", AudioConfig.latmToAsc(bits(smcWith(43, "1"))));