Skip to content

Commit 6f43336

Browse files
compot-devcompot-dev
committed
fix: address review findings for LATM and fragmented AAC
- latmToAsc() parses the whole StreamMuxConfig and accepts only the layout the RTP path handles: same time framing, one subframe, one program and layer, AAC-LC, a standard sampling rate, mono or stereo, frameLengthType 0, no other data, no CRC. Anything else disables LATM instead of reaching the splitter or MediaCodec. - ascInfo() rejects more than two channels; finish() disables LATM when the ASC gives no valid rate/channels. - LatmAssembler joins an AudioMuxElement that spans several RTP packets (marker on the last) and drops an element with a missing piece. - AuAssembler tracks RTP sequence numbers: after a loss the pending AAC fragment is dropped, so bytes of two AUs are never joined.
1 parent dad8392 commit 6f43336

3 files changed

Lines changed: 150 additions & 32 deletions

File tree

‎android/app/src/main/java/com/openipc/decoder/AudioConfig.java‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -122,9 +122,11 @@ void finish() {
122122
if (asc == null) {
123123
codec = NONE;
124124
} else {
125-
aacConfig = asc;
126125
int[] info = ascInfo(asc);
127-
if (info != null) {
126+
if (info == null) {
127+
codec = NONE;
128+
} else {
129+
aacConfig = asc;
128130
sampleRate = info[0];
129131
channels = info[1];
130132
}
@@ -138,25 +140,35 @@ void finish() {
138140
};
139141

140142
/**
141-
* StreamMuxConfig (audioMuxVersion 0, one program, one layer) to a 2-byte
142-
* AudioSpecificConfig, or null if the layout is not supported.
143+
* StreamMuxConfig to a 2-byte AudioSpecificConfig, or null unless it is the one layout
144+
* the RTP path handles: audioMuxVersion 0, all streams same time framing, one subframe,
145+
* one program, one layer, AAC-LC, a standard sampling rate, mono or stereo,
146+
* frameLengthType 0 (byte-aligned PayloadLengthInfo), no other data and no CRC.
143147
*/
144148
static byte[] latmToAsc(byte[] smc) {
145149
BitReader r = new BitReader(smc);
146-
if (r.bits(1) != 0) return null; // audioMuxVersion 1 is not supported
147-
r.bits(1); // allStreamsSameTimeFraming
148-
r.bits(6); // numSubFrames
149-
if (r.bits(4) != 0) return null; // numProgram + 1 must be 1
150-
if (r.bits(3) != 0) return null; // numLayer + 1 must be 1
150+
if (r.bits(1) != 0) return null; // audioMuxVersion 1
151+
if (r.bits(1) != 1) return null; // allStreamsSameTimeFraming
152+
if (r.bits(6) != 0) return null; // numSubFrames: exactly one
153+
if (r.bits(4) != 0) return null; // numProgram: exactly one
154+
if (r.bits(3) != 0) return null; // numLayer: exactly one
155+
// AudioSpecificConfig
151156
int objectType = r.bits(5);
152-
if (objectType == 31) objectType = 32 + r.bits(6);
153157
int freqIndex = r.bits(4);
154-
if (freqIndex == 15) return null; // explicit frequency: not supported
155158
int channelConfig = r.bits(4);
156-
if (r.overrun() || objectType < 1 || objectType > 4 || channelConfig < 1
157-
|| channelConfig > 7) {
158-
return null;
159-
}
159+
if (objectType != 2) return null; // AAC-LC only (decoder profile)
160+
if (freqIndex >= AAC_RATES.length) return null; // reserved / explicit rate
161+
if (channelConfig < 1 || channelConfig > 2) return null; // playback is mono/stereo
162+
// GASpecificConfig
163+
r.bits(1); // frameLengthFlag
164+
if (r.bits(1) == 1) r.bits(14); // dependsOnCoreCoder
165+
if (r.bits(1) != 0) return null; // extensionFlag
166+
// Back in StreamMuxConfig
167+
if (r.bits(3) != 0) return null; // frameLengthType 0: PayloadLengthInfo in bytes
168+
r.bits(8); // latmBufferFullness
169+
if (r.bits(1) != 0) return null; // otherDataPresent
170+
if (r.bits(1) != 0) return null; // crcCheckPresent
171+
if (r.overrun()) return null;
160172
return new byte[] {
161173
(byte) ((objectType << 3) | (freqIndex >> 1)),
162174
(byte) (((freqIndex & 1) << 7) | (channelConfig << 3))
@@ -168,8 +180,8 @@ static int[] ascInfo(byte[] asc) {
168180
if (asc == null || asc.length < 2) return null;
169181
int freqIndex = ((asc[0] & 0x07) << 1) | ((asc[1] & 0x80) >> 7);
170182
int channelConfig = (asc[1] & 0x78) >> 3;
171-
if (freqIndex >= AAC_RATES.length || channelConfig < 1) return null;
172-
return new int[] {AAC_RATES[freqIndex], Math.min(channelConfig, 2)};
183+
if (freqIndex >= AAC_RATES.length || channelConfig < 1 || channelConfig > 2) return null;
184+
return new int[] {AAC_RATES[freqIndex], channelConfig};
173185
}
174186

175187
private static final class BitReader {

‎android/app/src/main/java/com/openipc/decoder/AudioManager.java‎

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ private void configure(AudioConfig audio) {
130130
releaseAacDecoder();
131131
aacQueue.clear();
132132
auAssembler.reset();
133+
latmAssembler.reset();
133134
if (!AudioConfig.AAC.equals(audio.codec)) {
134135
pcmSampleRate = audio.sampleRate;
135136
pcmChannels = audio.channels;
@@ -145,15 +146,20 @@ void onAudioFrame(Frame frame, int r) {
145146
int end = RtspSession.rtpPayloadEnd(data, frame.length());
146147
if (end <= start) return;
147148

149+
boolean marker = (data[1] & 0x80) != 0;
150+
int seq = ((data[2] & 0xFF) << 8) | (data[3] & 0xFF);
148151
AudioConfig c = config;
149152
switch (c.codec) {
150153
case AudioConfig.AAC:
151-
processAac(c, data, start, end, (data[1] & 0x80) != 0, r);
154+
processAac(c, data, start, end, marker, seq, r);
152155
break;
153156
case AudioConfig.LATM:
154157
ensureAacDecoder(c, false);
155-
for (int[] u : splitLatm(data, start, end)) {
156-
offerAac(Arrays.copyOfRange(data, u[0], u[0] + u[1]), r);
158+
byte[] element = latmAssembler.push(data, start, end, marker, seq);
159+
if (element != null) {
160+
for (int[] u : splitLatm(element, 0, element.length)) {
161+
offerAac(Arrays.copyOfRange(element, u[0], u[0] + u[1]), r);
162+
}
157163
}
158164
break;
159165
case AudioConfig.OPUS:
@@ -370,13 +376,14 @@ private void releaseAacDecoder() {
370376
}
371377

372378
private final AuAssembler auAssembler = new AuAssembler();
379+
private final LatmAssembler latmAssembler = new LatmAssembler();
373380

374381
private void processAac(AudioConfig c, byte[] data, int start, int end, boolean marker,
375-
int r) {
382+
int seq, int r) {
376383
List<int[]> units;
377384
if (c.sizeLength > 0) {
378385
ensureAacDecoder(c, false);
379-
for (byte[] au : auAssembler.push(data, start, end, c, marker)) offerAac(au, r);
386+
for (byte[] au : auAssembler.push(data, start, end, c, marker, seq)) offerAac(au, r);
380387
return;
381388
} else if (end - start >= 7 && isAdtsSync(data, start)) {
382389
ensureAacDecoder(c, true);
@@ -426,11 +433,19 @@ static List<int[]> splitAuHeaders(byte[] data, int start, int end,
426433
static final class AuAssembler {
427434
private byte[] fragment;
428435
private int have;
436+
private int lastSeq = -1;
429437

430-
void reset() { fragment = null; }
438+
void reset() {
439+
fragment = null;
440+
lastSeq = -1;
441+
}
431442

432-
List<byte[]> push(byte[] data, int start, int end, AudioConfig c, boolean marker) {
443+
List<byte[]> push(byte[] data, int start, int end, AudioConfig c, boolean marker,
444+
int seq) {
433445
List<byte[]> out = new ArrayList<>();
446+
// A lost packet in the middle of an AU: the pending fragment is not trustworthy
447+
if (lastSeq >= 0 && seq != ((lastSeq + 1) & 0xFFFF)) fragment = null;
448+
lastSeq = seq;
434449
if (end - start < 2) return out;
435450
int headerBits = ((data[start] & 0xFF) << 8) | (data[start + 1] & 0xFF);
436451
int pos = start + 2 + (headerBits + 7) / 8;
@@ -470,6 +485,36 @@ List<byte[]> push(byte[] data, int start, int end, AudioConfig c, boolean marker
470485
}
471486
}
472487

488+
/**
489+
* RFC 6416: an AudioMuxElement may span several RTP packets, the marker bit is set on
490+
* the last one. Returns the complete element, or null while it is incomplete or after
491+
* a loss (then the rest of that element is skipped).
492+
*/
493+
static final class LatmAssembler {
494+
private final java.io.ByteArrayOutputStream pending = new java.io.ByteArrayOutputStream();
495+
private int lastSeq = -1;
496+
private boolean broken;
497+
498+
void reset() {
499+
pending.reset();
500+
lastSeq = -1;
501+
broken = false;
502+
}
503+
504+
byte[] push(byte[] data, int start, int end, boolean marker, int seq) {
505+
if (lastSeq >= 0 && seq != ((lastSeq + 1) & 0xFFFF) && pending.size() > 0) {
506+
broken = true; // a piece of the current element is missing
507+
}
508+
lastSeq = seq;
509+
if (!broken) pending.write(data, start, end - start);
510+
if (!marker) return null;
511+
byte[] element = broken ? null : pending.toByteArray();
512+
pending.reset();
513+
broken = false;
514+
return element;
515+
}
516+
}
517+
473518
/**
474519
* RFC 6416 MP4A-LATM with out-of-band config: each AudioMuxElement is a
475520
* PayloadLengthInfo (bytes of 255 plus a final byte, summed) and the AU.

‎android/app/src/test/java/com/openipc/decoder/AudioFormatsTest.java‎

Lines changed: 69 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
import static org.junit.Assert.assertArrayEquals;
1111
import static org.junit.Assert.assertEquals;
12+
import static org.junit.Assert.assertNotNull;
1213
import static org.junit.Assert.assertNull;
1314

1415
import org.junit.Test;
@@ -28,8 +29,16 @@ private static byte[] bits(String s) {
2829

2930
// StreamMuxConfig: version 0, sameTimeFraming 1, numSubFrames 0, numProgram 0,
3031
// numLayer 0, then ASC: AAC LC (2), 16 kHz (index 8), mono (1), GASpecific 000
32+
// ... GASpecificConfig 000, frameLengthType 000, latmBufferFullness 0xFF,
33+
// otherDataPresent 0, crcCheckPresent 0
3134
private static final String SMC_16K_MONO =
32-
"0 1 000000 0000 000 00010 1000 0001 000 000 00000000";
35+
"0 1 000000 0000 000 00010 1000 0001 000 000 11111111 0 0";
36+
37+
/** SMC_16K_MONO with the bits from {@code from} replaced by {@code bits}. */
38+
private static String smcWith(int from, String bits) {
39+
String b = SMC_16K_MONO.replaceAll("[^01]", "");
40+
return b.substring(0, from) + bits + b.substring(from + bits.length());
41+
}
3342

3443
private static AudioConfig parse(String sdp) {
3544
return Sdp.parse(sdp, "rtsp://x").audio;
@@ -124,9 +133,9 @@ private static byte[] auPacket(int auSize, int pieceLen, int fill) {
124133
public void fragmentedAuIsReassembled() {
125134
AudioManager.AuAssembler a = new AudioManager.AuAssembler();
126135
AudioConfig c = aacHbr();
127-
assertEquals(0, a.push(auPacket(10, 4, 1), 0, 8, c, false).size());
128-
assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, false).size());
129-
List<byte[]> out = a.push(auPacket(10, 2, 3), 0, 6, c, true);
136+
assertEquals(0, a.push(auPacket(10, 4, 1), 0, 8, c, false, 1).size());
137+
assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, false, 2).size());
138+
List<byte[]> out = a.push(auPacket(10, 2, 3), 0, 6, c, true, 3);
130139
assertEquals(1, out.size());
131140
assertArrayEquals(new byte[] {1, 1, 1, 1, 2, 2, 2, 2, 3, 3}, out.get(0));
132141
}
@@ -135,11 +144,11 @@ public void fragmentedAuIsReassembled() {
135144
public void incompleteFragmentIsDropped() {
136145
AudioManager.AuAssembler a = new AudioManager.AuAssembler();
137146
AudioConfig c = aacHbr();
138-
a.push(auPacket(10, 4, 1), 0, 8, c, false);
147+
a.push(auPacket(10, 4, 1), 0, 8, c, false, 4);
139148
// Marker on the next piece, but 2 bytes are still missing
140-
assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, true).size());
149+
assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, true, 5).size());
141150
// A normal, complete AU afterwards is delivered
142-
List<byte[]> out = a.push(auPacket(3, 3, 7), 0, 7, c, true);
151+
List<byte[]> out = a.push(auPacket(3, 3, 7), 0, 7, c, true, 6);
143152
assertEquals(1, out.size());
144153
assertArrayEquals(new byte[] {7, 7, 7}, out.get(0));
145154
}
@@ -149,8 +158,60 @@ public void wholeAusPassThrough() {
149158
AudioManager.AuAssembler a = new AudioManager.AuAssembler();
150159
// Two AUs (3 and 2 bytes) in one packet
151160
byte[] p = {0x00, 0x20, 0x00, 0x18, 0x00, 0x10, 1, 1, 1, 2, 2};
152-
List<byte[]> out = a.push(p, 0, p.length, aacHbr(), true);
161+
List<byte[]> out = a.push(p, 0, p.length, aacHbr(), true, 7);
153162
assertEquals(2, out.size());
154163
assertArrayEquals(new byte[] {2, 2}, out.get(1));
155164
}
165+
166+
@Test
167+
public void lossBetweenFragmentsDropsTheStaleAu() {
168+
AudioManager.AuAssembler a = new AudioManager.AuAssembler();
169+
AudioConfig c = aacHbr();
170+
a.push(auPacket(10, 4, 1), 0, 8, c, false, 100);
171+
// Packet 101 lost; 102 is the start of a new AU of the same size
172+
assertEquals(0, a.push(auPacket(10, 4, 2), 0, 8, c, false, 102).size());
173+
assertEquals(0, a.push(auPacket(10, 4, 3), 0, 8, c, false, 103).size());
174+
List<byte[]> out = a.push(auPacket(10, 2, 4), 0, 6, c, true, 104);
175+
assertEquals(1, out.size());
176+
assertArrayEquals("only bytes of the new AU",
177+
new byte[] {2, 2, 2, 2, 3, 3, 3, 3, 4, 4}, out.get(0));
178+
}
179+
180+
@Test
181+
public void latmElementAcrossPackets() {
182+
AudioManager.LatmAssembler a = new AudioManager.LatmAssembler();
183+
byte[] part1 = {5, 1, 1}; // length 5, first 2 bytes
184+
byte[] part2 = {1, 1, 1}; // remaining 3 bytes, marker set
185+
assertNull(a.push(part1, 0, 3, false, 10));
186+
byte[] element = a.push(part2, 0, 3, true, 11);
187+
assertArrayEquals(new byte[] {5, 1, 1, 1, 1, 1}, element);
188+
List<int[]> u = AudioManager.splitLatm(element, 0, element.length);
189+
assertArrayEquals(new int[] {1, 5}, u.get(0));
190+
}
191+
192+
@Test
193+
public void latmElementWithALostPieceIsDropped() {
194+
AudioManager.LatmAssembler a = new AudioManager.LatmAssembler();
195+
assertNull(a.push(new byte[] {5, 1}, 0, 2, false, 10));
196+
assertNull("piece 11 lost", a.push(new byte[] {1, 1}, 0, 2, true, 12));
197+
// The next element is fine
198+
assertArrayEquals(new byte[] {1, 9}, a.push(new byte[] {1, 9}, 0, 2, true, 13));
199+
}
200+
201+
@Test
202+
public void latmRejectsLayoutsTheRtpPathCannotHandle() {
203+
// bit offsets in SMC_16K_MONO: 1 sameTimeFraming, 2..7 numSubFrames,
204+
// 15..19 objectType, 20..23 freqIndex, 24..27 channels, 31..33 frameLengthType,
205+
// 42 otherDataPresent, 43 crcCheckPresent
206+
assertNotNull(AudioConfig.latmToAsc(bits(SMC_16K_MONO)));
207+
assertNull("not same time framing", AudioConfig.latmToAsc(bits(smcWith(1, "0"))));
208+
assertNull("two subframes", AudioConfig.latmToAsc(bits(smcWith(2, "000001"))));
209+
assertNull("AAC Main", AudioConfig.latmToAsc(bits(smcWith(15, "00001"))));
210+
assertNull("reserved rate", AudioConfig.latmToAsc(bits(smcWith(20, "1101"))));
211+
assertNull("5.1", AudioConfig.latmToAsc(bits(smcWith(24, "0110"))));
212+
assertNull("frameLengthType 1", AudioConfig.latmToAsc(bits(smcWith(31, "001"))));
213+
assertNull("other data", AudioConfig.latmToAsc(bits(smcWith(42, "1"))));
214+
assertNull("CRC", AudioConfig.latmToAsc(bits(smcWith(43, "1"))));
215+
assertNull("multichannel ASC info", AudioConfig.ascInfo(new byte[] {0x14, 0x30}));
216+
}
156217
}

0 commit comments

Comments
 (0)