From 7298207ca3aae859aa71ff4c067b320d8308c203 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Mon, 12 Oct 2026 12:58:05 +0000 Subject: [PATCH 1/3] feat: audio in Quad mode for the selected cell Every Quad cell now sets up its audio track, but only the selected cell passes its audio to the AudioManager, so switching needs no reconnect (fixes #12). - Long press on a cell selects it; a long press on the selected cell mutes. The cell shows an AUDIO badge, and the choice is saved ("quad_audio"). - A dedicated playback thread runs while Quad is shown; it is stopped with Quad and on pause. - A reconnect of the audible cell reconfigures the AudioManager with the new audio parameters. - README (EN/RU/ZH): Quad audio documented, #12 removed from the limitations. Tests: audio of a cell reaches the sink only while selected, never video packets; the audio configuration is reported on connect. --- README.md | 3 +- README.ru.md | 3 +- README.zh-CN.md | 3 +- .../java/com/openipc/decoder/Decoder.java | 117 +++++++++++++++--- .../java/com/openipc/decoder/QuadCell.java | 33 ++++- android/app/src/main/res/values/strings.xml | 1 + .../com/openipc/decoder/QuadAudioTest.java | 76 ++++++++++++ 7 files changed, 213 insertions(+), 23 deletions(-) create mode 100644 android/app/src/test/java/com/openipc/decoder/QuadAudioTest.java diff --git a/README.md b/README.md index df41e509..7ff94bcb 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ The app accepts plain-text shares (`ACTION_SEND`, `text/plain`). Sharing an `rts ### Quad mode -Shows all configured slots in a 2×2 grid. Each cell has its own connection and decoder. Audio is off in this mode. Tap any cell to open the menu. +Shows all configured slots in a 2×2 grid. Each cell has its own connection and decoder. Long-press a cell to play its audio (long-press it again to mute); the selected cell shows an AUDIO badge and is remembered. Tap any cell to open the menu. ## Known limitations @@ -92,7 +92,6 @@ Shows all configured slots in a 2×2 grid. Each cell has its own connection and > These are known gaps that we plan to address. Each one has an issue — comments, test results and pull requests are welcome. > > - **RTSP over TCP only.** No UDP or multicast — [#7](https://github.com/OpenIPC/decoder/issues/7) -> - **No audio in Quad mode** — [#12](https://github.com/OpenIPC/decoder/issues/12) > - **Only 4 camera slots** — [#13](https://github.com/OpenIPC/decoder/issues/13) > - **Audio formats not supported:** MP4A-LATM, Opus, G.726, and AAC frames split across RTP packets — [#14](https://github.com/OpenIPC/decoder/issues/14) > - **Possible dependency on Google services (GAPPS)** on some devices; under investigation, so disable GAPPS with care — [#15](https://github.com/OpenIPC/decoder/issues/15) diff --git a/README.ru.md b/README.ru.md index 50208585..0fda88f6 100644 --- a/README.ru.md +++ b/README.ru.md @@ -84,7 +84,7 @@ ### Режим Quad -Показывает все настроенные слоты сеткой 2×2. У каждой ячейки своё подключение и свой декодер. Звук в этом режиме выключен. Коснитесь любой ячейки, чтобы открыть меню. +Показывает все настроенные слоты сеткой 2×2. У каждой ячейки своё подключение и свой декодер. Долгое нажатие на ячейку включает её звук (повторное — выключает); у выбранной ячейки виден значок AUDIO, выбор запоминается. Коснитесь любой ячейки, чтобы открыть меню. ## Известные ограничения @@ -92,7 +92,6 @@ > Это известные пробелы, которые мы планируем закрыть. На каждый заведён issue — комментарии, результаты проверок и pull request приветствуются. > > - **Только RTSP поверх TCP.** UDP и multicast не поддерживаются — [#7](https://github.com/OpenIPC/decoder/issues/7) -> - **В режиме Quad нет звука** — [#12](https://github.com/OpenIPC/decoder/issues/12) > - **Только 4 слота для камер** — [#13](https://github.com/OpenIPC/decoder/issues/13) > - **Не поддерживаются форматы звука** MP4A-LATM, Opus, G.726, а также AAC-кадры, разбитые на несколько RTP-пакетов — [#14](https://github.com/OpenIPC/decoder/issues/14) > - **Возможная зависимость от сервисов Google (GAPPS)** на некоторых устройствах; изучается, поэтому отключайте GAPPS осознанно — [#15](https://github.com/OpenIPC/decoder/issues/15) diff --git a/README.zh-CN.md b/README.zh-CN.md index e1f6bfbd..2e79fcaa 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -84,7 +84,7 @@ ### Quad 四分屏模式 -以 2×2 网格显示所有已配置的槽位。每个画面都有独立的连接和解码器。此模式下没有声音。点击任意画面即可打开菜单。 +以 2×2 网格显示所有已配置的槽位。每个画面都有独立的连接和解码器。长按某个画面即可播放其声音（再次长按则静音）；被选中的画面显示 AUDIO 标记，选择会被记住。点击任意画面即可打开菜单。 ## 已知限制 @@ -92,7 +92,6 @@ > 以下是我们计划解决的已知不足，每一项都有对应的 issue。欢迎留言、提交测试结果和 pull request。 > > - **仅支持 RTSP over TCP**，不支持 UDP 和组播 — [#7](https://github.com/OpenIPC/decoder/issues/7) -> - **Quad 模式下没有声音** — [#12](https://github.com/OpenIPC/decoder/issues/12) > - **仅有 4 个摄像机槽位** — [#13](https://github.com/OpenIPC/decoder/issues/13) > - **不支持的音频格式：** MP4A-LATM、Opus、G.726，以及跨多个 RTP 包分片的 AAC 帧 — [#14](https://github.com/OpenIPC/decoder/issues/14) > - **部分设备上可能依赖 Google 服务（GAPPS）**，仍在调查中，请谨慎禁用 GAPPS — [#15](https://github.com/OpenIPC/decoder/issues/15) diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index 06ffc3c7..759a940a 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -83,6 +83,11 @@ public class Decoder extends Activity private QuadCell[] quadCells; private LinearLayout quadContainer; private final TextureView[] quadViews = new TextureView[4]; + // Quad audio (#12): the cell whose sound plays, -1 for none + private int quadAudio = -1; + private final TextView[] quadAudioBadges = new TextView[4]; + private volatile int quadAudioGen; + private ExecutorService quadAudioExecutor; // Camera settings private static final int CAM_COUNT = 4; @@ -236,6 +241,24 @@ public void onCodecChanged(boolean h265, AudioConfig audio) { cell.addView(quadViews[idx], new FrameLayout.LayoutParams( FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT)); + // Long press selects the cell whose audio plays (#12) + quadViews[idx].setOnLongClickListener(v -> { + toggleQuadAudio(idx); + return true; + }); + TextView badge = new TextView(this); + badge.setText(R.string.quad_audio_badge); + badge.setTextColor(Color.WHITE); + badge.setTextSize(11); + badge.setBackgroundColor(0x99000000); + badge.setPadding(dp(6), dp(2), dp(6), dp(2)); + badge.setVisibility(View.GONE); + FrameLayout.LayoutParams bp = new FrameLayout.LayoutParams( + FrameLayout.LayoutParams.WRAP_CONTENT, + FrameLayout.LayoutParams.WRAP_CONTENT, Gravity.TOP | Gravity.END); + bp.setMargins(dp(4), dp(4), dp(4), dp(4)); + cell.addView(badge, bp); + quadAudioBadges[idx] = badge; lr.addView(cell, new LinearLayout.LayoutParams( 0, LinearLayout.LayoutParams.MATCH_PARENT, 1)); } @@ -487,13 +510,7 @@ private void startQuad() { ensureQuadLaidOut(); removeClearOverlay(); - quadCells = new QuadCell[4]; - for (int i = 0; i < 4; i++) { - String url = mHosts[i]; - if (url == null || url.isEmpty() || url.equals(DEFAULT_URL)) continue; - quadCells[i] = new QuadCell(i, url, quadViews[i], framePool, mUserAgent, index -> runOnUiThread(() -> removeQuadOverlay(index))); - quadCells[i].start(); - } + createQuadCells(); quadEnabled = true; getSharedPreferences("settings", MODE_PRIVATE).edit() @@ -501,6 +518,7 @@ private void startQuad() { } private void stopQuad() { + stopQuadAudio(); quadEnabled = false; getSharedPreferences("settings", MODE_PRIVATE).edit() .putBoolean("quad_enabled", false).apply(); @@ -536,6 +554,82 @@ private void ensureQuadLaidOut() { }, 500); } + private void createQuadCells() { + quadCells = new QuadCell[4]; + for (int i = 0; i < 4; i++) { + String url = mHosts[i]; + if (url == null || url.isEmpty() || url.equals(DEFAULT_URL)) continue; + quadCells[i] = new QuadCell(i, url, quadViews[i], framePool, mUserAgent, + new QuadCell.Listener() { + @Override public void onRemoveOverlay(int index) { + runOnUiThread(() -> removeQuadOverlay(index)); + } + @Override public void onAudioConfig(int index, AudioConfig audio) { + // A (re)connect of the audible cell brings its audio parameters + if (index == quadAudio) audioManager.configure(audio); + } + }); + quadCells[i].start(); + } + quadAudio = getSharedPreferences("settings", MODE_PRIVATE).getInt("quad_audio", -1); + startQuadAudio(); + } + + // ====================== Quad audio (#12) ====================== + + private void toggleQuadAudio(int index) { + quadAudio = quadAudio == index ? -1 : index; + getSharedPreferences("settings", MODE_PRIVATE).edit() + .putInt("quad_audio", quadAudio).apply(); + applyQuadAudio(); + Toast.makeText(this, quadAudio < 0 ? "Audio off" + : "Audio: camera " + (quadAudio + 1), Toast.LENGTH_SHORT).show(); + } + + /** Route audio of the selected cell to the AudioManager and update the badges. */ + private void applyQuadAudio() { + audioManager.closeAudio(); + QuadCell[] cells = quadCells; + for (int i = 0; i < 4; i++) { + QuadCell cell = cells != null ? cells[i] : null; + boolean on = i == quadAudio && cell != null; + if (cell != null) cell.setAudioSink(on ? audioManager : null); + if (on) audioManager.configure(cell.getAudioConfig()); + if (quadAudioBadges[i] != null) { + quadAudioBadges[i].setVisibility(on ? View.VISIBLE : View.GONE); + } + } + } + + /** The playback thread for Quad audio (single view has its own in startListener). */ + private void startQuadAudio() { + stopQuadAudio(); + applyQuadAudio(); + final int gen = ++quadAudioGen; + quadAudioExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "quad-audio")); + quadAudioExecutor.execute(() -> { + while (gen == quadAudioGen) audioManager.playPcmFrame(); + }); + } + + private void stopQuadAudio() { + quadAudioGen++; + if (quadAudioExecutor != null) { + quadAudioExecutor.shutdownNow(); + quadAudioExecutor = null; + } + QuadCell[] cells = quadCells; + if (cells != null) { + for (QuadCell cell : cells) { + if (cell != null) cell.setAudioSink(null); + } + } + audioManager.closeAudio(); + for (TextView badge : quadAudioBadges) { + if (badge != null) badge.setVisibility(View.GONE); + } + } + /** Clear the video view so no stale frame lingers. */ private void clearVideo() { if (mSurface == null) return; @@ -764,6 +858,7 @@ private void stopStreams() { quadCells = null; } + stopQuadAudio(); listenerGen++; if (rtspClient != null) { rtspClient.stop(); @@ -830,13 +925,7 @@ private void startStreams() { quadCells = null; } clearQuadViews(); - quadCells = new QuadCell[4]; - for (int i = 0; i < 4; i++) { - String url = mHosts[i]; - if (url == null || url.isEmpty() || url.equals(DEFAULT_URL)) continue; - quadCells[i] = new QuadCell(i, url, quadViews[i], framePool, mUserAgent, index -> runOnUiThread(() -> removeQuadOverlay(index))); - quadCells[i].start(); - } + createQuadCells(); getSharedPreferences("settings", MODE_PRIVATE).edit() .putBoolean("quad_enabled", true).apply(); } else { diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index 0253337b..b65a9090 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -48,8 +48,20 @@ class QuadCell implements RtspSession.Callback { interface Listener { void onRemoveOverlay(int index); + + /** Audio parameters of this cell's stream (network thread). */ + default void onAudioConfig(int index, AudioConfig audio) {} } + // Audio (#12): every cell sets up its audio track, only the selected one plays it + private volatile AudioManager audioSink; + private volatile AudioConfig audioConfig = new AudioConfig(); + + /** Route this cell's audio to {@code sink}, or mute it with null. */ + void setAudioSink(AudioManager sink) { this.audioSink = sink; } + + AudioConfig getAudioConfig() { return audioConfig; } + private final Listener listener; QuadCell(int index, String host, TextureView view, FramePool pool, String userAgent, Listener listener) { @@ -160,8 +172,8 @@ private void startThreads() { }); Log.i(TAG, tag + ": surface ready, starting session"); - // Video only: Quad mode has no audio (#12) - session.configure(host, userAgent, false); + // Audio is set up too, so switching the audible cell needs no reconnect (#12) + session.configure(host, userAgent, true); session.start(); } @@ -169,6 +181,7 @@ void stop() { started = false; threadsStarted = false; running = false; + audioSink = null; session.stop(); nalAssembler.reset(); // Disabling releases the decoder and stops the decode thread from creating another, @@ -193,11 +206,25 @@ public void onSdp(Sdp sdp) { lastUnknownPayload = -1; videoPt = sdp.videoPt; codecH265 = sdp.videoH265; + audioConfig = sdp.audio; + if (listener != null) listener.onAudioConfig(index, sdp.audio); } @Override public boolean onPacket(int channel, byte[] buf, int offset, int length) { - if (channel != RtspSession.CHANNEL_VIDEO || length < 12) return false; + if (length < 12) return false; + if (channel == RtspSession.CHANNEL_AUDIO) { + AudioManager sink = audioSink; + if (sink != null) { + Frame frame = cellFramePool.obtain(length); + System.arraycopy(buf, offset, frame.data(), 0, length); + frame.setLength(length); + sink.onAudioFrame(frame); + cellFramePool.recycle(frame); + } + return false; + } + if (channel != RtspSession.CHANNEL_VIDEO) return false; int pt = buf[offset + 1] & 0x7F; if (pt != videoPt) { if (pt != lastUnknownPayload) { diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index edfc7079..74dac101 100755 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -5,4 +5,5 @@ Disconnected Buffering… No camera configured + AUDIO diff --git a/android/app/src/test/java/com/openipc/decoder/QuadAudioTest.java b/android/app/src/test/java/com/openipc/decoder/QuadAudioTest.java new file mode 100644 index 00000000..32185ee6 --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/QuadAudioTest.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * QuadAudioTest.java — audio routing of a Quad cell (#12) + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class QuadAudioTest { + /** Counts the audio frames it is given. */ + private static final class CountingAudio extends AudioManager { + final AtomicInteger frames = new AtomicInteger(); + + @Override void onAudioFrame(Frame frame) { frames.incrementAndGet(); } + } + + private static byte[] rtp(int pt) { + byte[] p = new byte[20]; + p[0] = (byte) 0x80; + p[1] = (byte) pt; + return p; + } + + @Test + public void onlyTheSelectedCellPlaysAudio() { + QuadCell cell = new QuadCell(1, "rtsp://cam/s", null, new FramePool(4), "", index -> {}); + CountingAudio audio = new CountingAudio(); + byte[] a = rtp(98); + + cell.onPacket(RtspSession.CHANNEL_AUDIO, a, 0, a.length); + assertEquals("muted by default", 0, audio.frames.get()); + + cell.setAudioSink(audio); + cell.onPacket(RtspSession.CHANNEL_AUDIO, a, 0, a.length); + cell.onPacket(RtspSession.CHANNEL_AUDIO, a, 0, a.length); + assertEquals(2, audio.frames.get()); + + // Video packets never reach the audio sink + byte[] v = rtp(96); + cell.onPacket(RtspSession.CHANNEL_VIDEO, v, 0, v.length); + assertEquals(2, audio.frames.get()); + + cell.setAudioSink(null); + cell.onPacket(RtspSession.CHANNEL_AUDIO, a, 0, a.length); + assertEquals(2, audio.frames.get()); + } + + @Test + public void audioConfigIsReportedOnConnect() { + AtomicReference reported = new AtomicReference<>(); + AtomicInteger reportedIndex = new AtomicInteger(-1); + QuadCell cell = new QuadCell(2, "rtsp://cam/s", null, new FramePool(4), "", + new QuadCell.Listener() { + @Override public void onRemoveOverlay(int index) {} + @Override public void onAudioConfig(int index, AudioConfig audio) { + reportedIndex.set(index); + reported.set(audio); + } + }); + Sdp sdp = Sdp.parse("m=video 0 RTP/AVP 96\r\nm=audio 0 RTP/AVP 0\r\n", "rtsp://cam/s"); + cell.onSdp(sdp); + assertEquals(2, reportedIndex.get()); + assertSame(sdp.audio, reported.get()); + assertSame(sdp.audio, cell.getAudioConfig()); + assertEquals(AudioConfig.PCMU, cell.getAudioConfig().codec); + } +} From eac5cc250f7d87c419d1a310189f0e39a3774ffd Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Mon, 12 Oct 2026 13:04:14 +0000 Subject: [PATCH 2/3] fix: audio routes, so a stopped source cannot leak into the next one Review findings on Quad audio: an in-flight SDP callback of a stopped cell could reconfigure the audio, a track created during shutdown could stay open, and a late Quad packet could be played in single view. AudioManager now has routes: newRoute() bumps the route and clears the queues under one lock and closes audio. Every producer carries its route: onAudioFrame(frame, route), configure(config, route), the AAC decode loop (route of its codec), and track creation (route at start). Anything from an older route is dropped. - Single view: startListener() gives RtspClient a new route, and RtspClient configures the AudioManager itself on that route. - Quad: applyQuadAudio() starts a route for the selected cell; the cell configures audio on its own route; stopQuadAudio() invalidates it. Tests: stale config and frames are ignored, a new route clears the queue; a cell's config reaches the sink with the cell's route. --- .../com/openipc/decoder/AudioManager.java | 92 ++++++++++++++----- .../java/com/openipc/decoder/Decoder.java | 26 +++--- .../java/com/openipc/decoder/QuadCell.java | 17 ++-- .../java/com/openipc/decoder/RtspClient.java | 6 +- .../com/openipc/decoder/QuadAudioTest.java | 59 ++++++++---- 5 files changed, 138 insertions(+), 62 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/AudioManager.java b/android/app/src/main/java/com/openipc/decoder/AudioManager.java index 7ae496f9..4a0739ad 100644 --- a/android/app/src/main/java/com/openipc/decoder/AudioManager.java +++ b/android/app/src/main/java/com/openipc/decoder/AudioManager.java @@ -88,8 +88,43 @@ void shutdown() { } } + // Audio routes: every source (single view, the audible Quad cell) gets a route number. + // Frames, configs and track creation of an older route are ignored, so a stopped source + // cannot leak into the next one. + private final Object routeLock = new Object(); + private int route; + + /** Start a new route: closes current audio and invalidates all older producers. */ + int newRoute() { + int r; + synchronized (routeLock) { + r = ++route; + pcmQueue.clear(); + aacQueue.clear(); + } + closeAudio(); + return r; + } + + /** Number of decoded frames waiting for playback (for tests and diagnostics). */ + int queuedPcmFrames() { return pcmQueue.size(); } + + private boolean isRoute(int r) { + synchronized (routeLock) { + return r == route; + } + } + + /** Audio parameters of route {@code r}; ignored if that route is no longer current. */ + void configure(AudioConfig audio, int r) { + synchronized (routeLock) { + if (r != route) return; + configure(audio); + } + } + /** Called on every RTSP (re)connect, before PLAY. */ - void configure(AudioConfig audio) { + private void configure(AudioConfig audio) { config = audio; // Parameters may differ after a reconnect: the AAC decoder is recreated on the next packet releaseAacDecoder(); @@ -100,8 +135,9 @@ void configure(AudioConfig audio) { } } - /** Called from the network thread to deliver an audio RTP packet. */ - void onAudioFrame(Frame frame) { + /** Called from a network thread to deliver an audio RTP packet of route {@code r}. */ + void onAudioFrame(Frame frame, int r) { + if (!isRoute(r)) return; byte[] data = frame.data(); int start = RtspSession.rtpHeaderLength(data, frame.length()); if (start < 0) return; @@ -111,16 +147,16 @@ void onAudioFrame(Frame frame) { AudioConfig c = config; switch (c.codec) { case AudioConfig.AAC: - processAac(c, data, start, end); + processAac(c, data, start, end, r); break; case AudioConfig.PCMU: - offerPcm(decodeG711(data, start, end, ULAW_TABLE)); + offerPcm(decodeG711(data, start, end, ULAW_TABLE), r); break; case AudioConfig.PCMA: - offerPcm(decodeG711(data, start, end, ALAW_TABLE)); + offerPcm(decodeG711(data, start, end, ALAW_TABLE), r); break; case AudioConfig.PCM: - offerPcm(decodeL16(data, start, end, c.bigEndian)); + offerPcm(decodeL16(data, start, end, c.bigEndian), r); break; default: if (!c.codec.equals(lastUnsupported)) { @@ -165,10 +201,14 @@ void playPcmFrame() { // ====================== PCM ====================== - private void offerPcm(byte[] pcm) { + private void offerPcm(byte[] pcm, int r) { if (pcm.length == 0) return; - if (!pcmQueue.offer(new Frame(pcm, pcm.length))) { - Log.w(TAG, "Audio queue full, frame dropped"); + synchronized (routeLock) { + // Checked under the lock that newRoute() clears the queue with: no leftovers + if (r != route) return; + if (!pcmQueue.offer(new Frame(pcm, pcm.length))) { + Log.w(TAG, "Audio queue full, frame dropped"); + } } } @@ -242,6 +282,10 @@ private void playAudio(Frame data) { private AudioTrack createPcmAudio(int sample, int channels) { if (!enabled) return null; + final int trackRoute; + synchronized (routeLock) { + trackRoute = route; + } int mask = channels == 2 ? AudioFormat.CHANNEL_OUT_STEREO : AudioFormat.CHANNEL_OUT_MONO; Log.d(TAG, "Create PCM audio (" + sample + "hz, " + channels + "ch)"); @@ -283,8 +327,8 @@ private AudioTrack createPcmAudio(int sample, int channels) { AudioTrack old; synchronized (trackLock) { - // Paused while the track was being created: do not publish it - if (!enabled) { + // Paused or switched to another source while the track was being created + if (!enabled || !isRoute(trackRoute)) { try { track.stop(); } catch (Exception ignored) {} track.release(); return null; @@ -313,7 +357,7 @@ private void releaseAacDecoder() { } } - private void processAac(AudioConfig c, byte[] data, int start, int end) { + private void processAac(AudioConfig c, byte[] data, int start, int end, int r) { List units; if (c.sizeLength > 0) { ensureAacDecoder(c, false); @@ -328,7 +372,7 @@ private void processAac(AudioConfig c, byte[] data, int start, int end) { units = Collections.singletonList(new int[] {start, end - start}); } for (int[] u : units) { - offerAac(Arrays.copyOfRange(data, u[0], u[0] + u[1])); + offerAac(Arrays.copyOfRange(data, u[0], u[0] + u[1]), r); } } @@ -388,9 +432,12 @@ static int readBits(byte[] data, int bitPos, int count) { return value; } - private void offerAac(byte[] au) { - if (!aacQueue.offer(au)) { - Log.w(TAG, "AAC audio queue full, frame dropped"); + private void offerAac(byte[] au, int r) { + synchronized (routeLock) { + if (r != route) return; + if (!aacQueue.offer(au)) { + Log.w(TAG, "AAC audio queue full, frame dropped"); + } } } @@ -450,11 +497,12 @@ private void ensureAacDecoder(AudioConfig c, boolean adts) { aacExecutor = Executors.newSingleThreadExecutor( r -> new Thread(r, "rtsp-aac-decode")); } - aacExecutor.execute(() -> aacLoop(codec)); + final int codecRoute = route; + aacExecutor.execute(() -> aacLoop(codec, codecRoute)); } } - private void aacLoop(MediaCodec codec) { + private void aacLoop(MediaCodec codec, int codecRoute) { MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); byte[] pending = null; while (aacRunning && !Thread.currentThread().isInterrupted()) { @@ -486,7 +534,7 @@ private void aacLoop(MediaCodec codec) { } } // Drain on idle iterations too, so the last decoded frame is not held back - drainAacOutput(codec, info); + drainAacOutput(codec, info, codecRoute); } catch (Exception e) { Log.e(TAG, "AAC decode error", e); pending = null; @@ -495,7 +543,7 @@ private void aacLoop(MediaCodec codec) { } } - private void drainAacOutput(MediaCodec codec, MediaCodec.BufferInfo info) { + private void drainAacOutput(MediaCodec codec, MediaCodec.BufferInfo info, int codecRoute) { int outId; while ((outId = codec.dequeueOutputBuffer(info, 0)) != MediaCodec.INFO_TRY_AGAIN_LATER) { if (outId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { @@ -510,7 +558,7 @@ private void drainAacOutput(MediaCodec codec, MediaCodec.BufferInfo info) { byte[] pcm = new byte[info.size]; outBuf.position(info.offset); outBuf.get(pcm); - offerPcm(pcm); + offerPcm(pcm, codecRoute); } codec.releaseOutputBuffer(outId, false); } diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index 759a940a..e222a70c 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -156,7 +156,7 @@ protected void onCreate(Bundle savedInstanceState) { public void onCodecChanged(boolean h265, AudioConfig audio) { codecH265 = h265; audioCodec = audio.codec; - audioManager.configure(audio); + // RtspClient configures the AudioManager itself, on its own audio route } @Override public void onJitterSample(int avg) { menuManager.updateQuality(avg); @@ -560,15 +560,7 @@ private void createQuadCells() { String url = mHosts[i]; if (url == null || url.isEmpty() || url.equals(DEFAULT_URL)) continue; quadCells[i] = new QuadCell(i, url, quadViews[i], framePool, mUserAgent, - new QuadCell.Listener() { - @Override public void onRemoveOverlay(int index) { - runOnUiThread(() -> removeQuadOverlay(index)); - } - @Override public void onAudioConfig(int index, AudioConfig audio) { - // A (re)connect of the audible cell brings its audio parameters - if (index == quadAudio) audioManager.configure(audio); - } - }); + index -> runOnUiThread(() -> removeQuadOverlay(index))); quadCells[i].start(); } quadAudio = getSharedPreferences("settings", MODE_PRIVATE).getInt("quad_audio", -1); @@ -588,13 +580,14 @@ private void toggleQuadAudio(int index) { /** Route audio of the selected cell to the AudioManager and update the badges. */ private void applyQuadAudio() { - audioManager.closeAudio(); + // A new route: audio of a previously selected cell can no longer reach playback + int route = audioManager.newRoute(); QuadCell[] cells = quadCells; for (int i = 0; i < 4; i++) { QuadCell cell = cells != null ? cells[i] : null; boolean on = i == quadAudio && cell != null; - if (cell != null) cell.setAudioSink(on ? audioManager : null); - if (on) audioManager.configure(cell.getAudioConfig()); + if (cell != null) cell.setAudioSink(on ? audioManager : null, route); + if (on) audioManager.configure(cell.getAudioConfig(), route); if (quadAudioBadges[i] != null) { quadAudioBadges[i].setVisibility(on ? View.VISIBLE : View.GONE); } @@ -621,10 +614,11 @@ private void stopQuadAudio() { QuadCell[] cells = quadCells; if (cells != null) { for (QuadCell cell : cells) { - if (cell != null) cell.setAudioSink(null); + if (cell != null) cell.setAudioSink(null, 0); } } - audioManager.closeAudio(); + // Invalidates in-flight Quad frames and track creation, clears the queues + audioManager.newRoute(); for (TextView badge : quadAudioBadges) { if (badge != null) badge.setVisibility(View.GONE); } @@ -806,6 +800,8 @@ private void startListener() { // generation for the new threads must be taken after it stopExecutor(); final int gen = listenerGen; + // The single-view stream gets its own audio route (nothing from Quad leaks in) + rtspClient.setAudioRoute(audioManager.newRoute()); executor = Executors.newFixedThreadPool(2); // Video decode thread diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index b65a9090..5dfe90c3 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -48,17 +48,18 @@ class QuadCell implements RtspSession.Callback { interface Listener { void onRemoveOverlay(int index); - - /** Audio parameters of this cell's stream (network thread). */ - default void onAudioConfig(int index, AudioConfig audio) {} } // Audio (#12): every cell sets up its audio track, only the selected one plays it private volatile AudioManager audioSink; + private volatile int audioRoute; private volatile AudioConfig audioConfig = new AudioConfig(); - /** Route this cell's audio to {@code sink}, or mute it with null. */ - void setAudioSink(AudioManager sink) { this.audioSink = sink; } + /** Route this cell's audio to {@code sink} on route {@code route}, or mute it with null. */ + void setAudioSink(AudioManager sink, int route) { + this.audioRoute = route; + this.audioSink = sink; + } AudioConfig getAudioConfig() { return audioConfig; } @@ -207,7 +208,9 @@ public void onSdp(Sdp sdp) { videoPt = sdp.videoPt; codecH265 = sdp.videoH265; audioConfig = sdp.audio; - if (listener != null) listener.onAudioConfig(index, sdp.audio); + // A stopped or deselected cell holds an old route: the AudioManager ignores it + AudioManager sink = audioSink; + if (sink != null) sink.configure(sdp.audio, audioRoute); } @Override @@ -219,7 +222,7 @@ public boolean onPacket(int channel, byte[] buf, int offset, int length) { Frame frame = cellFramePool.obtain(length); System.arraycopy(buf, offset, frame.data(), 0, length); frame.setLength(length); - sink.onAudioFrame(frame); + sink.onAudioFrame(frame, audioRoute); cellFramePool.recycle(frame); } return false; diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java index f4086bb1..2bc3b6d5 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java @@ -33,6 +33,8 @@ class RtspClient implements RtspSession.Callback { private int lastUnknownPayload = -1; private AudioManager audioManager; + /** Audio route of the current stream (see AudioManager.newRoute()). */ + private volatile int audioRoute; interface Listener { void onStatusChanged(String status); @@ -60,6 +62,7 @@ interface Listener { void setListener(Listener cb) { this.listenerCb = cb; } void setAudioManager(AudioManager am) { this.audioManager = am; } + void setAudioRoute(int route) { this.audioRoute = route; } void configure(String host, String userAgent) { session.configure(host, userAgent, true); @@ -98,6 +101,7 @@ public void onSdp(Sdp sdp) { codecH265 = sdp.videoH265; Log.i(TAG, "Video pt " + sdp.videoPt + (sdp.videoH265 ? " H.265" : " H.264") + ", audio " + sdp.audio.codec); + if (audioManager != null) audioManager.configure(sdp.audio, audioRoute); Listener l = listenerCb; if (l != null) l.onCodecChanged(codecH265, sdp.audio); } @@ -115,7 +119,7 @@ public boolean onPacket(int channel, byte[] buf, int offset, int length) { Frame frame = framePool.obtain(length); System.arraycopy(buf, offset, frame.data(), 0, length); frame.setLength(length); - audioManager.onAudioFrame(frame); + audioManager.onAudioFrame(frame, audioRoute); framePool.recycle(frame); } return false; diff --git a/android/app/src/test/java/com/openipc/decoder/QuadAudioTest.java b/android/app/src/test/java/com/openipc/decoder/QuadAudioTest.java index 32185ee6..8ed19e60 100644 --- a/android/app/src/test/java/com/openipc/decoder/QuadAudioTest.java +++ b/android/app/src/test/java/com/openipc/decoder/QuadAudioTest.java @@ -20,7 +20,15 @@ public class QuadAudioTest { private static final class CountingAudio extends AudioManager { final AtomicInteger frames = new AtomicInteger(); - @Override void onAudioFrame(Frame frame) { frames.incrementAndGet(); } + final AtomicReference configured = new AtomicReference<>(); + volatile int configuredRoute = -1; + + @Override void onAudioFrame(Frame frame, int route) { frames.incrementAndGet(); } + + @Override void configure(AudioConfig audio, int route) { + configured.set(audio); + configuredRoute = route; + } } private static byte[] rtp(int pt) { @@ -39,7 +47,7 @@ public void onlyTheSelectedCellPlaysAudio() { cell.onPacket(RtspSession.CHANNEL_AUDIO, a, 0, a.length); assertEquals("muted by default", 0, audio.frames.get()); - cell.setAudioSink(audio); + cell.setAudioSink(audio, 7); cell.onPacket(RtspSession.CHANNEL_AUDIO, a, 0, a.length); cell.onPacket(RtspSession.CHANNEL_AUDIO, a, 0, a.length); assertEquals(2, audio.frames.get()); @@ -49,28 +57,45 @@ public void onlyTheSelectedCellPlaysAudio() { cell.onPacket(RtspSession.CHANNEL_VIDEO, v, 0, v.length); assertEquals(2, audio.frames.get()); - cell.setAudioSink(null); + cell.setAudioSink(null, 0); cell.onPacket(RtspSession.CHANNEL_AUDIO, a, 0, a.length); assertEquals(2, audio.frames.get()); } @Test - public void audioConfigIsReportedOnConnect() { - AtomicReference reported = new AtomicReference<>(); - AtomicInteger reportedIndex = new AtomicInteger(-1); - QuadCell cell = new QuadCell(2, "rtsp://cam/s", null, new FramePool(4), "", - new QuadCell.Listener() { - @Override public void onRemoveOverlay(int index) {} - @Override public void onAudioConfig(int index, AudioConfig audio) { - reportedIndex.set(index); - reported.set(audio); - } - }); + public void audioConfigGoesToTheSinkWithTheCellsRoute() { + QuadCell cell = new QuadCell(2, "rtsp://cam/s", null, new FramePool(4), "", index -> {}); + CountingAudio audio = new CountingAudio(); Sdp sdp = Sdp.parse("m=video 0 RTP/AVP 96\r\nm=audio 0 RTP/AVP 0\r\n", "rtsp://cam/s"); - cell.onSdp(sdp); - assertEquals(2, reportedIndex.get()); - assertSame(sdp.audio, reported.get()); + + cell.onSdp(sdp); // not selected: nothing configured + assertSame(null, audio.configured.get()); assertSame(sdp.audio, cell.getAudioConfig()); + + cell.setAudioSink(audio, 5); + cell.onSdp(sdp); + assertSame(sdp.audio, audio.configured.get()); + assertEquals(5, audio.configuredRoute); assertEquals(AudioConfig.PCMU, cell.getAudioConfig().codec); } + + @Test + public void staleRoutesAreIgnored() { + AudioManager audio = new AudioManager(); + int first = audio.newRoute(); + int second = audio.newRoute(); + AudioConfig pcmu = Sdp.parse("m=audio 0 RTP/AVP 0\r\n", "rtsp://x").audio; + // A G.711 packet: 12-byte header + 4 samples + byte[] p = {(byte) 0x80, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4}; + audio.configure(pcmu, first); // from the old route: ignored + audio.onAudioFrame(new Frame(p, p.length), second); + assertEquals("not configured for G.711 by the stale route", 0, audio.queuedPcmFrames()); + audio.configure(pcmu, second); + audio.onAudioFrame(new Frame(p, p.length), first); + assertEquals("frame of the old route dropped", 0, audio.queuedPcmFrames()); + audio.onAudioFrame(new Frame(p, p.length), second); + assertEquals(1, audio.queuedPcmFrames()); + audio.newRoute(); + assertEquals("a new route clears the queue", 0, audio.queuedPcmFrames()); + } } From 6c7da5b10c3c727381873c305f66a2baaf24a2dd Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Mon, 12 Oct 2026 14:26:43 +0000 Subject: [PATCH 3/3] fix: Quad audio loop also stops when interrupted --- android/app/src/main/java/com/openipc/decoder/Decoder.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index e222a70c..0850793f 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -601,7 +601,9 @@ private void startQuadAudio() { final int gen = ++quadAudioGen; quadAudioExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "quad-audio")); quadAudioExecutor.execute(() -> { - while (gen == quadAudioGen) audioManager.playPcmFrame(); + while (gen == quadAudioGen && !Thread.currentThread().isInterrupted()) { + audioManager.playPcmFrame(); + } }); }