From 36755bdf80227a2c0b993e08e0ea07b56ad2bcb7 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Tue, 13 Oct 2026 15:45:59 +0000 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20U=20cell=20/=20udp://=20slots=20?= =?UTF-8?q?=E2=80=94=20a=20pushed=20stream=20that=20loses=20a=20piece=20of?= =?UTF-8?q?=20every=20key=20frame=20is=20decoded=20damaged=20instead=20of?= =?UTF-8?q?=20staying=20black=20(#133)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UDP stall check now covers pushed streams (no RTSP, so it never ran for them). After 8 s of packets without a decoded frame the cell passes NAL units with a lost piece on cut short at the gap; the decoder conceals the rest. Corner: "· damaged"; /stats.json: nal_damaged. Co-Authored-By: Claude --- .../java/com/openipc/decoder/Decoder.java | 3 ++ .../com/openipc/decoder/NalAssembler.java | 36 +++++++++++++++++-- .../java/com/openipc/decoder/QuadCell.java | 25 +++++++++++-- .../java/com/openipc/decoder/RtspSession.java | 3 ++ .../com/openipc/decoder/NalAssemblerTest.java | 30 ++++++++++++++++ 5 files changed, 91 insertions(+), 6 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index ac407c9a..8521dc7e 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -1407,6 +1407,9 @@ private Map debugState() { c.put("lost_fps", cell.lostFps()); // NAL units dropped because one of their packets was lost (#113) c.put("nal_dropped", cell.nalDropped()); + // Decoded damaged, cut short at a lost piece; absent while that is off (#133) + int salvaged = cell.nalSalvaged(); + if (salvaged >= 0) c.put("nal_damaged", salvaged); // Every frame, or key frames only while the decoder has no room (#85) c.put("decode", cell.isKeyFramesOnly() ? "key frames" : "full"); if (cell.knowsLoad()) c.put("load", cell.isKeyFramesOnly() diff --git a/android/app/src/main/java/com/openipc/decoder/NalAssembler.java b/android/app/src/main/java/com/openipc/decoder/NalAssembler.java index 7c960561..5806f7b5 100644 --- a/android/app/src/main/java/com/openipc/decoder/NalAssembler.java +++ b/android/app/src/main/java/com/openipc/decoder/NalAssembler.java @@ -44,6 +44,13 @@ interface Sink { private int dropped; /** Of those, dropped because a packet of them was lost (not for size) (#113). */ private int lostDropped; + /** + * Pass a NAL unit with a lost piece on cut short at the gap instead of dropping it (#133): + * for a stream that loses a piece of every key frame, where dropping them leaves nothing to + * decode. The decoder conceals the missing part; a damaged picture beats a black one. + */ + private volatile boolean salvage; + private int salvaged; NalAssembler(int bufferSize, Runnable onCodecSwitch, FramePool framePool) { this.nalBuffer = new byte[bufferSize]; @@ -63,6 +70,11 @@ void reset() { /** NAL units dropped because one of their packets was lost (#113). */ int lossCount() { return lostDropped; } + void setSalvage(boolean on) { salvage = on; } + boolean salvaging() { return salvage; } + /** NAL units passed on cut short at a lost piece (#133). */ + int salvagedCount() { return salvaged; } + /** * Process one RTP packet ({@code length} bytes at {@code offset}, header included). * Returns false if the RTP header is malformed. @@ -89,7 +101,7 @@ boolean assemble(byte[] buf, int offset, int length, boolean codecH265, Sink sin lastSeq = seq; if (gap && inFragment) { lostDropped++; - drop("packet loss inside a fragmented NAL unit"); + dropOrSalvage("packet loss inside a fragmented NAL unit", sink); orphan = true; // its remaining pieces belong to the unit just counted } @@ -104,7 +116,7 @@ boolean assemble(byte[] buf, int offset, int length, boolean codecH265, Sink sin // Any other packet ends a fragmented NAL unit that did not finish; its remaining // pieces, if any arrive, belong to the unit counted here if (inFragment) { - drop("fragmented NAL unit interrupted by another packet"); + dropOrSalvage("fragmented NAL unit interrupted by another packet", sink); orphan = true; } else { orphan = false; @@ -131,7 +143,7 @@ private void fragmentUnit(byte[] buf, int start, int end, boolean h265, Sink sin if (first) { if (inFragment) { lostDropped++; - drop("fragmented NAL unit without its end"); + dropOrSalvage("fragmented NAL unit without its end", sink); } orphan = false; nalSize = 0; @@ -201,6 +213,24 @@ private void emit(byte[] buf, int pos, int size, Sink sink) { sink.onNal(out); } + /** The unit collected so far, cut short, when salvaging (#133); dropped otherwise. */ + private void dropOrSalvage(String reason, Sink sink) { + // Start code and NAL header plus some slice data, or there is nothing to decode + if (!salvage || nalSize < 16) { + drop(reason); + return; + } + Frame out = framePool.obtain(nalSize); + System.arraycopy(nalBuffer, 0, out.data(), 0, nalSize); + out.setLength(nalSize); + nalSize = 0; + inFragment = false; + if (salvaged++ % 100 == 0) { + Log.w(TAG, "NAL unit passed on cut short (" + salvaged + "): " + reason); + } + sink.onNal(out); + } + private void drop(String reason) { nalSize = 0; inFragment = false; diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index 287ff8d0..5217c17a 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -302,9 +302,15 @@ String sample(long nowMs) { String rate = outAvg < 10 ? String.format(java.util.Locale.ROOT, "%.1f", outAvg) : String.valueOf(Math.round(outAvg)); return size + codec + " key frames · " + rate + " fps" - + (decoder.isSoftware() ? " · software" : ""); + + (decoder.isSoftware() ? " · software" : "") + damaged(); } - return size + codec + " " + Math.round(outFps) + "/" + Math.round(inFps) + " fps"; + return size + codec + " " + Math.round(outFps) + "/" + Math.round(inFps) + " fps" + + damaged(); + } + + /** Corner note while damaged key frames are decoded (#133). */ + private String damaged() { + return nalAssembler.salvaging() ? " · damaged" : ""; } boolean isKeyFramesOnly() { return keyFramesOnly; } @@ -373,7 +379,9 @@ private void checkUdpStall() { lastVideoPackets = packets; lastLossCount = loss; Surface s = surface; - boolean stalled = session.playsOverUdp() && "connected".equals(state) + // A pushed stream (udp:// slot, the U cell) counts too: no TCP to fall back to (#133) + boolean stalled = (session.playsOverUdp() || session.listens()) + && "connected".equals(state) && pausedReason == null && problem == null && s != null && s.isValid() && arriving && outFps == 0; if (!stalled) { @@ -385,6 +393,12 @@ private void checkUdpStall() { if (losing || udpStallSamples > 0) udpStallSamples++; if (udpStallSamples >= UDP_STALL_SAMPLES) { udpStallSamples = 0; + // Every key frame loses a piece: decode them damaged rather than not at all. A + // pushed stream has no TCP to go to; over RTSP, Auto goes to TCP anyway (#133) + if (!nalAssembler.salvaging()) { + nalAssembler.setSalvage(true); + Log.w(TAG, tag + ": no complete key frame over UDP, decoding damaged ones"); + } session.reportUdpBroken("packets but no decoded frame for " + UDP_STALL_SAMPLES + " s (" + loss + " NAL units lost)"); } @@ -393,6 +407,9 @@ private void checkUdpStall() { /** NAL units dropped so far because a packet of them was lost (#113). */ int nalDropped() { return nalAssembler.lossCount(); } + /** NAL units decoded cut short at a lost piece, or -1 while that is off (#133). */ + int nalSalvaged() { return nalAssembler.salvaging() ? nalAssembler.salvagedCount() : -1; } + float inFps() { return inFps; } float outFps() { return outFps; } float lostFps() { return lostFps; } @@ -441,6 +458,8 @@ public void onSdp(Sdp sdp) { } udpStallSamples = 0; nalAssembler.reset(); + // A new stream gets a chance to be complete again (#133) + nalAssembler.setSalvage(false); lastUnknownPayload = -1; videoPt = sdp.videoPt; codecH265 = sdp.videoH265; diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index 3377f89d..c35743a7 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -140,6 +140,9 @@ private void markUdpFailed(Transport.Mode mode) { /** RTSP is playing with RTP over UDP or multicast (not TCP, not a udp:// URL). */ boolean playsOverUdp() { return udpPlaying; } + /** A udp:// slot listening for a pushed stream (#133): UDP too, without RTSP. */ + boolean listens() { return listening; } + /** * Packets arrive over UDP but the stream cannot be decoded (#113: key frames keep losing * a piece). With Auto or Multicast the session goes over to TCP, as when no video comes diff --git a/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java b/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java index 7de17ef0..665f39a3 100644 --- a/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java +++ b/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java @@ -138,6 +138,36 @@ public void lostMiddleFragmentDropsTheNal() { assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xD1, 0xD2), out.get(0)); } + @Test + public void salvagePassesTheNalOnCutShortAtTheGap() { + nal.setSalvage(true); + // 16 slice bytes before the gap: enough to be worth decoding (#133) + feed(rtp(0x7C, 0x85, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12), false); + feed(rtp(0x7C, 0x05, 13, 14, 15, 16), false); + seq++; // one packet lost + feed(rtp(0x7C, 0x05, 0xB1), false); + assertEquals(1, out.size()); + assertArrayEquals(b(0, 0, 0, 1, 0x65, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16), out.get(0)); + assertEquals(1, nal.salvagedCount()); + assertEquals(1, nal.lossCount()); + // The rest of that unit is skipped; the next one is complete again + feed(rtp(0x7C, 0x45, 0xC1), false); + feed(rtp(0x7C, 0x85, 0xD1), false); + feed(rtp(0x7C, 0x45, 0xD2), false); + assertEquals(2, out.size()); + assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xD1, 0xD2), out.get(1)); + } + + @Test + public void salvageOffDropsAsBefore() { + feed(rtp(0x7C, 0x85, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), false); + seq++; + feed(rtp(0x7C, 0x45, 0xC1), false); + assertEquals(0, out.size()); + assertEquals(0, nal.salvagedCount()); + } + @Test public void continuationWithoutStartIsIgnoredAndCountedOnce() { feed(rtp(0x7C, 0x05, 0xB1), false); From 3161d14cf17f22a36aa2d3b137e6be1389c44085 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Tue, 13 Oct 2026 15:50:32 +0000 Subject: [PATCH 2/4] fix: the single view of a udp:// slot decodes a lossy pushed stream damaged too (#133) Co-Authored-By: Claude --- .../app/src/main/java/com/openipc/decoder/Decoder.java | 9 ++++++++- .../src/main/java/com/openipc/decoder/RtspClient.java | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index 8521dc7e..ad4ce6bd 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -1778,13 +1778,20 @@ public void run() { int frames = videoDecoder.framesOut() + jpegFramesOut; long bytes = client != null ? client.getStreamBytes() : 0; boolean stalled = !quadEnabled && !udpViewShown && client != null - && client.playsOverUdp() && "connected".equals(singleState) + && (client.playsOverUdp() || client.listens()) + && "connected".equals(singleState) && bytes - lastSingleBytes > 10_000 && frames == lastSingleFrames; lastSingleFrames = frames; lastSingleBytes = bytes; singleStallSeconds = stalled ? singleStallSeconds + 1 : 0; if (singleStallSeconds >= UDP_STALL_SECONDS) { singleStallSeconds = 0; + // Every key frame loses a piece: decode them damaged rather than not at all; + // a pushed stream has no TCP to go to (#133) + if (!nalAssembler.salvaging()) { + nalAssembler.setSalvage(true); + Log.w(TAG, "No complete key frame over UDP, decoding damaged ones"); + } client.reportUdpBroken("packets but no decoded frame for " + UDP_STALL_SECONDS + " s"); } diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java index a2fb1248..c71f86a6 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java @@ -75,6 +75,8 @@ void configure(String host, String userAgent, Transport.Mode transport) { boolean isActive() { return session.isActive(); } /** RTP over UDP or multicast at the moment (#128). */ boolean playsOverUdp() { return session.playsOverUdp(); } + /** A udp:// slot listening for a pushed stream (#133). */ + boolean listens() { return session.listens(); } /** Packets arrive but nothing decodes: with Auto the session goes over to TCP. */ void reportUdpBroken(String why) { session.reportUdpBroken(why); } /** Transport the stream really uses now, or null (#115). */ @@ -108,6 +110,8 @@ public void onStatus(String status) { @Override public void onSdp(Sdp sdp) { nalAssembler.reset(); + // A new stream gets a chance to be complete again (#133) + nalAssembler.setSalvage(false); lastUnknownPayload = -1; lastRtpTimestamp = -1; lastRtpArrivalNs = -1; From 0c64fcc06cbde9c303b06fb10008c42909fe31e0 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Tue, 13 Oct 2026 16:09:32 +0000 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20Qodo=20on=20#146=20=E2=80=94=20only?= =?UTF-8?q?=20slices=20are=20passed=20on=20cut=20short,=20damaged=20decodi?= =?UTF-8?q?ng=20ends=20when=20a=20pushed=20stream=20goes=20quiet,=20the=20?= =?UTF-8?q?counter=20says=20what=20it=20counts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude --- .../main/java/com/openipc/decoder/Decoder.java | 3 ++- .../java/com/openipc/decoder/NalAssembler.java | 14 +++++++++++--- .../main/java/com/openipc/decoder/QuadCell.java | 5 ++++- .../java/com/openipc/decoder/RtspClient.java | 2 ++ .../com/openipc/decoder/NalAssemblerTest.java | 16 ++++++++++++++++ 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index ad4ce6bd..fa1bd66e 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -1407,7 +1407,8 @@ private Map debugState() { c.put("lost_fps", cell.lostFps()); // NAL units dropped because one of their packets was lost (#113) c.put("nal_dropped", cell.nalDropped()); - // Decoded damaged, cut short at a lost piece; absent while that is off (#133) + // Slices passed on cut short at a lost piece (some may still be dropped by + // a full queue); absent while that is off (#133) int salvaged = cell.nalSalvaged(); if (salvaged >= 0) c.put("nal_damaged", salvaged); // Every frame, or key frames only while the decoder has no room (#85) diff --git a/android/app/src/main/java/com/openipc/decoder/NalAssembler.java b/android/app/src/main/java/com/openipc/decoder/NalAssembler.java index 5806f7b5..d5c478ab 100644 --- a/android/app/src/main/java/com/openipc/decoder/NalAssembler.java +++ b/android/app/src/main/java/com/openipc/decoder/NalAssembler.java @@ -72,7 +72,7 @@ void reset() { void setSalvage(boolean on) { salvage = on; } boolean salvaging() { return salvage; } - /** NAL units passed on cut short at a lost piece (#133). */ + /** Slices passed on to the sink cut short at a lost piece (#133); the queue may drop some. */ int salvagedCount() { return salvaged; } /** @@ -215,8 +215,9 @@ private void emit(byte[] buf, int pos, int size, Sink sink) { /** The unit collected so far, cut short, when salvaging (#133); dropped otherwise. */ private void dropOrSalvage(String reason, Sink sink) { - // Start code and NAL header plus some slice data, or there is nothing to decode - if (!salvage || nalSize < 16) { + // Start code and NAL header plus some slice data, or there is nothing to decode; only + // slices: a cut parameter set would be taken for a valid configuration (Qodo on #146) + if (!salvage || nalSize < 16 || !isSlice(nalBuffer[4], lastCodec == 1)) { drop(reason); return; } @@ -231,6 +232,13 @@ private void dropOrSalvage(String reason, Sink sink) { sink.onNal(out); } + /** A coded slice (VCL NAL unit): H.264 types 1–5, H.265 types 0–31. */ + static boolean isSlice(byte nalHeader, boolean h265) { + if (h265) return ((nalHeader >> 1) & 0x3F) <= 31; + int type = nalHeader & 0x1F; + return type >= 1 && type <= 5; + } + private void drop(String reason) { nalSize = 0; inFragment = false; diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index 5217c17a..fe004f98 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -232,6 +232,9 @@ void stop() { public void onStatus(String status) { // A new connection (or a lost one) starts the UDP stall count afresh (Qodo on #125) if (!status.equals(state)) udpStallSamples = 0; + // A pushed stream that went quiet may come back whole: no damaged decoding until it + // stalls again (Qodo on #146) + if ("disconnected".equals(status)) nalAssembler.setSalvage(false); state = status; // A connection that works again clears the last connection error if ("connected".equals(status) && problem != null && !problem.startsWith("Decoder")) { @@ -407,7 +410,7 @@ private void checkUdpStall() { /** NAL units dropped so far because a packet of them was lost (#113). */ int nalDropped() { return nalAssembler.lossCount(); } - /** NAL units decoded cut short at a lost piece, or -1 while that is off (#133). */ + /** Slices passed on cut short at a lost piece, or -1 while that is off (#133). */ int nalSalvaged() { return nalAssembler.salvaging() ? nalAssembler.salvagedCount() : -1; } float inFps() { return inFps; } diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java index c71f86a6..6b45af0d 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java @@ -103,6 +103,8 @@ void stop() { @Override public void onStatus(String status) { + // A pushed stream that went quiet may come back whole (Qodo on #146) + if ("disconnected".equals(status)) nalAssembler.setSalvage(false); Listener l = listenerCb; if (l != null) l.onStatusChanged(status); } diff --git a/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java b/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java index 665f39a3..c89fb985 100644 --- a/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java +++ b/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java @@ -159,6 +159,22 @@ public void salvagePassesTheNalOnCutShortAtTheGap() { assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xD1, 0xD2), out.get(1)); } + @Test + public void salvageNeverPassesACutParameterSet() { + nal.setSalvage(true); + // FU-A of an SPS (type 7), then a lost packet (Qodo on #146) + feed(rtp(0x7C, 0x87, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), false); + seq++; + feed(rtp(0x7C, 0x47, 0xC1), false); + assertEquals(0, out.size()); + assertEquals(0, nal.salvagedCount()); + assertTrue(NalAssembler.isSlice((byte) 0x65, false)); + assertFalse(NalAssembler.isSlice((byte) 0x68, false)); + // H.265: IDR_W_RADL (19) is a slice, VPS (32) is not + assertTrue(NalAssembler.isSlice((byte) (19 << 1), true)); + assertFalse(NalAssembler.isSlice((byte) (32 << 1), true)); + } + @Test public void salvageOffDropsAsBefore() { feed(rtp(0x7C, 0x85, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), false); From c9d36855b9c5b23900dfea431a93b107a7dcfb93 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Tue, 13 Oct 2026 16:30:25 +0000 Subject: [PATCH 4/4] fix: no NAL salvage and no "damaged" note for an MJPEG cell (#133 after #139) Co-Authored-By: Claude --- android/app/src/main/java/com/openipc/decoder/QuadCell.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index b130f406..5c596444 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -357,7 +357,7 @@ String sample(long nowMs) { /** Corner note while damaged key frames are decoded (#133). */ private String damaged() { - return nalAssembler.salvaging() ? " · damaged" : ""; + return !jpeg && nalAssembler.salvaging() ? " · damaged" : ""; } boolean isKeyFramesOnly() { return keyFramesOnly; } @@ -446,7 +446,7 @@ private void checkUdpStall() { udpStallSamples = 0; // Every key frame loses a piece: decode them damaged rather than not at all. A // pushed stream has no TCP to go to; over RTSP, Auto goes to TCP anyway (#133) - if (!nalAssembler.salvaging()) { + if (!jpeg && !nalAssembler.salvaging()) { nalAssembler.setSalvage(true); Log.w(TAG, tag + ": no complete key frame over UDP, decoding damaged ones"); }