diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index fa3d4a46..5cc00cae 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -1402,6 +1402,10 @@ private Map debugState() { c.put("lost_fps", cell.lostFps()); // NAL units dropped because one of their packets was lost (#113) c.put("nal_dropped", cell.nalDropped()); + // Slices passed on cut short at a lost piece (some may still be dropped by + // a full queue); absent while that is off (#133) + int salvaged = cell.nalSalvaged(); + if (salvaged >= 0) c.put("nal_damaged", salvaged); // Every frame, or key frames only while the decoder has no room (#85) c.put("decode", cell.isKeyFramesOnly() ? "key frames" : "full"); if (cell.knowsLoad()) c.put("load", cell.isKeyFramesOnly() @@ -1715,13 +1719,20 @@ public void run() { int frames = videoDecoder.framesOut() + jpegFramesOut; long bytes = client != null ? client.getStreamBytes() : 0; boolean stalled = !quadEnabled && !udpViewShown && client != null - && client.playsOverUdp() && "connected".equals(singleState) + && (client.playsOverUdp() || client.listens()) + && "connected".equals(singleState) && bytes - lastSingleBytes > 10_000 && frames == lastSingleFrames; lastSingleFrames = frames; lastSingleBytes = bytes; singleStallSeconds = stalled ? singleStallSeconds + 1 : 0; if (singleStallSeconds >= UDP_STALL_SECONDS) { singleStallSeconds = 0; + // Every key frame loses a piece: decode them damaged rather than not at all; + // a pushed stream has no TCP to go to (#133) + if (!nalAssembler.salvaging()) { + nalAssembler.setSalvage(true); + Log.w(TAG, "No complete key frame over UDP, decoding damaged ones"); + } client.reportUdpBroken("packets but no decoded frame for " + UDP_STALL_SECONDS + " s"); } diff --git a/android/app/src/main/java/com/openipc/decoder/NalAssembler.java b/android/app/src/main/java/com/openipc/decoder/NalAssembler.java index 7c960561..d5c478ab 100644 --- a/android/app/src/main/java/com/openipc/decoder/NalAssembler.java +++ b/android/app/src/main/java/com/openipc/decoder/NalAssembler.java @@ -44,6 +44,13 @@ interface Sink { private int dropped; /** Of those, dropped because a packet of them was lost (not for size) (#113). */ private int lostDropped; + /** + * Pass a NAL unit with a lost piece on cut short at the gap instead of dropping it (#133): + * for a stream that loses a piece of every key frame, where dropping them leaves nothing to + * decode. The decoder conceals the missing part; a damaged picture beats a black one. + */ + private volatile boolean salvage; + private int salvaged; NalAssembler(int bufferSize, Runnable onCodecSwitch, FramePool framePool) { this.nalBuffer = new byte[bufferSize]; @@ -63,6 +70,11 @@ void reset() { /** NAL units dropped because one of their packets was lost (#113). */ int lossCount() { return lostDropped; } + void setSalvage(boolean on) { salvage = on; } + boolean salvaging() { return salvage; } + /** Slices passed on to the sink cut short at a lost piece (#133); the queue may drop some. */ + int salvagedCount() { return salvaged; } + /** * Process one RTP packet ({@code length} bytes at {@code offset}, header included). * Returns false if the RTP header is malformed. @@ -89,7 +101,7 @@ boolean assemble(byte[] buf, int offset, int length, boolean codecH265, Sink sin lastSeq = seq; if (gap && inFragment) { lostDropped++; - drop("packet loss inside a fragmented NAL unit"); + dropOrSalvage("packet loss inside a fragmented NAL unit", sink); orphan = true; // its remaining pieces belong to the unit just counted } @@ -104,7 +116,7 @@ boolean assemble(byte[] buf, int offset, int length, boolean codecH265, Sink sin // Any other packet ends a fragmented NAL unit that did not finish; its remaining // pieces, if any arrive, belong to the unit counted here if (inFragment) { - drop("fragmented NAL unit interrupted by another packet"); + dropOrSalvage("fragmented NAL unit interrupted by another packet", sink); orphan = true; } else { orphan = false; @@ -131,7 +143,7 @@ private void fragmentUnit(byte[] buf, int start, int end, boolean h265, Sink sin if (first) { if (inFragment) { lostDropped++; - drop("fragmented NAL unit without its end"); + dropOrSalvage("fragmented NAL unit without its end", sink); } orphan = false; nalSize = 0; @@ -201,6 +213,32 @@ private void emit(byte[] buf, int pos, int size, Sink sink) { sink.onNal(out); } + /** The unit collected so far, cut short, when salvaging (#133); dropped otherwise. */ + private void dropOrSalvage(String reason, Sink sink) { + // Start code and NAL header plus some slice data, or there is nothing to decode; only + // slices: a cut parameter set would be taken for a valid configuration (Qodo on #146) + if (!salvage || nalSize < 16 || !isSlice(nalBuffer[4], lastCodec == 1)) { + drop(reason); + return; + } + Frame out = framePool.obtain(nalSize); + System.arraycopy(nalBuffer, 0, out.data(), 0, nalSize); + out.setLength(nalSize); + nalSize = 0; + inFragment = false; + if (salvaged++ % 100 == 0) { + Log.w(TAG, "NAL unit passed on cut short (" + salvaged + "): " + reason); + } + sink.onNal(out); + } + + /** A coded slice (VCL NAL unit): H.264 types 1–5, H.265 types 0–31. */ + static boolean isSlice(byte nalHeader, boolean h265) { + if (h265) return ((nalHeader >> 1) & 0x3F) <= 31; + int type = nalHeader & 0x1F; + return type >= 1 && type <= 5; + } + private void drop(String reason) { nalSize = 0; inFragment = false; diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index a6526c07..5c596444 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -274,6 +274,9 @@ void stop() { public void onStatus(String status) { // A new connection (or a lost one) starts the UDP stall count afresh (Qodo on #125) if (!status.equals(state)) udpStallSamples = 0; + // A pushed stream that went quiet may come back whole: no damaged decoding until it + // stalls again (Qodo on #146) + if ("disconnected".equals(status)) nalAssembler.setSalvage(false); state = status; // A connection that works again clears the last connection error if ("connected".equals(status) && problem != null && !problem.startsWith("Decoder")) { @@ -346,9 +349,15 @@ String sample(long nowMs) { // MJPEG has no key frames: its cheap mode shows one frame per second (#132) return size + codec + (jpeg ? " reduced" : " key frames") + " · " + rate + " fps" + ((jpeg ? jpegRenderer.isSoftware() : decoder.isSoftware()) - ? " · software" : ""); + ? " · software" : "") + damaged(); } - return size + codec + " " + Math.round(outFps) + "/" + Math.round(inFps) + " fps"; + return size + codec + " " + Math.round(outFps) + "/" + Math.round(inFps) + " fps" + + damaged(); + } + + /** Corner note while damaged key frames are decoded (#133). */ + private String damaged() { + return !jpeg && nalAssembler.salvaging() ? " · damaged" : ""; } boolean isKeyFramesOnly() { return keyFramesOnly; } @@ -421,7 +430,9 @@ private void checkUdpStall() { lastVideoPackets = packets; lastLossCount = loss; Surface s = surface; - boolean stalled = session.playsOverUdp() && "connected".equals(state) + // A pushed stream (udp:// slot, the U cell) counts too: no TCP to fall back to (#133) + boolean stalled = (session.playsOverUdp() || session.listens()) + && "connected".equals(state) && pausedReason == null && problem == null && s != null && s.isValid() && arriving && outFps == 0; if (!stalled) { @@ -433,6 +444,12 @@ private void checkUdpStall() { if (losing || udpStallSamples > 0) udpStallSamples++; if (udpStallSamples >= UDP_STALL_SAMPLES) { udpStallSamples = 0; + // Every key frame loses a piece: decode them damaged rather than not at all. A + // pushed stream has no TCP to go to; over RTSP, Auto goes to TCP anyway (#133) + if (!jpeg && !nalAssembler.salvaging()) { + nalAssembler.setSalvage(true); + Log.w(TAG, tag + ": no complete key frame over UDP, decoding damaged ones"); + } session.reportUdpBroken("packets but no decoded frame for " + UDP_STALL_SAMPLES + " s (" + loss + (jpeg ? " JPEG frames" : " NAL units") + " lost)"); } @@ -441,6 +458,9 @@ private void checkUdpStall() { /** NAL units dropped so far because a packet of them was lost (#113). */ int nalDropped() { return nalAssembler.lossCount(); } + /** Slices passed on cut short at a lost piece, or -1 while that is off (#133). */ + int nalSalvaged() { return nalAssembler.salvaging() ? nalAssembler.salvagedCount() : -1; } + float inFps() { return inFps; } float outFps() { return outFps; } float lostFps() { return lostFps; } @@ -517,6 +537,8 @@ public void onSdp(Sdp sdp) { lastLossCount = sdp.videoJpeg ? jpegAssembler.droppedCount() : nalAssembler.lossCount(); udpStallSamples = 0; nalAssembler.reset(); + // A new stream gets a chance to be complete again (#133) + nalAssembler.setSalvage(false); jpegAssembler.reset(); lastUnknownPayload = -1; videoPt = sdp.videoPt; diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java index a2fb1248..6b45af0d 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java @@ -75,6 +75,8 @@ void configure(String host, String userAgent, Transport.Mode transport) { boolean isActive() { return session.isActive(); } /** RTP over UDP or multicast at the moment (#128). */ boolean playsOverUdp() { return session.playsOverUdp(); } + /** A udp:// slot listening for a pushed stream (#133). */ + boolean listens() { return session.listens(); } /** Packets arrive but nothing decodes: with Auto the session goes over to TCP. */ void reportUdpBroken(String why) { session.reportUdpBroken(why); } /** Transport the stream really uses now, or null (#115). */ @@ -101,6 +103,8 @@ void stop() { @Override public void onStatus(String status) { + // A pushed stream that went quiet may come back whole (Qodo on #146) + if ("disconnected".equals(status)) nalAssembler.setSalvage(false); Listener l = listenerCb; if (l != null) l.onStatusChanged(status); } @@ -108,6 +112,8 @@ public void onStatus(String status) { @Override public void onSdp(Sdp sdp) { nalAssembler.reset(); + // A new stream gets a chance to be complete again (#133) + nalAssembler.setSalvage(false); lastUnknownPayload = -1; lastRtpTimestamp = -1; lastRtpArrivalNs = -1; diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index 3377f89d..c35743a7 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -140,6 +140,9 @@ private void markUdpFailed(Transport.Mode mode) { /** RTSP is playing with RTP over UDP or multicast (not TCP, not a udp:// URL). */ boolean playsOverUdp() { return udpPlaying; } + /** A udp:// slot listening for a pushed stream (#133): UDP too, without RTSP. */ + boolean listens() { return listening; } + /** * Packets arrive over UDP but the stream cannot be decoded (#113: key frames keep losing * a piece). With Auto or Multicast the session goes over to TCP, as when no video comes diff --git a/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java b/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java index 7de17ef0..c89fb985 100644 --- a/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java +++ b/android/app/src/test/java/com/openipc/decoder/NalAssemblerTest.java @@ -138,6 +138,52 @@ public void lostMiddleFragmentDropsTheNal() { assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xD1, 0xD2), out.get(0)); } + @Test + public void salvagePassesTheNalOnCutShortAtTheGap() { + nal.setSalvage(true); + // 16 slice bytes before the gap: enough to be worth decoding (#133) + feed(rtp(0x7C, 0x85, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12), false); + feed(rtp(0x7C, 0x05, 13, 14, 15, 16), false); + seq++; // one packet lost + feed(rtp(0x7C, 0x05, 0xB1), false); + assertEquals(1, out.size()); + assertArrayEquals(b(0, 0, 0, 1, 0x65, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16), out.get(0)); + assertEquals(1, nal.salvagedCount()); + assertEquals(1, nal.lossCount()); + // The rest of that unit is skipped; the next one is complete again + feed(rtp(0x7C, 0x45, 0xC1), false); + feed(rtp(0x7C, 0x85, 0xD1), false); + feed(rtp(0x7C, 0x45, 0xD2), false); + assertEquals(2, out.size()); + assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xD1, 0xD2), out.get(1)); + } + + @Test + public void salvageNeverPassesACutParameterSet() { + nal.setSalvage(true); + // FU-A of an SPS (type 7), then a lost packet (Qodo on #146) + feed(rtp(0x7C, 0x87, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), false); + seq++; + feed(rtp(0x7C, 0x47, 0xC1), false); + assertEquals(0, out.size()); + assertEquals(0, nal.salvagedCount()); + assertTrue(NalAssembler.isSlice((byte) 0x65, false)); + assertFalse(NalAssembler.isSlice((byte) 0x68, false)); + // H.265: IDR_W_RADL (19) is a slice, VPS (32) is not + assertTrue(NalAssembler.isSlice((byte) (19 << 1), true)); + assertFalse(NalAssembler.isSlice((byte) (32 << 1), true)); + } + + @Test + public void salvageOffDropsAsBefore() { + feed(rtp(0x7C, 0x85, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), false); + seq++; + feed(rtp(0x7C, 0x45, 0xC1), false); + assertEquals(0, out.size()); + assertEquals(0, nal.salvagedCount()); + } + @Test public void continuationWithoutStartIsIgnoredAndCountedOnce() { feed(rtp(0x7C, 0x05, 0xB1), false);