diff --git a/android/app/src/main/java/com/openipc/decoder/DebugServer.java b/android/app/src/main/java/com/openipc/decoder/DebugServer.java index 96919e9d..924261d2 100644 --- a/android/app/src/main/java/com/openipc/decoder/DebugServer.java +++ b/android/app/src/main/java/com/openipc/decoder/DebugServer.java @@ -512,8 +512,11 @@ private static void closeQuietly(Socket s) { + "up(s.uptime_s):'';\n" // Orange: the UI thread is stuck, or there is no activity behind the server at all + "var ui=s.ui_thread&&s.ui_thread!=='ok',none=!s.ui_thread;\n" - + "$('err').textContent=ui?'UI '+s.ui_thread:none?(s.mode||'no data'):'';\n" - + "$('dot').className='dot '+(ui||none?'warn':'ok');\n" + // Red: frames arrive, the device's decoder gives none; only a restart helps (#145) + + "var stuck=s.decoder_stuck===true;\n" + + "$('err').textContent=stuck?'video decoder not responding \u2014 restart the device'" + + ":ui?'UI '+s.ui_thread:none?(s.mode||'no data'):'';\n" + + "$('dot').className='dot '+(stuck?'bad':ui||none?'warn':'ok');\n" + "var list,side=1;\n" + "if(s.mode==='paused'||!(s.cells||s.single))" + "list=[{text:s.mode==='paused'?'App in background':(s.mode||'')}];\n" diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index abcfbf33..bab1efb8 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -26,6 +26,7 @@ import android.os.SystemClock; import android.provider.MediaStore; import android.util.Log; +import android.util.TypedValue; import android.view.GestureDetector; import android.view.Gravity; import android.view.KeyEvent; @@ -478,6 +479,7 @@ private boolean leaveUdpView() { menuManager.setUdpShown(false); stopQuadCells(); quadContainer.setVisibility(View.GONE); + resetDecoderStuck(); mSurface.setVisibility(View.VISIBLE); return true; } @@ -731,6 +733,7 @@ private void stopQuad() { } quadContainer.setVisibility(View.GONE); + resetDecoderStuck(); mSurface.setVisibility(View.VISIBLE); } @@ -1017,6 +1020,7 @@ public void run() { v.setVisibility(showStats ? View.VISIBLE : View.GONE); if (log) Log.i(TAG, "Quad-" + i + ": " + line); } + checkDecoderStuck(cells); planDecoding(cells, now); updateDebugUrls(); refreshUdpHint(); @@ -1024,6 +1028,83 @@ public void run() { } }; + // ====================== The device's decoder stands still (#145) ====================== + + /** Time with frames arriving in several cells and none decoded: the device is stuck. */ + static final long STUCK_MS = 20_000; + /** Since when the condition holds, or -1 (monotonic ms; Qodo on #147: time, not ticks). */ + private long stuckSinceMs = -1; + private TextView stuckBanner; + + /** + * One observation of the detector: since when at least two cells have received whole + * frames and no cell at all decoded one (#145), or -1 when that does not hold now. + */ + static long stuckSince(long sinceMs, long nowMs, int receiving, boolean anyOutput) { + if (receiving < 2 || anyOutput) return -1; + return sinceMs >= 0 ? sinceMs : nowMs; + } + + static boolean isStuck(long sinceMs, long nowMs) { + return sinceMs >= 0 && nowMs - sinceMs >= STUCK_MS; + } + + private boolean decoderStuck() { + return isStuck(stuckSinceMs, System.nanoTime() / 1_000_000); + } + + /** + * Seen on the AG-216: every cell receives but no decoder gives a frame, and only a reboot + * of the device helps (#145). Said on screen instead of black cells and "0.0 fps". + */ + private void checkDecoderStuck(QuadCell[] cells) { + int receiving = 0; + boolean anyOutput = false; + for (QuadCell c : cells) { + if (c == null) continue; + // Output of any cell, a lossy one too, shows the decoder works (Qodo on #147) + // Only H.264/H.265 cells: MJPEG may be drawn without the video decoder at all + if (!c.usesVideoDecoder()) continue; + if (c.outFps() > 0) anyOutput = true; + if (c.isReceiving()) receiving++; + } + long now = System.nanoTime() / 1_000_000; + stuckSinceMs = stuckSince(stuckSinceMs, now, receiving, anyOutput); + showDecoderStuck(isStuck(stuckSinceMs, now)); + } + + /** Leaving or rebuilding the grid: a fresh count, no banner (Qodo on #147). */ + private void resetDecoderStuck() { + stuckSinceMs = -1; + showDecoderStuck(false); + } + + private void showDecoderStuck(boolean on) { + if (!on) { + if (stuckBanner != null) stuckBanner.setVisibility(View.GONE); + return; + } + if (stuckBanner == null) { + stuckBanner = new TextView(this); + stuckBanner.setText("The video decoder of this device does not respond.\n" + + "Frames arrive, none is decoded. Restart the device."); + stuckBanner.setTextColor(Color.WHITE); + stuckBanner.setTextSize(TypedValue.COMPLEX_UNIT_SP, 20); + stuckBanner.setGravity(Gravity.CENTER); + stuckBanner.setBackgroundColor(0xCC000000); + int pad = (int) (16 * getResources().getDisplayMetrics().density); + stuckBanner.setPadding(pad, pad, pad, pad); + ((FrameLayout) findViewById(R.id.decoder)).addView(stuckBanner, new FrameLayout.LayoutParams( + FrameLayout.LayoutParams.WRAP_CONTENT, FrameLayout.LayoutParams.WRAP_CONTENT, + Gravity.CENTER)); + } + if (stuckBanner.getVisibility() != View.VISIBLE) { + Log.w(TAG, "No cell decodes although frames arrive: the device's decoder is stuck"); + } + stuckBanner.setVisibility(View.VISIBLE); + stuckBanner.bringToFront(); + } + private void startStatsTick() { quadContainer.removeCallbacks(quadStatsTick); decodePlanner = newPlanner(true); @@ -1363,6 +1444,8 @@ private Map debugState() { dec.put("learnt", planner.isLearnt()); dec.put("failed_load", planner.failedLoad()); m.put("decoder", dec); + // Frames arrive in several cells, none decodes: restart the device (#145) + m.put("decoder_stuck", decoderStuck()); } if (udpViewShown) { Map udp = new LinkedHashMap<>(); @@ -1753,6 +1836,7 @@ public void run() { private void stopStreams() { lastJpegFrameMs = 0; + resetDecoderStuck(); mSurface.removeCallbacks(singleUdpCheck); // The single view's stream ends here: it is no longer "connected" for the debug page singleState = "disconnected"; @@ -1847,6 +1931,7 @@ private void startStreams() { } else { // The views may still be in Quad layout from before the pause quadContainer.setVisibility(View.GONE); + resetDecoderStuck(); mSurface.setVisibility(View.VISIBLE); boolean configured = mHost != null && !mHost.isEmpty() && !mHost.equals(DEFAULT_URL); if (configured) { diff --git a/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java b/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java index 89134142..06ebfbcf 100644 --- a/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java +++ b/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java @@ -83,7 +83,8 @@ interface Sink { /** Offset the next fragment must have; -1 while no frame is being collected. */ private int expectedOffset = -1; private byte[] header; - private int dropped; + /** Written by the receive thread, read by the UI thread (Qodo on #147). */ + private volatile int dropped; /** In-band tables by Q (128..254): a sender may send them once and then only the Q. */ private final java.util.Map tablesByQ = new java.util.HashMap<>(); private final java.util.Map precisionByQ = new java.util.HashMap<>(); diff --git a/android/app/src/main/java/com/openipc/decoder/NalAssembler.java b/android/app/src/main/java/com/openipc/decoder/NalAssembler.java index d5c478ab..8f2755ee 100644 --- a/android/app/src/main/java/com/openipc/decoder/NalAssembler.java +++ b/android/app/src/main/java/com/openipc/decoder/NalAssembler.java @@ -44,6 +44,8 @@ interface Sink { private int dropped; /** Of those, dropped because a packet of them was lost (not for size) (#113). */ private int lostDropped; + /** Written by the receive thread, read by the UI thread (Qodo on #147). */ + private volatile int gaps; /** * Pass a NAL unit with a lost piece on cut short at the gap instead of dropping it (#133): * for a stream that loses a piece of every key frame, where dropping them leaves nothing to @@ -69,6 +71,8 @@ void reset() { int droppedCount() { return dropped; } /** NAL units dropped because one of their packets was lost (#113). */ int lossCount() { return lostDropped; } + /** Gaps in the RTP sequence, any packet lost (#145). */ + int gapCount() { return gaps; } void setSalvage(boolean on) { salvage = on; } boolean salvaging() { return salvage; } @@ -98,6 +102,7 @@ boolean assemble(byte[] buf, int offset, int length, boolean codecH265, Sink sin int seq = ((buf[offset + 2] & 0xFF) << 8) | (buf[offset + 3] & 0xFF); boolean gap = lastSeq >= 0 && seq != ((lastSeq + 1) & 0xFFFF); + if (gap) gaps++; lastSeq = seq; if (gap && inFragment) { lostDropped++; diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index 83204945..e4e776d9 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -142,6 +142,14 @@ void setAudioSink(AudioManager sink, int route) { } }); this.nalSink = nal -> { + // Picture slices out of the assembler, before the queue: what the video decoder + // would get (#145); parameter sets alone are no picture (Qodo on #147) + // On key frames only, just the slices decode() really passes on (Qodo on #147) + if (!jpeg && nal.length() > 4 && NalAssembler.isSlice(nal.data()[4], codecH265) + && (!(keyFramesOnly || waitingKeyFrame) || isKeyNal( + NalAssembler.fragment(nal.data()[4], codecH265), codecH265))) { + assembled++; + } if (!nalQueue.offer(nal)) { // Rate-limited: one aggregation packet can carry many NAL units if (queueFullDrops++ % 100 == 0) { @@ -328,6 +336,13 @@ String sample(long nowMs) { lastIn = in; lastOut = out; lastLost = lost; + // Any lost packet, not only one inside a fragmented NAL unit (Qodo on #147) + int netLoss = jpeg ? jpegAssembler.droppedCount() : nalAssembler.gapCount(); + networkLoss = netLoss > lastNetLoss; + int units = assembled; + unitsArrived = units != lastAssembled; + lastAssembled = units; + lastNetLoss = netLoss; lastSampleMs = nowMs; checkUdpStall(); @@ -479,6 +494,31 @@ boolean showsVideo(long ms) { return pausedReason == null && "connected".equals(state) && renderedWithin(window); } + /** NAL units were lost on the network in the last second (#145). */ + private volatile boolean networkLoss; + /** + * NAL units or JPEG frames out of the assembler: unlike marker bits (inFps), only data the + * decoder could take; queue drops included (Qodo on #147). + */ + private volatile int assembled; + private int lastAssembled; + /** Some arrived since the last sample, whatever the sampling interval. */ + private volatile boolean unitsArrived; + private int lastNetLoss; + + /** + * Connected, not paused, and whole frames arrive (#145): a cell losing packets on the + * network may decode nothing for that reason alone (#113, #133). + */ + /** H.264/H.265, decoded by the device's video decoder (not MJPEG, #145). */ + boolean usesVideoDecoder() { return !jpeg; } + + boolean isReceiving() { + // A cell with its own decoder problem (error, limit) says so itself (Qodo on #147) + return pausedReason == null && problem == null && "connected".equals(state) + && unitsArrived && !networkLoss; + } + private boolean renderedWithin(long ms) { if (!jpeg) return decoder.renderedWithin(ms); long shown = lastJpegShownMs; diff --git a/android/app/src/test/java/com/openipc/decoder/QuadGridTest.java b/android/app/src/test/java/com/openipc/decoder/QuadGridTest.java index 041e7664..1a5cbad6 100644 --- a/android/app/src/test/java/com/openipc/decoder/QuadGridTest.java +++ b/android/app/src/test/java/com/openipc/decoder/QuadGridTest.java @@ -76,4 +76,23 @@ public void udpHintListsEveryAddress() { java.util.Arrays.asList("172.17.32.144", "192.168.1.20"), 5600); assertTrue(two, two.endsWith("output to\n\n172.17.32.144:5600\n192.168.1.20:5600")); } + + @Test + public void decoderStuckIsReportedAfterTwentySecondsOfTime() { + long since = -1; + // Ticks a little late: the time counts, not the ticks (Qodo on #147) + long t = 1000; + for (; t < 1000 + Decoder.STUCK_MS; t += 1300) { + since = Decoder.stuckSince(since, t, 9, false); + assertEquals(1000, since); + assertFalse(Decoder.isStuck(since, t)); + } + since = Decoder.stuckSince(since, t, 9, false); + assertTrue(Decoder.isStuck(since, t)); + // Any output, from any cell, ends it + assertEquals(-1, Decoder.stuckSince(since, t, 9, true)); + // One camera alone may just wait for a key frame + assertEquals(-1, Decoder.stuckSince(-1, t, 1, false)); + assertFalse(Decoder.isStuck(-1, t)); + } }