feat: say when the device's video decoder stands still (#145)

android/app/src/main/java/com/openipc/decoder/DebugServer.java

@@ -512,8 +512,11 @@ private static void closeQuietly(Socket s) {
             + "up(s.uptime_s):'';\n"
             // Orange: the UI thread is stuck, or there is no activity behind the server at all
             + "var ui=s.ui_thread&&s.ui_thread!=='ok',none=!s.ui_thread;\n"
-            + "$('err').textContent=ui?'UI '+s.ui_thread:none?(s.mode||'no data'):'';\n"
-            + "$('dot').className='dot '+(ui||none?'warn':'ok');\n"
+            // Red: frames arrive, the device's decoder gives none; only a restart helps (#145)
+            + "var stuck=s.decoder_stuck===true;\n"
+            + "$('err').textContent=stuck?'video decoder not responding \u2014 restart the device'"
+            + ":ui?'UI '+s.ui_thread:none?(s.mode||'no data'):'';\n"
+            + "$('dot').className='dot '+(stuck?'bad':ui||none?'warn':'ok');\n"
             + "var list,side=1;\n"
             + "if(s.mode==='paused'||!(s.cells||s.single))"
             + "list=[{text:s.mode==='paused'?'App in background':(s.mode||'')}];\n"

android/app/src/main/java/com/openipc/decoder/Decoder.java

@@ -26,6 +26,7 @@
 import android.os.SystemClock;
 import android.provider.MediaStore;
 import android.util.Log;
+import android.util.TypedValue;
 import android.view.GestureDetector;
 import android.view.Gravity;
 import android.view.KeyEvent;
@@ -478,6 +479,7 @@ private boolean leaveUdpView() {
         menuManager.setUdpShown(false);
         stopQuadCells();
         quadContainer.setVisibility(View.GONE);
+        resetDecoderStuck();
         mSurface.setVisibility(View.VISIBLE);
         return true;
     }
@@ -731,6 +733,7 @@ private void stopQuad() {
         }
 
         quadContainer.setVisibility(View.GONE);
+        resetDecoderStuck();
         mSurface.setVisibility(View.VISIBLE);
     }
 
@@ -1017,13 +1020,91 @@ public void run() {
                 v.setVisibility(showStats ? View.VISIBLE : View.GONE);
                 if (log) Log.i(TAG, "Quad-" + i + ": " + line);
             }
+            checkDecoderStuck(cells);
             planDecoding(cells, now);
             updateDebugUrls();
             refreshUdpHint();
             quadContainer.postDelayed(this, 1000);
         }
     };
 
+    // ====================== The device's decoder stands still (#145) ======================
+
+    /** Time with frames arriving in several cells and none decoded: the device is stuck. */
+    static final long STUCK_MS = 20_000;
+    /** Since when the condition holds, or -1 (monotonic ms; Qodo on #147: time, not ticks). */
+    private long stuckSinceMs = -1;
+    private TextView stuckBanner;
+
+    /**
+     * One observation of the detector: since when at least two cells have received whole
+     * frames and no cell at all decoded one (#145), or -1 when that does not hold now.
+     */
+    static long stuckSince(long sinceMs, long nowMs, int receiving, boolean anyOutput) {
+        if (receiving < 2 || anyOutput) return -1;
+        return sinceMs >= 0 ? sinceMs : nowMs;
+    }
+
+    static boolean isStuck(long sinceMs, long nowMs) {
+        return sinceMs >= 0 && nowMs - sinceMs >= STUCK_MS;
+    }
+
+    private boolean decoderStuck() {
+        return isStuck(stuckSinceMs, System.nanoTime() / 1_000_000);
+    }
+
+    /**
+     * Seen on the AG-216: every cell receives but no decoder gives a frame, and only a reboot
+     * of the device helps (#145). Said on screen instead of black cells and "0.0 fps".
+     */
+    private void checkDecoderStuck(QuadCell[] cells) {
+        int receiving = 0;
+        boolean anyOutput = false;
+        for (QuadCell c : cells) {
+            if (c == null) continue;
+            // Output of any cell, a lossy one too, shows the decoder works (Qodo on #147)
+            // Only H.264/H.265 cells: MJPEG may be drawn without the video decoder at all
+            if (!c.usesVideoDecoder()) continue;
+            if (c.outFps() > 0) anyOutput = true;
+            if (c.isReceiving()) receiving++;
+        }
+        long now = System.nanoTime() / 1_000_000;
+        stuckSinceMs = stuckSince(stuckSinceMs, now, receiving, anyOutput);
+        showDecoderStuck(isStuck(stuckSinceMs, now));
+    }
+
+    /** Leaving or rebuilding the grid: a fresh count, no banner (Qodo on #147). */
+    private void resetDecoderStuck() {
+        stuckSinceMs = -1;
+        showDecoderStuck(false);
+    }
+
+    private void showDecoderStuck(boolean on) {
+        if (!on) {
+            if (stuckBanner != null) stuckBanner.setVisibility(View.GONE);
+            return;
+        }
+        if (stuckBanner == null) {
+            stuckBanner = new TextView(this);
+            stuckBanner.setText("The video decoder of this device does not respond.\n"
+                    + "Frames arrive, none is decoded. Restart the device.");
+            stuckBanner.setTextColor(Color.WHITE);
+            stuckBanner.setTextSize(TypedValue.COMPLEX_UNIT_SP, 20);
+            stuckBanner.setGravity(Gravity.CENTER);
+            stuckBanner.setBackgroundColor(0xCC000000);
+            int pad = (int) (16 * getResources().getDisplayMetrics().density);
+            stuckBanner.setPadding(pad, pad, pad, pad);
+            ((FrameLayout) findViewById(R.id.decoder)).addView(stuckBanner, new FrameLayout.LayoutParams(
+                    FrameLayout.LayoutParams.WRAP_CONTENT, FrameLayout.LayoutParams.WRAP_CONTENT,
+                    Gravity.CENTER));
+        }
+        if (stuckBanner.getVisibility() != View.VISIBLE) {
+            Log.w(TAG, "No cell decodes although frames arrive: the device's decoder is stuck");
+        }
+        stuckBanner.setVisibility(View.VISIBLE);
+        stuckBanner.bringToFront();
+    }
+
     private void startStatsTick() {
         quadContainer.removeCallbacks(quadStatsTick);
         decodePlanner = newPlanner(true);
@@ -1363,6 +1444,8 @@ private Map<String, Object> debugState() {
             dec.put("learnt", planner.isLearnt());
             dec.put("failed_load", planner.failedLoad());
             m.put("decoder", dec);
+            // Frames arrive in several cells, none decodes: restart the device (#145)
+            m.put("decoder_stuck", decoderStuck());
         }
         if (udpViewShown) {
             Map<String, Object> udp = new LinkedHashMap<>();
@@ -1753,6 +1836,7 @@ public void run() {
 
     private void stopStreams() {
         lastJpegFrameMs = 0;
+        resetDecoderStuck();
         mSurface.removeCallbacks(singleUdpCheck);
         // The single view's stream ends here: it is no longer "connected" for the debug page
         singleState = "disconnected";
@@ -1847,6 +1931,7 @@ private void startStreams() {
         } else {
             // The views may still be in Quad layout from before the pause
             quadContainer.setVisibility(View.GONE);
+            resetDecoderStuck();
             mSurface.setVisibility(View.VISIBLE);
             boolean configured = mHost != null && !mHost.isEmpty() && !mHost.equals(DEFAULT_URL);
             if (configured) {

android/app/src/main/java/com/openipc/decoder/JpegAssembler.java

@@ -83,7 +83,8 @@ interface Sink {
     /** Offset the next fragment must have; -1 while no frame is being collected. */
     private int expectedOffset = -1;
     private byte[] header;
-    private int dropped;
+    /** Written by the receive thread, read by the UI thread (Qodo on #147). */
+    private volatile int dropped;
     /** In-band tables by Q (128..254): a sender may send them once and then only the Q. */
     private final java.util.Map<Integer, byte[]> tablesByQ = new java.util.HashMap<>();
     private final java.util.Map<Integer, Integer> precisionByQ = new java.util.HashMap<>();

android/app/src/main/java/com/openipc/decoder/NalAssembler.java

@@ -44,6 +44,8 @@ interface Sink {
     private int dropped;
     /** Of those, dropped because a packet of them was lost (not for size) (#113). */
     private int lostDropped;
+    /** Written by the receive thread, read by the UI thread (Qodo on #147). */
+    private volatile int gaps;
     /**
      * Pass a NAL unit with a lost piece on cut short at the gap instead of dropping it (#133):
      * for a stream that loses a piece of every key frame, where dropping them leaves nothing to
@@ -69,6 +71,8 @@ void reset() {
     int droppedCount() { return dropped; }
     /** NAL units dropped because one of their packets was lost (#113). */
     int lossCount() { return lostDropped; }
+    /** Gaps in the RTP sequence, any packet lost (#145). */
+    int gapCount() { return gaps; }
 
     void setSalvage(boolean on) { salvage = on; }
     boolean salvaging() { return salvage; }
@@ -98,6 +102,7 @@ boolean assemble(byte[] buf, int offset, int length, boolean codecH265, Sink sin
 
         int seq = ((buf[offset + 2] & 0xFF) << 8) | (buf[offset + 3] & 0xFF);
         boolean gap = lastSeq >= 0 && seq != ((lastSeq + 1) & 0xFFFF);
+        if (gap) gaps++;
         lastSeq = seq;
         if (gap && inFragment) {
             lostDropped++;

android/app/src/main/java/com/openipc/decoder/QuadCell.java

@@ -142,6 +142,14 @@ void setAudioSink(AudioManager sink, int route) {
             }
         });
         this.nalSink = nal -> {
+            // Picture slices out of the assembler, before the queue: what the video decoder
+            // would get (#145); parameter sets alone are no picture (Qodo on #147)
+            // On key frames only, just the slices decode() really passes on (Qodo on #147)
+            if (!jpeg && nal.length() > 4 && NalAssembler.isSlice(nal.data()[4], codecH265)
+                    && (!(keyFramesOnly || waitingKeyFrame) || isKeyNal(
+                            NalAssembler.fragment(nal.data()[4], codecH265), codecH265))) {
+                assembled++;
+            }
             if (!nalQueue.offer(nal)) {
                 // Rate-limited: one aggregation packet can carry many NAL units
                 if (queueFullDrops++ % 100 == 0) {
@@ -328,6 +336,13 @@ String sample(long nowMs) {
         lastIn = in;
         lastOut = out;
         lastLost = lost;
+        // Any lost packet, not only one inside a fragmented NAL unit (Qodo on #147)
+        int netLoss = jpeg ? jpegAssembler.droppedCount() : nalAssembler.gapCount();
+        networkLoss = netLoss > lastNetLoss;
+        int units = assembled;
+        unitsArrived = units != lastAssembled;
+        lastAssembled = units;
+        lastNetLoss = netLoss;
         lastSampleMs = nowMs;
         checkUdpStall();
 
@@ -479,6 +494,31 @@ boolean showsVideo(long ms) {
         return pausedReason == null && "connected".equals(state) && renderedWithin(window);
     }
 
+    /** NAL units were lost on the network in the last second (#145). */
+    private volatile boolean networkLoss;
+    /**
+     * NAL units or JPEG frames out of the assembler: unlike marker bits (inFps), only data the
+     * decoder could take; queue drops included (Qodo on #147).
+     */
+    private volatile int assembled;
+    private int lastAssembled;
+    /** Some arrived since the last sample, whatever the sampling interval. */
+    private volatile boolean unitsArrived;
+    private int lastNetLoss;
+
+    /**
+     * Connected, not paused, and whole frames arrive (#145): a cell losing packets on the
+     * network may decode nothing for that reason alone (#113, #133).
+     */
+    /** H.264/H.265, decoded by the device's video decoder (not MJPEG, #145). */
+    boolean usesVideoDecoder() { return !jpeg; }
+
+    boolean isReceiving() {
+        // A cell with its own decoder problem (error, limit) says so itself (Qodo on #147)
+        return pausedReason == null && problem == null && "connected".equals(state)
+                && unitsArrived && !networkLoss;
+    }
+
     private boolean renderedWithin(long ms) {
         if (!jpeg) return decoder.renderedWithin(ms);
         long shown = lastJpegShownMs;

android/app/src/test/java/com/openipc/decoder/QuadGridTest.java

@@ -76,4 +76,23 @@ public void udpHintListsEveryAddress() {
                 java.util.Arrays.asList("172.17.32.144", "192.168.1.20"), 5600);
         assertTrue(two, two.endsWith("output to\n\n172.17.32.144:5600\n192.168.1.20:5600"));
     }
+
+    @Test
+    public void decoderStuckIsReportedAfterTwentySecondsOfTime() {
+        long since = -1;
+        // Ticks a little late: the time counts, not the ticks (Qodo on #147)
+        long t = 1000;
+        for (; t < 1000 + Decoder.STUCK_MS; t += 1300) {
+            since = Decoder.stuckSince(since, t, 9, false);
+            assertEquals(1000, since);
+            assertFalse(Decoder.isStuck(since, t));
+        }
+        since = Decoder.stuckSince(since, t, 9, false);
+        assertTrue(Decoder.isStuck(since, t));
+        // Any output, from any cell, ends it
+        assertEquals(-1, Decoder.stuckSince(since, t, 9, true));
+        // One camera alone may just wait for a key frame
+        assertEquals(-1, Decoder.stuckSince(-1, t, 1, false));
+        assertFalse(Decoder.isStuck(-1, t));
+    }
 }