diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index 46e3d7d2..ed8593b7 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -92,6 +92,13 @@ public class Decoder extends Activity // Quad audio (#12): the cell whose sound plays, -1 for none private int quadAudio = -1; private final TextView[] quadAudioBadges = new TextView[CAM_COUNT]; + /** Small text in each cell's corner: stats while playing, otherwise the status (#80, #82). */ + private final TextView[] quadStats = new TextView[CAM_COUNT]; + /** Samples (seconds) a cell may fall behind before the grid is relieved (#83). */ + private static final int OVERLOAD_SAMPLES = 5; + /** After pausing a cell, time for the load to settle before the next decision. */ + private static final long RELIEF_COOLDOWN_MS = 10_000; + private long lastReliefMs; private volatile int quadAudioGen; private ExecutorService quadAudioExecutor; @@ -571,6 +578,7 @@ private void startQuad() { } private void stopQuad() { + quadContainer.removeCallbacks(quadStatsTick); stopQuadAudio(); quadEnabled = false; getSharedPreferences("settings", MODE_PRIVATE).edit() @@ -621,6 +629,7 @@ private void buildQuadGrid(int size) { quadContainer.removeAllViews(); Arrays.fill(quadViews, null); Arrays.fill(quadAudioBadges, null); + Arrays.fill(quadStats, null); for (int row = 0; row < size; row++) { LinearLayout lr = new LinearLayout(this); lr.setOrientation(LinearLayout.HORIZONTAL); @@ -662,6 +671,18 @@ private void buildQuadGrid(int size) { bp.setMargins(dp(4), dp(4), dp(4), dp(4)); cell.addView(badge, bp); quadAudioBadges[idx] = badge; + TextView stats = new TextView(this); + stats.setTextColor(Color.WHITE); + stats.setTextSize(10); + stats.setBackgroundColor(0x80000000); + stats.setPadding(dp(4), dp(1), dp(4), dp(1)); + stats.setVisibility(View.GONE); + FrameLayout.LayoutParams sp = new FrameLayout.LayoutParams( + FrameLayout.LayoutParams.WRAP_CONTENT, + FrameLayout.LayoutParams.WRAP_CONTENT, Gravity.BOTTOM | Gravity.START); + sp.setMargins(dp(2), dp(2), dp(2), dp(2)); + cell.addView(stats, sp); + quadStats[idx] = stats; lr.addView(cell, new LinearLayout.LayoutParams( 0, LinearLayout.LayoutParams.MATCH_PARENT, 1)); } @@ -723,6 +744,7 @@ private void createQuadCells() { } quadAudio = getSharedPreferences("settings", MODE_PRIVATE).getInt("quad_audio", -1); startQuadAudio(); + startStatsTick(); } // ====================== Quad audio (#12) ====================== @@ -807,22 +829,92 @@ private void removeClearOverlay() { } private void clearQuadViews() { - for (int i = 0; i < CAM_COUNT; i++) { - SurfaceView t = quadViews[i]; - if (t == null) continue; - FrameLayout cell = (FrameLayout) t.getParent(); - View overlay = cell.findViewWithTag("quad_overlay_" + i); - if (overlay == null) { - overlay = new View(this); - overlay.setTag("quad_overlay_" + i); - overlay.setLayoutParams(new FrameLayout.LayoutParams( - FrameLayout.LayoutParams.MATCH_PARENT, - FrameLayout.LayoutParams.MATCH_PARENT)); - overlay.setBackgroundColor(Color.BLACK); - cell.addView(overlay); + for (int i = 0; i < CAM_COUNT; i++) showQuadOverlay(i); + } + + /** Black overlay over a cell; its corner text stays visible above it. */ + private void showQuadOverlay(int i) { + SurfaceView t = quadViews[i]; + if (t == null) return; + FrameLayout cell = (FrameLayout) t.getParent(); + View overlay = cell.findViewWithTag("quad_overlay_" + i); + if (overlay == null) { + overlay = new View(this); + overlay.setTag("quad_overlay_" + i); + overlay.setLayoutParams(new FrameLayout.LayoutParams( + FrameLayout.LayoutParams.MATCH_PARENT, + FrameLayout.LayoutParams.MATCH_PARENT)); + overlay.setBackgroundColor(Color.BLACK); + cell.addView(overlay); + } + overlay.setVisibility(View.VISIBLE); + overlay.bringToFront(); + if (quadStats[i] != null) quadStats[i].bringToFront(); + } + + // ====================== Cell stats and decoder load (#80, #82, #83) ====================== + + private final Runnable quadStatsTick = new Runnable() { + private int ticks; + + @Override + public void run() { + QuadCell[] cells = quadCells; + if (!quadEnabled || cells == null) return; + long now = System.nanoTime() / 1_000_000; + boolean log = ++ticks % 5 == 0; + for (int i = 0; i < cells.length; i++) { + QuadCell c = cells[i]; + TextView v = quadStats[i]; + if (c == null || v == null) continue; + String line = c.sample(now); + v.setText(line); + v.setVisibility(View.VISIBLE); + if (log) Log.i(TAG, "Quad-" + i + ": " + line); } - overlay.setVisibility(View.VISIBLE); - overlay.bringToFront(); + relieveDecoder(cells, now); + quadContainer.postDelayed(this, 1000); + } + }; + + private void startStatsTick() { + quadContainer.removeCallbacks(quadStatsTick); + quadContainer.postDelayed(quadStatsTick, 1000); + } + + /** + * Load is judged by measurement, not by the declared budget: a cell falls behind when the + * decoder makes it lose frames (its queue overflows, or the decoder has no free input + * buffer). Frames lost on the network do not count: they lower decoded/received in the + * corner text but say nothing about the decoder. When a cell keeps falling behind, the + * highest-numbered playing cell is paused, then the load is given time to settle (#83). + */ + private void relieveDecoder(QuadCell[] cells, long now) { + boolean overloaded = false; + int active = 0; + int last = -1; + for (int i = 0; i < cells.length; i++) { + QuadCell c = cells[i]; + if (c == null || c.isPaused()) continue; + active++; + last = i; + if (c.isSettled(now) && c.inFps() >= 5) { + boolean behind = c.lostFps() >= Math.max(2f, 0.1f * c.inFps()); + c.behindSamples = behind ? c.behindSamples + 1 : 0; + if (c.behindSamples >= OVERLOAD_SAMPLES) overloaded = true; + } else { + c.behindSamples = 0; + } + } + if (!overloaded || active < 2 || now - lastReliefMs < RELIEF_COOLDOWN_MS) return; + lastReliefMs = now; + String reason = "Paused: decoder overloaded"; + Log.w(TAG, "Decoder overloaded with " + active + " streams: pausing Quad-" + last); + cells[last].pause(reason); + showQuadOverlay(last); + if (quadStats[last] != null) quadStats[last].setText(reason); + for (QuadCell c : cells) { + if (c != null) c.behindSamples = 0; } } @@ -1058,6 +1150,7 @@ protected void onPause() { menuManager.onPauseDismissBrowser(); // The UDP view is not a saved mode: after a resume the slot or grid comes back leaveUdpView(); + quadContainer.removeCallbacks(quadStatsTick); stopStreams(); } diff --git a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java index 161eb05a..44b131dc 100644 --- a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java +++ b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java @@ -8,11 +8,15 @@ package com.openipc.decoder; import android.media.MediaCodec; +import android.media.MediaCodecInfo; import android.media.MediaFormat; +import android.os.Build; import android.util.Log; import android.view.Surface; import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; class MediaCodecManager { private static final String TAG = "OpenIPCDecoder"; @@ -44,10 +48,92 @@ class MediaCodecManager { // Surface set by the owning activity private volatile Surface surface; + /** Frames rendered so far (#82): with the frames received this gives the real decode rate. */ + private volatile int framesOut; + /** NAL units not fed because the decoder had no free input buffer: it is not keeping up. */ + private volatile int inputDropped; + private volatile int outWidth; + private volatile int outHeight; + /** MIME type whose instance slot this manager holds while it has a decoder (#83). */ + private String heldMime; + interface Listener { void onResolutionChanged(int width, int height); void onDecoderStarted(); void onDecoderFailed(); + /** No decoder was created: {@code limit} decoders of {@code mime} already exist (#83). */ + default void onDecoderLimit(String mime, int limit) {} + } + + int framesOut() { return framesOut; } + int inputDropped() { return inputDropped; } + int outWidth() { return outWidth; } + int outHeight() { return outHeight; } + boolean hasDecoder() { return decoder != null; } + + /** + * Decoders alive in this process per MIME type, against the declared limit of the codec + * (MediaCodecInfo.CodecCapabilities#getMaxSupportedInstances, Android 6+). The limit is + * learnt from the first decoder of a type, so it is the codec that createDecoderByType + * really picks (#83). + */ + static final class InstanceBudget { + private static final Map used = new HashMap<>(); + private static final Map limit = new HashMap<>(); + + static synchronized boolean tryAcquire(String mime) { + int u = used.containsKey(mime) ? used.get(mime) : 0; + Integer max = limit.get(mime); + if (max != null && u >= max) return false; + used.put(mime, u + 1); + return true; + } + + static synchronized void release(String mime) { + int u = used.containsKey(mime) ? used.get(mime) : 0; + if (u > 0) used.put(mime, u - 1); + } + + static synchronized void learn(String mime, int max) { + if (max > 0 && !limit.containsKey(mime)) limit.put(mime, max); + } + + /** Declared limit, or -1 while unknown. */ + static synchronized int limit(String mime) { + Integer max = limit.get(mime); + return max != null ? max : -1; + } + + static synchronized void reset() { + used.clear(); + limit.clear(); + } + } + + /** Give the instance slot back; called wherever the decoder is detached (decoderLock held). */ + private void releaseSlot() { + if (heldMime != null) { + InstanceBudget.release(heldMime); + heldMime = null; + } + } + + /** Learn the codec's instance limit and log its declared throughput, once per type. */ + private static void learnLimits(MediaCodec codec, String mime) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) return; + if (InstanceBudget.limit(mime) >= 0) return; + try { + MediaCodecInfo.CodecCapabilities caps = codec.getCodecInfo().getCapabilitiesForType(mime); + int max = caps.getMaxSupportedInstances(); + InstanceBudget.learn(mime, max); + MediaCodecInfo.VideoCapabilities video = caps.getVideoCapabilities(); + String fps1080 = video != null && video.isSizeSupported(1920, 1080) + ? String.valueOf(video.getSupportedFrameRatesFor(1920, 1080).getUpper()) : "?"; + Log.i(TAG, "Decoder " + codec.getName() + " (" + mime + "): up to " + max + + " instances, declared 1080p frame rate in total " + fps1080); + } catch (Exception e) { + Log.w(TAG, "Decoder limits unknown: " + e.getMessage()); + } } private Listener listener; @@ -76,6 +162,7 @@ void setEnabled(boolean enabled) { synchronized (decoderLock) { if (decoder != target) return; // already replaced or released decoder = null; + releaseSlot(); retry.onSuccess(); } releaseNow(target); @@ -98,6 +185,7 @@ void closeDecoder() { MediaCodec codec = decoder; if (codec == null) return; decoder = null; + releaseSlot(); Log.i(TAG, "Close video decoder"); try { codec.stop(); @@ -131,10 +219,21 @@ void createDecoder(boolean codecH265) { // Must hold the largest NAL unit the assembler can deliver format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, MAX_NAL_SIZE); + if (!InstanceBudget.tryAcquire(type)) { + int limit = InstanceBudget.limit(type); + synchronized (decoderLock) { + retry.onFailure(nowMs()); + } + Log.w(TAG, "Decoder limit reached: " + limit + " × " + type); + if (listener != null) listener.onDecoderLimit(type, limit); + return; + } + MediaCodec local; try { Log.i(TAG, "Start video decoder (" + type + ")"); local = MediaCodec.createDecoderByType(type); + learnLimits(local, type); try { local.configure(format, s, null, 0); local.start(); @@ -144,6 +243,7 @@ void createDecoder(boolean codecH265) { throw e; } } catch (Exception e) { + InstanceBudget.release(type); synchronized (decoderLock) { // A creation cancelled by a pause must not delay the next start if (gen != lifecycle || !enabled) return; @@ -156,10 +256,12 @@ void createDecoder(boolean codecH265) { synchronized (decoderLock) { // Disabled meanwhile (pause/stop) or another thread was faster if (decoder != null || !enabled || gen != lifecycle) { + InstanceBudget.release(type); releaseAsync(local); return; } decoder = local; + heldMime = type; awaitingConfig = true; configSeen = 0; } @@ -213,6 +315,8 @@ boolean decodeFrame(Frame buffer, boolean codecH265, long watchdogRef) { codec.queueInputBuffer(inputBufferId, 0, buffer.length(), System.nanoTime() / 1000, flag); } + } else { + inputDropped++; } MediaCodec.BufferInfo info = bufferInfo; @@ -223,16 +327,20 @@ boolean decodeFrame(Frame buffer, boolean codecH265, long watchdogRef) { MediaFormat format = codec.getOutputFormat(); int mWidth = format.getInteger(MediaFormat.KEY_WIDTH); int mHeight = format.getInteger(MediaFormat.KEY_HEIGHT); + outWidth = mWidth; + outHeight = mHeight; if (listener != null) { listener.onResolutionChanged(mWidth, mHeight); } } else { codec.releaseOutputBuffer(outputBufferId, true); + framesOut++; retry.onSuccess(); // a decoded frame: the decoder works } } } catch (Exception e) { decoder = null; + releaseSlot(); retry.onFailure(nowMs()); Log.e(TAG, "Codec exception, retry in " + DecoderRetry.delayMs(retry.failures()) + " ms: " + e.getMessage()); diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index 45cecc86..6d3a1e53 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -43,11 +43,33 @@ class QuadCell implements RtspSession.Callback { private final BlockingQueue nalQueue = new ArrayBlockingQueue<>(30); private final NalAssembler nalAssembler; private final NalAssembler.Sink nalSink; - private int queueFullDrops; + private volatile int queueFullDrops; private int lastUnknownPayload = -1; private ExecutorService executor; + // Stats and status shown in the cell's corner (#80, #82) + /** Complete video frames received (RTP marker bit). */ + private volatile int framesIn; + /** Session state: "connecting", "buffering", "connected" or "disconnected". */ + private volatile String state = "connecting"; + /** What keeps the cell from playing (connection error, decoder limit), or null. */ + private volatile String problem; + /** Why the cell was paused to relieve the decoder (#83), or null. */ + private volatile String pausedReason; + private volatile long decoderStartedMs; + // Sampled on the UI thread + private int lastIn; + private int lastOut; + private long lastSampleMs; + private int lastLost; + private float inFps; + private float outFps; + /** Frames per second lost because the decoder did not keep up (queue full, no input buffer). */ + private float lostFps; + /** Consecutive samples in which the decoder fell behind (UI thread, #83). */ + int behindSamples; + interface Listener { void onRemoveOverlay(int index); } @@ -81,9 +103,15 @@ void setAudioSink(AudioManager sink, int route) { this.decoder = new MediaCodecManager(new MediaCodecManager.Listener() { @Override public void onResolutionChanged(int w, int h) {} @Override public void onDecoderStarted() { + problem = null; + decoderStartedMs = System.nanoTime() / 1_000_000; if (listener != null) listener.onRemoveOverlay(index); } @Override public void onDecoderFailed() {} + @Override public void onDecoderLimit(String mime, int limit) { + problem = "Decoder limit: " + (limit > 0 ? limit + " × " : "") + + (mime.contains("hevc") ? "H.265" : "H.264"); + } }); this.nalSink = nal -> { if (!nalQueue.offer(nal)) { @@ -189,7 +217,88 @@ void stop() { // ====================== RtspSession.Callback ====================== @Override - public void onStatus(String status) {} + public void onStatus(String status) { + state = status; + // A connection that works again clears the last connection error + if ("connected".equals(status) && problem != null && !problem.startsWith("Decoder")) { + problem = null; + } + } + + @Override + public void onFailure(String message) { + problem = shortError(message); + } + + /** A connection error short enough for a cell corner. */ + static String shortError(String message) { + if (message == null) return "Connection failed"; + String m = message; + if (m.contains("EADDRINUSE")) return "Port in use"; + if (m.contains("ECONNREFUSED")) return "Connection refused"; + if (m.contains("EHOSTUNREACH") || m.contains("ENETUNREACH")) return "Camera unreachable"; + if (m.contains("ETIMEDOUT") || m.contains("timed out") || m.contains("after ")) { + return "No answer from camera"; + } + if (m.contains("401") || m.toLowerCase(java.util.Locale.ROOT).contains("authentication")) { + return "Wrong login or password"; + } + if (m.contains("404")) return "Stream not found"; + return m.length() > 40 ? m.substring(0, 40) + "…" : m; + } + + /** + * One line for the cell's corner, sampled about once per second on the UI thread: + * resolution, codec and decoded/received frames per second while playing, otherwise + * what the cell is waiting for or why it does not play (#80, #82). + */ + String sample(long nowMs) { + int in = framesIn; + int out = decoder.framesOut(); + int lost = queueFullDrops + decoder.inputDropped(); + if (lastSampleMs > 0 && nowMs > lastSampleMs) { + float sec = (nowMs - lastSampleMs) / 1000f; + inFps = Math.max(0, in - lastIn) / sec; + outFps = Math.max(0, out - lastOut) / sec; + lostFps = Math.max(0, lost - lastLost) / sec; + } + lastIn = in; + lastOut = out; + lastLost = lost; + lastSampleMs = nowMs; + + if (pausedReason != null) return pausedReason; + String p = problem; + if (p != null) return p; + String st = state; + if (!"connected".equals(st)) { + if ("disconnected".equals(st)) return "Disconnected"; + return "buffering".equals(st) ? "Buffering…" : "Connecting…"; + } + int w = decoder.outWidth(); + int h = decoder.outHeight(); + String size = w > 0 ? w + "×" + h + " " : ""; + return size + (codecH265 ? "H.265" : "H.264") + " " + + Math.round(outFps) + "/" + Math.round(inFps) + " fps"; + } + + float inFps() { return inFps; } + float outFps() { return outFps; } + float lostFps() { return lostFps; } + boolean isPaused() { return pausedReason != null; } + + /** Playing long enough for its decode rate to mean something (#83). */ + boolean isSettled(long nowMs) { + long started = decoderStartedMs; + return pausedReason == null && "connected".equals(state) && decoder.hasDecoder() + && started > 0 && nowMs - started > 5000; + } + + /** Stop the stream to relieve the decoder; the reason stays visible in the cell (#83). */ + void pause(String reason) { + pausedReason = reason; + stop(); + } @Override public void onSdp(Sdp sdp) { @@ -226,6 +335,8 @@ public boolean onPacket(int channel, byte[] buf, int offset, int length) { } return false; } + // The marker bit ends an access unit: one received frame + if ((buf[offset + 1] & 0x80) != 0) framesIn++; return nalAssembler.assemble(buf, offset, length, codecH265, nalSink); } diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index 67596809..e35c6424 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -54,6 +54,9 @@ interface Callback { /** "disconnected", "buffering" (after PLAY) or "connected" (first video packet). */ void onStatus(String status); + /** Why the last connection attempt failed, for display (#80). */ + default void onFailure(String message) {} + /** Session parameters, before PLAY. Audio is NONE when it was not set up. */ void onSdp(Sdp sdp); @@ -192,6 +195,7 @@ private void networkLoop(int gen) { // stop() closes the socket on purpose: not a connection failure if (gen != generation) break; failures++; + callback.onFailure(e.getMessage()); if (failures <= 3 || failures % 10 == 0) { Log.w(TAG, tag + ": RTSP connection failed (" + failures + "): " + e.getMessage()); diff --git a/android/app/src/test/java/com/openipc/decoder/CellStatsTest.java b/android/app/src/test/java/com/openipc/decoder/CellStatsTest.java new file mode 100644 index 00000000..d2f73cd6 --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/CellStatsTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * CellStatsTest.java — multi-view cell status text and decoder instance limits (#80, #82, #83) + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.Test; + +public class CellStatsTest { + @After + public void resetBudget() { + MediaCodecManager.InstanceBudget.reset(); + } + + private static QuadCell cell() { + return new QuadCell(0, "rtsp://cam/s", Transport.Mode.TCP, null, new FramePool(4), "", + index -> {}); + } + + @Test + public void connectionErrorsAreShortened() { + assertEquals("Port in use", QuadCell.shortError("bind failed: EADDRINUSE (Address already in use)")); + assertEquals("Connection refused", QuadCell.shortError( + "failed to connect to /10.0.0.2 (port 554) after 5000ms: isConnected failed: ECONNREFUSED")); + assertEquals("No answer from camera", QuadCell.shortError( + "failed to connect to /10.0.0.2 (port 554) after 5000ms")); + assertEquals("Wrong login or password", QuadCell.shortError("RTSP/1.0 401 Unauthorized")); + assertEquals("Connection failed", QuadCell.shortError(null)); + assertTrue(QuadCell.shortError("x".repeat(80)).length() <= 41); + } + + @Test + public void statusBeforePlaying() { + QuadCell c = cell(); + assertEquals("Connecting…", c.sample(1000)); + c.onStatus("buffering"); + assertEquals("Buffering…", c.sample(2000)); + c.onStatus("disconnected"); + assertEquals("Disconnected", c.sample(3000)); + c.onFailure("bind failed: EADDRINUSE (Address already in use)"); + assertEquals("Port in use", c.sample(4000)); + } + + @Test + public void playingShowsCodecAndRates() { + QuadCell c = cell(); + c.onFailure("ECONNREFUSED"); + c.onStatus("connected"); + // A working connection clears the connection error; no frames yet + assertEquals("H.264 0/0 fps", c.sample(1000)); + assertFalse(c.isPaused()); + } + + @Test + public void pausedReasonWins() { + QuadCell c = cell(); + c.onStatus("connected"); + c.pause("Paused: decoder overloaded"); + assertTrue(c.isPaused()); + assertEquals("Paused: decoder overloaded", c.sample(1000)); + } + + @Test + public void instanceBudgetHoldsTheLearntLimit() { + String hevc = "video/hevc"; + // Unknown limit: no restriction yet + assertTrue(MediaCodecManager.InstanceBudget.tryAcquire(hevc)); + MediaCodecManager.InstanceBudget.learn(hevc, 2); + assertEquals(2, MediaCodecManager.InstanceBudget.limit(hevc)); + assertTrue(MediaCodecManager.InstanceBudget.tryAcquire(hevc)); + assertFalse("third H.265 decoder over a limit of 2", + MediaCodecManager.InstanceBudget.tryAcquire(hevc)); + MediaCodecManager.InstanceBudget.release(hevc); + assertTrue(MediaCodecManager.InstanceBudget.tryAcquire(hevc)); + // Other codecs are counted separately + assertTrue(MediaCodecManager.InstanceBudget.tryAcquire("video/avc")); + assertEquals(-1, MediaCodecManager.InstanceBudget.limit("video/avc")); + } +}