feat: software decoder for key-frame cells beyond the hardware limit (#88, step 1)

android/app/src/main/java/com/openipc/decoder/DecodePlanner.java

@@ -58,6 +58,8 @@ static final class Cell {
         boolean lossy;
         /** Connected and decoding: only such cells are demoted or paused (Qodo on #84). */
         boolean playing = true;
+        /** On a software decoder beyond the hardware limit: never promoted (#88). */
+        boolean pinned;
 
         Cell(int index, boolean full, boolean known, long fullLoad, boolean lossy) {
             this.index = index;
@@ -150,7 +152,8 @@ Decision step(List<Cell> cells, long nowMs) {
         long load = 0;
         boolean anyLossy = false;
         for (Cell c : cells) {
-            if (c.paused) continue;
+            // A software cell neither loads the hardware decoder nor shows its limits (#88)
+            if (c.paused || c.pinned) continue;
             if (c.full && c.known) load += c.fullLoad;
             else if (!c.full) load += c.keyLoad;
             anyLossy |= c.lossy;
@@ -198,15 +201,17 @@ Decision step(List<Cell> cells, long nowMs) {
             if (countPlaying(cells) < 2) return Decision.NONE;
             for (int i = cells.size() - 1; i >= 0; i--) {
                 Cell c = cells.get(i);
-                if (!c.paused && c.playing) return change(nowMs, Action.PAUSE, c.index);
+                if (!c.paused && c.playing && !c.pinned) {
+                    return change(nowMs, Action.PAUSE, c.index);
+                }
             }
             return Decision.NONE;
         }
         if (anyLossy) return Decision.NONE;
 
         // Promote the highest-priority key-frame cell whose load is known
         for (Cell c : cells) {
-            if (c.full || c.paused || !c.playing) {
+            if (c.full || c.paused || !c.playing || c.pinned) {
                 // Not connected (an offline camera) or paused: nothing to wait for
                 unknownSince.remove(c.index);
                 continue;

android/app/src/main/java/com/openipc/decoder/Decoder.java

@@ -1159,6 +1159,7 @@ private static DecodePlanner.Cell plannerCell(int index, QuadCell c, long now) {
                 c.knowsLoad(), c.knowsLoad() ? c.fullLoad() : 0, lossy);
         pc.paused = c.isPaused();
         pc.playing = c.isPlaying();
+        pc.pinned = c.isSoftwareDecoder();
         if (c.knowsLoad()) pc.keyLoad = c.keyLoad();
         return pc;
     }

android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java

@@ -69,6 +69,14 @@ class MediaCodecManager {
     private volatile int outHeight;
     /** MIME type whose instance slot this manager holds while it has a decoder (#83). */
     private String heldMime;
+    /**
+     * Beyond the hardware instance limit, a software decoder may be used (#88): set while the
+     * cell decodes key frames only, about one frame per second, which a software decoder
+     * manages at little CPU cost.
+     */
+    private volatile boolean allowSoftware;
+    private volatile boolean softwareInUse;
+    private static final java.util.Map<String, String> SOFTWARE = new java.util.HashMap<>();
 
     interface Listener {
         void onResolutionChanged(int width, int height);
@@ -95,6 +103,64 @@ boolean renderedWithin(long ms) {
      * learnt from the first decoder of a type, so it is the codec that createDecoderByType
      * really picks (#83).
      */
+    /** Allow a software decoder beyond the hardware limit (key-frame cells, #88). */
+    void setSoftwareFallback(boolean allow) { allowSoftware = allow; }
+
+    /** The current decoder is a software one (it is never promoted to full rate). */
+    boolean isSoftware() { return softwareInUse && decoder != null; }
+
+    /** Budget key of the software decoders of a mime type: they have their own limit. */
+    static String softwareSlot(String mime) { return "software:" + mime; }
+
+    /**
+     * A software decoder for the mime type (c2.android.* first), or null. Looked up once, with
+     * its instance limit; a failed codec list is not cached, so a later call tries again.
+     */
+    static String softwareDecoder(String mime) {
+        synchronized (SOFTWARE) {
+            if (SOFTWARE.containsKey(mime)) return SOFTWARE.get(mime);
+            MediaCodecInfo[] infos;
+            try {
+                infos = new android.media.MediaCodecList(
+                        android.media.MediaCodecList.ALL_CODECS).getCodecInfos();
+            } catch (RuntimeException e) {
+                Log.w(TAG, "Codec list unavailable: " + e.getMessage());
+                return null;
+            }
+            String found = null;
+            int max = Integer.MAX_VALUE;
+            for (MediaCodecInfo ci : infos) {
+                // One broken entry must not hide the others
+                try {
+                    if (ci.isEncoder()) continue;
+                    String n = ci.getName();
+                    boolean sw = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q
+                            ? ci.isSoftwareOnly()
+                            : n.startsWith("c2.android.") || n.startsWith("OMX.google.");
+                    if (!sw) continue;
+                    for (String t : ci.getSupportedTypes()) {
+                        if (!t.equalsIgnoreCase(mime)
+                                || (found != null && !n.startsWith("c2.android."))) continue;
+                        // The limit first: a failed query keeps the previous candidate
+                        int m = Build.VERSION.SDK_INT >= Build.VERSION_CODES.M
+                                ? ci.getCapabilitiesForType(t).getMaxSupportedInstances()
+                                : Integer.MAX_VALUE;
+                        found = n;
+                        max = m;
+                    }
+                } catch (RuntimeException ignored) {
+                }
+            }
+            if (found != null) {
+                InstanceBudget.learn(softwareSlot(mime), max > 0 ? max : Integer.MAX_VALUE);
+                Log.i(TAG, "Software decoder " + found + " (" + mime + "): up to " + max
+                        + " instances");
+            }
+            SOFTWARE.put(mime, found);
+            return found;
+        }
+    }
+
     static final class InstanceBudget {
         private static final Map<String, Integer> used = new HashMap<>();
         private static final Map<String, Integer> limit = new HashMap<>();
@@ -276,23 +342,39 @@ void createDecoder(boolean codecH265) {
         // Must hold the largest NAL unit the assembler can deliver
         format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, MAX_NAL_SIZE);
 
+        String softwareName = null;
         if (!InstanceBudget.tryAcquire(type)) {
             int limit = InstanceBudget.limit(type);
-            synchronized (decoderLock) {
-                retry.onFailure(nowMs());
+            softwareName = limit >= 0 && allowSoftware ? softwareDecoder(type) : null;
+            // Software decoders have their own limit: beyond it, the hardware one is reported
+            if (softwareName != null && !InstanceBudget.tryAcquire(softwareSlot(type))) {
+                softwareName = null;
+            }
+            if (softwareName == null) {
+                synchronized (decoderLock) {
+                    retry.onFailure(nowMs());
+                }
+                // Limit still unknown: another cell is creating the first decoder, just retry
+                if (limit < 0) return;
+                Log.w(TAG, "Decoder limit reached: " + limit + " × " + type);
+                if (listener != null) listener.onDecoderLimit(type, limit);
+                return;
             }
-            // Limit still unknown: another cell is creating the first decoder, just retry
-            if (limit < 0) return;
-            Log.w(TAG, "Decoder limit reached: " + limit + " × " + type);
-            if (listener != null) listener.onDecoderLimit(type, limit);
-            return;
         }
+        final boolean software = softwareName != null;
+        final String slot = software ? softwareSlot(type) : type;
 
         MediaCodec local;
         try {
-            Log.i(TAG, "Start video decoder (" + type + ")");
-            local = MediaCodec.createDecoderByType(type);
-            learnLimits(local, type);
+            if (software) {
+                Log.i(TAG, "Start software video decoder " + softwareName
+                        + " (hardware limit reached, key frames only)");
+                local = MediaCodec.createByCodecName(softwareName);
+            } else {
+                Log.i(TAG, "Start video decoder (" + type + ")");
+                local = MediaCodec.createDecoderByType(type);
+                learnLimits(local, type);
+            }
             try {
                 local.configure(format, s, null, 0);
                 local.start();
@@ -302,7 +384,7 @@ void createDecoder(boolean codecH265) {
                 throw e;
             }
         } catch (Exception e) {
-            InstanceBudget.release(type);
+            InstanceBudget.release(slot);
             synchronized (decoderLock) {
                 // A creation cancelled by a pause must not delay the next start
                 if (gen != lifecycle || !enabled) return;
@@ -316,12 +398,14 @@ void createDecoder(boolean codecH265) {
         synchronized (decoderLock) {
             // Disabled meanwhile (pause/stop) or another thread was faster
             if (decoder != null || !enabled || gen != lifecycle) {
-                InstanceBudget.release(type);
+                InstanceBudget.release(slot);
                 releaseAsync(local);
                 return;
             }
             decoder = local;
-            heldMime = type;
+            // A software decoder holds a slot of its own budget, not of the hardware one
+            heldMime = slot;
+            softwareInUse = software;
             awaitingConfig = true;
             configSeen = 0;
         }

android/app/src/main/java/com/openipc/decoder/QuadCell.java

@@ -141,6 +141,7 @@ void setAudioSink(AudioManager sink, int route) {
     }
 
     void start() {
+        decoder.setSoftwareFallback(keyFramesOnly);
         if (started) return;
         started = true;
         running = true;
@@ -300,15 +301,21 @@ String sample(long nowMs) {
         if (keyFramesOnly) {
             String rate = outAvg < 10 ? String.format(java.util.Locale.ROOT, "%.1f", outAvg)
                     : String.valueOf(Math.round(outAvg));
-            return size + codec + " key frames · " + rate + " fps";
+            return size + codec + " key frames · " + rate + " fps"
+                    + (decoder.isSoftware() ? " · software" : "");
         }
         return size + codec + " " + Math.round(outFps) + "/" + Math.round(inFps) + " fps";
     }
 
     boolean isKeyFramesOnly() { return keyFramesOnly; }
 
+    /** Decoded by a software decoder beyond the hardware limit: stays on key frames (#88). */
+    boolean isSoftwareDecoder() { return decoder.isSoftware(); }
+
     /** Full rate or key frames only (#85). Going to full rate waits for the next key frame. */
     void setKeyFramesOnly(boolean on) {
+        // Key frames only may run on a software decoder beyond the hardware limit (#88)
+        decoder.setSoftwareFallback(on);
         if (on == keyFramesOnly) return;
         if (!on) waitingKeyFrame = true;
         keyFramesOnly = on;

android/app/src/test/java/com/openipc/decoder/DecodePlannerTest.java

@@ -329,4 +329,30 @@ public void aDeclaredBudgetBelowTheFloorIsRaised() {
         p.offerDeclared(P1080_30 / 3);
         assertEquals(P1080_30, p.capacity());
     }
+
+    @Test
+    public void aCellOnASoftwareDecoderIsNeverPromoted() {
+        DecodePlanner p = new DecodePlanner(BUDGET, false);
+        List<DecodePlanner.Cell> cells = keyCells(2, P1080_30);
+        cells.get(0).pinned = true; // over the hardware limit (#88)
+        long t = 0;
+        for (int i = 0; i < 10; i++, t += 1000) apply(cells, p.step(cells, t));
+        assertTrue(!cells.get(0).full);
+        assertTrue("the next cell is not held back", cells.get(1).full);
+    }
+
+    @Test
+    public void aSoftwareCellNeitherLoadsNorPausesTheHardware() {
+        DecodePlanner p = new DecodePlanner(P1080_30, false, 0, P1080_30);
+        List<DecodePlanner.Cell> cells = keyCells(2, P1080_30);
+        DecodePlanner.Cell sw = cells.get(0);
+        sw.pinned = true;
+        sw.keyLoad = P1080_30 / 2;
+        sw.lossy = true; // the software decoder drops frames: not a hardware overload
+        long t = 0;
+        for (int i = 0; i < 20; i++, t += 1000) apply(cells, p.step(cells, t));
+        assertTrue("the hardware cell fits without the software load", cells.get(1).full);
+        assertTrue(!cells.get(0).paused && !cells.get(1).paused);
+        assertEquals(P1080_30, p.capacity());
+    }
 }