feat: MJPEG cells in multi-view (#132)

android/app/src/main/java/com/openipc/decoder/Decoder.java

@@ -15,12 +15,8 @@
 import android.content.SharedPreferences;
 import android.content.pm.PackageManager;
 import android.graphics.Bitmap;
-import android.graphics.BitmapFactory;
-import android.graphics.Canvas;
 import android.graphics.Color;
 import android.graphics.Matrix;
-import android.graphics.Paint;
-import android.graphics.Rect;
 import android.net.Uri;
 import android.os.Build;
 import android.os.Bundle;
@@ -204,12 +200,11 @@ public void onCodecChanged(boolean h265, AudioConfig audio) {
                     // video decoder must not hold the surface
                     closeDecoder();
                 } else {
-                    jpegCodec.release();
+                    jpegRenderer.release();
                 }
-                if (!jpeg && jpegDrawn) {
+                if (!jpeg && jpegRenderer.takeCanvasDrawn()) {
                     // A surface drawn with lockCanvas() cannot take a decoder any more: no
                     // decoder until the UI thread has put a new one in place (Qodo on #131)
-                    jpegDrawn = false;
                     surfacePending = true;
                     runOnUiThread(() -> {
                         recreateVideoSurface();
@@ -1577,7 +1572,7 @@ private void resetZoom() {
 
     private void replaceSurface(Surface next) {
         // The MJPEG decoder renders into the old surface: let go of it first (#132)
-        jpegCodec.release();
+        jpegRenderer.release();
         Surface prev = mVideoSurface;
         mVideoSurface = next;
         videoDecoder.setSurface(next);
@@ -1594,77 +1589,22 @@ private int dp(float dp) {
 
     // ====================== MJPEG (#119) ======================
 
-    /** Hardware MJPEG decoding (#132), unless it failed once in this run. */
-    private final JpegCodec jpegCodec = new JpegCodec();
-    private volatile boolean hwJpegFailed;
-
-    /** Something was drawn into the video surface with lockCanvas(). */
-    private volatile boolean jpegDrawn;
+    /** MJPEG frames onto the video surface: hardware decoder or BitmapFactory (#119, #132). */
+    private final JpegRenderer jpegRenderer = new JpegRenderer();
     /** A new video surface is being made after MJPEG: no decoder until it is there. */
     private volatile boolean surfacePending;
     /** JPEG frames drawn, the MJPEG counterpart of the decoder's output count. */
     private volatile int jpegFramesOut;
     private volatile long lastJpegFrameMs;
-    private Bitmap jpegBitmap;
-    private final Paint jpegPaint = new Paint(Paint.FILTER_BITMAP_FLAG);
 
     /**
-     * One JPEG frame of an MJPEG stream into the single view, on the decode thread: decoded
-     * with BitmapFactory (into the previous bitmap when the size is the same) and drawn over
+     * One JPEG frame of an MJPEG stream into the single view, on the decode thread, drawn over
      * the whole surface; the view's fit transform keeps the aspect ratio, as for video.
      */
     private void drawJpeg(Frame frame, int gen) {
         Surface s = mVideoSurface;
-        if (s == null || !s.isValid() || gen != listenerGen) return;
-        // The hardware JPEG decoder first, where there is one (#132)
-        if (!hwJpegFailed && !jpegDrawn && JpegCodec.candidate() != null) {
-            int[] size = JpegAssembler.size(frame.data(), frame.length());
-            if (size != null) {
-                try {
-                    if (jpegCodec.decode(frame, s, size[0], size[1],
-                            () -> gen == listenerGen && s == mVideoSurface)) {
-                        jpegShown(size[0], size[1]);
-                    }
-                    return;
-                } catch (Exception e) {
-                    jpegCodec.release();
-                    // The surface was replaced meanwhile: not the decoder's fault, try again
-                    if (s != mVideoSurface || gen != listenerGen) return;
-                    Log.w(TAG, "Hardware MJPEG decoder failed, decoding in software", e);
-                    hwJpegFailed = true;
-                }
-            }
-        }
-        BitmapFactory.Options o = new BitmapFactory.Options();
-        o.inMutable = true;
-        Bitmap reuse = jpegBitmap;
-        if (reuse != null) o.inBitmap = reuse;
-        Bitmap b;
-        try {
-            b = BitmapFactory.decodeByteArray(frame.data(), 0, frame.length(), o);
-        } catch (IllegalArgumentException e) {
-            // Another size than the reusable bitmap
-            o.inBitmap = null;
-            b = BitmapFactory.decodeByteArray(frame.data(), 0, frame.length(), o);
-        }
-        if (b == null) return;
-        jpegBitmap = b;
-        final int w = b.getWidth();
-        final int h = b.getHeight();
-        try {
-            Canvas c = s.lockCanvas(null);
-            try {
-                c.drawColor(Color.BLACK);
-                c.drawBitmap(b, null, new Rect(0, 0, c.getWidth(), c.getHeight()), jpegPaint);
-            } finally {
-                s.unlockCanvasAndPost(c);
-            }
-        } catch (RuntimeException e) {
-            // The surface went away meanwhile (view or activity change)
-            return;
-        }
-        jpegDrawn = true;
-        jpegShown(w, h);
+        int[] size = jpegRenderer.render(frame, s, () -> gen == listenerGen && s == mVideoSurface);
+        if (size != null) jpegShown(size[0], size[1]);
     }
 
     /** A JPEG frame is on screen (hardware or software path). */
@@ -1812,7 +1752,7 @@ private void stopStreams() {
         stopExecutor();
         // After the worker's generation is gone: a frame still in flight sees that and can no
         // longer create the JPEG decoder again (Qodo on #135)
-        jpegCodec.release();
+        jpegRenderer.release();
         nalQueue.clear();
     }
 

android/app/src/main/java/com/openipc/decoder/JpegAssembler.java

@@ -92,10 +92,13 @@ interface Sink {
         this.pool = pool;
     }
 
+    /** A new stream: its frames must not use the tables of the previous one (Qodo on #139). */
     void reset() {
         expectedOffset = -1;
         scanLength = 0;
         header = null;
+        tablesByQ.clear();
+        precisionByQ.clear();
     }
 
     /** Frames dropped because a fragment was missing or malformed. */

android/app/src/main/java/com/openipc/decoder/JpegCodec.java

@@ -33,7 +33,8 @@ final class JpegCodec {
      * 10 does not list video/jpeg (#132). Tried once when the list has none.
      */
     private static final String[][] BY_NAME = {{"OMX.allwinner.video.decoder.mjpeg", "video/jpeg"}};
-    private static volatile boolean byNameTried;
+    /** The BY_NAME decoder could not be created: no hardware JPEG decoder on this device. */
+    private static volatile boolean byNameFailed;
     /** The decoder that worked, once one did. */
     private static volatile String[] chosen;
 
@@ -51,7 +52,7 @@ static String[] find() {
         try {
             for (MediaCodecInfo ci : new MediaCodecList(MediaCodecList.ALL_CODECS)
                     .getCodecInfos()) {
-                if (ci.isEncoder()) continue;
+                if (ci.isEncoder() || softwareOnly(ci)) continue;
                 for (String type : ci.getSupportedTypes()) {
                     for (String m : MIMES) {
                         if (m.equalsIgnoreCase(type) && result.length == 0) {
@@ -85,6 +86,16 @@ static String[] find() {
         return result.length == 0 ? null : result;
     }
 
+    /**
+     * A software codec does no better than BitmapFactory and its load is not the hardware's:
+     * the software path takes the stream instead (Qodo on #139).
+     */
+    private static boolean softwareOnly(MediaCodecInfo ci) {
+        if (android.os.Build.VERSION.SDK_INT >= 29) return ci.isSoftwareOnly();
+        String n = ci.getName().toLowerCase(java.util.Locale.ROOT);
+        return n.startsWith("omx.google.") || n.startsWith("c2.android.");
+    }
+
     /**
      * Decodes one JPEG of {@code w}×{@code h} to {@code target}; (re)creates the decoder when
      * the size or surface changes. Returns true when a frame was rendered. Throws when the
@@ -95,12 +106,17 @@ static String[] candidate() {
         if (chosen != null) return chosen;
         String[] dec = find();
         if (dec != null) return dec;
-        return byNameTried ? null : BY_NAME[0];
+        // Also while another stream is still trying it (Qodo on #139)
+        return byNameFailed ? null : BY_NAME[0];
     }
 
     /** Frames queued in a row without any output: the decoder is not working (Qodo on #135). */
     static final int MAX_SILENT_FRAMES = 15;
     private int silentFrames;
+    /** Frames the decoder had no input buffer for: decoder losses for the planner (#85). */
+    private volatile int inputDropped;
+
+    int inputDropped() { return inputDropped; }
 
     /** Checked under the lock: false once the stream that called decode() has stopped. */
     interface Current {
@@ -110,27 +126,40 @@ interface Current {
     synchronized boolean decode(Frame frame, Surface target, int w, int h, Current current)
             throws Exception {
         if (!current.holds()) return false;
-        String[] dec = chosen != null ? chosen : find();
-        if (dec == null) {
-            if (byNameTried) throw new IllegalStateException("no JPEG decoder");
-            byNameTried = true;
-            dec = BY_NAME[0];
-        }
         if (codec == null || target != surface || w != width || h != height) {
             releaseLocked();
-            MediaCodec c = MediaCodec.createByCodecName(dec[0]);
-            try {
-                MediaFormat format = MediaFormat.createVideoFormat(dec[1], w, h);
-                format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, Math.max(frame.length() * 2,
-                        512 * 1024));
-                c.configure(format, target, null, 0);
-                c.start();
-            } catch (Exception e) {
-                c.release();
-                throw e;
+            MediaCodec c;
+            String[] dec;
+            // One stream at a time chooses the decoder, so the cells that start together do
+            // not take a probe in progress for "no decoder" (Qodo on #139)
+            synchronized (JpegCodec.class) {
+                dec = chosen != null ? chosen : find();
+                boolean byName = dec == null;
+                if (byName) {
+                    if (byNameFailed) throw new IllegalStateException("no JPEG decoder");
+                    dec = BY_NAME[0];
+                }
+                try {
+                    c = MediaCodec.createByCodecName(dec[0]);
+                } catch (Exception e) {
+                    if (byName) byNameFailed = true;
+                    throw e;
+                }
+                try {
+                    MediaFormat format = MediaFormat.createVideoFormat(dec[1], w, h);
+                    format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE,
+                            Math.max(frame.length() * 2, 512 * 1024));
+                    c.configure(format, target, null, 0);
+                    c.start();
+                } catch (Exception e) {
+                    c.release();
+                    // Never worked on this device: the name is not a usable decoder
+                    if (byName && chosen == null) byNameFailed = true;
+                    throw e;
+                }
+                chosen = dec;
             }
             codec = c;
-            chosen = dec;
             silentFrames = 0;
             surface = target;
             width = w;
@@ -149,6 +178,8 @@ synchronized boolean decode(Frame frame, Surface target, int w, int h, Current c
             b.clear();
             b.put(frame.data(), 0, frame.length());
             codec.queueInputBuffer(in, 0, frame.length(), System.nanoTime() / 1000, 0);
+        } else {
+            inputDropped++;
         }
         boolean rendered = false;
         int out;

android/app/src/main/java/com/openipc/decoder/JpegRenderer.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) OpenIPC  https://openipc.org  Prosperity Public License 3.0.0
+ *
+ * JpegRenderer.java — one MJPEG stream onto a surface: hardware decoder first, else BitmapFactory
+ *
+ */
+
+package com.openipc.decoder;
+
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Paint;
+import android.graphics.Rect;
+import android.util.Log;
+import android.view.Surface;
+
+/**
+ * Shows the JPEG frames of one MJPEG stream on a surface, for the single view (#119) and for a
+ * multi-view cell (#132): through the hardware JPEG decoder where the device has one, otherwise
+ * decoded with BitmapFactory (into the previous bitmap when the size is the same) and drawn over
+ * the whole surface with lockCanvas(). A surface drawn that way cannot take a decoder any more:
+ * the owner makes a new one when the stream changes to H.264/H.265 ({@link #drewWithCanvas}).
+ * Used by one decode thread; release() may come from others.
+ */
+final class JpegRenderer {
+    private static final String TAG = "OpenIPCDecoder";
+
+    /**
+     * This stream's hardware decoder failed: the software path from then on. Per renderer, so
+     * one cell at a decoder limit does not send the others to lockCanvas() (Qodo on #139).
+     */
+    private volatile boolean hwFailed;
+    /** The last frame was decoded with BitmapFactory. */
+    private volatile boolean software;
+
+    private final JpegCodec codec = new JpegCodec();
+    private final Paint paint = new Paint(Paint.FILTER_BITMAP_FLAG);
+    private volatile boolean drawn;
+    private Bitmap bitmap;
+
+    /**
+     * Shows one frame on {@code s}. Returns its {width, height} when it is on screen, null when
+     * not (yet). {@code current} is false once the stream or the surface has been replaced.
+     */
+    int[] render(Frame frame, Surface s, JpegCodec.Current current) {
+        if (s == null || !s.isValid() || !current.holds()) return null;
+        // The hardware JPEG decoder first, where there is one (#132)
+        if (!hwFailed && !drawn && JpegCodec.candidate() != null) {
+            int[] size = JpegAssembler.size(frame.data(), frame.length());
+            if (size != null) {
+                try {
+                    software = false;
+                    return codec.decode(frame, s, size[0], size[1], current) ? size : null;
+                } catch (Exception e) {
+                    codec.release();
+                    // The surface was replaced meanwhile: not the decoder's fault, try again
+                    if (!current.holds()) return null;
+                    Log.w(TAG, "Hardware MJPEG decoder failed, decoding in software", e);
+                    hwFailed = true;
+                }
+            }
+        }
+        // A decoder still connected to the surface would make lockCanvas() fail
+        codec.release();
+        software = true;
+        BitmapFactory.Options o = new BitmapFactory.Options();
+        o.inMutable = true;
+        Bitmap reuse = bitmap;
+        if (reuse != null) o.inBitmap = reuse;
+        Bitmap b;
+        try {
+            b = BitmapFactory.decodeByteArray(frame.data(), 0, frame.length(), o);
+        } catch (IllegalArgumentException e) {
+            // Another size than the reusable bitmap
+            o.inBitmap = null;
+            b = BitmapFactory.decodeByteArray(frame.data(), 0, frame.length(), o);
+        }
+        if (b == null) return null;
+        bitmap = b;
+        try {
+            Canvas c = s.lockCanvas(null);
+            try {
+                c.drawColor(Color.BLACK);
+                c.drawBitmap(b, null, new Rect(0, 0, c.getWidth(), c.getHeight()), paint);
+            } finally {
+                s.unlockCanvasAndPost(c);
+            }
+        } catch (RuntimeException e) {
+            // The surface went away meanwhile (view or activity change)
+            return null;
+        }
+        drawn = true;
+        return new int[] {b.getWidth(), b.getHeight()};
+    }
+
+    /**
+     * Something was drawn with lockCanvas() since the last call: the surface cannot take a
+     * decoder any more. Clears the flag, for the owner who replaces the surface.
+     */
+    boolean takeCanvasDrawn() {
+        boolean d = drawn;
+        drawn = false;
+        return d;
+    }
+
+    /** The surface holds frames drawn with lockCanvas(). */
+    boolean drewWithCanvas() { return drawn; }
+
+    /** Frames are decoded with BitmapFactory: CPU load, not the hardware decoder's. */
+    boolean isSoftware() { return software; }
+
+    /** Frames the hardware decoder had no input buffer for. */
+    int dropped() { return codec.inputDropped(); }
+
+    /** Releases the hardware decoder (the stream stopped or is no MJPEG any more). */
+    void release() {
+        codec.release();
+    }
+}