feat: hardware MJPEG decoding through MediaCodec where available (#132, single view)

android/app/src/main/java/com/openipc/decoder/Decoder.java

@@ -198,9 +198,13 @@ public void onCodecChanged(boolean h265, AudioConfig audio) {
                 audioCodec = audio.codec;
                 boolean jpeg = rtspClient.isJpeg();
                 if (jpeg) {
-                    // MJPEG is drawn by the app (#119): no MediaCodec may hold the surface
+                    // MJPEG has its own decoder or is drawn by the app (#119, #132): the
+                    // video decoder must not hold the surface
                     closeDecoder();
-                } else if (jpegDrawn) {
+                } else {
+                    jpegCodec.release();
+                }
+                if (!jpeg && jpegDrawn) {
                     // A surface drawn with lockCanvas() cannot take a decoder any more: no
                     // decoder until the UI thread has put a new one in place (Qodo on #131)
                     jpegDrawn = false;
@@ -1566,6 +1570,8 @@ private void resetZoom() {
     }
 
     private void replaceSurface(Surface next) {
+        // The MJPEG decoder renders into the old surface: let go of it first (#132)
+        jpegCodec.release();
         Surface prev = mVideoSurface;
         mVideoSurface = next;
         videoDecoder.setSurface(next);
@@ -1582,6 +1588,10 @@ private int dp(float dp) {
 
     // ====================== MJPEG (#119) ======================
 
+    /** Hardware MJPEG decoding (#132), unless it failed once in this run. */
+    private final JpegCodec jpegCodec = new JpegCodec();
+    private volatile boolean hwJpegFailed;
+
     /** Something was drawn into the video surface with lockCanvas(). */
     private volatile boolean jpegDrawn;
     /** A new video surface is being made after MJPEG: no decoder until it is there. */
@@ -1597,9 +1607,28 @@ private int dp(float dp) {
      * with BitmapFactory (into the previous bitmap when the size is the same) and drawn over
      * the whole surface; the view's fit transform keeps the aspect ratio, as for video.
      */
-    private void drawJpeg(Frame frame) {
+    private void drawJpeg(Frame frame, int gen) {
         Surface s = mVideoSurface;
-        if (s == null || !s.isValid()) return;
+        if (s == null || !s.isValid() || gen != listenerGen) return;
+        // The hardware JPEG decoder first, where there is one (#132)
+        if (!hwJpegFailed && !jpegDrawn && JpegCodec.candidate() != null) {
+            int[] size = JpegAssembler.size(frame.data(), frame.length());
+            if (size != null) {
+                try {
+                    if (jpegCodec.decode(frame, s, size[0], size[1],
+                            () -> gen == listenerGen && s == mVideoSurface)) {
+                        jpegShown(size[0], size[1]);
+                    }
+                    return;
+                } catch (Exception e) {
+                    jpegCodec.release();
+                    // The surface was replaced meanwhile: not the decoder's fault, try again
+                    if (s != mVideoSurface || gen != listenerGen) return;
+                    Log.w(TAG, "Hardware MJPEG decoder failed, decoding in software", e);
+                    hwJpegFailed = true;
+                }
+            }
+        }
         BitmapFactory.Options o = new BitmapFactory.Options();
         o.inMutable = true;
         Bitmap reuse = jpegBitmap;
@@ -1629,6 +1658,11 @@ private void drawJpeg(Frame frame) {
             return;
         }
         jpegDrawn = true;
+        jpegShown(w, h);
+    }
+
+    /** A JPEG frame is on screen (hardware or software path). */
+    private void jpegShown(int w, int h) {
         jpegFramesOut++;
         boolean first = lastJpegFrameMs == 0;
         lastJpegFrameMs = SystemClock.elapsedRealtime();
@@ -1671,7 +1705,7 @@ private void startListener() {
                 try {
                     Frame buffer = nalQueue.poll(5, TimeUnit.MILLISECONDS);
                     if (buffer != null && rtspClient.isJpeg()) {
-                        drawJpeg(buffer);
+                        drawJpeg(buffer, gen);
                         framePool.recycle(buffer);
                     } else if (buffer != null && surfacePending) {
                         // Waiting for a fresh surface after MJPEG; the next key frame follows
@@ -1770,6 +1804,9 @@ private void stopStreams() {
         // for on the UI thread (#32: this used to block it for seconds)
         audioManager.setEnabled(false);
         stopExecutor();
+        // After the worker's generation is gone: a frame still in flight sees that and can no
+        // longer create the JPEG decoder again (Qodo on #135)
+        jpegCodec.release();
         nalQueue.clear();
     }
 

android/app/src/main/java/com/openipc/decoder/JpegAssembler.java

@@ -220,6 +220,27 @@ private void drop(String reason) {
         }
     }
 
+    /** {width, height} from the SOF0 header of a JPEG, or null (#132). */
+    static int[] size(byte[] d, int length) {
+        // Walk the marker segments by their lengths: table bytes may contain FF C0 (Qodo on #135)
+        int i = 2;
+        while (i + 4 <= length) {
+            if ((d[i] & 0xFF) != 0xFF) return null;
+            int marker = d[i + 1] & 0xFF;
+            if (marker == 0xDA || marker == 0xD9) return null; // scan or end before any SOF
+            int len = ((d[i + 2] & 0xFF) << 8) | (d[i + 3] & 0xFF);
+            if (len < 2) return null;
+            if (marker == 0xC0) {
+                if (i + 9 > length) return null;
+                int h = ((d[i + 5] & 0xFF) << 8) | (d[i + 6] & 0xFF);
+                int w = ((d[i + 7] & 0xFF) << 8) | (d[i + 8] & 0xFF);
+                return new int[] {w, h};
+            }
+            i += 2 + len;
+        }
+        return null;
+    }
+
     /** Quantization tables for Q 1..99 (RFC 2435 Appendix A), luminance then chrominance. */
     static byte[] tables(int q) {
         q = Math.max(1, Math.min(99, q));

android/app/src/main/java/com/openipc/decoder/JpegCodec.java

@@ -0,0 +1,188 @@
+/*
+ * Copyright (c) OpenIPC  https://openipc.org  Prosperity Public License 3.0.0
+ *
+ * JpegCodec.java — hardware MJPEG decoding through MediaCodec, where the device has it (#132)
+ *
+ */
+
+package com.openipc.decoder;
+
+import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
+import android.media.MediaCodecList;
+import android.media.MediaFormat;
+import android.util.Log;
+import android.view.Surface;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A JPEG decoder from the device's codec list ({@code video/jpeg} on Allwinner boxes such as
+ * the AG-216, {@code video/mjpeg} elsewhere) fed with the whole frames of JpegAssembler and
+ * rendering to the video surface like the H.264/H.265 decoder. The caller falls back to
+ * BitmapFactory when there is none or it fails. Used by one decode thread; release() may come
+ * from others, hence synchronized.
+ */
+final class JpegCodec {
+    private static final String TAG = "OpenIPCDecoder";
+    private static final String[] MIMES = {"video/jpeg", "video/mjpeg", "video/x-motion-jpeg"};
+    private static volatile String[] found;
+    /**
+     * Vendor JPEG decoders a platform may leave out of its codec list although it can create
+     * them by name: the AG-216 declares this one in /vendor/etc/media_codecs.xml, but Android
+     * 10 does not list video/jpeg (#132). Tried once when the list has none.
+     */
+    private static final String[][] BY_NAME = {{"OMX.allwinner.video.decoder.mjpeg", "video/jpeg"}};
+    private static volatile boolean byNameTried;
+    /** The decoder that worked, once one did. */
+    private static volatile String[] chosen;
+
+    private MediaCodec codec;
+    private Surface surface;
+    private int width;
+    private int height;
+    private final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
+
+    /** {name, mime} of a hardware-or-vendor JPEG decoder, or null; looked up once. */
+    static String[] find() {
+        String[] f = found;
+        if (f != null) return f.length == 0 ? null : f;
+        String[] result = new String[0];
+        try {
+            for (MediaCodecInfo ci : new MediaCodecList(MediaCodecList.ALL_CODECS)
+                    .getCodecInfos()) {
+                if (ci.isEncoder()) continue;
+                for (String type : ci.getSupportedTypes()) {
+                    for (String m : MIMES) {
+                        if (m.equalsIgnoreCase(type) && result.length == 0) {
+                            result = new String[] {ci.getName(), type};
+                        }
+                    }
+                }
+            }
+        } catch (RuntimeException e) {
+            Log.w(TAG, "Codec list: " + e.getMessage());
+        }
+        found = result;
+        if (result.length > 0) {
+            Log.i(TAG, "MJPEG decoder: " + result[0] + " (" + result[1] + ")");
+        } else {
+            // Which video decoders the codec list offers instead (the vendor list may name a
+            // JPEG decoder that the platform does not expose)
+            StringBuilder all = new StringBuilder();
+            try {
+                for (MediaCodecInfo ci : new MediaCodecList(MediaCodecList.ALL_CODECS)
+                        .getCodecInfos()) {
+                    if (ci.isEncoder()) continue;
+                    for (String type : ci.getSupportedTypes()) {
+                        if (type.startsWith("video/")) all.append(' ').append(type);
+                    }
+                }
+            } catch (RuntimeException ignored) {
+            }
+            Log.i(TAG, "No MJPEG decoder; video decoders:" + all);
+        }
+        return result.length == 0 ? null : result;
+    }
+
+    /**
+     * Decodes one JPEG of {@code w}×{@code h} to {@code target}; (re)creates the decoder when
+     * the size or surface changes. Returns true when a frame was rendered. Throws when the
+     * decoder cannot be used: the caller then switches to the software path.
+     */
+    /** A listed decoder, or one of BY_NAME once; null when there is none at all. */
+    static String[] candidate() {
+        if (chosen != null) return chosen;
+        String[] dec = find();
+        if (dec != null) return dec;
+        return byNameTried ? null : BY_NAME[0];
+    }
+
+    /** Frames queued in a row without any output: the decoder is not working (Qodo on #135). */
+    static final int MAX_SILENT_FRAMES = 15;
+    private int silentFrames;
+
+    /** Checked under the lock: false once the stream that called decode() has stopped. */
+    interface Current {
+        boolean holds();
+    }
+
+    synchronized boolean decode(Frame frame, Surface target, int w, int h, Current current)
+            throws Exception {
+        if (!current.holds()) return false;
+        String[] dec = chosen != null ? chosen : find();
+        if (dec == null) {
+            if (byNameTried) throw new IllegalStateException("no JPEG decoder");
+            byNameTried = true;
+            dec = BY_NAME[0];
+        }
+        if (codec == null || target != surface || w != width || h != height) {
+            releaseLocked();
+            MediaCodec c = MediaCodec.createByCodecName(dec[0]);
+            try {
+                MediaFormat format = MediaFormat.createVideoFormat(dec[1], w, h);
+                format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, Math.max(frame.length() * 2,
+                        512 * 1024));
+                c.configure(format, target, null, 0);
+                c.start();
+            } catch (Exception e) {
+                c.release();
+                throw e;
+            }
+            codec = c;
+            chosen = dec;
+            silentFrames = 0;
+            surface = target;
+            width = w;
+            height = h;
+            Log.i(TAG, "Start MJPEG decoder " + dec[0] + " " + w + "x" + h);
+        }
+        int in = codec.dequeueInputBuffer(40_000);
+        if (in >= 0) {
+            ByteBuffer b = codec.getInputBuffer(in);
+            if (b == null || b.capacity() < frame.length()) {
+                // A frame larger than the decoder's input buffers: let the software path have it
+                codec.queueInputBuffer(in, 0, 0, 0, 0);
+                throw new IllegalStateException("JPEG frame of " + frame.length()
+                        + " bytes does not fit the input buffer");
+            }
+            b.clear();
+            b.put(frame.data(), 0, frame.length());
+            codec.queueInputBuffer(in, 0, frame.length(), System.nanoTime() / 1000, 0);
+        }
+        boolean rendered = false;
+        int out;
+        while ((out = codec.dequeueOutputBuffer(info, in >= 0 ? 20_000 : 0)) >= 0
+                || out == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
+            if (out >= 0) {
+                codec.releaseOutputBuffer(out, true);
+                rendered = true;
+            }
+            in = -1; // after the first output, only take what is ready
+        }
+        silentFrames = rendered ? 0 : silentFrames + 1;
+        if (silentFrames > MAX_SILENT_FRAMES) {
+            throw new IllegalStateException(MAX_SILENT_FRAMES + " frames without output");
+        }
+        return rendered;
+    }
+
+    synchronized void release() {
+        releaseLocked();
+    }
+
+    private void releaseLocked() {
+        MediaCodec c = codec;
+        codec = null;
+        surface = null;
+        if (c == null) return;
+        try {
+            c.stop();
+        } catch (Exception ignored) {
+        }
+        try {
+            c.release();
+        } catch (Exception ignored) {
+        }
+    }
+}

android/app/src/test/java/com/openipc/decoder/JpegAssemblerTest.java

@@ -170,4 +170,26 @@ public void sixteenBitTables() {
         assertEquals(3 + 128, ((h[dqt + 2] & 0xFF) << 8) | (h[dqt + 3] & 0xFF));
         assertEquals(0x10, h[dqt + 4] & 0xFF);
     }
+
+    @Test
+    public void sizeFromTheFrameHeader() throws Exception {
+        byte[] jpeg = resource("jpeg_q50_64x32.jpg");
+        int[] size = JpegAssembler.size(jpeg, jpeg.length);
+        assertNotNull(size);
+        assertEquals(64, size[0]);
+        assertEquals(32, size[1]);
+    }
+
+    @Test
+    public void sizeSkipsTableBytesThatLookLikeSof() {
+        // A DQT whose table contains FF C0 before the real SOF0 (64x32)
+        byte[] tables = new byte[128];
+        tables[10] = (byte) 0xFF;
+        tables[11] = (byte) 0xC0;
+        byte[] h = JpegAssembler.headers(0, 64, 32, tables, 0);
+        int[] size = JpegAssembler.size(h, h.length);
+        assertNotNull(size);
+        assertEquals(64, size[0]);
+        assertEquals(32, size[1]);
+    }
 }