diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index 0373d096..c97230d9 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -198,9 +198,13 @@ public void onCodecChanged(boolean h265, AudioConfig audio) { audioCodec = audio.codec; boolean jpeg = rtspClient.isJpeg(); if (jpeg) { - // MJPEG is drawn by the app (#119): no MediaCodec may hold the surface + // MJPEG has its own decoder or is drawn by the app (#119, #132): the + // video decoder must not hold the surface closeDecoder(); - } else if (jpegDrawn) { + } else { + jpegCodec.release(); + } + if (!jpeg && jpegDrawn) { // A surface drawn with lockCanvas() cannot take a decoder any more: no // decoder until the UI thread has put a new one in place (Qodo on #131) jpegDrawn = false; @@ -1566,6 +1570,8 @@ private void resetZoom() { } private void replaceSurface(Surface next) { + // The MJPEG decoder renders into the old surface: let go of it first (#132) + jpegCodec.release(); Surface prev = mVideoSurface; mVideoSurface = next; videoDecoder.setSurface(next); @@ -1582,6 +1588,10 @@ private int dp(float dp) { // ====================== MJPEG (#119) ====================== + /** Hardware MJPEG decoding (#132), unless it failed once in this run. */ + private final JpegCodec jpegCodec = new JpegCodec(); + private volatile boolean hwJpegFailed; + /** Something was drawn into the video surface with lockCanvas(). */ private volatile boolean jpegDrawn; /** A new video surface is being made after MJPEG: no decoder until it is there. */ @@ -1597,9 +1607,28 @@ private int dp(float dp) { * with BitmapFactory (into the previous bitmap when the size is the same) and drawn over * the whole surface; the view's fit transform keeps the aspect ratio, as for video. */ - private void drawJpeg(Frame frame) { + private void drawJpeg(Frame frame, int gen) { Surface s = mVideoSurface; - if (s == null || !s.isValid()) return; + if (s == null || !s.isValid() || gen != listenerGen) return; + // The hardware JPEG decoder first, where there is one (#132) + if (!hwJpegFailed && !jpegDrawn && JpegCodec.candidate() != null) { + int[] size = JpegAssembler.size(frame.data(), frame.length()); + if (size != null) { + try { + if (jpegCodec.decode(frame, s, size[0], size[1], + () -> gen == listenerGen && s == mVideoSurface)) { + jpegShown(size[0], size[1]); + } + return; + } catch (Exception e) { + jpegCodec.release(); + // The surface was replaced meanwhile: not the decoder's fault, try again + if (s != mVideoSurface || gen != listenerGen) return; + Log.w(TAG, "Hardware MJPEG decoder failed, decoding in software", e); + hwJpegFailed = true; + } + } + } BitmapFactory.Options o = new BitmapFactory.Options(); o.inMutable = true; Bitmap reuse = jpegBitmap; @@ -1629,6 +1658,11 @@ private void drawJpeg(Frame frame) { return; } jpegDrawn = true; + jpegShown(w, h); + } + + /** A JPEG frame is on screen (hardware or software path). */ + private void jpegShown(int w, int h) { jpegFramesOut++; boolean first = lastJpegFrameMs == 0; lastJpegFrameMs = SystemClock.elapsedRealtime(); @@ -1671,7 +1705,7 @@ private void startListener() { try { Frame buffer = nalQueue.poll(5, TimeUnit.MILLISECONDS); if (buffer != null && rtspClient.isJpeg()) { - drawJpeg(buffer); + drawJpeg(buffer, gen); framePool.recycle(buffer); } else if (buffer != null && surfacePending) { // Waiting for a fresh surface after MJPEG; the next key frame follows @@ -1770,6 +1804,9 @@ private void stopStreams() { // for on the UI thread (#32: this used to block it for seconds) audioManager.setEnabled(false); stopExecutor(); + // After the worker's generation is gone: a frame still in flight sees that and can no + // longer create the JPEG decoder again (Qodo on #135) + jpegCodec.release(); nalQueue.clear(); } diff --git a/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java b/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java index 3318f974..9c3ce706 100644 --- a/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java +++ b/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java @@ -220,6 +220,27 @@ private void drop(String reason) { } } + /** {width, height} from the SOF0 header of a JPEG, or null (#132). */ + static int[] size(byte[] d, int length) { + // Walk the marker segments by their lengths: table bytes may contain FF C0 (Qodo on #135) + int i = 2; + while (i + 4 <= length) { + if ((d[i] & 0xFF) != 0xFF) return null; + int marker = d[i + 1] & 0xFF; + if (marker == 0xDA || marker == 0xD9) return null; // scan or end before any SOF + int len = ((d[i + 2] & 0xFF) << 8) | (d[i + 3] & 0xFF); + if (len < 2) return null; + if (marker == 0xC0) { + if (i + 9 > length) return null; + int h = ((d[i + 5] & 0xFF) << 8) | (d[i + 6] & 0xFF); + int w = ((d[i + 7] & 0xFF) << 8) | (d[i + 8] & 0xFF); + return new int[] {w, h}; + } + i += 2 + len; + } + return null; + } + /** Quantization tables for Q 1..99 (RFC 2435 Appendix A), luminance then chrominance. */ static byte[] tables(int q) { q = Math.max(1, Math.min(99, q)); diff --git a/android/app/src/main/java/com/openipc/decoder/JpegCodec.java b/android/app/src/main/java/com/openipc/decoder/JpegCodec.java new file mode 100644 index 00000000..68e9ee5e --- /dev/null +++ b/android/app/src/main/java/com/openipc/decoder/JpegCodec.java @@ -0,0 +1,188 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * JpegCodec.java — hardware MJPEG decoding through MediaCodec, where the device has it (#132) + * + */ + +package com.openipc.decoder; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaCodecList; +import android.media.MediaFormat; +import android.util.Log; +import android.view.Surface; + +import java.nio.ByteBuffer; + +/** + * A JPEG decoder from the device's codec list ({@code video/jpeg} on Allwinner boxes such as + * the AG-216, {@code video/mjpeg} elsewhere) fed with the whole frames of JpegAssembler and + * rendering to the video surface like the H.264/H.265 decoder. The caller falls back to + * BitmapFactory when there is none or it fails. Used by one decode thread; release() may come + * from others, hence synchronized. + */ +final class JpegCodec { + private static final String TAG = "OpenIPCDecoder"; + private static final String[] MIMES = {"video/jpeg", "video/mjpeg", "video/x-motion-jpeg"}; + private static volatile String[] found; + /** + * Vendor JPEG decoders a platform may leave out of its codec list although it can create + * them by name: the AG-216 declares this one in /vendor/etc/media_codecs.xml, but Android + * 10 does not list video/jpeg (#132). Tried once when the list has none. + */ + private static final String[][] BY_NAME = {{"OMX.allwinner.video.decoder.mjpeg", "video/jpeg"}}; + private static volatile boolean byNameTried; + /** The decoder that worked, once one did. */ + private static volatile String[] chosen; + + private MediaCodec codec; + private Surface surface; + private int width; + private int height; + private final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + + /** {name, mime} of a hardware-or-vendor JPEG decoder, or null; looked up once. */ + static String[] find() { + String[] f = found; + if (f != null) return f.length == 0 ? null : f; + String[] result = new String[0]; + try { + for (MediaCodecInfo ci : new MediaCodecList(MediaCodecList.ALL_CODECS) + .getCodecInfos()) { + if (ci.isEncoder()) continue; + for (String type : ci.getSupportedTypes()) { + for (String m : MIMES) { + if (m.equalsIgnoreCase(type) && result.length == 0) { + result = new String[] {ci.getName(), type}; + } + } + } + } + } catch (RuntimeException e) { + Log.w(TAG, "Codec list: " + e.getMessage()); + } + found = result; + if (result.length > 0) { + Log.i(TAG, "MJPEG decoder: " + result[0] + " (" + result[1] + ")"); + } else { + // Which video decoders the codec list offers instead (the vendor list may name a + // JPEG decoder that the platform does not expose) + StringBuilder all = new StringBuilder(); + try { + for (MediaCodecInfo ci : new MediaCodecList(MediaCodecList.ALL_CODECS) + .getCodecInfos()) { + if (ci.isEncoder()) continue; + for (String type : ci.getSupportedTypes()) { + if (type.startsWith("video/")) all.append(' ').append(type); + } + } + } catch (RuntimeException ignored) { + } + Log.i(TAG, "No MJPEG decoder; video decoders:" + all); + } + return result.length == 0 ? null : result; + } + + /** + * Decodes one JPEG of {@code w}×{@code h} to {@code target}; (re)creates the decoder when + * the size or surface changes. Returns true when a frame was rendered. Throws when the + * decoder cannot be used: the caller then switches to the software path. + */ + /** A listed decoder, or one of BY_NAME once; null when there is none at all. */ + static String[] candidate() { + if (chosen != null) return chosen; + String[] dec = find(); + if (dec != null) return dec; + return byNameTried ? null : BY_NAME[0]; + } + + /** Frames queued in a row without any output: the decoder is not working (Qodo on #135). */ + static final int MAX_SILENT_FRAMES = 15; + private int silentFrames; + + /** Checked under the lock: false once the stream that called decode() has stopped. */ + interface Current { + boolean holds(); + } + + synchronized boolean decode(Frame frame, Surface target, int w, int h, Current current) + throws Exception { + if (!current.holds()) return false; + String[] dec = chosen != null ? chosen : find(); + if (dec == null) { + if (byNameTried) throw new IllegalStateException("no JPEG decoder"); + byNameTried = true; + dec = BY_NAME[0]; + } + if (codec == null || target != surface || w != width || h != height) { + releaseLocked(); + MediaCodec c = MediaCodec.createByCodecName(dec[0]); + try { + MediaFormat format = MediaFormat.createVideoFormat(dec[1], w, h); + format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, Math.max(frame.length() * 2, + 512 * 1024)); + c.configure(format, target, null, 0); + c.start(); + } catch (Exception e) { + c.release(); + throw e; + } + codec = c; + chosen = dec; + silentFrames = 0; + surface = target; + width = w; + height = h; + Log.i(TAG, "Start MJPEG decoder " + dec[0] + " " + w + "x" + h); + } + int in = codec.dequeueInputBuffer(40_000); + if (in >= 0) { + ByteBuffer b = codec.getInputBuffer(in); + if (b == null || b.capacity() < frame.length()) { + // A frame larger than the decoder's input buffers: let the software path have it + codec.queueInputBuffer(in, 0, 0, 0, 0); + throw new IllegalStateException("JPEG frame of " + frame.length() + + " bytes does not fit the input buffer"); + } + b.clear(); + b.put(frame.data(), 0, frame.length()); + codec.queueInputBuffer(in, 0, frame.length(), System.nanoTime() / 1000, 0); + } + boolean rendered = false; + int out; + while ((out = codec.dequeueOutputBuffer(info, in >= 0 ? 20_000 : 0)) >= 0 + || out == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + if (out >= 0) { + codec.releaseOutputBuffer(out, true); + rendered = true; + } + in = -1; // after the first output, only take what is ready + } + silentFrames = rendered ? 0 : silentFrames + 1; + if (silentFrames > MAX_SILENT_FRAMES) { + throw new IllegalStateException(MAX_SILENT_FRAMES + " frames without output"); + } + return rendered; + } + + synchronized void release() { + releaseLocked(); + } + + private void releaseLocked() { + MediaCodec c = codec; + codec = null; + surface = null; + if (c == null) return; + try { + c.stop(); + } catch (Exception ignored) { + } + try { + c.release(); + } catch (Exception ignored) { + } + } +} diff --git a/android/app/src/test/java/com/openipc/decoder/JpegAssemblerTest.java b/android/app/src/test/java/com/openipc/decoder/JpegAssemblerTest.java index c465f50c..5e38053f 100644 --- a/android/app/src/test/java/com/openipc/decoder/JpegAssemblerTest.java +++ b/android/app/src/test/java/com/openipc/decoder/JpegAssemblerTest.java @@ -170,4 +170,26 @@ public void sixteenBitTables() { assertEquals(3 + 128, ((h[dqt + 2] & 0xFF) << 8) | (h[dqt + 3] & 0xFF)); assertEquals(0x10, h[dqt + 4] & 0xFF); } + + @Test + public void sizeFromTheFrameHeader() throws Exception { + byte[] jpeg = resource("jpeg_q50_64x32.jpg"); + int[] size = JpegAssembler.size(jpeg, jpeg.length); + assertNotNull(size); + assertEquals(64, size[0]); + assertEquals(32, size[1]); + } + + @Test + public void sizeSkipsTableBytesThatLookLikeSof() { + // A DQT whose table contains FF C0 before the real SOF0 (64x32) + byte[] tables = new byte[128]; + tables[10] = (byte) 0xFF; + tables[11] = (byte) 0xC0; + byte[] h = JpegAssembler.headers(0, 64, 32, tables, 0); + int[] size = JpegAssembler.size(h, h.length); + assertNotNull(size); + assertEquals(64, size[0]); + assertEquals(32, size[1]); + } }