From d3ea50cc47866cb1f81786a4c16c9444a2690ad3 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Tue, 13 Oct 2026 14:54:07 +0000 Subject: [PATCH 1/4] feat: MJPEG cells in multi-view, one frame per second in the cheap mode (#132) --- .../java/com/openipc/decoder/Decoder.java | 78 ++--------- .../com/openipc/decoder/JpegRenderer.java | 106 +++++++++++++++ .../java/com/openipc/decoder/QuadCell.java | 128 +++++++++++++++--- 3 files changed, 222 insertions(+), 90 deletions(-) create mode 100644 android/app/src/main/java/com/openipc/decoder/JpegRenderer.java diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index ffab4504..486f7cd0 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -15,12 +15,8 @@ import android.content.SharedPreferences; import android.content.pm.PackageManager; import android.graphics.Bitmap; -import android.graphics.BitmapFactory; -import android.graphics.Canvas; import android.graphics.Color; import android.graphics.Matrix; -import android.graphics.Paint; -import android.graphics.Rect; import android.net.Uri; import android.os.Build; import android.os.Bundle; @@ -204,12 +200,11 @@ public void onCodecChanged(boolean h265, AudioConfig audio) { // video decoder must not hold the surface closeDecoder(); } else { - jpegCodec.release(); + jpegRenderer.release(); } - if (!jpeg && jpegDrawn) { + if (!jpeg && jpegRenderer.takeCanvasDrawn()) { // A surface drawn with lockCanvas() cannot take a decoder any more: no // decoder until the UI thread has put a new one in place (Qodo on #131) - jpegDrawn = false; surfacePending = true; runOnUiThread(() -> { recreateVideoSurface(); @@ -1577,7 +1572,7 @@ private void resetZoom() { private void replaceSurface(Surface next) { // The MJPEG decoder renders into the old surface: let go of it first (#132) - jpegCodec.release(); + jpegRenderer.release(); Surface prev = mVideoSurface; mVideoSurface = next; videoDecoder.setSurface(next); @@ -1594,77 +1589,22 @@ private int dp(float dp) { // ====================== MJPEG (#119) ====================== - /** Hardware MJPEG decoding (#132), unless it failed once in this run. */ - private final JpegCodec jpegCodec = new JpegCodec(); - private volatile boolean hwJpegFailed; - - /** Something was drawn into the video surface with lockCanvas(). */ - private volatile boolean jpegDrawn; + /** MJPEG frames onto the video surface: hardware decoder or BitmapFactory (#119, #132). */ + private final JpegRenderer jpegRenderer = new JpegRenderer(); /** A new video surface is being made after MJPEG: no decoder until it is there. */ private volatile boolean surfacePending; /** JPEG frames drawn, the MJPEG counterpart of the decoder's output count. */ private volatile int jpegFramesOut; private volatile long lastJpegFrameMs; - private Bitmap jpegBitmap; - private final Paint jpegPaint = new Paint(Paint.FILTER_BITMAP_FLAG); /** - * One JPEG frame of an MJPEG stream into the single view, on the decode thread: decoded - * with BitmapFactory (into the previous bitmap when the size is the same) and drawn over + * One JPEG frame of an MJPEG stream into the single view, on the decode thread, drawn over * the whole surface; the view's fit transform keeps the aspect ratio, as for video. */ private void drawJpeg(Frame frame, int gen) { Surface s = mVideoSurface; - if (s == null || !s.isValid() || gen != listenerGen) return; - // The hardware JPEG decoder first, where there is one (#132) - if (!hwJpegFailed && !jpegDrawn && JpegCodec.candidate() != null) { - int[] size = JpegAssembler.size(frame.data(), frame.length()); - if (size != null) { - try { - if (jpegCodec.decode(frame, s, size[0], size[1], - () -> gen == listenerGen && s == mVideoSurface)) { - jpegShown(size[0], size[1]); - } - return; - } catch (Exception e) { - jpegCodec.release(); - // The surface was replaced meanwhile: not the decoder's fault, try again - if (s != mVideoSurface || gen != listenerGen) return; - Log.w(TAG, "Hardware MJPEG decoder failed, decoding in software", e); - hwJpegFailed = true; - } - } - } - BitmapFactory.Options o = new BitmapFactory.Options(); - o.inMutable = true; - Bitmap reuse = jpegBitmap; - if (reuse != null) o.inBitmap = reuse; - Bitmap b; - try { - b = BitmapFactory.decodeByteArray(frame.data(), 0, frame.length(), o); - } catch (IllegalArgumentException e) { - // Another size than the reusable bitmap - o.inBitmap = null; - b = BitmapFactory.decodeByteArray(frame.data(), 0, frame.length(), o); - } - if (b == null) return; - jpegBitmap = b; - final int w = b.getWidth(); - final int h = b.getHeight(); - try { - Canvas c = s.lockCanvas(null); - try { - c.drawColor(Color.BLACK); - c.drawBitmap(b, null, new Rect(0, 0, c.getWidth(), c.getHeight()), jpegPaint); - } finally { - s.unlockCanvasAndPost(c); - } - } catch (RuntimeException e) { - // The surface went away meanwhile (view or activity change) - return; - } - jpegDrawn = true; - jpegShown(w, h); + int[] size = jpegRenderer.render(frame, s, () -> gen == listenerGen && s == mVideoSurface); + if (size != null) jpegShown(size[0], size[1]); } /** A JPEG frame is on screen (hardware or software path). */ @@ -1812,7 +1752,7 @@ private void stopStreams() { stopExecutor(); // After the worker's generation is gone: a frame still in flight sees that and can no // longer create the JPEG decoder again (Qodo on #135) - jpegCodec.release(); + jpegRenderer.release(); nalQueue.clear(); } diff --git a/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java b/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java new file mode 100644 index 00000000..1fba6882 --- /dev/null +++ b/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * JpegRenderer.java — one MJPEG stream onto a surface: hardware decoder first, else BitmapFactory + * + */ + +package com.openipc.decoder; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.graphics.Rect; +import android.util.Log; +import android.view.Surface; + +/** + * Shows the JPEG frames of one MJPEG stream on a surface, for the single view (#119) and for a + * multi-view cell (#132): through the hardware JPEG decoder where the device has one, otherwise + * decoded with BitmapFactory (into the previous bitmap when the size is the same) and drawn over + * the whole surface with lockCanvas(). A surface drawn that way cannot take a decoder any more: + * the owner makes a new one when the stream changes to H.264/H.265 ({@link #drewWithCanvas}). + * Used by one decode thread; release() may come from others. + */ +final class JpegRenderer { + private static final String TAG = "OpenIPCDecoder"; + + /** The hardware decoder failed once in this run: the software path from then on. */ + private static volatile boolean hwFailed; + + private final JpegCodec codec = new JpegCodec(); + private final Paint paint = new Paint(Paint.FILTER_BITMAP_FLAG); + private volatile boolean drawn; + private Bitmap bitmap; + + /** + * Shows one frame on {@code s}. Returns its {width, height} when it is on screen, null when + * not (yet). {@code current} is false once the stream or the surface has been replaced. + */ + int[] render(Frame frame, Surface s, JpegCodec.Current current) { + if (s == null || !s.isValid() || !current.holds()) return null; + // The hardware JPEG decoder first, where there is one (#132) + if (!hwFailed && !drawn && JpegCodec.candidate() != null) { + int[] size = JpegAssembler.size(frame.data(), frame.length()); + if (size != null) { + try { + return codec.decode(frame, s, size[0], size[1], current) ? size : null; + } catch (Exception e) { + codec.release(); + // The surface was replaced meanwhile: not the decoder's fault, try again + if (!current.holds()) return null; + Log.w(TAG, "Hardware MJPEG decoder failed, decoding in software", e); + hwFailed = true; + } + } + } + BitmapFactory.Options o = new BitmapFactory.Options(); + o.inMutable = true; + Bitmap reuse = bitmap; + if (reuse != null) o.inBitmap = reuse; + Bitmap b; + try { + b = BitmapFactory.decodeByteArray(frame.data(), 0, frame.length(), o); + } catch (IllegalArgumentException e) { + // Another size than the reusable bitmap + o.inBitmap = null; + b = BitmapFactory.decodeByteArray(frame.data(), 0, frame.length(), o); + } + if (b == null) return null; + bitmap = b; + try { + Canvas c = s.lockCanvas(null); + try { + c.drawColor(Color.BLACK); + c.drawBitmap(b, null, new Rect(0, 0, c.getWidth(), c.getHeight()), paint); + } finally { + s.unlockCanvasAndPost(c); + } + } catch (RuntimeException e) { + // The surface went away meanwhile (view or activity change) + return null; + } + drawn = true; + return new int[] {b.getWidth(), b.getHeight()}; + } + + /** + * Something was drawn with lockCanvas() since the last call: the surface cannot take a + * decoder any more. Clears the flag, for the owner who replaces the surface. + */ + boolean takeCanvasDrawn() { + boolean d = drawn; + drawn = false; + return d; + } + + /** The surface holds frames drawn with lockCanvas(). */ + boolean drewWithCanvas() { return drawn; } + + /** Releases the hardware decoder (the stream stopped or is no MJPEG any more). */ + void release() { + codec.release(); + } +} diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index 287ff8d0..5343ae09 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -73,8 +73,24 @@ class QuadCell implements RtspSession.Callback { private float outAvg; /** Decode only key frames and parameter sets (#85): a fraction of the load. */ private volatile boolean keyFramesOnly = true; - /** The camera sends MJPEG, which the grid does not decode yet (#119). */ + /** The camera sends MJPEG (#119): whole JPEG frames go to the queue instead of NAL units. */ private volatile boolean jpeg; + // MJPEG in multi-view (#132) + private final JpegAssembler jpegAssembler; + private final JpegRenderer jpegRenderer = new JpegRenderer(); + /** JPEG frames shown, the MJPEG counterpart of the decoder's output count. */ + private volatile int jpegFramesOut; + private volatile int jpegWidth; + private volatile int jpegHeight; + private volatile long lastJpegShownMs; + /** A JPEG frame was shown since the stream started. */ + private volatile boolean jpegShowing; + /** The last JPEG decoded on key frames only: then one per JPEG_REDUCED_MS (#85). */ + private long lastJpegDecodeMs; + /** MJPEG has no key frames: its cheap mode decodes one frame per second instead. */ + static final long JPEG_REDUCED_MS = 1000; + /** A new surface is being made after MJPEG was drawn with lockCanvas(): no decoder yet. */ + private volatile boolean surfacePending; /** Just promoted to full rate: skip until the next key frame, the references are missing. */ private volatile boolean waitingKeyFrame; @@ -138,6 +154,7 @@ void setAudioSink(AudioManager sink, int route) { nalQueue.clear(); decoder.closeDecoder(); }, pool); + this.jpegAssembler = new JpegAssembler(pool); } void start() { @@ -162,6 +179,9 @@ void start() { public void surfaceCreated(SurfaceHolder holder) { // The surface belongs to the SurfaceView: it is used, never released, here surface = holder.getSurface(); + // A new surface: nothing drawn into it with lockCanvas() yet (#132) + jpegRenderer.takeCanvasDrawn(); + surfacePending = false; if (!threadsStarted) startThreads(); } @Override @@ -173,6 +193,7 @@ public void surfaceDestroyed(SurfaceHolder holder) { // a new one when the next surface is there surface = null; decoder.closeDecoder(); + jpegRenderer.release(); } }; @@ -214,6 +235,10 @@ void stop() { audioSink = null; session.stop(); nalAssembler.reset(); + jpegAssembler.reset(); + jpegRenderer.release(); + jpegShowing = false; + lastJpegShownMs = 0; // Disabling releases the decoder and stops the decode thread from creating another, // so there is nothing to wait for on the UI thread (#32) decoder.setEnabled(false); @@ -270,7 +295,7 @@ static String shortError(String message) { */ String sample(long nowMs) { int in = framesIn; - int out = decoder.framesOut(); + int out = decoder.framesOut() + jpegFramesOut; int lost = queueFullDrops + decoder.inputDropped(); if (lastSampleMs > 0 && nowMs > lastSampleMs) { float sec = (nowMs - lastSampleMs) / 1000f; @@ -294,14 +319,15 @@ String sample(long nowMs) { if ("disconnected".equals(st)) return "Disconnected"; return "buffering".equals(st) ? "Buffering…" : "Connecting…"; } - int w = decoder.outWidth(); - int h = decoder.outHeight(); + int w = width(); + int h = height(); String size = w > 0 ? w + "×" + h + " " : ""; - String codec = codecH265 ? "H.265" : "H.264"; + String codec = jpeg ? "MJPEG" : codecH265 ? "H.265" : "H.264"; if (keyFramesOnly) { String rate = outAvg < 10 ? String.format(java.util.Locale.ROOT, "%.1f", outAvg) : String.valueOf(Math.round(outAvg)); - return size + codec + " key frames · " + rate + " fps" + // MJPEG has no key frames: its cheap mode shows one frame per second (#132) + return size + codec + (jpeg ? " reduced" : " key frames") + " · " + rate + " fps" + (decoder.isSoftware() ? " · software" : ""); } return size + codec + " " + Math.round(outFps) + "/" + Math.round(inFps) + " fps"; @@ -323,17 +349,17 @@ void setKeyFramesOnly(boolean on) { /** Resolution and frame rate are known, so the full-rate load can be estimated. */ boolean knowsLoad() { - return decoder.outWidth() > 0 && inAvg > 0; + return width() > 0 && inAvg > 0; } /** Macroblocks per second at full rate (#85). */ long fullLoad() { - return DecodePlanner.fullLoad(decoder.outWidth(), decoder.outHeight(), inAvg); + return DecodePlanner.fullLoad(width(), height(), inAvg); } /** Macroblocks per second it costs now on key frames only (#85). */ long keyLoad() { - return DecodePlanner.fullLoad(decoder.outWidth(), decoder.outHeight(), outAvg); + return DecodePlanner.fullLoad(width(), height(), outAvg); } /** A parameter set or a key frame (H.264 IDR; H.265 IRAP). */ @@ -408,18 +434,33 @@ private void checkUdpStall() { */ boolean showsVideo(long ms) { long window = keyFramesOnly ? Math.max(ms, KEY_FRAMES_LIVE_MS) : ms; - return pausedReason == null && "connected".equals(state) && decoder.renderedWithin(window); + return pausedReason == null && "connected".equals(state) && renderedWithin(window); } + private boolean renderedWithin(long ms) { + if (!jpeg) return decoder.renderedWithin(ms); + long shown = lastJpegShownMs; + return shown > 0 && System.nanoTime() / 1_000_000 - shown < ms; + } + + /** A decoder is set up, or MJPEG frames are being shown (#132). */ + private boolean decoding() { + return jpeg ? jpegShowing : decoder.hasDecoder(); + } + + /** Size of the picture shown (video decoder or MJPEG). */ + private int width() { return jpeg ? jpegWidth : decoder.outWidth(); } + private int height() { return jpeg ? jpegHeight : decoder.outHeight(); } + /** Connected and decoding: its load is real, so demoting or pausing it helps (#85). */ boolean isPlaying() { - return pausedReason == null && "connected".equals(state) && decoder.hasDecoder(); + return pausedReason == null && "connected".equals(state) && decoding(); } /** Playing long enough for its decode rate to mean something (#83). */ boolean isSettled(long nowMs) { long started = decoderStartedMs; - return pausedReason == null && "connected".equals(state) && decoder.hasDecoder() + return pausedReason == null && "connected".equals(state) && decoding() && started > 0 && nowMs - started > 5000; } @@ -431,16 +472,34 @@ void pause(String reason) { @Override public void onSdp(Sdp sdp) { - // MJPEG plays in the single view only for now (#119) - jpeg = sdp.videoJpeg; - if (jpeg) { - // Paused rather than pulling a stream it cannot show (Qodo on #131) - problem = "Decoder: MJPEG in the single view only"; - View v = view; - if (v != null) v.post(() -> pause("MJPEG: single view only")); + boolean wasJpeg = jpeg; + if (sdp.videoJpeg != wasJpeg || sdp.videoH265 != codecH265) { + // Frames of the previous codec must not reach the new path (Qodo on #131) + Frame f; + while ((f = nalQueue.poll()) != null) cellFramePool.recycle(f); } + if (sdp.videoJpeg) { + // MJPEG has its own decoder or is drawn by the app (#132): the video decoder must + // not hold the surface + decoder.closeDecoder(); + } else { + jpegRenderer.release(); + if (jpegRenderer.takeCanvasDrawn()) { + // A surface drawn with lockCanvas() cannot take a decoder any more: hide and + // show the view for a new one, no decoder until it is there (Qodo on #131) + surfacePending = true; + SurfaceView v = view; + if (v != null) v.post(() -> { + v.setVisibility(View.GONE); + v.post(() -> v.setVisibility(View.VISIBLE)); + }); + } + } + jpeg = sdp.videoJpeg; + lastJpegDecodeMs = 0; udpStallSamples = 0; nalAssembler.reset(); + jpegAssembler.reset(); lastUnknownPayload = -1; videoPt = sdp.videoPt; codecH265 = sdp.videoH265; @@ -465,7 +524,6 @@ public boolean onPacket(int channel, byte[] buf, int offset, int length) { return false; } if (channel != RtspSession.CHANNEL_VIDEO) return false; - if (jpeg) return true; // dropped, but no reconnect loop before the pause int pt = buf[offset + 1] & 0x7F; if (pt != videoPt) { if (pt != lastUnknownPayload) { @@ -478,12 +536,17 @@ public boolean onPacket(int channel, byte[] buf, int offset, int length) { videoPackets++; if ((buf[offset + 1] & 0x80) != 0) framesIn++; + if (jpeg) return jpegAssembler.assemble(buf, offset, length, nalSink::onNal); return nalAssembler.assemble(buf, offset, length, codecH265, nalSink); } private void decode(Frame buffer) { - if (buffer.length() < 5) return; + if (jpeg) { + decodeJpeg(buffer); + return; + } + if (buffer.length() < 5 || surfacePending) return; if (keyFramesOnly || waitingKeyFrame) { int type = NalAssembler.fragment(buffer.data()[4], codecH265); if (!isKeyNal(type, codecH265)) return; @@ -498,4 +561,27 @@ private void decode(Frame buffer) { decoder.decodeFrame(buffer, codecH265, 0); } } + + /** One JPEG frame onto the cell's surface; on key frames only, one per second (#132). */ + private void decodeJpeg(Frame frame) { + long now = System.nanoTime() / 1_000_000; + if (keyFramesOnly && lastJpegDecodeMs > 0 && now - lastJpegDecodeMs < JPEG_REDUCED_MS) { + return; + } + Surface s = surface; + int[] size = jpegRenderer.render(frame, s, + () -> running && jpeg && s == surface); + if (size == null) return; + lastJpegDecodeMs = now; + jpegWidth = size[0]; + jpegHeight = size[1]; + lastJpegShownMs = now; + jpegFramesOut++; + if (!jpegShowing) { + jpegShowing = true; + problem = null; + decoderStartedMs = now; + if (listener != null) listener.onRemoveOverlay(index); + } + } } From 9758984cb711148eefb116ac978f208726ccfa7b Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Tue, 13 Oct 2026 15:18:53 +0000 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20Qodo=20on=20#139=20=E2=80=94=20per-s?= =?UTF-8?q?tream=20hardware=20JPEG=20failure,=20frames=20checked=20against?= =?UTF-8?q?=20the=20path,=20surface=20recreated=20after=20a=20layout=20pas?= =?UTF-8?q?s,=20JPEG=20losses=20and=20UDP=20stalls=20counted,=20software?= =?UTF-8?q?=20JPEG=20off=20the=20hardware=20budget,=20software=20codecs=20?= =?UTF-8?q?skipped?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude --- .../java/com/openipc/decoder/JpegCodec.java | 18 +++- .../com/openipc/decoder/JpegRenderer.java | 19 +++- .../java/com/openipc/decoder/QuadCell.java | 92 ++++++++++++++----- 3 files changed, 104 insertions(+), 25 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/JpegCodec.java b/android/app/src/main/java/com/openipc/decoder/JpegCodec.java index 68e9ee5e..d54a03f6 100644 --- a/android/app/src/main/java/com/openipc/decoder/JpegCodec.java +++ b/android/app/src/main/java/com/openipc/decoder/JpegCodec.java @@ -51,7 +51,7 @@ static String[] find() { try { for (MediaCodecInfo ci : new MediaCodecList(MediaCodecList.ALL_CODECS) .getCodecInfos()) { - if (ci.isEncoder()) continue; + if (ci.isEncoder() || softwareOnly(ci)) continue; for (String type : ci.getSupportedTypes()) { for (String m : MIMES) { if (m.equalsIgnoreCase(type) && result.length == 0) { @@ -85,6 +85,16 @@ static String[] find() { return result.length == 0 ? null : result; } + /** + * A software codec does no better than BitmapFactory and its load is not the hardware's: + * the software path takes the stream instead (Qodo on #139). + */ + private static boolean softwareOnly(MediaCodecInfo ci) { + if (android.os.Build.VERSION.SDK_INT >= 29) return ci.isSoftwareOnly(); + String n = ci.getName().toLowerCase(java.util.Locale.ROOT); + return n.startsWith("omx.google.") || n.startsWith("c2.android."); + } + /** * Decodes one JPEG of {@code w}×{@code h} to {@code target}; (re)creates the decoder when * the size or surface changes. Returns true when a frame was rendered. Throws when the @@ -101,6 +111,10 @@ static String[] candidate() { /** Frames queued in a row without any output: the decoder is not working (Qodo on #135). */ static final int MAX_SILENT_FRAMES = 15; private int silentFrames; + /** Frames the decoder had no input buffer for: decoder losses for the planner (#85). */ + private volatile int inputDropped; + + int inputDropped() { return inputDropped; } /** Checked under the lock: false once the stream that called decode() has stopped. */ interface Current { @@ -149,6 +163,8 @@ synchronized boolean decode(Frame frame, Surface target, int w, int h, Current c b.clear(); b.put(frame.data(), 0, frame.length()); codec.queueInputBuffer(in, 0, frame.length(), System.nanoTime() / 1000, 0); + } else { + inputDropped++; } boolean rendered = false; int out; diff --git a/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java b/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java index 1fba6882..999c02c0 100644 --- a/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java +++ b/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java @@ -27,8 +27,13 @@ final class JpegRenderer { private static final String TAG = "OpenIPCDecoder"; - /** The hardware decoder failed once in this run: the software path from then on. */ - private static volatile boolean hwFailed; + /** + * This stream's hardware decoder failed: the software path from then on. Per renderer, so + * one cell at a decoder limit does not send the others to lockCanvas() (Qodo on #139). + */ + private volatile boolean hwFailed; + /** The last frame was decoded with BitmapFactory. */ + private volatile boolean software; private final JpegCodec codec = new JpegCodec(); private final Paint paint = new Paint(Paint.FILTER_BITMAP_FLAG); @@ -46,6 +51,7 @@ int[] render(Frame frame, Surface s, JpegCodec.Current current) { int[] size = JpegAssembler.size(frame.data(), frame.length()); if (size != null) { try { + software = false; return codec.decode(frame, s, size[0], size[1], current) ? size : null; } catch (Exception e) { codec.release(); @@ -56,6 +62,9 @@ int[] render(Frame frame, Surface s, JpegCodec.Current current) { } } } + // A decoder still connected to the surface would make lockCanvas() fail + codec.release(); + software = true; BitmapFactory.Options o = new BitmapFactory.Options(); o.inMutable = true; Bitmap reuse = bitmap; @@ -99,6 +108,12 @@ boolean takeCanvasDrawn() { /** The surface holds frames drawn with lockCanvas(). */ boolean drewWithCanvas() { return drawn; } + /** Frames are decoded with BitmapFactory: CPU load, not the hardware decoder's. */ + boolean isSoftware() { return software; } + + /** Frames the hardware decoder had no input buffer for. */ + int dropped() { return codec.inputDropped(); } + /** Releases the hardware decoder (the stream stopped or is no MJPEG any more). */ void release() { codec.release(); diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index 5343ae09..87b65ba6 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -168,6 +168,8 @@ void start() { // the VM, which allocated about 1 MB/s of garbage with four streams view.getHolder().addCallback(holderCallback); Surface s = view.getHolder().getSurface(); + // Drawn with lockCanvas() before stop(): the video path asks for a new one (decode()) + surfacePending = false; if (s != null && s.isValid()) { surface = s; if (!threadsStarted) startThreads(); @@ -194,6 +196,11 @@ public void surfaceDestroyed(SurfaceHolder holder) { surface = null; decoder.closeDecoder(); jpegRenderer.release(); + if (recreating) { + recreating = false; + SurfaceView v = view; + if (v != null) v.post(() -> v.setVisibility(View.VISIBLE)); + } } }; @@ -239,6 +246,12 @@ void stop() { jpegRenderer.release(); jpegShowing = false; lastJpegShownMs = 0; + surfacePending = false; + if (recreating) { + recreating = false; + SurfaceView v = view; + if (v != null) v.post(() -> v.setVisibility(View.VISIBLE)); + } // Disabling releases the decoder and stops the decode thread from creating another, // so there is nothing to wait for on the UI thread (#32) decoder.setEnabled(false); @@ -296,7 +309,7 @@ static String shortError(String message) { String sample(long nowMs) { int in = framesIn; int out = decoder.framesOut() + jpegFramesOut; - int lost = queueFullDrops + decoder.inputDropped(); + int lost = queueFullDrops + decoder.inputDropped() + jpegRenderer.dropped(); if (lastSampleMs > 0 && nowMs > lastSampleMs) { float sec = (nowMs - lastSampleMs) / 1000f; inFps = Math.max(0, in - lastIn) / sec; @@ -328,7 +341,8 @@ String sample(long nowMs) { : String.valueOf(Math.round(outAvg)); // MJPEG has no key frames: its cheap mode shows one frame per second (#132) return size + codec + (jpeg ? " reduced" : " key frames") + " · " + rate + " fps" - + (decoder.isSoftware() ? " · software" : ""); + + ((jpeg ? jpegRenderer.isSoftware() : decoder.isSoftware()) + ? " · software" : ""); } return size + codec + " " + Math.round(outFps) + "/" + Math.round(inFps) + " fps"; } @@ -354,11 +368,14 @@ boolean knowsLoad() { /** Macroblocks per second at full rate (#85). */ long fullLoad() { + // MJPEG drawn with BitmapFactory loads the CPU, not the hardware decoder (Qodo on #139) + if (jpeg && jpegRenderer.isSoftware()) return 0; return DecodePlanner.fullLoad(width(), height(), inAvg); } /** Macroblocks per second it costs now on key frames only (#85). */ long keyLoad() { + if (jpeg && jpegRenderer.isSoftware()) return 0; return DecodePlanner.fullLoad(width(), height(), outAvg); } @@ -382,7 +399,7 @@ static boolean isKeyFrameNal(int type, boolean h265) { /** Video RTP packets received, for the stall check (frames need a marker bit). */ private volatile int videoPackets; private int lastVideoPackets; - private int lastLossCount; + private volatile int lastLossCount; /** * Over UDP a lost packet drops the whole NAL unit; on key frames only (#85) a camera that @@ -393,7 +410,8 @@ static boolean isKeyFrameNal(int type, boolean h265) { */ private void checkUdpStall() { int packets = videoPackets; - int loss = nalAssembler.lossCount(); + // The assembler of the stream: JPEG frames lost to missing fragments count too + int loss = jpeg ? jpegAssembler.droppedCount() : nalAssembler.lossCount(); boolean arriving = packets - lastVideoPackets >= 10; boolean losing = loss > lastLossCount; lastVideoPackets = packets; @@ -412,7 +430,7 @@ private void checkUdpStall() { if (udpStallSamples >= UDP_STALL_SAMPLES) { udpStallSamples = 0; session.reportUdpBroken("packets but no decoded frame for " + UDP_STALL_SAMPLES - + " s (" + loss + " NAL units lost)"); + + " s (" + loss + (jpeg ? " JPEG frames" : " NAL units") + " lost)"); } } @@ -472,31 +490,27 @@ void pause(String reason) { @Override public void onSdp(Sdp sdp) { - boolean wasJpeg = jpeg; - if (sdp.videoJpeg != wasJpeg || sdp.videoH265 != codecH265) { - // Frames of the previous codec must not reach the new path (Qodo on #131) - Frame f; - while ((f = nalQueue.poll()) != null) cellFramePool.recycle(f); - } + // First the path, so a JPEG frame still being drawn no longer counts as current + jpeg = sdp.videoJpeg; + // Frames of the previous connection must not reach the new one (Qodo on #131, #139); + // one the decode thread already holds is checked against the path in decode() + Frame f; + while ((f = nalQueue.poll()) != null) cellFramePool.recycle(f); if (sdp.videoJpeg) { // MJPEG has its own decoder or is drawn by the app (#132): the video decoder must // not hold the surface decoder.closeDecoder(); } else { + // A surface drawn with lockCanvas() is replaced by the decode thread (decode()) jpegRenderer.release(); - if (jpegRenderer.takeCanvasDrawn()) { - // A surface drawn with lockCanvas() cannot take a decoder any more: hide and - // show the view for a new one, no decoder until it is there (Qodo on #131) - surfacePending = true; - SurfaceView v = view; - if (v != null) v.post(() -> { - v.setVisibility(View.GONE); - v.post(() -> v.setVisibility(View.VISIBLE)); - }); - } } - jpeg = sdp.videoJpeg; + // Nothing of the new stream is on screen yet (Qodo on #139) + jpegShowing = false; + jpegWidth = 0; + jpegHeight = 0; + lastJpegShownMs = 0; lastJpegDecodeMs = 0; + lastLossCount = sdp.videoJpeg ? jpegAssembler.droppedCount() : nalAssembler.lossCount(); udpStallSamples = 0; nalAssembler.reset(); jpegAssembler.reset(); @@ -542,11 +556,22 @@ public boolean onPacket(int channel, byte[] buf, int offset, int length) { private void decode(Frame buffer) { + // The frame's own type, not only the current path: a frame taken from the queue before + // a codec change must not reach the other path (Qodo on #139) + boolean isJpeg = buffer.length() >= 2 && (buffer.data()[0] & 0xFF) == 0xFF + && (buffer.data()[1] & 0xFF) == 0xD8; + if (isJpeg != jpeg) return; if (jpeg) { decodeJpeg(buffer); return; } if (buffer.length() < 5 || surfacePending) return; + if (jpegRenderer.takeCanvasDrawn()) { + // MJPEG was drawn into this surface with lockCanvas(), which cannot take a decoder + // any more: no decoder until there is a new surface (Qodo on #131, #139) + recreateSurface(); + return; + } if (keyFramesOnly || waitingKeyFrame) { int type = NalAssembler.fragment(buffer.data()[4], codecH265); if (!isKeyNal(type, codecH265)) return; @@ -562,6 +587,29 @@ private void decode(Frame buffer) { } } + /** The view hides the surface to get a new one; surfaceDestroyed() shows it again. */ + private volatile boolean recreating; + + /** + * A new surface for the view after MJPEG was drawn with lockCanvas(). GONE has to reach a + * layout pass to destroy the surface, so VISIBLE follows in surfaceDestroyed(), not right + * after it (Qodo on #139). + */ + private void recreateSurface() { + surfacePending = true; + SurfaceView v = view; + if (v == null) return; + v.post(() -> { + if (!running || recreating) return; + if (!v.getHolder().getSurface().isValid()) { + // No surface now: the next one is new anyway (surfaceCreated) + return; + } + recreating = true; + v.setVisibility(View.GONE); + }); + } + /** One JPEG frame onto the cell's surface; on key frames only, one per second (#132). */ private void decodeJpeg(Frame frame) { long now = System.nanoTime() / 1_000_000; From a3dab50b3fd8e47eb65435c3deda6a2da6724500 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Tue, 13 Oct 2026 15:32:08 +0000 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20Qodo=20on=20#139=20=E2=80=94=20JPEG?= =?UTF-8?q?=20tables=20cleared=20for=20a=20new=20stream,=20one=20stream=20?= =?UTF-8?q?at=20a=20time=20picks=20the=20JPEG=20decoder,=20a=20stale=20vid?= =?UTF-8?q?eo=20decoder=20is=20closed=20before=20MJPEG=20is=20drawn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude --- .../com/openipc/decoder/JpegAssembler.java | 3 ++ .../java/com/openipc/decoder/JpegCodec.java | 53 ++++++++++++------- .../java/com/openipc/decoder/QuadCell.java | 3 ++ .../openipc/decoder/JpegAssemblerTest.java | 27 ++++++++++ 4 files changed, 67 insertions(+), 19 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java b/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java index 9c3ce706..89134142 100644 --- a/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java +++ b/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java @@ -92,10 +92,13 @@ interface Sink { this.pool = pool; } + /** A new stream: its frames must not use the tables of the previous one (Qodo on #139). */ void reset() { expectedOffset = -1; scanLength = 0; header = null; + tablesByQ.clear(); + precisionByQ.clear(); } /** Frames dropped because a fragment was missing or malformed. */ diff --git a/android/app/src/main/java/com/openipc/decoder/JpegCodec.java b/android/app/src/main/java/com/openipc/decoder/JpegCodec.java index d54a03f6..40acf0c7 100644 --- a/android/app/src/main/java/com/openipc/decoder/JpegCodec.java +++ b/android/app/src/main/java/com/openipc/decoder/JpegCodec.java @@ -33,7 +33,8 @@ final class JpegCodec { * 10 does not list video/jpeg (#132). Tried once when the list has none. */ private static final String[][] BY_NAME = {{"OMX.allwinner.video.decoder.mjpeg", "video/jpeg"}}; - private static volatile boolean byNameTried; + /** The BY_NAME decoder could not be created: no hardware JPEG decoder on this device. */ + private static volatile boolean byNameFailed; /** The decoder that worked, once one did. */ private static volatile String[] chosen; @@ -105,7 +106,8 @@ static String[] candidate() { if (chosen != null) return chosen; String[] dec = find(); if (dec != null) return dec; - return byNameTried ? null : BY_NAME[0]; + // Also while another stream is still trying it (Qodo on #139) + return byNameFailed ? null : BY_NAME[0]; } /** Frames queued in a row without any output: the decoder is not working (Qodo on #135). */ @@ -124,27 +126,40 @@ interface Current { synchronized boolean decode(Frame frame, Surface target, int w, int h, Current current) throws Exception { if (!current.holds()) return false; - String[] dec = chosen != null ? chosen : find(); - if (dec == null) { - if (byNameTried) throw new IllegalStateException("no JPEG decoder"); - byNameTried = true; - dec = BY_NAME[0]; - } if (codec == null || target != surface || w != width || h != height) { releaseLocked(); - MediaCodec c = MediaCodec.createByCodecName(dec[0]); - try { - MediaFormat format = MediaFormat.createVideoFormat(dec[1], w, h); - format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, Math.max(frame.length() * 2, - 512 * 1024)); - c.configure(format, target, null, 0); - c.start(); - } catch (Exception e) { - c.release(); - throw e; + MediaCodec c; + String[] dec; + // One stream at a time chooses the decoder, so the cells that start together do + // not take a probe in progress for "no decoder" (Qodo on #139) + synchronized (JpegCodec.class) { + dec = chosen != null ? chosen : find(); + boolean byName = dec == null; + if (byName) { + if (byNameFailed) throw new IllegalStateException("no JPEG decoder"); + dec = BY_NAME[0]; + } + try { + c = MediaCodec.createByCodecName(dec[0]); + } catch (Exception e) { + if (byName) byNameFailed = true; + throw e; + } + try { + MediaFormat format = MediaFormat.createVideoFormat(dec[1], w, h); + format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, + Math.max(frame.length() * 2, 512 * 1024)); + c.configure(format, target, null, 0); + c.start(); + } catch (Exception e) { + c.release(); + // Never worked on this device: the name is not a usable decoder + if (byName && chosen == null) byNameFailed = true; + throw e; + } + chosen = dec; } codec = c; - chosen = dec; silentFrames = 0; surface = target; width = w; diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index 87b65ba6..cba6f63a 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -612,6 +612,9 @@ private void recreateSurface() { /** One JPEG frame onto the cell's surface; on key frames only, one per second (#132). */ private void decodeJpeg(Frame frame) { + // A video decoder made from a frame taken just before the switch to MJPEG would hold + // the surface: this thread closes it before drawing (Qodo on #139) + if (decoder.hasDecoder()) decoder.closeDecoder(); long now = System.nanoTime() / 1_000_000; if (keyFramesOnly && lastJpegDecodeMs > 0 && now - lastJpegDecodeMs < JPEG_REDUCED_MS) { return; diff --git a/android/app/src/test/java/com/openipc/decoder/JpegAssemblerTest.java b/android/app/src/test/java/com/openipc/decoder/JpegAssemblerTest.java index 5e38053f..a66b7094 100644 --- a/android/app/src/test/java/com/openipc/decoder/JpegAssemblerTest.java +++ b/android/app/src/test/java/com/openipc/decoder/JpegAssemblerTest.java @@ -149,6 +149,33 @@ public void tablesSentOnceAreReusedForTheSameQ() throws Exception { assertTrue(Arrays.equals(frames.get(0), frames.get(1))); } + @Test + public void aNewStreamDoesNotReuseTheTablesOfTheOldOne() throws Exception { + List first = packets("jpeg_q255_64x32.rtp"); + for (byte[] p : first) p[12 + 5] = (byte) 200; + JpegAssembler a = new JpegAssembler(new FramePool(4)); + List frames = new ArrayList<>(); + for (byte[] p : first) { + a.assemble(p, 0, p.length, f -> frames.add(new byte[f.length()])); + } + assertEquals(1, frames.size()); + // A new stream (Qodo on #139): the same frame with "length 0" tables has none to use + a.reset(); + for (int i = 0; i < first.size(); i++) { + byte[] p = first.get(i).clone(); + if (i == 0) { + byte[] shorter = new byte[p.length - 128]; + System.arraycopy(p, 0, shorter, 0, 12 + 8 + 4); + System.arraycopy(p, 12 + 8 + 4 + 128, shorter, 12 + 8 + 4, p.length - 12 - 8 - 4 - 128); + shorter[12 + 8 + 2] = 0; + shorter[12 + 8 + 3] = 0; + p = shorter; + } + a.assemble(p, 0, p.length, f -> frames.add(new byte[f.length()])); + } + assertEquals(1, frames.size()); + } + @Test public void zeroSizeMeans2048() throws Exception { List p = packets("jpeg_q50_64x32.rtp"); From 1c3fb86bda4630f43824fad5771b950e557eb813 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Tue, 13 Oct 2026 15:39:09 +0000 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20Qodo=20on=20#139=20=E2=80=94=20no=20?= =?UTF-8?q?JPEG=20assembler=20reset=20from=20stop()=20while=20a=20packet?= =?UTF-8?q?=20may=20be=20in=20it,=20one=20JPEG=20sink=20per=20cell?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude --- .../app/src/main/java/com/openipc/decoder/QuadCell.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index cba6f63a..a6526c07 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -77,6 +77,7 @@ class QuadCell implements RtspSession.Callback { private volatile boolean jpeg; // MJPEG in multi-view (#132) private final JpegAssembler jpegAssembler; + private final JpegAssembler.Sink jpegSink; private final JpegRenderer jpegRenderer = new JpegRenderer(); /** JPEG frames shown, the MJPEG counterpart of the decoder's output count. */ private volatile int jpegFramesOut; @@ -155,6 +156,8 @@ void setAudioSink(AudioManager sink, int route) { decoder.closeDecoder(); }, pool); this.jpegAssembler = new JpegAssembler(pool); + // Once, not a method reference per packet: the receive path makes no garbage (#61) + this.jpegSink = nalSink::onNal; } void start() { @@ -242,7 +245,8 @@ void stop() { audioSink = null; session.stop(); nalAssembler.reset(); - jpegAssembler.reset(); + // Not the JPEG assembler: a packet may still be in it on the receive thread; the next + // stream resets it in onSdp() (Qodo on #139) jpegRenderer.release(); jpegShowing = false; lastJpegShownMs = 0; @@ -550,7 +554,7 @@ public boolean onPacket(int channel, byte[] buf, int offset, int length) { videoPackets++; if ((buf[offset + 1] & 0x80) != 0) framesIn++; - if (jpeg) return jpegAssembler.assemble(buf, offset, length, nalSink::onNal); + if (jpeg) return jpegAssembler.assemble(buf, offset, length, jpegSink); return nalAssembler.assemble(buf, offset, length, codecH265, nalSink); }