From 5e67e01b2bf2289f25fd9bc65907b3b275a69b12 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 14:50:23 +0000 Subject: [PATCH 01/21] feat: FLOW-UFO toy drones over UDP (#187) A flowufo:// slot (flowufo://192.168.169.1:8800, host and port optional) plays the video of Wi-Fi toy drones with a FLOW-UFO-xxxxxx access point, in single view and in grid cells like an http:// MJPEG slot. Video only: no flight commands are sent. The client probes the drone until fragments arrive, collects the JPEG fragments of the last four frames, acknowledges them every 25 ms (masks of missing fragments, whole-frame requests for lagging slots and for the next frame after a pause) and probes again after 3 s of silence. The frames carry only the scan data: the headers are generated (IJG tables at the frame's quality, baseline 4:4:4, standard Huffman tables). Not yet checked on a drone: the log shows the raw header bytes of the first fragments and the start of the first frame, /stats.json the protocol counters under "flowufo". The socket is bound to the network whose subnet holds the drone, so that a Wi-Fi without internet is used even with mobile data on (ACCESS_NETWORK_STATE). Version 2.10 (35). --- android/app/build.gradle | 4 +- android/app/src/main/AndroidManifest.xml | 2 + .../java/com/openipc/decoder/Decoder.java | 19 + .../java/com/openipc/decoder/FlowUfo.java | 810 ++++++++++++++++++ .../com/openipc/decoder/JpegAssembler.java | 22 +- .../java/com/openipc/decoder/NetInfo.java | 50 ++ .../java/com/openipc/decoder/QuadCell.java | 12 + .../java/com/openipc/decoder/RtspClient.java | 10 + .../java/com/openipc/decoder/RtspSession.java | 82 +- .../com/openipc/decoder/StreamProbes.java | 2 + .../java/com/openipc/decoder/FlowUfoTest.java | 738 ++++++++++++++++ 11 files changed, 1746 insertions(+), 5 deletions(-) create mode 100644 android/app/src/main/java/com/openipc/decoder/FlowUfo.java create mode 100644 android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java diff --git a/android/app/build.gradle b/android/app/build.gradle index c517b841..a725c471 100755 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -23,8 +23,8 @@ android { applicationId "com.openipc.decoder" minSdk 21 targetSdk 36 - versionCode 34 - versionName "2.09" + versionCode 35 + versionName "2.10" buildConfigField "String", "GIT_HASH", "\"${getGitHash()}\"" } diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 922a4fe1..28a826da 100755 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -2,6 +2,8 @@ + + { + if (statusText == null || "connected".equals(singleState)) return; + statusText.setText(text); + statusText.setVisibility(View.VISIBLE); + }); + } + private void updateStatus(String status) { singleState = status; runOnUiThread(() -> { @@ -1542,6 +1555,9 @@ private Map debugState() { // Frames shown in the last second and the playback mode of S (#153) single.put("out_fps", singleOutFps); single.put("decode", slides ? "key frames" : "full"); + // The toy drone's protocol counters (#187) + Map ufo = rtspClient != null ? rtspClient.flowUfoStats() : null; + if (ufo != null) single.put("flowufo", ufo); m.put("single", single); } QuadCell[] cells = quadCells; @@ -1580,6 +1596,8 @@ private Map debugState() { ? cell.keyLoad() : cell.fullLoad()); c.put("paused", cell.isPaused()); c.put("audio", i == quadAudio); + Map ufo = cell.flowUfoStats(); + if (ufo != null) c.put("flowufo", ufo); } list.add(c); } @@ -1647,6 +1665,7 @@ private String slotTransport(int i, String url) { } if (RtpListener.Url.matches(url)) return "UDP listen"; if (HttpMjpeg.matches(url)) return "HTTP"; + if (FlowUfo.Url.matches(url)) return "FLOW-UFO UDP"; Transport.Mode m = StreamProbe.probeMode(mTransports[i]); return "check over " + (m == Transport.Mode.MULTICAST ? "multicast" : m.name()); } diff --git a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java new file mode 100644 index 00000000..55e64837 --- /dev/null +++ b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java @@ -0,0 +1,810 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * FlowUfo.java — video of FLOW-UFO toy drones: JPEG frames in UDP fragments, with acks (#187) + * + */ + +package com.openipc.decoder; + +import android.util.Log; + +import java.io.Closeable; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketTimeoutException; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +/** + * Cheap Wi-Fi toy drones with a "FLOW-UFO-xxxxxx" access point (a BL60x camera) send MJPEG + * over UDP from 192.168.169.1:8800 in a protocol of their own (#187). A short probe wakes the + * drone; it then sends every JPEG frame, cut into fragments of 1024 bytes, to the address the + * probe came from, and the client acknowledges the last four frames every 25 ms: what arrived, + * what is missing, and which frame to send whole. A frame carries only the entropy-coded data + * of the scan: the client puts the JPEG headers in front (standard tables at the quality the + * fragment names). Video only: flight commands are never sent. Written from a description of + * the client side and not yet checked against a drone: what is a guess says so, and the log + * shows the raw bytes that would settle it. Pure Java apart from the log. + */ +final class FlowUfo { + private static final String TAG = "OpenIPCDecoder"; + + static final String SCHEME = "flowufo://"; + static final String DEFAULT_HOST = "192.168.169.1"; + static final int DEFAULT_PORT = 8800; + + /** Wakes the drone up; sent every PROBE_MS until the first fragment. */ + static final byte[] PROBE = {(byte) 0xEF, 0x00, 0x04, 0x00}; + static final long PROBE_MS = 100; + static final long ACK_MS = 25; + /** No fragment for longer: the last frame is over, the acknowledgement asks for the next. */ + static final long QUIET_MS = 19; + /** + * No fragment for this long: everything is reset and the probes start again. Not final so + * unit tests can shorten it. + */ + static volatile long resetMs = 3000; + + // Drone packets: 0x93, type, length (u16 LE, the whole datagram) + static final int MAGIC = 0x93; + static final int TYPE_FRAGMENT = 1; + static final int TYPE_SERVICE = 4; + static final int MAX_PACKET = 0x438; + /** Header of a fragment; its data follows. */ + static final int FRAGMENT_HEADER = 0x38; + /** Fragment i holds the body bytes from i * FRAGMENT_DATA on. */ + static final int FRAGMENT_DATA = 1024; + /** More fragments than this (1 MiB) are not a frame of these cameras. */ + static final int MAX_FRAGMENTS = 1024; + + // The acknowledgement: a fixed header, then one record per frame + static final int ACK_HEADER = 0x58; + /** Quality parameters the vendor app sends by default (25, 50, 35, 75). */ + static final byte[] QPARA = {0x19, 0x32, 0x23, 0x4B}; + static final int SLOTS = 4; + /** A slot this many frames behind the newest asks for its next frame whole. */ + static final int LAG = 5; + static final int STATUS_COLLECTING = 0; + static final int STATUS_COMPLETE = 1; + static final int STATUS_WHOLE = 3; + private static final int MAX_MASK_WORDS = (MAX_FRAGMENTS + 31) / 32; + static final int ACK_CAPACITY = ACK_HEADER + (SLOTS + 1) * (16 + 4 * MAX_MASK_WORDS); + + // The JPEG headers put in front of a body. Unverified choices, easy to change here: the + // luma sampling (0x11 = 4:4:4, like the chroma), no restart interval. With these the headers + // are 605 bytes and SOF0 has the height at 0x91 and the width at 0x93, the layout the vendor + // app is known to use. Cb and Cr share the second quantization and Huffman tables + static final int LUMA_SAMPLING = 0x11; + static final int RESTART_INTERVAL = 0; + /** A frame that names no usable quality gets this one. */ + static final int DEFAULT_QUALITY = 50; + static final int DEFAULT_WIDTH = 640; + static final int DEFAULT_HEIGHT = 360; + + private FlowUfo() {} + + // ====================== URL ====================== + + /** flowufo://[host][:port][/...]: the drone's address, 192.168.169.1:8800 when left out. */ + static final class Url { + String host = DEFAULT_HOST; + int port = DEFAULT_PORT; + + static boolean matches(String url) { + return url != null && url.trim().toLowerCase(Locale.ROOT).startsWith(SCHEME); + } + + static Url parse(String url) throws IOException { + if (!matches(url)) throw new IOException("Not a flowufo:// URL"); + String rest = url.trim().substring(SCHEME.length()); + int end = rest.length(); + for (int i = 0; i < rest.length(); i++) { + char ch = rest.charAt(i); + if (ch == '/' || ch == '?' || ch == '#') { + end = i; + break; + } + } + String authority = rest.substring(0, end).trim(); + Url u = new Url(); + String host = authority; + int colon = authority.lastIndexOf(':'); + if (authority.startsWith("[")) { + int close = authority.indexOf(']'); + if (close < 0) throw new IOException("Bad host in URL"); + host = authority.substring(1, close); + colon = authority.indexOf(':', close); + } else if (colon >= 0) { + host = authority.substring(0, colon); + } + if (colon >= 0) { + try { + u.port = Integer.parseInt(authority.substring(colon + 1)); + } catch (NumberFormatException e) { + throw new IOException("Bad port in URL"); + } + if (u.port <= 0 || u.port > 65535) throw new IOException("Bad port in URL"); + } + if (!host.isEmpty()) u.host = host; + for (int i = 0; i < u.host.length(); i++) { + char ch = u.host.charAt(i); + if (ch <= 0x20 || ch == 0x7F || ch == '@') throw new IOException("Bad host in URL"); + } + return u; + } + + String endpoint() { + return (host.indexOf(':') >= 0 ? "[" + host + "]" : host) + ":" + port; + } + } + + // ====================== Drone packets ====================== + + /** Type of a drone packet, or -1 when it is none: wrong first byte or length. */ + static int packetType(byte[] b, int off, int len) { + if (len < 4 || len > MAX_PACKET || (b[off] & 0xFF) != MAGIC) return -1; + if (u16(b, off + 2) != len) return -1; + return b[off + 1] & 0xFF; + } + + /** One frame fragment (type 1); fields as little-endian numbers. */ + static final class Fragment { + /** + * Frame number, counting from 1. Read as u32 at 0x08: whether 0x08 is a u64 or a u32 + * followed by another field at 0x0C is not known, and both read the same while the + * upper half is zero. The log shows 0x08..0x17 of the first fragments. + */ + long seq; + /** Frames up to this one are not acknowledged any more (low 32 bits of 0x18). */ + long base; + int index; + int total; + int bodyLength; + int width; + int height; + int quality; + /** 01 01 at 0x34: the camera family whose layout this is. */ + boolean gl; + /** The packet parsed last; its data starts at dataOffset. */ + byte[] buf; + int dataOffset; + int dataLength; + + /** Fills the fields from a type-1 packet; false when it is malformed. */ + boolean parse(byte[] b, int off, int len) { + if (len < FRAGMENT_HEADER || packetType(b, off, len) != TYPE_FRAGMENT) return false; + seq = u32(b, off + 0x08); + base = u32(b, off + 0x18); + long idx = u32(b, off + 0x20); + long tot = u32(b, off + 0x24); + long body = u32(b, off + 0x28); + width = u16(b, off + 0x2C); + height = u16(b, off + 0x2E); + quality = b[off + 0x30] & 0xFF; + gl = b[off + 0x34] == 1 && b[off + 0x35] == 1; + buf = b; + dataOffset = off + FRAGMENT_HEADER; + dataLength = len - FRAGMENT_HEADER; + if (seq == 0 || tot < 1 || tot > MAX_FRAGMENTS || idx >= tot) return false; + if (body < 1 || body > tot * FRAGMENT_DATA || dataLength > FRAGMENT_DATA) return false; + index = (int) idx; + total = (int) tot; + bodyLength = (int) body; + return true; + } + } + + // ====================== Frames and acknowledgements ====================== + + /** One frame being collected, in slot (seq - 1) & 3. */ + static final class Slot { + long seq; + int status; + int total; + int received; + int bodyLength; + int width; + int height; + int quality; + int[] mask = new int[1]; + byte[] body = new byte[64 * 1024]; + + void clear() { + seq = 0; + status = STATUS_COLLECTING; + received = 0; + } + + void start(Fragment f) { + seq = f.seq; + status = STATUS_COLLECTING; + total = f.total; + received = 0; + bodyLength = f.bodyLength; + width = f.width; + height = f.height; + quality = f.quality; + int words = (total + 31) / 32; + if (mask.length < words) mask = new int[words]; + Arrays.fill(mask, 0); + if (body.length < total * FRAGMENT_DATA) body = new byte[total * FRAGMENT_DATA]; + } + + boolean has(int i) { + return (mask[i >> 5] & (1 << (i & 31))) != 0; + } + + /** Mask word w as acknowledged: fragments past the frame's last count as received. */ + int ackWord(int w) { + int v = mask[w]; + int first = w * 32; + if (first + 32 > total) v |= -1 << (total - first); + return v; + } + } + + /** + * The four frames in flight and the acknowledgement built from them. A frame goes into + * slot (seq - 1) & 3; a newer frame of the same slot replaces the one there, which counts + * as lost unless it was complete (the drone should only send seq + 4 there, frames skipped + * on the way are lost too). Single-threaded; the counters are read by the stats. + */ + static final class Window { + final Slot[] slots = new Slot[SLOTS]; + long maxSeq; + long base; + /** The frame whose fragments arrive now: not acknowledged while it is coming in. */ + long current; + long lastDelivered; + long lastFragmentMs; + volatile int fragments; + volatile int duplicates; + volatile int stale; + volatile int mismatched; + volatile int completed; + volatile int delivered; + volatile int lost; + volatile int late; + volatile int wholeRequests; + volatile int nextRequests; + /** The frame numbers started from 1 again (the drone restarted). */ + volatile int restarts; + + Window() { + for (int i = 0; i < SLOTS; i++) slots[i] = new Slot(); + } + + void reset() { + for (Slot s : slots) s.clear(); + maxSeq = 0; + base = 0; + current = 0; + lastDelivered = 0; + } + + static int slotOf(long seq) { + return (int) ((seq - 1) & (SLOTS - 1)); + } + + /** Adds a fragment; returns its slot when the fragment completed a frame to show. */ + Slot add(Fragment f, long nowMs) { + lastFragmentMs = nowMs; + // Far behind the newest frame: the drone counts from 1 again + if (maxSeq > 64 && f.seq + 64 < maxSeq) { + restarts++; + reset(); + } + // Only a base behind the frame itself makes sense + if (f.base > base && f.base < f.seq) base = f.base; + Slot s = slots[slotOf(f.seq)]; + if (f.seq < s.seq) { + stale++; + return null; + } + if (f.seq > s.seq) { + if (s.seq > 0) { + if (s.status != STATUS_COMPLETE) lost++; + // Frames of this slot that never came at all + lost += (int) Math.min(1000, (f.seq - s.seq) / SLOTS - 1); + } + s.start(f); + } + fragments++; + current = f.seq; + if (f.seq > maxSeq) maxSeq = f.seq; + if (s.status == STATUS_COMPLETE || (f.total == s.total && s.has(f.index))) { + duplicates++; + return null; + } + if (f.total != s.total) { + mismatched++; + return null; + } + if (f.bodyLength != s.bodyLength) { + // Not seen as such; the longest is taken, the fragments fit either way + mismatched++; + s.bodyLength = Math.max(s.bodyLength, f.bodyLength); + } + int at = f.index * FRAGMENT_DATA; + int n = Math.min(f.dataLength, s.total * FRAGMENT_DATA - at); + System.arraycopy(f.buf, f.dataOffset, s.body, at, n); + s.mask[f.index >> 5] |= 1 << (f.index & 31); + s.received++; + if (s.received < s.total) return null; + s.status = STATUS_COMPLETE; + completed++; + if (s.seq <= lastDelivered) { + // Completed after a newer frame was shown + late++; + return null; + } + lastDelivered = s.seq; + delivered++; + return s; + } + + /** + * The acknowledgement for now, into {@code out} (ACK_CAPACITY bytes); returns its + * length. Per slot: complete (status 1) or still collecting (0) with the mask of the + * fragments received; a slot LAG or more frames behind the newest asks for its next + * frame whole instead (3, no mask). The frame coming in now is left out until the + * fragments pause; after QUIET_MS without any, when every frame is complete, a record + * asks for the frame after the newest. Unverified: whether the drone needs or uses any + * of it, the record order, and that the paused frame is included again. + */ + int buildAck(byte[] out, long nowMs) { + boolean quiet = nowMs - lastFragmentMs > QUIET_MS; + int pos = ACK_HEADER; + int records = 0; + boolean any = false; + boolean allComplete = true; + for (Slot s : slots) { + if (s.seq == 0) continue; + any = true; + if (s.status != STATUS_COMPLETE) allComplete = false; + if (s.seq == current && !quiet) continue; + if (s.seq <= base) continue; + if (maxSeq - s.seq >= LAG) { + pos = wholeRecord(out, pos, s.seq + SLOTS); + records++; + wholeRequests++; + continue; + } + int words = (s.total + 31) / 32; + le64(out, pos, s.seq); + boolean complete = s.status == STATUS_COMPLETE; + le32(out, pos + 8, complete ? STATUS_COMPLETE : STATUS_COLLECTING); + le32(out, pos + 12, 16 + 4 * words); + pos += 16; + for (int w = 0; w < words; w++) { + le32(out, pos, complete ? -1 : s.ackWord(w)); + pos += 4; + } + records++; + } + if (quiet && any && allComplete) { + pos = wholeRecord(out, pos, maxSeq + 1); + records++; + nextRequests++; + } + ackHeader(out, pos, records); + return pos; + } + + private static int wholeRecord(byte[] out, int pos, long seq) { + le64(out, pos, seq); + le32(out, pos + 8, STATUS_WHOLE); + le32(out, pos + 12, 16); + return pos + 16; + } + } + + /** + * The fixed part of an acknowledgement: EF 02, length, 02 02 00 01, record count, then no + * command (counter 0, length 0, 0x40 zero bytes), QPARA and active camera 0. + */ + static void ackHeader(byte[] out, int length, int records) { + out[0] = (byte) 0xEF; + out[1] = 0x02; + le16(out, 2, length); + out[4] = 0x02; + out[5] = 0x02; + out[6] = 0x00; + out[7] = 0x01; + le32(out, 0x08, records); + le32(out, 0x0C, 0); + le16(out, 0x10, 0); + Arrays.fill(out, 0x12, 0x52, (byte) 0); + System.arraycopy(QPARA, 0, out, 0x52, QPARA.length); + out[0x56] = 0; + out[0x57] = 0; + } + + // ====================== JPEG ====================== + + /** The headers in front of a body: SOI, DQT, SOF0, DHT, SOS (605 bytes as configured). */ + static byte[] jpegHeader(int quality, int width, int height) { + int q = quality >= 1 && quality <= 100 ? quality : DEFAULT_QUALITY; + return JpegAssembler.headersSampled(LUMA_SAMPLING, width > 0 ? width : DEFAULT_WIDTH, + height > 0 ? height : DEFAULT_HEIGHT, JpegAssembler.ijgTables(q), 0, + RESTART_INTERVAL); + } + + /** + * Header, body and EOI into {@code out} (grown when too small, returned in out[0]); + * returns the length. A body that is a whole JPEG already goes out as it is. + */ + static int jpeg(byte[] header, byte[] body, int bodyLength, byte[][] out) { + boolean whole = bodyLength >= 2 && (body[0] & 0xFF) == 0xFF && (body[1] & 0xFF) == 0xD8; + boolean eoi = bodyLength >= 2 && (body[bodyLength - 2] & 0xFF) == 0xFF + && (body[bodyLength - 1] & 0xFF) == 0xD9; + int head = whole ? 0 : header.length; + int total = head + bodyLength + (eoi ? 0 : 2); + if (out[0] == null || out[0].length < total) out[0] = new byte[total + 64 * 1024]; + byte[] d = out[0]; + if (!whole) System.arraycopy(header, 0, d, 0, head); + System.arraycopy(body, 0, d, head, bodyLength); + if (!eoi) { + d[total - 2] = (byte) 0xFF; + d[total - 1] = (byte) 0xD9; + } + return total; + } + + // ====================== Client ====================== + + interface Listener { + /** Probing (no fragments, or none for resetMs) or receiving; text for the status. */ + void onState(boolean receiving, String text); + + /** One whole JPEG in {@code buf}, valid until the call returns. */ + void onFrame(byte[] buf, int length); + } + + /** Called with the socket before the first send: binds it to the drone's Wi-Fi. */ + interface SocketSetup { + String prepare(DatagramSocket socket, InetAddress target); + } + + /** + * The receive loop on the calling thread: probes, fragments, acknowledgements and the + * timers between them, until {@code keepGoing} fails or close(). + */ + static final class Client implements Closeable { + private final String tag; + private final Url url; + private final SocketSetup setup; + private volatile DatagramSocket socket; + private volatile boolean closed; + + final Window window = new Window(); + private final Fragment fragment = new Fragment(); + private final byte[] ack = new byte[ACK_CAPACITY]; + private final byte[][] jpegOut = new byte[1][]; + private byte[] header; + /** Quality and size the header was made for. */ + private int headerQuality; + private int headerWidth; + private int headerHeight; + + // For the stats; written by the loop thread only + private volatile boolean receiving; + private volatile boolean playing; + private volatile String network = "default"; + private volatile int probes; + private volatile int acks; + private volatile int packets; + private volatile long bytes; + private volatile int badPackets; + private volatile int foreign; + private volatile int resets; + private volatile int sendErrors; + private volatile String lastError; + private volatile int services; + private volatile String lastService; + private volatile String firstHeader; + private volatile String size; + private volatile int quality; + private volatile boolean gl = true; + private volatile float fragmentsPerS; + private volatile float framesPerS; + + Client(String tag, Url url, SocketSetup setup) { + this.tag = tag; + this.url = url; + this.setup = setup; + } + + @Override + public void close() { + closed = true; + DatagramSocket s = socket; + if (s != null) s.close(); + } + + String probingText(boolean again) { + return (again ? "The toy went quiet: probing " : "Waiting for the toy: probing ") + + url.endpoint() + (again ? " again" : ""); + } + + private DatagramSocket open(InetAddress target) throws IOException { + DatagramSocket s = new DatagramSocket(); + try { + s.setReceiveBufferSize(1024 * 1024); + } catch (IOException ignored) {} + if (setup != null) network = setup.prepare(s, target); + socket = s; + if (closed) s.close(); + return s; + } + + void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException { + InetAddress address = InetAddress.getByName(url.host); + InetSocketAddress target = new InetSocketAddress(address, url.port); + DatagramSocket s = open(address); + Log.i(TAG, tag + ": FLOW-UFO: probing " + url.endpoint() + " from local port " + + s.getLocalPort() + " (network: " + network + ")"); + listener.onState(false, probingText(false)); + byte[] rx = new byte[2048]; + DatagramPacket in = new DatagramPacket(rx, rx.length); + DatagramPacket probe = new DatagramPacket(PROBE, PROBE.length, target); + DatagramPacket out = new DatagramPacket(ack, 0, target); + long nextProbe = 0; + long nextAck = 0; + long rateMark = now(); + int rateFragments = 0; + int rateFrames = 0; + long nextSummary = rateMark + 30_000; + int failedSends = 0; + try { + while (!closed && keepGoing.holds()) { + long t = now(); + if (t - rateMark >= 1000) { + float sec = (t - rateMark) / 1000f; + fragmentsPerS = (window.fragments - rateFragments) / sec; + framesPerS = (window.delivered - rateFrames) / sec; + rateFragments = window.fragments; + rateFrames = window.delivered; + rateMark = t; + } + if (receiving && t - window.lastFragmentMs > resetMs) { + resets++; + receiving = false; + playing = false; + window.reset(); + nextProbe = t; + Log.w(TAG, tag + ": FLOW-UFO: no fragment for " + resetMs + + " ms, probing again (" + summary() + ")"); + listener.onState(false, probingText(true)); + } + if (receiving && t >= nextSummary) { + nextSummary = t + 30_000; + Log.i(TAG, tag + ": FLOW-UFO: " + summary()); + } + boolean sendFailed = false; + if (!receiving && t >= nextProbe) { + nextProbe = t + PROBE_MS; + sendFailed = !send(s, probe); + if (!sendFailed) probes++; + } else if (receiving && t >= nextAck) { + // Behind schedule (a slow thread) is caught up, not sent in a burst + nextAck = Math.max(nextAck + ACK_MS, t); + int len = window.buildAck(ack, t); + out.setLength(len); + sendFailed = !send(s, out); + if (!sendFailed) acks++; + } + if (sendFailed) { + // Off the toy's Wi-Fi, or its network went away: a new socket after + // 3 s of failures, bound again to whatever network reaches it now + if (++failedSends >= 30) { + failedSends = 0; + s.close(); + s = open(address); + } + } else { + failedSends = 0; + } + long wake = receiving ? nextAck : nextProbe; + int timeout = (int) Math.max(1, Math.min(100, wake - now())); + try { + s.setSoTimeout(timeout); + in.setLength(rx.length); + s.receive(in); + } catch (SocketTimeoutException e) { + continue; + } + if (!in.getAddress().equals(address)) { + if (++foreign <= 3) { + Log.w(TAG, tag + ": FLOW-UFO: packet from " + in.getAddress() + + ", not the toy: ignored"); + } + continue; + } + handle(rx, in.getLength(), listener); + } + } catch (IOException e) { + if (closed) return; + throw e; + } finally { + s.close(); + } + } + + private boolean send(DatagramSocket s, DatagramPacket p) { + try { + s.send(p); + return true; + } catch (IOException e) { + sendErrors++; + lastError = e.getMessage(); + if (sendErrors <= 3 || sendErrors % 100 == 0) { + Log.w(TAG, tag + ": FLOW-UFO: cannot send to " + url.endpoint() + " (" + + sendErrors + "): " + e.getMessage() + + "; is this device on the toy's Wi-Fi?"); + } + return false; + } + } + + private void handle(byte[] b, int len, Listener listener) { + packets++; + bytes += len; + int type = packetType(b, 0, len); + if (type == TYPE_SERVICE) { + // Logged only: the vendor app answers some of them with commands, we do not + services++; + lastService = hex(b, 0, Math.min(len, 64)); + if (services <= 5 || services % 100 == 0) { + Log.i(TAG, tag + ": FLOW-UFO: service message (" + services + "): " + + lastService); + } + return; + } + if (type != TYPE_FRAGMENT || !fragment.parse(b, 0, len)) { + badPackets++; + if (badPackets <= 5 || badPackets % 500 == 0) { + Log.w(TAG, tag + ": FLOW-UFO: unknown or malformed packet (" + badPackets + + ", " + len + " bytes): " + hex(b, 0, Math.min(len, 64))); + } + return; + } + long t = now(); + if (!receiving) { + receiving = true; + if (firstHeader == null) firstHeader = hex(b, 0, FRAGMENT_HEADER); + Log.i(TAG, tag + ": FLOW-UFO: first fragment after " + probes + " probes: " + + hex(b, 0, FRAGMENT_HEADER)); + listener.onState(true, "Receiving from the toy at " + url.endpoint() + "…"); + } + int n = window.fragments; + if (n < 5) { + Log.i(TAG, tag + ": FLOW-UFO: fragment seq " + fragment.seq + " base " + + fragment.base + " " + fragment.index + "/" + fragment.total + ", body " + + fragment.bodyLength + ", " + fragment.width + "x" + fragment.height + + " Q" + fragment.quality + ", bytes 0x08-0x17: " + hex(b, 0x08, 16)); + } + if (!fragment.gl && gl) { + gl = false; + Log.w(TAG, tag + ": FLOW-UFO: bytes 0x34/0x35 are " + hex(b, 0x34, 2) + + ", not 01 01: another camera family, frames may not decode"); + } + Slot done = window.add(fragment, t); + if (done == null) return; + if (header == null || done.quality != headerQuality || done.width != headerWidth + || done.height != headerHeight) { + header = jpegHeader(done.quality, done.width, done.height); + headerQuality = done.quality; + headerWidth = done.width; + headerHeight = done.height; + quality = done.quality; + size = done.width + "x" + done.height; + Log.i(TAG, tag + ": FLOW-UFO: frames " + size + " Q" + done.quality + + (done.quality >= 1 && done.quality <= 100 ? "" + : " (not a quality, Q" + DEFAULT_QUALITY + " tables)")); + } + int length = jpeg(header, done.body, done.bodyLength, jpegOut); + if (!playing) { + playing = true; + Log.i(TAG, tag + ": FLOW-UFO: first frame, seq " + done.seq + ", " + + done.total + " fragments, body " + done.bodyLength + " bytes: " + + hex(done.body, 0, Math.min(32, done.bodyLength))); + } + listener.onFrame(jpegOut[0], length); + } + + private String summary() { + Window w = window; + return String.format(Locale.ROOT, "%.1f fps, %.0f fragments/s, %d frames, %d lost," + + " %d late, %d whole requests, %d next requests, %d acks, %d bad", + framesPerS, fragmentsPerS, w.delivered, w.lost, w.late, w.wholeRequests, + w.nextRequests, acks, badPackets); + } + + /** For /stats.json. */ + Map stats() { + Window w = window; + Map m = new LinkedHashMap<>(); + m.put("state", playing ? "playing" : receiving ? "receiving" : "probing"); + m.put("target", url.endpoint()); + m.put("network", network); + m.put("frames_per_s", Math.round(framesPerS * 10) / 10f); + m.put("fragments_per_s", Math.round(fragmentsPerS)); + if (size != null) m.put("size", size); + if (size != null) m.put("quality", quality); + m.put("gl", gl); + m.put("probes", probes); + m.put("acks", acks); + m.put("packets", packets); + m.put("bytes", bytes); + m.put("fragments", w.fragments); + m.put("duplicates", w.duplicates); + m.put("stale", w.stale); + m.put("mismatched", w.mismatched); + m.put("bad_packets", badPackets); + m.put("frames", w.delivered); + m.put("frames_complete", w.completed); + m.put("frames_lost", w.lost); + m.put("frames_late", w.late); + m.put("whole_requests", w.wholeRequests); + m.put("next_requests", w.nextRequests); + m.put("resets", resets); + m.put("seq_restarts", w.restarts); + m.put("seq", w.maxSeq); + m.put("base", w.base); + m.put("service_messages", services); + if (lastService != null) m.put("last_service", lastService); + if (firstHeader != null) m.put("first_fragment_header", firstHeader); + if (foreign > 0) m.put("foreign_packets", foreign); + if (sendErrors > 0) { + m.put("send_errors", sendErrors); + m.put("last_error", lastError); + } + return m; + } + } + + // ====================== Bytes ====================== + + private static long now() { + return System.nanoTime() / 1_000_000; + } + + static String hex(byte[] b, int off, int len) { + StringBuilder sb = new StringBuilder(len * 3); + for (int i = 0; i < len; i++) { + if (i > 0) sb.append(' '); + sb.append(String.format(Locale.ROOT, "%02X", b[off + i] & 0xFF)); + } + return sb.toString(); + } + + static int u16(byte[] b, int off) { + return (b[off] & 0xFF) | (b[off + 1] & 0xFF) << 8; + } + + static long u32(byte[] b, int off) { + return (u16(b, off) | (long) u16(b, off + 2) << 16) & 0xFFFFFFFFL; + } + + static void le16(byte[] b, int off, int v) { + b[off] = (byte) v; + b[off + 1] = (byte) (v >> 8); + } + + static void le32(byte[] b, int off, int v) { + le16(b, off, v); + le16(b, off + 2, v >> 16); + } + + static void le64(byte[] b, int off, long v) { + le32(b, off, (int) v); + le32(b, off + 4, (int) (v >>> 32)); + } +} diff --git a/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java b/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java index 06ebfbcf..a46d6fb7 100644 --- a/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java +++ b/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java @@ -247,7 +247,15 @@ static int[] size(byte[] d, int length) { /** Quantization tables for Q 1..99 (RFC 2435 Appendix A), luminance then chrominance. */ static byte[] tables(int q) { - q = Math.max(1, Math.min(99, q)); + return ijgTables(Math.max(1, Math.min(99, q))); + } + + /** + * The standard tables scaled to an IJG quality 1..100 as libjpeg does it (100: all ones), + * zig-zag order, luminance then chrominance (#187). + */ + static byte[] ijgTables(int q) { + q = Math.max(1, Math.min(100, q)); int factor = q < 50 ? 5000 / q : 200 - q * 2; byte[] out = new byte[128]; for (int i = 0; i < 64; i++) { @@ -274,6 +282,16 @@ static byte[] headers(int type, int width, int height, byte[] tables, int restar static byte[] headers(int type, int width, int height, byte[] tables, int precision, int restartInterval) { + return headersSampled((type & 63) == 0 ? 0x21 : 0x22, width, height, tables, precision, + restartInterval); + } + + /** + * As headers(), with the luma sampling given ({@code 0x11} for 4:4:4, #187); chroma is + * always 1x1. + */ + static byte[] headersSampled(int lumaSampling, int width, int height, byte[] tables, + int precision, int restartInterval) { java.io.ByteArrayOutputStream o = new java.io.ByteArrayOutputStream(700); o.write(0xFF); o.write(0xD8); @@ -297,7 +315,7 @@ static byte[] headers(int type, int width, int height, byte[] tables, int precis o.write(3); int chromaTable = count > 1 ? 1 : 0; o.write(0); - o.write((type & 63) == 0 ? 0x21 : 0x22); + o.write(lumaSampling); o.write(0); o.write(1); o.write(0x11); diff --git a/android/app/src/main/java/com/openipc/decoder/NetInfo.java b/android/app/src/main/java/com/openipc/decoder/NetInfo.java index 2d53a3a1..c155e2e6 100644 --- a/android/app/src/main/java/com/openipc/decoder/NetInfo.java +++ b/android/app/src/main/java/com/openipc/decoder/NetInfo.java @@ -7,8 +7,15 @@ package com.openipc.decoder; +import android.content.Context; +import android.net.ConnectivityManager; +import android.net.LinkAddress; +import android.net.LinkProperties; +import android.net.Network; +import android.os.Build; import android.util.Log; +import java.net.DatagramSocket; import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; @@ -58,4 +65,47 @@ static boolean isLocalClient(InetAddress a) { if (a instanceof Inet4Address) return a.isSiteLocalAddress(); return a instanceof Inet6Address && (a.getAddress()[0] & 0xfe) == 0xfc; } + + /** + * Binds {@code socket} to the network whose own subnet holds {@code target} (#187). A toy's + * Wi-Fi has no internet: with mobile data on, Android may send even on-link traffic over the + * default network instead. Returns what was done, for the log: "default" when no such + * network is found (or before Android 5.1), else the interface bound to. + */ + static String bindToNetworkOf(DatagramSocket socket, InetAddress target) { + Context ctx = App.context; + if (ctx == null || Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP_MR1) { + return "default"; + } + try { + ConnectivityManager cm = + (ConnectivityManager) ctx.getSystemService(Context.CONNECTIVITY_SERVICE); + if (cm == null) return "default"; + for (Network n : cm.getAllNetworks()) { + LinkProperties lp = cm.getLinkProperties(n); + if (lp == null) continue; + for (LinkAddress la : lp.getLinkAddresses()) { + if (!sameSubnet(la.getAddress(), la.getPrefixLength(), target)) continue; + n.bindSocket(socket); + return "bound to " + lp.getInterfaceName(); + } + } + } catch (Exception e) { + Log.w(TAG, "No network bound for " + target.getHostAddress() + ": " + e.getMessage()); + } + return "default"; + } + + /** {@code a} and {@code b} share the first {@code prefix} bits (and the address family). */ + static boolean sameSubnet(InetAddress a, int prefix, InetAddress b) { + if (a == null || b == null) return false; + byte[] x = a.getAddress(); + byte[] y = b.getAddress(); + if (x.length != y.length || prefix < 0 || prefix > x.length * 8) return false; + for (int i = 0; i < prefix; i++) { + int mask = 0x80 >> (i & 7); + if ((x[i >> 3] & mask) != (y[i >> 3] & mask)) return false; + } + return true; + } } diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index e4e776d9..9c409dfa 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -56,6 +56,8 @@ class QuadCell implements RtspSession.Callback { private volatile String state = "connecting"; /** What keeps the cell from playing (connection error, decoder limit), or null. */ private volatile String problem; + /** What a flowufo:// cell waits for (#187), shown instead of "Buffering…", or null. */ + private volatile String detail; /** Why the cell was paused to relieve the decoder (#83), or null. */ private volatile String pausedReason; private volatile long decoderStartedMs; @@ -292,6 +294,11 @@ public void onStatus(String status) { } } + @Override + public void onDetail(String text) { + detail = text; + } + @Override public void onFailure(String message) { // A decoder problem stays visible; the connection retries on its own @@ -351,6 +358,8 @@ String sample(long nowMs) { if (p != null) return p; String st = state; if (!"connected".equals(st)) { + String d = detail; + if (d != null) return d; if ("disconnected".equals(st)) return "Disconnected"; return "buffering".equals(st) ? "Buffering…" : "Connecting…"; } @@ -417,6 +426,9 @@ static boolean isKeyFrameNal(int type, boolean h265) { return h265 ? type >= 16 && type <= 21 : type == 5; } + /** FLOW-UFO statistics of a flowufo:// cell (#187), else null. */ + java.util.Map flowUfoStats() { return session.flowUfoStats(); } + /** Transport in use, with the mode after a fallback (#112). */ String transportLabel() { return Transport.label(session.transportInUse(), transport); } diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java index 287e4a67..84a46e42 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java @@ -43,6 +43,8 @@ interface Listener { void onStatusChanged(String status); void onCodecChanged(boolean codecH265, AudioConfig audio); void onJitterSample(int avgJitter); + /** What a flowufo:// slot waits for (#187), shown instead of the plain status. */ + default void onDetail(String text) {} } private volatile Listener listenerCb; @@ -85,6 +87,8 @@ void configure(String host, String userAgent, Transport.Mode transport) { /** The stream is MJPEG: the queue carries complete JPEG frames (#119). */ boolean isJpeg() { return videoJpeg; } long getBitrateMarkNs() { return session.getBitrateMarkNs(); } + /** FLOW-UFO statistics while a flowufo:// slot plays (#187), else null. */ + java.util.Map flowUfoStats() { return session.flowUfoStats(); } void start() { nalAssembler.reset(); @@ -109,6 +113,12 @@ public void onStatus(String status) { if (l != null) l.onStatusChanged(status); } + @Override + public void onDetail(String text) { + Listener l = listenerCb; + if (l != null) l.onDetail(text); + } + @Override public void onSdp(Sdp sdp) { nalAssembler.reset(); diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index 40bd0581..4174bd03 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -71,6 +71,12 @@ default void onFailure(String message) {} * Returns true if it was usable; only those feed the watchdog. */ default boolean onJpegFrame(byte[] buf, int offset, int length) { return false; } + + /** + * What a flowufo:// slot waits for while it has no picture (#187), e.g. "Waiting for + * the toy: probing 192.168.169.1:8800"; shown instead of the plain status. + */ + default void onDetail(String text) {} } private final String tag; @@ -99,6 +105,8 @@ default void onFailure(String message) {} private RtpListener listener; /** A udp:// slot is listening: silence is reported, never "fixed" by a rebind. */ private volatile boolean listening; + /** Client of a flowufo:// slot (#187), else null: it probes on its own when silent. */ + private volatile FlowUfo.Client ufo; private Thread networkThread; private Thread watchdogThread; @@ -201,6 +209,7 @@ void stop() { */ private void closeSocketLocked() { if (listener != null) listener.close(); + if (ufo != null) ufo.close(); if (udp != null) { udp.close(); if (udpPlaying) return; @@ -273,7 +282,7 @@ private void watchdogLoop(int gen) { } } } - if (active && !listening + if (active && !listening && ufo == null && System.nanoTime() / 1_000_000 - lastPacketMs > watchdogMs) { Log.w(TAG, tag + ": stream is inactive, reconnecting"); active = false; @@ -293,6 +302,7 @@ private void watchdogLoop(int gen) { private boolean connect(int gen) throws IOException { if (RtpListener.Url.matches(url)) return listen(gen, RtpListener.Url.parse(url)); if (HttpMjpeg.matches(url)) return httpMjpeg(gen, HttpMjpeg.Url.parse(url)); + if (FlowUfo.Url.matches(url)) return flowUfo(gen, FlowUfo.Url.parse(url)); Url u = Url.parse(url); Socket s = new Socket(); synchronized (lock) { @@ -589,6 +599,76 @@ private boolean httpMjpeg(int gen, HttpMjpeg.Url u) throws IOException { throw new IOException("401 Unauthorized"); } + /** FLOW-UFO statistics for /stats.json while a flowufo:// slot runs (#187), else null. */ + Map flowUfoStats() { + FlowUfo.Client c = ufo; + return c != null ? c.stats() : null; + } + + /** + * A flowufo:// slot (#187): the toy drone's own protocol over UDP, JPEG frames to + * onJpegFrame. The client probes until the drone answers and again after it went quiet, + * so the watchdog leaves it alone. Returns true once it ran, like a PLAY that succeeded. + */ + private boolean flowUfo(int gen, FlowUfo.Url u) throws IOException { + FlowUfo.Client c = new FlowUfo.Client(tag, u, NetInfo::bindToNetworkOf); + synchronized (lock) { + if (gen != generation) return false; + ufo = c; + keepalive = null; + lastPacketMs = System.nanoTime() / 1_000_000; + streamBytes = 0; + bitrateMarkNs = System.nanoTime(); + active = true; + } + try { + Sdp sdp = new Sdp(); + sdp.videoJpeg = true; + sdp.videoPt = Sdp.PT_JPEG; + callback.onSdp(sdp); + final boolean[] first = {true}; + final boolean[] started = {false}; + c.run(() -> active && gen == generation, new FlowUfo.Listener() { + @Override + public void onState(boolean receiving, String text) { + if (gen != generation) return; + if (receiving) { + inUse = "FLOW-UFO UDP"; + } else { + inUse = null; + // The toy went quiet: say so, the next frame reports "connected" again + status(gen, started[0] ? "disconnected" : "buffering"); + started[0] = true; + first[0] = true; + } + callback.onDetail(text); + } + + @Override + public void onFrame(byte[] buf, int length) { + if (gen != generation) return; + streamBytes += length; + if (callback.onJpegFrame(buf, 0, length)) { + lastPacketMs = System.nanoTime() / 1_000_000; + if (first[0]) { + first[0] = false; + status(gen, "connected"); + } + } + } + }); + return true; + } finally { + synchronized (lock) { + if (ufo == c) { + ufo = null; + inUse = null; + } + } + c.close(); + } + } + private boolean listen(int gen, RtpListener.Url u) throws IOException { RtpListener l = new RtpListener(tag, u); synchronized (lock) { diff --git a/android/app/src/main/java/com/openipc/decoder/StreamProbes.java b/android/app/src/main/java/com/openipc/decoder/StreamProbes.java index 9e3d0946..4c1ddfd6 100644 --- a/android/app/src/main/java/com/openipc/decoder/StreamProbes.java +++ b/android/app/src/main/java/com/openipc/decoder/StreamProbes.java @@ -49,6 +49,8 @@ static String skipReason(String url, String defaultUrl, String screenUrl) { if (url == null || url.trim().isEmpty() || url.equals(defaultUrl)) return NOT_CONFIGURED; // A listening slot would take the port; the camera pushes to it, nothing to ask if (RtpListener.Url.matches(url)) return "UDP listen slot: not checked in single view"; + // A toy drone sends to one client; a check would take the stream from the screen + if (FlowUfo.Url.matches(url)) return "toy drone slot: not checked in single view"; String e = endpoint(url); if (e == null) return "invalid URL"; if (e.equals(endpoint(screenUrl))) return "same camera as on screen"; diff --git a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java new file mode 100644 index 00000000..6a6ac1bd --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java @@ -0,0 +1,738 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * FlowUfoTest.java — flowufo:// slots: packets, acknowledgements, JPEG headers, a fake drone + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.After; +import org.junit.Test; +import org.w3c.dom.NodeList; + +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.SocketAddress; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import javax.imageio.IIOImage; +import javax.imageio.ImageIO; +import javax.imageio.ImageTypeSpecifier; +import javax.imageio.ImageWriteParam; +import javax.imageio.ImageWriter; +import javax.imageio.metadata.IIOMetadata; +import javax.imageio.metadata.IIOMetadataNode; +import javax.imageio.plugins.jpeg.JPEGQTable; +import javax.imageio.stream.ImageOutputStream; + +public class FlowUfoTest { + /** Zig-zag position i to natural (row-major) position. */ + private static final int[] NATURAL = { + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63 + }; + + @After + public void restoreTimers() { + FlowUfo.resetMs = 3000; + } + + // ====================== Helpers ====================== + + private static byte[] bytes(String hex) { + String h = hex.replaceAll("\\s", ""); + byte[] b = new byte[h.length() / 2]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) Integer.parseInt(h.substring(2 * i, 2 * i + 2), 16); + } + return b; + } + + /** A type-1 packet as the drone sends it. */ + static byte[] fragment(long seq, long base, int index, int total, int bodyLength, int width, + int height, int quality, byte[] data, int dataOff, int dataLen) { + byte[] p = new byte[FlowUfo.FRAGMENT_HEADER + dataLen]; + p[0] = (byte) 0x93; + p[1] = 0x01; + FlowUfo.le16(p, 2, p.length); + FlowUfo.le64(p, 0x08, seq); + FlowUfo.le64(p, 0x18, base); + FlowUfo.le32(p, 0x20, index); + FlowUfo.le32(p, 0x24, total); + FlowUfo.le32(p, 0x28, bodyLength); + FlowUfo.le16(p, 0x2C, width); + FlowUfo.le16(p, 0x2E, height); + p[0x30] = (byte) quality; + p[0x34] = 1; + p[0x35] = 1; + System.arraycopy(data, dataOff, p, FlowUfo.FRAGMENT_HEADER, dataLen); + return p; + } + + /** Fragment {@code index} of {@code body} for frame {@code seq}. */ + static byte[] piece(long seq, byte[] body, int index) { + return piece(seq, body, index, 640, 360, 50); + } + + static byte[] piece(long seq, byte[] body, int index, int width, int height, int quality) { + int total = (body.length + FlowUfo.FRAGMENT_DATA - 1) / FlowUfo.FRAGMENT_DATA; + int at = index * FlowUfo.FRAGMENT_DATA; + int len = Math.min(FlowUfo.FRAGMENT_DATA, body.length - at); + return fragment(seq, 0, index, total, body.length, width, height, quality, body, at, + len); + } + + private static FlowUfo.Fragment parsed(byte[] p) { + FlowUfo.Fragment f = new FlowUfo.Fragment(); + assertTrue(f.parse(p, 0, p.length)); + return f; + } + + private static byte[] ack(FlowUfo.Window w, long nowMs) { + byte[] out = new byte[FlowUfo.ACK_CAPACITY]; + int len = w.buildAck(out, nowMs); + return Arrays.copyOf(out, len); + } + + /** {seq, status, record length} of each record of an acknowledgement. */ + private static List records(byte[] a) { + List list = new ArrayList<>(); + int n = (int) FlowUfo.u32(a, 0x08); + int pos = FlowUfo.ACK_HEADER; + for (int i = 0; i < n; i++) { + long seq = FlowUfo.u32(a, pos) | FlowUfo.u32(a, pos + 4) << 32; + long status = FlowUfo.u32(a, pos + 8); + long len = FlowUfo.u32(a, pos + 12); + list.add(new long[] {seq, status, len, pos}); + pos += (int) len; + } + assertEquals("records fill the packet", a.length, pos); + return list; + } + + // ====================== URL ====================== + + @Test + public void urlDefaultsAndPort() throws IOException { + FlowUfo.Url u = FlowUfo.Url.parse("flowufo://192.168.169.1:8800"); + assertEquals("192.168.169.1", u.host); + assertEquals(8800, u.port); + u = FlowUfo.Url.parse("flowufo://"); + assertEquals(FlowUfo.DEFAULT_HOST, u.host); + assertEquals(FlowUfo.DEFAULT_PORT, u.port); + u = FlowUfo.Url.parse(" FLOWUFO://10.0.0.5/ "); + assertEquals("10.0.0.5", u.host); + assertEquals(8800, u.port); + u = FlowUfo.Url.parse("flowufo://:9000"); + assertEquals(FlowUfo.DEFAULT_HOST, u.host); + assertEquals(9000, u.port); + assertEquals("192.168.169.1:9000", u.endpoint()); + assertTrue(FlowUfo.Url.matches("flowufo://")); + assertFalse(FlowUfo.Url.matches("udp://@:5600")); + } + + @Test + public void badUrlsAreRefused() { + for (String bad : new String[] {"flowufo://host:0", "flowufo://host:x", + "flowufo://host:70000", "flowufo://a b", "flowufo://u@host", "http://x"}) { + try { + FlowUfo.Url.parse(bad); + fail(bad); + } catch (IOException expected) { + // ok + } + } + } + + @Test + public void notCheckedBySingleViewProbes() { + assertEquals("toy drone slot: not checked in single view", + StreamProbes.skipReason("flowufo://", "rtsp://default", null)); + } + + // ====================== Packets ====================== + + @Test + public void packetTypeNeedsMagicAndLength() { + byte[] service = bytes("93 04 0E 00 01 00 00 00 02 00 00 00 55 5A"); + assertEquals(FlowUfo.TYPE_SERVICE, FlowUfo.packetType(service, 0, service.length)); + // The length field must be the datagram size + assertEquals(-1, FlowUfo.packetType(service, 0, service.length - 1)); + byte[] other = service.clone(); + other[0] = (byte) 0x94; + assertEquals(-1, FlowUfo.packetType(other, 0, other.length)); + byte[] big = new byte[FlowUfo.MAX_PACKET + 1]; + big[0] = (byte) 0x93; + FlowUfo.le16(big, 2, big.length); + assertEquals(-1, FlowUfo.packetType(big, 0, big.length)); + } + + @Test + public void fragmentFields() { + byte[] data = new byte[1024]; + for (int i = 0; i < data.length; i++) data[i] = (byte) i; + byte[] p = fragment(7, 3, 2, 5, 4500, 640, 360, 75, data, 0, 1024); + // An unknown field at 0x0C (if 0x08 is a u32): the frame number stays 7 + p[0x0C] = 0x55; + p[0x0F] = 0x66; + FlowUfo.Fragment f = parsed(p); + assertEquals(7, f.seq); + assertEquals(3, f.base); + assertEquals(2, f.index); + assertEquals(5, f.total); + assertEquals(4500, f.bodyLength); + assertEquals(640, f.width); + assertEquals(360, f.height); + assertEquals(75, f.quality); + assertTrue(f.gl); + assertEquals(FlowUfo.FRAGMENT_HEADER, f.dataOffset); + assertEquals(1024, f.dataLength); + assertSame(p, f.buf); + } + + @Test + public void malformedFragmentsAreRefused() { + byte[] d = new byte[1024]; + FlowUfo.Fragment f = new FlowUfo.Fragment(); + // Index beyond the count, no fragments, body longer than the fragments, no frame number + byte[][] bad = { + fragment(1, 0, 3, 3, 3000, 640, 360, 50, d, 0, 10), + fragment(1, 0, 0, 0, 3000, 640, 360, 50, d, 0, 10), + fragment(1, 0, 0, 2, 2049, 640, 360, 50, d, 0, 10), + fragment(0, 0, 0, 1, 10, 640, 360, 50, d, 0, 10), + fragment(1, 0, 0, FlowUfo.MAX_FRAGMENTS + 1, 10, 640, 360, 50, d, 0, 10), + }; + for (byte[] p : bad) assertFalse(f.parse(p, 0, p.length)); + byte[] shortOne = Arrays.copyOf(fragment(1, 0, 0, 1, 10, 0, 0, 50, d, 0, 10), 0x30); + FlowUfo.le16(shortOne, 2, shortOne.length); + assertFalse(f.parse(shortOne, 0, shortOne.length)); + byte[] service = fragment(1, 0, 0, 1, 10, 0, 0, 50, d, 0, 10); + service[1] = 4; + assertFalse(f.parse(service, 0, service.length)); + } + + // ====================== Window and acknowledgements ====================== + + @Test + public void ackBytesExact() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] d = new byte[3 * 1024]; + // Frame 1 of two fragments, complete; frame 2 of three with the middle one missing + assertNull(w.add(parsed(piece(1, Arrays.copyOf(d, 2000), 0)), 1000)); + assertNotNull(w.add(parsed(piece(1, Arrays.copyOf(d, 2000), 1)), 1000)); + assertNull(w.add(parsed(piece(2, Arrays.copyOf(d, 2100), 0)), 1010)); + assertNull(w.add(parsed(piece(2, Arrays.copyOf(d, 2100), 2)), 1010)); + + // Fragments still coming: frame 2 is the one arriving, left out + String header = "EF 02 %s 02 02 00 01 %s 00 00 00 00 00 00" + + " 00000000000000000000000000000000 00000000000000000000000000000000" + + " 00000000000000000000000000000000 00000000000000000000000000000000" + + " 19 32 23 4B 00 00"; + byte[] expected = bytes(String.format(header, "6C 00", "01 00 00 00") + + " 01 00 00 00 00 00 00 00 01 00 00 00 14 00 00 00 FF FF FF FF"); + assertArrayEquals(expected, ack(w, 1015)); + + // A pause: frame 2 is in again, collecting, fragment 1 missing (bits past 3 set) + expected = bytes(String.format(header, "80 00", "02 00 00 00") + + " 01 00 00 00 00 00 00 00 01 00 00 00 14 00 00 00 FF FF FF FF" + + " 02 00 00 00 00 00 00 00 00 00 00 00 14 00 00 00 FD FF FF FF"); + assertArrayEquals(expected, ack(w, 1030)); + + // All complete and quiet: frame 3 is asked for whole (status 3, no mask) + assertNotNull(w.add(parsed(piece(2, Arrays.copyOf(d, 2100), 1)), 1040)); + expected = bytes(String.format(header, "90 00", "03 00 00 00") + + " 01 00 00 00 00 00 00 00 01 00 00 00 14 00 00 00 FF FF FF FF" + + " 02 00 00 00 00 00 00 00 01 00 00 00 14 00 00 00 FF FF FF FF" + + " 03 00 00 00 00 00 00 00 03 00 00 00 10 00 00 00"); + assertArrayEquals(expected, ack(w, 1060)); + assertEquals(1, w.nextRequests); + } + + @Test + public void maskOfManyFragments() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[40 * 1024 - 100]; + // 40 fragments: two mask words; only 0 and 33 arrive + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(piece(1, body, 33)), 0); + byte[] a = ack(w, 100); + List r = records(a); + assertEquals(1, r.size()); + assertEquals(16 + 8, r.get(0)[2]); + int pos = (int) r.get(0)[3] + 16; + assertEquals(0x00000001L, FlowUfo.u32(a, pos)); + // Bit 1 is fragment 33; bits 8.. are past fragment 39 + assertEquals(0xFFFFFF02L, FlowUfo.u32(a, pos + 4)); + } + + @Test + public void lateSlotAsksForItsNextFrameWhole() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[100]; + // Frames 1..4 and 6, each one fragment; frame 5 (slot of frame 1) never comes + for (long seq : new long[] {1, 2, 3, 4, 6}) { + assertNotNull(w.add(parsed(piece(seq, body, 0)), 0)); + } + List r = records(ack(w, 5)); + // Slot 0 (frame 1) is 5 behind: frame 5 whole; 3 and 4 complete; 6 is arriving + assertEquals(3, r.size()); + assertArrayEquals(new long[] {5, FlowUfo.STATUS_WHOLE, 16}, Arrays.copyOf(r.get(0), 3)); + assertArrayEquals(new long[] {3, FlowUfo.STATUS_COMPLETE, 20}, Arrays.copyOf(r.get(1), 3)); + assertArrayEquals(new long[] {4, FlowUfo.STATUS_COMPLETE, 20}, Arrays.copyOf(r.get(2), 3)); + assertEquals(1, w.wholeRequests); + // Frame 9 takes slot 0: frame 1 was complete, frame 5 never came + w.add(parsed(piece(9, body, 0)), 10); + assertEquals(1, w.lost); + } + + @Test + public void baseHidesOldFrames() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[100]; + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(piece(2, body, 0)), 0); + byte[] p = fragment(3, 1, 0, 1, body.length, 640, 360, 50, body, 0, body.length); + w.add(parsed(p), 0); + List r = records(ack(w, 1)); + // 1 is at or below the base, 3 is arriving: only 2 + assertEquals(1, r.size()); + assertEquals(2, r.get(0)[0]); + } + + @Test + public void incompleteFrameReplacedCountsAsLost() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + w.add(parsed(piece(1, body, 0)), 0); + // Frame 5 takes slot 0 while frame 1 misses a fragment + assertNull(w.add(parsed(piece(5, body, 0)), 1)); + assertEquals(1, w.lost); + // A fragment of frame 1 now is stale + assertNull(w.add(parsed(piece(1, body, 1)), 2)); + assertEquals(1, w.stale); + assertNotNull(w.add(parsed(piece(5, body, 1)), 3)); + } + + @Test + public void duplicatesAndLateFrames() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(piece(1, body, 0)), 0); + assertEquals(1, w.duplicates); + // Frame 2 completes first, frame 1 after it is not shown + assertNotNull(w.add(parsed(piece(2, Arrays.copyOf(body, 10), 0)), 1)); + assertNull(w.add(parsed(piece(1, body, 1)), 2)); + assertEquals(1, w.late); + assertEquals(2, w.completed); + assertEquals(1, w.delivered); + } + + @Test + public void bodyIsAssembledInOrderOfIndex() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2500]; + for (int i = 0; i < body.length; i++) body[i] = (byte) (i * 7); + w.add(parsed(piece(1, body, 2)), 0); + w.add(parsed(piece(1, body, 0)), 0); + FlowUfo.Slot s = w.add(parsed(piece(1, body, 1)), 0); + assertNotNull(s); + assertArrayEquals(body, Arrays.copyOf(s.body, s.bodyLength)); + } + + @Test + public void restartedDroneCountsFromOne() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[10]; + assertNotNull(w.add(parsed(piece(500, body, 0)), 0)); + assertNotNull(w.add(parsed(piece(1, body, 0)), 1)); + assertEquals(1, w.restarts); + assertEquals(1, w.maxSeq); + } + + // ====================== JPEG headers ====================== + + @Test + public void headerLayout() { + byte[] h = FlowUfo.jpegHeader(50, 1280, 720); + assertEquals(605, h.length); + // Height and width in SOF0 at 0x91 and 0x93, big-endian + assertEquals(720, (h[0x91] & 0xFF) << 8 | h[0x92] & 0xFF); + assertEquals(1280, (h[0x93] & 0xFF) << 8 | h[0x94] & 0xFF); + assertEquals(0xFF, h[0] & 0xFF); + assertEquals(0xD8, h[1] & 0xFF); + // Segments by their lengths + List markers = new ArrayList<>(); + int i = 2; + while (i < h.length) { + assertEquals(0xFF, h[i] & 0xFF); + int marker = h[i + 1] & 0xFF; + int len = (h[i + 2] & 0xFF) << 8 | h[i + 3] & 0xFF; + markers.add(marker); + if (marker == 0xC0) { + assertEquals(17, len); + assertEquals(8, h[i + 4]); + assertEquals(3, h[i + 9]); + // Every component 1x1 (4:4:4); Y on table 0, Cb and Cr on table 1 + assertArrayEquals(bytes("00 11 00 01 11 01 02 11 01"), + Arrays.copyOfRange(h, i + 10, i + 19)); + } + if (marker == 0xDB) assertEquals(67, len); + if (marker == 0xDA) { + assertArrayEquals(bytes("03 00 00 01 11 02 11 00 3F 00"), + Arrays.copyOfRange(h, i + 4, i + 14)); + } + i += 2 + len; + } + assertEquals(h.length, i); + assertEquals(Arrays.asList(0xDB, 0xDB, 0xC0, 0xC4, 0xC4, 0xC4, 0xC4, 0xDA), markers); + // DHT: Y DC, Y AC, chroma DC, chroma AC + assertEquals(0x00, h[159 + 4]); + assertEquals(0x10, h[159 + 33 + 4]); + } + + @Test + public void ijgQualityScaling() { + // Zig-zag order: luma 16 11 12 14 ..., chroma 17 18 18 24 ... + byte[] q50 = JpegAssembler.ijgTables(50); + assertEquals(16, q50[0]); + assertEquals(11, q50[1]); + assertEquals(99, q50[63] & 0xFF); + assertEquals(17, q50[64]); + assertEquals(99, q50[127] & 0xFF); + // libjpeg: scale 5000/q below 50, else 200 - 2q; (v * scale + 50) / 100 in 1..255 + assertEquals(8, JpegAssembler.ijgTables(75)[0]); + assertEquals(6, JpegAssembler.ijgTables(75)[1]); + assertEquals(32, JpegAssembler.ijgTables(25)[0] & 0xFF); + assertEquals(80, JpegAssembler.ijgTables(10)[0] & 0xFF); + assertEquals(160, JpegAssembler.ijgTables(5)[0] & 0xFF); + assertEquals(170, JpegAssembler.ijgTables(5)[64] & 0xFF); + assertEquals(255, JpegAssembler.ijgTables(5)[127] & 0xFF); + for (byte v : JpegAssembler.ijgTables(100)) assertEquals(1, v); + // The headers carry them: luma after "FF DB 00 43 00", chroma after "FF DB 00 43 01" + byte[] h = FlowUfo.jpegHeader(75, 0, 0); + assertArrayEquals(Arrays.copyOf(JpegAssembler.ijgTables(75), 64), + Arrays.copyOfRange(h, 7, 71)); + assertArrayEquals(Arrays.copyOfRange(JpegAssembler.ijgTables(75), 64, 128), + Arrays.copyOfRange(h, 76, 140)); + // No size: 640x360; no usable quality: Q50 + assertEquals(360, (h[0x91] & 0xFF) << 8 | h[0x92] & 0xFF); + assertEquals(640, (h[0x93] & 0xFF) << 8 | h[0x94] & 0xFF); + assertArrayEquals(FlowUfo.jpegHeader(50, 640, 360), FlowUfo.jpegHeader(0, 640, 360)); + // RFC 2435 keeps its own clamp to 99 + assertArrayEquals(JpegAssembler.ijgTables(99), JpegAssembler.tables(100)); + } + + @Test + public void jpegAddsEoiOnceAndPassesWholeJpegs() { + byte[] header = FlowUfo.jpegHeader(50, 8, 8); + byte[][] out = new byte[1][]; + int n = FlowUfo.jpeg(header, new byte[] {1, 2, 3}, 3, out); + assertEquals(header.length + 5, n); + assertEquals(0xD9, out[0][n - 1] & 0xFF); + n = FlowUfo.jpeg(header, new byte[] {1, (byte) 0xFF, (byte) 0xD9}, 3, out); + assertEquals(header.length + 3, n); + byte[] whole = {(byte) 0xFF, (byte) 0xD8, 5, (byte) 0xFF, (byte) 0xD9}; + n = FlowUfo.jpeg(header, whole, whole.length, out); + assertArrayEquals(whole, Arrays.copyOf(out[0], n)); + } + + // ====================== Round trip through a real JPEG codec ====================== + + private static BufferedImage picture(int w, int h) { + BufferedImage img = new BufferedImage(w, h, BufferedImage.TYPE_INT_RGB); + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + int r = x * 255 / Math.max(1, w - 1); + int g = y * 255 / Math.max(1, h - 1); + int b = ((x / 8 + y / 8) & 1) != 0 ? 200 : 40; + img.setRGB(x, y, r << 16 | g << 8 | b); + } + } + return img; + } + + /** Baseline 4:4:4 JPEG with the IJG tables at {@code quality}, by ImageIO. */ + static byte[] encode444(BufferedImage img, int quality) throws IOException { + ImageWriter writer = ImageIO.getImageWritersByFormatName("jpeg").next(); + ImageWriteParam param = writer.getDefaultWriteParam(); + IIOMetadata meta = writer.getDefaultImageMetadata( + ImageTypeSpecifier.createFromRenderedImage(img), param); + String format = "javax_imageio_jpeg_image_1.0"; + IIOMetadataNode root = (IIOMetadataNode) meta.getAsTree(format); + byte[] tables = JpegAssembler.ijgTables(quality); + NodeList dqt = root.getElementsByTagName("dqtable"); + for (int i = 0; i < dqt.getLength(); i++) { + IIOMetadataNode n = (IIOMetadataNode) dqt.item(i); + int id = Integer.parseInt(n.getAttribute("qtableId")); + int[] natural = new int[64]; + for (int k = 0; k < 64; k++) natural[NATURAL[k]] = tables[id * 64 + k] & 0xFF; + n.setUserObject(new JPEGQTable(natural)); + } + NodeList comps = root.getElementsByTagName("componentSpec"); + for (int i = 0; i < comps.getLength(); i++) { + IIOMetadataNode n = (IIOMetadataNode) comps.item(i); + n.setAttribute("HsamplingFactor", "1"); + n.setAttribute("VsamplingFactor", "1"); + } + meta.setFromTree(format, root); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ImageOutputStream out = ImageIO.createImageOutputStream(bytes)) { + writer.setOutput(out); + writer.write(null, new IIOImage(img, null, meta), param); + } finally { + writer.dispose(); + } + return bytes.toByteArray(); + } + + /** The entropy-coded data between SOS and EOI: what the drone sends. */ + static byte[] scanOf(byte[] jpeg) { + int i = 2; + while (i + 4 <= jpeg.length) { + int marker = jpeg[i + 1] & 0xFF; + int len = (jpeg[i + 2] & 0xFF) << 8 | jpeg[i + 3] & 0xFF; + if (marker == 0xC0) { + // The encoder really wrote 4:4:4 + for (int c = 0; c < 3; c++) assertEquals(0x11, jpeg[i + 11 + 3 * c]); + } + if (marker == 0xDA) { + int start = i + 2 + len; + assertEquals(0xD9, jpeg[jpeg.length - 1] & 0xFF); + return Arrays.copyOfRange(jpeg, start, jpeg.length - 2); + } + i += 2 + len; + } + throw new AssertionError("no SOS"); + } + + /** + * ImageIO's own guess for a JPEG without JFIF or Adobe marker and no chroma subsampling is + * RGB; libjpeg(-turbo), as on Android, takes YCbCr. For ImageIO the headers get a JFIF + * APP0 after SOI, which says YCbCr; the rest is left as it is. + */ + static byte[] withJfif(byte[] jpeg, int length) { + byte[] app0 = bytes("FF E0 00 10 4A 46 49 46 00 01 01 00 00 01 00 01 00 00"); + byte[] out = new byte[length + app0.length]; + System.arraycopy(jpeg, 0, out, 0, 2); + System.arraycopy(app0, 0, out, 2, app0.length); + System.arraycopy(jpeg, 2, out, 2 + app0.length, length - 2); + return out; + } + + private static BufferedImage decode(byte[] jpeg, int length) throws IOException { + return ImageIO.read(new ByteArrayInputStream(withJfif(jpeg, length))); + } + + private static int[] pixels(BufferedImage img) { + return img.getRGB(0, 0, img.getWidth(), img.getHeight(), null, 0, img.getWidth()); + } + + @Test + public void scanWithOurHeadersDecodesLikeTheOriginal() throws IOException { + for (int q : new int[] {5, 10, 25, 50, 75, 100}) { + for (int[] size : new int[][] {{64, 48}, {50, 30}}) { + BufferedImage src = picture(size[0], size[1]); + byte[] original = encode444(src, q); + byte[] scan = scanOf(original); + byte[][] out = new byte[1][]; + int n = FlowUfo.jpeg(FlowUfo.jpegHeader(q, size[0], size[1]), scan, scan.length, + out); + BufferedImage a = ImageIO.read(new ByteArrayInputStream(original)); + BufferedImage b = decode(out[0], n); + assertNotNull("Q" + q, b); + assertEquals(size[0], b.getWidth()); + assertEquals(size[1], b.getHeight()); + assertArrayEquals("Q" + q + " " + size[0] + "x" + size[1], pixels(a), pixels(b)); + if (q >= 75) { + // And close to the picture itself: colours are not mixed up + int[] p = pixels(src); + int[] r = pixels(b); + long err = 0; + for (int k = 0; k < p.length; k++) { + for (int sh = 0; sh <= 16; sh += 8) { + err += Math.abs((p[k] >> sh & 0xFF) - (r[k] >> sh & 0xFF)); + } + } + assertTrue("Q" + q + " error " + err, err / (p.length * 3.0) < 8); + } + } + } + } + + // ====================== A fake drone on loopback ====================== + + /** Next packet from the client that starts with EF 02, skipping probes. */ + private static byte[] nextAck(DatagramSocket drone, long deadlineMs) throws IOException { + byte[] buf = new byte[4096]; + while (System.currentTimeMillis() < deadlineMs) { + DatagramPacket p = new DatagramPacket(buf, buf.length); + try { + drone.receive(p); + } catch (SocketTimeoutException e) { + continue; + } + if (p.getLength() >= FlowUfo.ACK_HEADER && (buf[0] & 0xFF) == 0xEF && buf[1] == 2) { + return Arrays.copyOf(buf, p.getLength()); + } + } + throw new AssertionError("no acknowledgement"); + } + + @Test + public void fakeDroneOnLoopback() throws Exception { + FlowUfo.resetMs = 400; + BufferedImage src = picture(160, 120); + byte[] original = encode444(src, 75); + byte[] body = scanOf(original); + int total = (body.length + 1023) / 1024; + assertTrue("several fragments: " + total, total >= 3); + + try (DatagramSocket drone = new DatagramSocket(0, InetAddress.getLoopbackAddress())) { + drone.setSoTimeout(200); + FlowUfo.Url u = FlowUfo.Url.parse("flowufo://127.0.0.1:" + drone.getLocalPort()); + FlowUfo.Client client = new FlowUfo.Client("test", u, null); + List frames = Collections.synchronizedList(new ArrayList<>()); + List states = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch frame = new CountDownLatch(1); + CountDownLatch again = new CountDownLatch(1); + Throwable[] error = new Throwable[1]; + Thread t = new Thread(() -> { + try { + client.run(() -> true, new FlowUfo.Listener() { + @Override + public void onState(boolean receiving, String text) { + states.add(text); + if (text.contains("went quiet")) again.countDown(); + } + + @Override + public void onFrame(byte[] buf, int length) { + frames.add(Arrays.copyOf(buf, length)); + frame.countDown(); + } + }); + } catch (Throwable e) { + error[0] = e; + } + }); + t.start(); + try { + // Probes first, every 100 ms + byte[] buf = new byte[64]; + DatagramPacket p = new DatagramPacket(buf, buf.length); + drone.setSoTimeout(2000); + drone.receive(p); + assertArrayEquals(FlowUfo.PROBE, Arrays.copyOf(buf, p.getLength())); + SocketAddress phone = p.getSocketAddress(); + drone.setSoTimeout(200); + assertTrue(states.get(0).startsWith("Waiting for the toy: probing 127.0.0.1:")); + + // Frame 1 without fragment 1 + for (int i = 0; i < total; i++) { + if (i == 1) continue; + byte[] f = piece(1, body, i, 160, 120, 75); + drone.send(new DatagramPacket(f, f.length, phone)); + } + // A service message is only logged + byte[] service = bytes("93 04 0E 00 01 00 00 00 02 00 00 00 55 5A"); + drone.send(new DatagramPacket(service, service.length, phone)); + + // After the pause the acknowledgement asks for the missing fragment + long deadline = System.currentTimeMillis() + 3000; + long[] rec; + byte[] a; + do { + a = nextAck(drone, deadline); + List r = records(a); + rec = r.isEmpty() ? null : r.get(0); + } while (rec == null); + assertEquals(1, rec[0]); + assertEquals(FlowUfo.STATUS_COLLECTING, rec[1]); + long mask = FlowUfo.u32(a, (int) rec[3] + 16); + assertEquals("fragment 1 missing", 0, mask & 2); + assertEquals(1, mask & 1); + assertEquals(1, frame.getCount()); + + byte[] f = piece(1, body, 1, 160, 120, 75); + drone.send(new DatagramPacket(f, f.length, phone)); + assertTrue(frame.await(3, TimeUnit.SECONDS)); + byte[] jpeg = frames.get(0); + byte[][] expected = new byte[1][]; + int n = FlowUfo.jpeg(FlowUfo.jpegHeader(75, 160, 120), body, body.length, expected); + assertArrayEquals(Arrays.copyOf(expected[0], n), jpeg); + BufferedImage got = decode(jpeg, jpeg.length); + assertEquals(160, got.getWidth()); + assertArrayEquals(pixels(ImageIO.read(new ByteArrayInputStream(original))), + pixels(got)); + + // Then: frame 1 complete, frame 2 asked for whole + boolean asked = false; + deadline = System.currentTimeMillis() + 3000; + while (!asked) { + for (long[] r : records(nextAck(drone, deadline))) { + if (r[0] == 2 && r[1] == FlowUfo.STATUS_WHOLE) asked = true; + } + } + assertEquals(1, client.window.delivered); + assertEquals(1, client.stats().get("service_messages")); + assertEquals("playing", client.stats().get("state")); + + // Silence: probing again + assertTrue(again.await(3, TimeUnit.SECONDS)); + deadline = System.currentTimeMillis() + 2000; + boolean probed = false; + while (!probed && System.currentTimeMillis() < deadline) { + DatagramPacket q = new DatagramPacket(buf, buf.length); + try { + drone.receive(q); + } catch (SocketTimeoutException e) { + continue; + } + probed = Arrays.equals(FlowUfo.PROBE, Arrays.copyOf(buf, q.getLength())); + } + assertTrue(probed); + assertEquals("probing", client.stats().get("state")); + } finally { + client.close(); + t.join(2000); + } + assertFalse(t.isAlive()); + assertNull(error[0]); + } + } + + @Test + public void sameSubnet() throws Exception { + InetAddress phone = InetAddress.getByName("192.168.169.2"); + InetAddress toy = InetAddress.getByName("192.168.169.1"); + assertTrue(NetInfo.sameSubnet(phone, 24, toy)); + assertFalse(NetInfo.sameSubnet(InetAddress.getByName("192.168.1.2"), 24, toy)); + assertTrue(NetInfo.sameSubnet(InetAddress.getByName("192.168.1.2"), 16, toy)); + assertFalse(NetInfo.sameSubnet(InetAddress.getByName("fe80::1"), 64, toy)); + assertFalse(NetInfo.sameSubnet(phone, 33, toy)); + } +} From 5e459b734598cbe5f9131e2c6b14a1b307a41340 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 14:53:01 +0000 Subject: [PATCH 02/21] test: ImageIO by reflection, unit tests compile against android.jar (#187) --- .../java/com/openipc/decoder/FlowUfoTest.java | 176 +++++++++++++----- 1 file changed, 127 insertions(+), 49 deletions(-) diff --git a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java index 6a6ac1bd..beb25be9 100644 --- a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java +++ b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java @@ -20,7 +20,6 @@ import org.junit.Test; import org.w3c.dom.NodeList; -import java.awt.image.BufferedImage; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -36,16 +35,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import javax.imageio.IIOImage; -import javax.imageio.ImageIO; -import javax.imageio.ImageTypeSpecifier; -import javax.imageio.ImageWriteParam; -import javax.imageio.ImageWriter; -import javax.imageio.metadata.IIOMetadata; -import javax.imageio.metadata.IIOMetadataNode; -import javax.imageio.plugins.jpeg.JPEGQTable; -import javax.imageio.stream.ImageOutputStream; - public class FlowUfoTest { /** Zig-zag position i to natural (row-major) position. */ private static final int[] NATURAL = { @@ -462,49 +451,142 @@ public void jpegAddsEoiOnceAndPassesWholeJpegs() { // ====================== Round trip through a real JPEG codec ====================== - private static BufferedImage picture(int w, int h) { - BufferedImage img = new BufferedImage(w, h, BufferedImage.TYPE_INT_RGB); + // Unit tests compile against android.jar, which has no java.awt or javax.imageio: the + // JDK's ImageIO (there at run time) is reached by reflection + + /** + * Calls the public method {@code name} whose parameters take {@code args}, through a + * public type: the classes behind ImageIO's interfaces are in packages of the JDK that are + * not exported, so their own Method objects cannot be invoked. + */ + private static Object call(Object target, String name, Object... args) throws Exception { + boolean statik = target instanceof Class; + Class c = statik ? (Class) target : target.getClass(); + List> types = new ArrayList<>(); + for (Class k = c; k != null; k = k.getSuperclass()) { + types.add(k); + Collections.addAll(types, k.getInterfaces()); + } + Exception last = new NoSuchMethodException(c.getName() + "." + name); + for (Class k : types) { + if (!java.lang.reflect.Modifier.isPublic(k.getModifiers())) continue; + for (java.lang.reflect.Method m : k.getMethods()) { + if (!m.getName().equals(name) || !fits(m.getParameterTypes(), args)) continue; + try { + return m.invoke(statik ? null : target, args); + } catch (IllegalAccessException e) { + last = e; + } + } + } + throw last; + } + + private static Object create(String className, Object... args) throws Exception { + for (java.lang.reflect.Constructor k : Class.forName(className).getConstructors()) { + if (fits(k.getParameterTypes(), args)) return k.newInstance(args); + } + throw new NoSuchMethodException(className); + } + + private static boolean fits(Class[] types, Object[] args) { + if (types.length != args.length) return false; + for (int i = 0; i < types.length; i++) { + if (args[i] == null) { + if (types[i].isPrimitive()) return false; + continue; + } + Class t = types[i] == int.class ? Integer.class + : types[i] == boolean.class ? Boolean.class : types[i]; + if (!t.isInstance(args[i])) return false; + } + return true; + } + + private static Class imageIo() throws Exception { + return Class.forName("javax.imageio.ImageIO"); + } + + /** A java.awt.image.BufferedImage. */ + static final class Img { + final Object image; + + Img(Object image) { + this.image = image; + } + + int width() throws Exception { + return (Integer) call(image, "getWidth"); + } + + int height() throws Exception { + return (Integer) call(image, "getHeight"); + } + + int[] pixels() throws Exception { + int w = width(); + return (int[]) call(image, "getRGB", 0, 0, w, height(), null, 0, w); + } + + /** Decoded by ImageIO, or null when it cannot. */ + static Img read(byte[] jpeg, int length) throws Exception { + Object img = call(imageIo(), "read", new ByteArrayInputStream(jpeg, 0, length)); + return img != null ? new Img(img) : null; + } + } + + private static Img picture(int w, int h) throws Exception { + // BufferedImage.TYPE_INT_RGB + Object img = create("java.awt.image.BufferedImage", w, h, 1); + int[] rgb = new int[w * h]; for (int y = 0; y < h; y++) { for (int x = 0; x < w; x++) { int r = x * 255 / Math.max(1, w - 1); int g = y * 255 / Math.max(1, h - 1); int b = ((x / 8 + y / 8) & 1) != 0 ? 200 : 40; - img.setRGB(x, y, r << 16 | g << 8 | b); + rgb[y * w + x] = r << 16 | g << 8 | b; } } - return img; + call(img, "setRGB", 0, 0, w, h, rgb, 0, w); + return new Img(img); } /** Baseline 4:4:4 JPEG with the IJG tables at {@code quality}, by ImageIO. */ - static byte[] encode444(BufferedImage img, int quality) throws IOException { - ImageWriter writer = ImageIO.getImageWritersByFormatName("jpeg").next(); - ImageWriteParam param = writer.getDefaultWriteParam(); - IIOMetadata meta = writer.getDefaultImageMetadata( - ImageTypeSpecifier.createFromRenderedImage(img), param); + static byte[] encode444(Img img, int quality) throws Exception { + Object writer = ((java.util.Iterator) call(imageIo(), "getImageWritersByFormatName", + "jpeg")).next(); + Object param = call(writer, "getDefaultWriteParam"); + Object type = call(Class.forName("javax.imageio.ImageTypeSpecifier"), + "createFromRenderedImage", img.image); + Object meta = call(writer, "getDefaultImageMetadata", type, param); String format = "javax_imageio_jpeg_image_1.0"; - IIOMetadataNode root = (IIOMetadataNode) meta.getAsTree(format); + org.w3c.dom.Element root = (org.w3c.dom.Element) call(meta, "getAsTree", format); byte[] tables = JpegAssembler.ijgTables(quality); NodeList dqt = root.getElementsByTagName("dqtable"); for (int i = 0; i < dqt.getLength(); i++) { - IIOMetadataNode n = (IIOMetadataNode) dqt.item(i); + org.w3c.dom.Element n = (org.w3c.dom.Element) dqt.item(i); int id = Integer.parseInt(n.getAttribute("qtableId")); int[] natural = new int[64]; for (int k = 0; k < 64; k++) natural[NATURAL[k]] = tables[id * 64 + k] & 0xFF; - n.setUserObject(new JPEGQTable(natural)); + call(n, "setUserObject", create("javax.imageio.plugins.jpeg.JPEGQTable", + (Object) natural)); } NodeList comps = root.getElementsByTagName("componentSpec"); for (int i = 0; i < comps.getLength(); i++) { - IIOMetadataNode n = (IIOMetadataNode) comps.item(i); + org.w3c.dom.Element n = (org.w3c.dom.Element) comps.item(i); n.setAttribute("HsamplingFactor", "1"); n.setAttribute("VsamplingFactor", "1"); } - meta.setFromTree(format, root); + call(meta, "setFromTree", format, root); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - try (ImageOutputStream out = ImageIO.createImageOutputStream(bytes)) { - writer.setOutput(out); - writer.write(null, new IIOImage(img, null, meta), param); + Object out = call(imageIo(), "createImageOutputStream", bytes); + try { + call(writer, "setOutput", out); + Object image = create("javax.imageio.IIOImage", img.image, null, meta); + call(writer, "write", null, image, param); } finally { - writer.dispose(); + call(out, "close"); + call(writer, "dispose"); } return bytes.toByteArray(); } @@ -543,34 +625,31 @@ static byte[] withJfif(byte[] jpeg, int length) { return out; } - private static BufferedImage decode(byte[] jpeg, int length) throws IOException { - return ImageIO.read(new ByteArrayInputStream(withJfif(jpeg, length))); - } - - private static int[] pixels(BufferedImage img) { - return img.getRGB(0, 0, img.getWidth(), img.getHeight(), null, 0, img.getWidth()); + private static Img decode(byte[] jpeg, int length) throws Exception { + byte[] j = withJfif(jpeg, length); + return Img.read(j, j.length); } @Test - public void scanWithOurHeadersDecodesLikeTheOriginal() throws IOException { + public void scanWithOurHeadersDecodesLikeTheOriginal() throws Exception { for (int q : new int[] {5, 10, 25, 50, 75, 100}) { for (int[] size : new int[][] {{64, 48}, {50, 30}}) { - BufferedImage src = picture(size[0], size[1]); + Img src = picture(size[0], size[1]); byte[] original = encode444(src, q); byte[] scan = scanOf(original); byte[][] out = new byte[1][]; int n = FlowUfo.jpeg(FlowUfo.jpegHeader(q, size[0], size[1]), scan, scan.length, out); - BufferedImage a = ImageIO.read(new ByteArrayInputStream(original)); - BufferedImage b = decode(out[0], n); + Img a = Img.read(original, original.length); + Img b = decode(out[0], n); assertNotNull("Q" + q, b); - assertEquals(size[0], b.getWidth()); - assertEquals(size[1], b.getHeight()); - assertArrayEquals("Q" + q + " " + size[0] + "x" + size[1], pixels(a), pixels(b)); + assertEquals(size[0], b.width()); + assertEquals(size[1], b.height()); + assertArrayEquals("Q" + q + " " + size[0] + "x" + size[1], a.pixels(), b.pixels()); if (q >= 75) { // And close to the picture itself: colours are not mixed up - int[] p = pixels(src); - int[] r = pixels(b); + int[] p = src.pixels(); + int[] r = b.pixels(); long err = 0; for (int k = 0; k < p.length; k++) { for (int sh = 0; sh <= 16; sh += 8) { @@ -605,7 +684,7 @@ private static byte[] nextAck(DatagramSocket drone, long deadlineMs) throws IOEx @Test public void fakeDroneOnLoopback() throws Exception { FlowUfo.resetMs = 400; - BufferedImage src = picture(160, 120); + Img src = picture(160, 120); byte[] original = encode444(src, 75); byte[] body = scanOf(original); int total = (body.length + 1023) / 1024; @@ -684,10 +763,9 @@ public void onFrame(byte[] buf, int length) { byte[][] expected = new byte[1][]; int n = FlowUfo.jpeg(FlowUfo.jpegHeader(75, 160, 120), body, body.length, expected); assertArrayEquals(Arrays.copyOf(expected[0], n), jpeg); - BufferedImage got = decode(jpeg, jpeg.length); - assertEquals(160, got.getWidth()); - assertArrayEquals(pixels(ImageIO.read(new ByteArrayInputStream(original))), - pixels(got)); + Img got = decode(jpeg, jpeg.length); + assertEquals(160, got.width()); + assertArrayEquals(Img.read(original, original.length).pixels(), got.pixels()); // Then: frame 1 complete, frame 2 asked for whole boolean asked = false; From 8aaf6660117179a1387b238e0d169460fe4be7e9 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 15:13:34 +0000 Subject: [PATCH 03/21] fix: FLOW-UFO window and client after review (Qodo on #188) A single fragment far behind no longer resets the window: only a run of them counts as a drone restart. Stale fragments and duplicates do not extend the quiet time; frames at or below the base do not hold up the request for the next frame; a short fragment inside the body stays missing, a short last one ends the body. Send errors reopen the socket after 3 s by time and probe again from the new port. A cell's toy status is cleared with every status change. Refs #187. --- .../java/com/openipc/decoder/FlowUfo.java | 79 ++++++++++++++----- .../java/com/openipc/decoder/QuadCell.java | 2 + .../java/com/openipc/decoder/FlowUfoTest.java | 70 +++++++++++++++- 3 files changed, 130 insertions(+), 21 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java index 55e64837..936baeed 100644 --- a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java +++ b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java @@ -70,6 +70,8 @@ final class FlowUfo { static final int SLOTS = 4; /** A slot this many frames behind the newest asks for its next frame whole. */ static final int LAG = 5; + /** Fragments in a row this far behind the newest frame: the drone counts from 1 again. */ + static final int RESTART_RUN = 8; static final int STATUS_COLLECTING = 0; static final int STATUS_COMPLETE = 1; static final int STATUS_WHOLE = 3; @@ -262,11 +264,16 @@ static final class Window { /** The frame whose fragments arrive now: not acknowledged while it is coming in. */ long current; long lastDelivered; + /** When a fragment last added data: the quiet time of the acknowledgement. */ long lastFragmentMs; + /** Fragments in a row far behind the newest frame. */ + private int behind; volatile int fragments; volatile int duplicates; volatile int stale; volatile int mismatched; + /** Fragments with less data than their place in the body needs. */ + volatile int shortFragments; volatile int completed; volatile int delivered; volatile int lost; @@ -286,20 +293,30 @@ void reset() { base = 0; current = 0; lastDelivered = 0; + behind = 0; } static int slotOf(long seq) { return (int) ((seq - 1) & (SLOTS - 1)); } - /** Adds a fragment; returns its slot when the fragment completed a frame to show. */ + /** + * Adds a fragment; returns its slot when the fragment completed a frame to show. Only a + * fragment that adds data counts for the quiet time: stale ones and duplicates do not + * (Qodo on #188). + */ Slot add(Fragment f, long nowMs) { - lastFragmentMs = nowMs; - // Far behind the newest frame: the drone counts from 1 again if (maxSeq > 64 && f.seq + 64 < maxSeq) { + // Far behind the newest frame: a straggler, or the drone counts from 1 again; + // only a run of them is taken for a restart (Qodo on #188) + if (++behind < RESTART_RUN) { + stale++; + return null; + } restarts++; reset(); } + behind = 0; // Only a base behind the frame itself makes sense if (f.base > base && f.base < f.seq) base = f.base; Slot s = slots[slotOf(f.seq)]; @@ -316,7 +333,6 @@ Slot add(Fragment f, long nowMs) { s.start(f); } fragments++; - current = f.seq; if (f.seq > maxSeq) maxSeq = f.seq; if (s.status == STATUS_COMPLETE || (f.total == s.total && s.has(f.index))) { duplicates++; @@ -332,10 +348,19 @@ Slot add(Fragment f, long nowMs) { s.bodyLength = Math.max(s.bodyLength, f.bodyLength); } int at = f.index * FRAGMENT_DATA; + if (f.dataLength < Math.min(FRAGMENT_DATA, s.bodyLength - at)) { + shortFragments++; + // A short one inside the body stays missing (Qodo on #188); the last one ends + // the body where its data ends + if (f.index < s.total - 1) return null; + s.bodyLength = at + f.dataLength; + } int n = Math.min(f.dataLength, s.total * FRAGMENT_DATA - at); System.arraycopy(f.buf, f.dataOffset, s.body, at, n); s.mask[f.index >> 5] |= 1 << (f.index & 31); s.received++; + lastFragmentMs = nowMs; + current = f.seq; if (s.received < s.total) return null; s.status = STATUS_COMPLETE; completed++; @@ -365,11 +390,12 @@ int buildAck(byte[] out, long nowMs) { boolean any = false; boolean allComplete = true; for (Slot s : slots) { - if (s.seq == 0) continue; + // At or below the base the drone is done with it: no record, and an incomplete + // one does not hold up the request for the next frame (Qodo on #188) + if (s.seq == 0 || s.seq <= base) continue; any = true; if (s.status != STATUS_COMPLETE) allComplete = false; if (s.seq == current && !quiet) continue; - if (s.seq <= base) continue; if (maxSeq - s.seq >= LAG) { pos = wholeRecord(out, pos, s.seq + SLOTS); records++; @@ -514,6 +540,8 @@ static final class Client implements Closeable { private volatile boolean gl = true; private volatile float fragmentsPerS; private volatile float framesPerS; + /** Any fragment from the drone, usable or not: it is still there. */ + private long lastFragmentAt; Client(String tag, Url url, SocketSetup setup) { this.tag = tag; @@ -561,7 +589,8 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException int rateFragments = 0; int rateFrames = 0; long nextSummary = rateMark + 30_000; - int failedSends = 0; + // Start of the current run of failed sends, or 0 + long failingSince = 0; try { while (!closed && keepGoing.holds()) { long t = now(); @@ -573,15 +602,11 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException rateFrames = window.delivered; rateMark = t; } - if (receiving && t - window.lastFragmentMs > resetMs) { - resets++; - receiving = false; - playing = false; - window.reset(); - nextProbe = t; + if (receiving && t - lastFragmentAt > resetMs) { Log.w(TAG, tag + ": FLOW-UFO: no fragment for " + resetMs + " ms, probing again (" + summary() + ")"); - listener.onState(false, probingText(true)); + probeAgain(listener, t); + nextProbe = t; } if (receiving && t >= nextSummary) { nextSummary = t + 30_000; @@ -602,14 +627,21 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException } if (sendFailed) { // Off the toy's Wi-Fi, or its network went away: a new socket after - // 3 s of failures, bound again to whatever network reaches it now - if (++failedSends >= 30) { - failedSends = 0; + // 3 s of failures, bound again to whatever network reaches it now. The + // drone sends to the port of the probe: the new port probes at once + // (Qodo on #188) + if (failingSince == 0) failingSince = t; + if (t - failingSince >= 3000) { + failingSince = 0; s.close(); s = open(address); + Log.w(TAG, tag + ": FLOW-UFO: new socket after send errors, local port " + + s.getLocalPort() + " (network: " + network + ")"); + if (receiving) probeAgain(listener, t); + nextProbe = t; } } else { - failedSends = 0; + failingSince = 0; } long wake = receiving ? nextAck : nextProbe; int timeout = (int) Math.max(1, Math.min(100, wake - now())); @@ -637,6 +669,15 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException } } + /** Back to probing: the stream is gone. */ + private void probeAgain(Listener listener, long t) { + resets++; + receiving = false; + playing = false; + window.reset(); + listener.onState(false, probingText(true)); + } + private boolean send(DatagramSocket s, DatagramPacket p) { try { s.send(p); @@ -676,6 +717,7 @@ private void handle(byte[] b, int len, Listener listener) { return; } long t = now(); + lastFragmentAt = t; if (!receiving) { receiving = true; if (firstHeader == null) firstHeader = hex(b, 0, FRAGMENT_HEADER); @@ -747,6 +789,7 @@ Map stats() { m.put("duplicates", w.duplicates); m.put("stale", w.stale); m.put("mismatched", w.mismatched); + m.put("short_fragments", w.shortFragments); m.put("bad_packets", badPackets); m.put("frames", w.delivered); m.put("frames_complete", w.completed); diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index 9c409dfa..5c3f3699 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -288,6 +288,8 @@ public void onStatus(String status) { // stalls again (Qodo on #146) if ("disconnected".equals(status)) nalAssembler.setSalvage(false); state = status; + // A flowufo:// session sends its detail after the status it belongs to (Qodo on #188) + detail = null; // A connection that works again clears the last connection error if ("connected".equals(status) && problem != null && !problem.startsWith("Decoder")) { problem = null; diff --git a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java index beb25be9..84ac2841 100644 --- a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java +++ b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java @@ -355,11 +355,75 @@ public void bodyIsAssembledInOrderOfIndex() { @Test public void restartedDroneCountsFromOne() { FlowUfo.Window w = new FlowUfo.Window(); - byte[] body = new byte[10]; - assertNotNull(w.add(parsed(piece(500, body, 0)), 0)); - assertNotNull(w.add(parsed(piece(1, body, 0)), 1)); + byte[] small = new byte[10]; + assertNotNull(w.add(parsed(piece(500, small, 0)), 0)); + // A run of fragments far behind: the drone started again from 1 + byte[] body = new byte[FlowUfo.RESTART_RUN * 1024]; + for (int i = 0; i < FlowUfo.RESTART_RUN; i++) w.add(parsed(piece(1, body, i)), 1); assertEquals(1, w.restarts); assertEquals(1, w.maxSeq); + assertNotNull(w.add(parsed(piece(2, small, 0)), 2)); + } + + @Test + public void oneStragglerDoesNotReset() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + assertNotNull(w.add(parsed(piece(100, Arrays.copyOf(body, 10), 0)), 0)); + assertNull(w.add(parsed(piece(101, body, 0)), 10)); + // A delayed fragment of frame 35 is only stale + assertNull(w.add(parsed(piece(35, body, 0)), 15)); + assertEquals(0, w.restarts); + assertEquals(1, w.stale); + assertEquals(101, w.maxSeq); + assertNotNull(w.add(parsed(piece(101, body, 1)), 20)); + } + + @Test + public void staleFragmentsDoNotHoldBackTheMask() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(piece(5, body, 0)), 0); + // Fragments of the replaced frame 1 keep coming: frame 5 is still due its mask + for (int t = 5; t <= 40; t += 5) w.add(parsed(piece(1, body, 1)), t); + List r = records(ack(w, 40)); + assertEquals(1, r.size()); + assertEquals(5, r.get(0)[0]); + assertEquals(FlowUfo.STATUS_COLLECTING, r.get(0)[1]); + } + + @Test + public void baseHiddenFrameDoesNotHoldUpTheNextRequest() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + byte[] one = Arrays.copyOf(body, 10); + // Frame 1 incomplete; frame 2 complete and says base 1 + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(fragment(2, 1, 0, 1, one.length, 640, 360, 50, one, 0, one.length)), 0); + List r = records(ack(w, 100)); + assertEquals(2, r.size()); + assertArrayEquals(new long[] {2, FlowUfo.STATUS_COMPLETE, 20}, Arrays.copyOf(r.get(0), 3)); + assertArrayEquals(new long[] {3, FlowUfo.STATUS_WHOLE, 16}, Arrays.copyOf(r.get(1), 3)); + } + + @Test + public void shortFragments() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2500]; + // Fragment 0 with 100 bytes instead of 1024: stays missing + byte[] p = fragment(1, 0, 0, 3, body.length, 640, 360, 50, body, 0, 100); + assertNull(w.add(parsed(p), 0)); + assertEquals(1, w.shortFragments); + assertEquals(0, w.slots[0].received); + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(piece(1, body, 1)), 0); + // The last one with 2 of its 452 bytes: the body ends there + p = fragment(1, 0, 2, 3, body.length, 640, 360, 50, body, 2048, 2); + FlowUfo.Slot s = w.add(parsed(p), 0); + assertNotNull(s); + assertEquals(2050, s.bodyLength); + assertEquals(2, w.shortFragments); } // ====================== JPEG headers ====================== From 417c46ce912d12ae94de661e120e01e431853930 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 16:04:14 +0000 Subject: [PATCH 04/21] chore: version 2.11 (2.10 went to #190) --- android/app/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/app/build.gradle b/android/app/build.gradle index a725c471..d2f10b2d 100755 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -23,8 +23,8 @@ android { applicationId "com.openipc.decoder" minSdk 21 targetSdk 36 - versionCode 35 - versionName "2.10" + versionCode 36 + versionName "2.11" buildConfigField "String", "GIT_HASH", "\"${getGitHash()}\"" } From b898ca1c27c76f654f3ccae4da88f73664de1c7e Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 18:27:57 +0000 Subject: [PATCH 05/21] feat: FLOW-UFO after turbodrone (both ports, quality bytes, lagging slots, dqt switch) and the KY UFO 7099 heartbeat (#187) Compared with marshallrichards/turbodrone (Apache-2.0, wifi_uav and cooingdv backends; no code taken over, credited in docs/toys.md and the comments): - ACK quality parameters 32 4B 14 2D by default; ?qpara= (or ?q=) with 8 hex digits - bytes 0x34/0x35 are main/flow camera status: shown in the stats, no family warning - slots 5 or more frames behind are left out of the ACK instead of a seq+4 request - port 8800 (default) probes 8801 too, one socket per port; locks onto the socket the first 0x93 packet arrives on (source address only, the drone sends from port 1234); BL618 timing on 8801: probes every 200 ms, ACKs every 50 ms only with a record - ?dqt=zigzag|raster|q50 for the JPEG quantisation tables, shown in the stats - no flight command in the ACK, with the reason in a comment - RTSP: 01 01 to UDP 7099 every second for rtsp://:7070/webcam, or with ?heartbeat= (off with ?heartbeat=off; the option is not sent to the camera); heartbeat object in /stats.json - subnet hint in the status when a flowufo:// slot targets the phone itself or another toy family's network - docs/toys.md: families by Wi-Fi name, the new options, KY UFO RTSP, credits (#196) --- .../java/com/openipc/decoder/Decoder.java | 4 + .../java/com/openipc/decoder/FlowUfo.java | 567 ++++++++++++++---- .../java/com/openipc/decoder/QuadCell.java | 3 + .../java/com/openipc/decoder/RtspClient.java | 2 + .../java/com/openipc/decoder/RtspSession.java | 102 ++++ .../com/openipc/decoder/UdpHeartbeat.java | 136 +++++ .../java/com/openipc/decoder/FlowUfoTest.java | 293 ++++++++- .../com/openipc/decoder/RtspSessionTest.java | 45 ++ .../com/openipc/decoder/UdpHeartbeatTest.java | 126 ++++ docs/debug.md | 2 +- docs/toys.md | 55 +- docs/transports.md | 4 +- 12 files changed, 1218 insertions(+), 121 deletions(-) create mode 100644 android/app/src/main/java/com/openipc/decoder/UdpHeartbeat.java create mode 100644 android/app/src/test/java/com/openipc/decoder/UdpHeartbeatTest.java diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index 1acd5d7b..d00efd27 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -1558,6 +1558,8 @@ private Map debugState() { // The toy drone's protocol counters (#187) Map ufo = rtspClient != null ? rtspClient.flowUfoStats() : null; if (ufo != null) single.put("flowufo", ufo); + Map hb = rtspClient != null ? rtspClient.heartbeatStats() : null; + if (hb != null) single.put("heartbeat", hb); m.put("single", single); } QuadCell[] cells = quadCells; @@ -1598,6 +1600,8 @@ private Map debugState() { c.put("audio", i == quadAudio); Map ufo = cell.flowUfoStats(); if (ufo != null) c.put("flowufo", ufo); + Map hb = cell.heartbeatStats(); + if (hb != null) c.put("heartbeat", hb); } list.add(c); } diff --git a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java index 936baeed..f5a2869f 100644 --- a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java +++ b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java @@ -13,24 +13,33 @@ import java.io.IOException; import java.net.DatagramPacket; import java.net.DatagramSocket; +import java.net.Inet4Address; import java.net.InetAddress; -import java.net.InetSocketAddress; +import java.net.InterfaceAddress; +import java.net.NetworkInterface; import java.net.SocketTimeoutException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Enumeration; import java.util.LinkedHashMap; +import java.util.List; import java.util.Locale; import java.util.Map; /** - * Cheap Wi-Fi toy drones with a "FLOW-UFO-xxxxxx" access point (a BL60x camera) send MJPEG - * over UDP from 192.168.169.1:8800 in a protocol of their own (#187). A short probe wakes the - * drone; it then sends every JPEG frame, cut into fragments of 1024 bytes, to the address the - * probe came from, and the client acknowledges the last four frames every 25 ms: what arrived, - * what is missing, and which frame to send whole. A frame carries only the entropy-coded data - * of the scan: the client puts the JPEG headers in front (standard tables at the quality the - * fragment names). Video only: flight commands are never sent. Written from a description of - * the client side and not yet checked against a drone: what is a guess says so, and the log - * shows the raw bytes that would settle it. Pure Java apart from the log. + * Cheap Wi-Fi toy drones with a "FLOW-UFO-xxxxxx" / "FLOW_xxxxxx" access point (a BL60x + * camera) send MJPEG over UDP from 192.168.169.1 in a protocol of their own (#187). A short + * probe to port 8800 (BL608 engine) or 8801 (BL618 engine) wakes the drone; it then sends + * every JPEG frame, cut into fragments of 1024 bytes, from source port 1234 to the address the + * probe came from, and the client acknowledges the last four frames: what arrived, what is + * missing, and which frame to send next. A frame carries only the entropy-coded data of the + * scan: the client puts the JPEG headers in front. Video only: flight commands are never sent. + * + * The same protocol is implemented by the open-source project turbodrone (Apache-2.0, + * github.com/marshallrichards/turbodrone, "wifi_uav") and works there on real drones; the + * values taken over from its observations (quality parameters, both ports, which slots go into + * the acknowledgement, the raster-order tables of ?dqt=raster) say so where they are used. + * No code of it is used. Pure Java apart from the log. */ final class FlowUfo { private static final String TAG = "OpenIPCDecoder"; @@ -38,11 +47,20 @@ final class FlowUfo { static final String SCHEME = "flowufo://"; static final String DEFAULT_HOST = "192.168.169.1"; static final int DEFAULT_PORT = 8800; + /** + * The port of the BL618 engine. The vendor library starts both engines, on 8800 and 8801, + * and keeps the one that delivers frames (turbodrone docs/research/wifi_uav.md): a slot on + * 8800 (the default) probes both and locks onto the first that answers. + */ + static final int BL618_PORT = 8801; - /** Wakes the drone up; sent every PROBE_MS until the first fragment. */ + /** Wakes the drone up; sent every PROBE_MS (BL618: PROBE_MS_BL618) until it answers. */ static final byte[] PROBE = {(byte) 0xEF, 0x00, 0x04, 0x00}; static final long PROBE_MS = 100; + static final long PROBE_MS_BL618 = 200; static final long ACK_MS = 25; + /** The BL618 engine acknowledges every 50 ms, and only when there is a record to send. */ + static final long ACK_MS_BL618 = 50; /** No fragment for longer: the last frame is over, the acknowledgement asks for the next. */ static final long QUIET_MS = 19; /** @@ -65,10 +83,17 @@ final class FlowUfo { // The acknowledgement: a fixed header, then one record per frame static final int ACK_HEADER = 0x58; - /** Quality parameters the vendor app sends by default (25, 50, 35, 75). */ - static final byte[] QPARA = {0x19, 0x32, 0x23, 0x4B}; + /** + * Quality parameters at 0x52 of the acknowledgement: 32 4B 14 2D, the bytes turbodrone + * sends to real drones (#187). The vendor app's own default was read as 19 32 23 4B; that + * one, or any other, can be given with ?qpara= (8 hex digits). + */ + static final byte[] QPARA = {0x32, 0x4B, 0x14, 0x2D}; static final int SLOTS = 4; - /** A slot this many frames behind the newest asks for its next frame whole. */ + /** + * A slot this many frames behind the newest is left out of the acknowledgement, as + * turbodrone does: the drone has moved on, asking for it again only holds it up. + */ static final int LAG = 5; /** Fragments in a row this far behind the newest frame: the drone counts from 1 again. */ static final int RESTART_RUN = 8; @@ -89,14 +114,44 @@ final class FlowUfo { static final int DEFAULT_WIDTH = 640; static final int DEFAULT_HEIGHT = 360; + /** + * Quantization tables of the header, ?dqt= (#187). ZIGZAG (default): the Annex K tables + * scaled to the quality byte of the fragment, in zig-zag order as the JPEG standard has + * DQT. RASTER: unscaled Annex K tables in raster (row-major) order, quality ignored: what + * turbodrone writes, and it shows real drones' frames. Q50: unscaled, zig-zag. Which one + * matches the drone's encoder is not known yet; the tester compares. + */ + static final int DQT_ZIGZAG = 0; + static final int DQT_RASTER = 1; + static final int DQT_Q50 = 2; + static final String[] DQT_NAMES = {"zigzag", "raster", "q50"}; + + /** Zig-zag position i to raster (row-major) position. */ + private static final int[] RASTER_OF_ZIGZAG = { + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63 + }; + private FlowUfo() {} // ====================== URL ====================== - /** flowufo://[host][:port][/...]: the drone's address, 192.168.169.1:8800 when left out. */ + /** + * flowufo://[host][:port][/...][?option=value&...]: the drone's address, 192.168.169.1:8800 + * when left out. Port 8800 (also when left out) probes 8801 as well; any other port only + * itself. Options, as in udp:// URLs (unknown ones are ignored, bad values refused): + * qpara=8 hex digits (the quality parameters of the acknowledgement, default 324B142D), + * dqt=zigzag|raster|q50 (the quantization tables of the JPEG header, default zigzag). + */ static final class Url { String host = DEFAULT_HOST; int port = DEFAULT_PORT; + /** Probed as well as port until one answers, or 0. Not final so tests can move it. */ + int alternatePort; + byte[] qpara = QPARA.clone(); + int dqt = DQT_ZIGZAG; static boolean matches(String url) { return url != null && url.trim().toLowerCase(Locale.ROOT).startsWith(SCHEME); @@ -114,6 +169,12 @@ static Url parse(String url) throws IOException { } } String authority = rest.substring(0, end).trim(); + String query = ""; + int q = rest.indexOf('?'); + int hash = rest.indexOf('#'); + if (q >= 0 && (hash < 0 || q < hash)) { + query = rest.substring(q + 1, hash >= 0 ? hash : rest.length()); + } Url u = new Url(); String host = authority; int colon = authority.lastIndexOf(':'); @@ -138,11 +199,55 @@ static Url parse(String url) throws IOException { char ch = u.host.charAt(i); if (ch <= 0x20 || ch == 0x7F || ch == '@') throw new IOException("Bad host in URL"); } + u.alternatePort = u.port == DEFAULT_PORT ? BL618_PORT : 0; + for (String param : query.split("&")) { + int eq = param.indexOf('='); + if (eq < 0) continue; + String name = param.substring(0, eq).trim().toLowerCase(Locale.ROOT); + String value = param.substring(eq + 1).trim().toLowerCase(Locale.ROOT); + if (name.equals("qpara") || name.equals("q")) { + u.qpara = parseQpara(value); + } else if (name.equals("dqt")) { + u.dqt = Arrays.asList(DQT_NAMES).indexOf(value); + if (u.dqt < 0) { + throw new IOException("Unknown dqt: " + value + " (zigzag, raster or q50)"); + } + } + } return u; } + /** Exactly 8 hex digits, e.g. 19322348, as 4 bytes. */ + static byte[] parseQpara(String v) throws IOException { + if (v.length() != 8) throw new IOException("qpara needs 8 hex digits: " + v); + byte[] b = new byte[4]; + for (int i = 0; i < 4; i++) { + int hi = Character.digit(v.charAt(2 * i), 16); + int lo = Character.digit(v.charAt(2 * i + 1), 16); + if (hi < 0 || lo < 0) throw new IOException("qpara needs 8 hex digits: " + v); + b[i] = (byte) (hi << 4 | lo); + } + return b; + } + + /** The ports probed: the given one, then the alternate one if any. */ + int[] ports() { + return alternatePort > 0 && alternatePort != port + ? new int[] {port, alternatePort} : new int[] {port}; + } + String endpoint() { - return (host.indexOf(':') >= 0 ? "[" + host + "]" : host) + ":" + port; + return endpoint(port); + } + + String endpoint(int p) { + return (host.indexOf(':') >= 0 ? "[" + host + "]" : host) + ":" + p; + } + + /** "192.168.169.1:8800 and :8801", or the one endpoint. */ + String probed() { + int[] p = ports(); + return p.length > 1 ? endpoint(p[0]) + " and :" + p[1] : endpoint(p[0]); } } @@ -171,8 +276,12 @@ static final class Fragment { int width; int height; int quality; - /** 01 01 at 0x34: the camera family whose layout this is. */ - boolean gl; + /** + * Main and flow camera status at 0x34 and 0x35 (turbodrone docs: the vendor library + * passes them on with the frame). Shown in the stats, not used. + */ + int mainCamera; + int flowCamera; /** The packet parsed last; its data starts at dataOffset. */ byte[] buf; int dataOffset; @@ -189,7 +298,8 @@ boolean parse(byte[] b, int off, int len) { width = u16(b, off + 0x2C); height = u16(b, off + 0x2E); quality = b[off + 0x30] & 0xFF; - gl = b[off + 0x34] == 1 && b[off + 0x35] == 1; + mainCamera = b[off + 0x34] & 0xFF; + flowCamera = b[off + 0x35] & 0xFF; buf = b; dataOffset = off + FRAGMENT_HEADER; dataLength = len - FRAGMENT_HEADER; @@ -278,8 +388,11 @@ static final class Window { volatile int delivered; volatile int lost; volatile int late; - volatile int wholeRequests; + /** Records left out because their slot lagged LAG or more frames behind. */ + volatile int lagging; volatile int nextRequests; + /** Quality parameters of the acknowledgement. */ + byte[] qpara = QPARA; /** The frame numbers started from 1 again (the drone restarted). */ volatile int restarts; @@ -377,11 +490,11 @@ Slot add(Fragment f, long nowMs) { /** * The acknowledgement for now, into {@code out} (ACK_CAPACITY bytes); returns its * length. Per slot: complete (status 1) or still collecting (0) with the mask of the - * fragments received; a slot LAG or more frames behind the newest asks for its next - * frame whole instead (3, no mask). The frame coming in now is left out until the - * fragments pause; after QUIET_MS without any, when every frame is complete, a record - * asks for the frame after the newest. Unverified: whether the drone needs or uses any - * of it, the record order, and that the paused frame is included again. + * fragments received; a slot LAG or more frames behind the newest is left out (as in + * turbodrone), and does not hold up the request for the next frame either. The frame + * coming in now is left out until the fragments pause; after QUIET_MS without any, when + * every frame is complete, a record asks for the frame after the newest (status 3). + * Unverified: the record order, and that the paused frame is included again. */ int buildAck(byte[] out, long nowMs) { boolean quiet = nowMs - lastFragmentMs > QUIET_MS; @@ -393,15 +506,13 @@ int buildAck(byte[] out, long nowMs) { // At or below the base the drone is done with it: no record, and an incomplete // one does not hold up the request for the next frame (Qodo on #188) if (s.seq == 0 || s.seq <= base) continue; - any = true; - if (s.status != STATUS_COMPLETE) allComplete = false; - if (s.seq == current && !quiet) continue; if (maxSeq - s.seq >= LAG) { - pos = wholeRecord(out, pos, s.seq + SLOTS); - records++; - wholeRequests++; + lagging++; continue; } + any = true; + if (s.status != STATUS_COMPLETE) allComplete = false; + if (s.seq == current && !quiet) continue; int words = (s.total + 31) / 32; le64(out, pos, s.seq); boolean complete = s.status == STATUS_COMPLETE; @@ -419,7 +530,7 @@ int buildAck(byte[] out, long nowMs) { records++; nextRequests++; } - ackHeader(out, pos, records); + ackHeader(out, pos, records, qpara); return pos; } @@ -433,9 +544,12 @@ private static int wholeRecord(byte[] out, int pos, long seq) { /** * The fixed part of an acknowledgement: EF 02, length, 02 02 00 01, record count, then no - * command (counter 0, length 0, 0x40 zero bytes), QPARA and active camera 0. + * command (counter 0, length 0, 0x40 zero bytes), the quality parameters and active camera + * 0. The vendor app and turbodrone put the current stick command in there, a neutral one + * when idle; this app never does: a viewer must not be able to move a toy, and a neutral + * command is still a command (it can hold a drone that the remote is flying). */ - static void ackHeader(byte[] out, int length, int records) { + static void ackHeader(byte[] out, int length, int records, byte[] qpara) { out[0] = (byte) 0xEF; out[1] = 0x02; le16(out, 2, length); @@ -447,7 +561,7 @@ static void ackHeader(byte[] out, int length, int records) { le32(out, 0x0C, 0); le16(out, 0x10, 0); Arrays.fill(out, 0x12, 0x52, (byte) 0); - System.arraycopy(QPARA, 0, out, 0x52, QPARA.length); + System.arraycopy(qpara, 0, out, 0x52, 4); out[0x56] = 0; out[0x57] = 0; } @@ -456,12 +570,107 @@ static void ackHeader(byte[] out, int length, int records) { /** The headers in front of a body: SOI, DQT, SOF0, DHT, SOS (605 bytes as configured). */ static byte[] jpegHeader(int quality, int width, int height) { - int q = quality >= 1 && quality <= 100 ? quality : DEFAULT_QUALITY; + return jpegHeader(quality, width, height, DQT_ZIGZAG); + } + + static byte[] jpegHeader(int quality, int width, int height, int dqt) { return JpegAssembler.headersSampled(LUMA_SAMPLING, width > 0 ? width : DEFAULT_WIDTH, - height > 0 ? height : DEFAULT_HEIGHT, JpegAssembler.ijgTables(q), 0, + height > 0 ? height : DEFAULT_HEIGHT, dqtTables(dqt, quality), 0, RESTART_INTERVAL); } + /** Luma then chroma table, 64 bytes each, in the order DQT carries them for {@code dqt}. */ + static byte[] dqtTables(int dqt, int quality) { + if (dqt == DQT_Q50) return JpegAssembler.ijgTables(50); + if (dqt == DQT_RASTER) { + // Q50 is the unscaled Annex K table; put each value at its raster position + byte[] zz = JpegAssembler.ijgTables(50); + byte[] out = new byte[128]; + for (int t = 0; t < 128; t += 64) { + for (int k = 0; k < 64; k++) out[t + RASTER_OF_ZIGZAG[k]] = zz[t + k]; + } + return out; + } + return JpegAssembler.ijgTables(quality >= 1 && quality <= 100 ? quality : DEFAULT_QUALITY); + } + + /** For the stats and the log: which tables, e.g. "zigzag Q75" or "raster (unscaled)". */ + static String dqtLabel(int dqt, int quality) { + if (dqt == DQT_ZIGZAG) { + return "zigzag Q" + (quality >= 1 && quality <= 100 ? quality : DEFAULT_QUALITY); + } + return DQT_NAMES[dqt] + " (unscaled)"; + } + + // ====================== Subnet hint ====================== + + /** + * A hint when the slot cannot be right for the phone's networks (#187), or null: the + * target is the phone itself, or the phone is on a network of another toy family than the + * address. {@code local}: the phone's IPv4 addresses. + */ + static String subnetHint(InetAddress target, List local) { + if (!(target instanceof Inet4Address) || local == null) return null; + boolean own = local.contains(target); + boolean on1 = false; + boolean on169 = false; + boolean on80 = false; + for (InetAddress a : local) { + on1 |= in24(a, 1); + on169 |= in24(a, 169); + on80 |= in24(a, 80); + } + StringBuilder parts = new StringBuilder(); + byte[] b = target.getAddress(); + if (own) { + sentence(parts, target.getHostAddress() + " is this phone; the toy is usually " + + (b[0] & 0xFF) + "." + (b[1] & 0xFF) + "." + (b[2] & 0xFF) + ".1"); + } + if (on1 && !on169 && (own || in24(target, 169))) { + sentence(parts, "This network looks like a KY UFO RTSP toy: try" + + " rtsp://192.168.1.1:7070/webcam"); + } + if (on169 && !on1 && in24(target, 1)) { + sentence(parts, "This phone is on 192.168.169.x: the toy is usually" + + " flowufo://192.168.169.1"); + } + if (on80 && !on169 && !on1) { + sentence(parts, "Toys on 192.168.80.x (HF_ Wi-Fi) use another protocol, not supported" + + " yet"); + } + return parts.length() > 0 ? parts.toString() : null; + } + + private static void sentence(StringBuilder sb, String s) { + if (sb.length() > 0) sb.append(". "); + sb.append(s); + } + + /** {@code a} is in 192.168.{@code third}.0/24. */ + private static boolean in24(InetAddress a, int third) { + if (!(a instanceof Inet4Address)) return false; + byte[] b = a.getAddress(); + return (b[0] & 0xFF) == 192 && (b[1] & 0xFF) == 168 && (b[2] & 0xFF) == third; + } + + /** The phone's IPv4 addresses on interfaces that are up. */ + static List localAddresses() { + List list = new ArrayList<>(); + try { + Enumeration e = NetworkInterface.getNetworkInterfaces(); + while (e != null && e.hasMoreElements()) { + NetworkInterface n = e.nextElement(); + if (!n.isUp() || n.isLoopback()) continue; + for (InterfaceAddress ia : n.getInterfaceAddresses()) { + if (ia.getAddress() instanceof Inet4Address) list.add(ia.getAddress()); + } + } + } catch (Exception ignored) { + // No hint then + } + return list; + } + /** * Header, body and EOI into {@code out} (grown when too small, returned in out[0]); * returns the length. A body that is a whole JPEG already goes out as it is. @@ -500,18 +709,30 @@ interface SocketSetup { /** * The receive loop on the calling thread: probes, fragments, acknowledgements and the - * timers between them, until {@code keepGoing} fails or close(). + * timers between them, until {@code keepGoing} fails or close(). Each probed port has a + * socket of its own: the drone answers from source port 1234 whichever port was probed, + * so only the socket the packets arrive on tells which engine answered. The first 0x93 + * packet from the toy's address locks the client onto that socket and port (acks go + * there); the other socket is closed, and opened again when the drone goes quiet and the + * probes start over. */ static final class Client implements Closeable { private final String tag; private final Url url; private final SocketSetup setup; - private volatile DatagramSocket socket; + private final int[] ports; + /** One socket per probed port; null once another port is locked. Guarded by itself. */ + private final DatagramSocket[] sockets; private volatile boolean closed; + /** Index of the port the drone answers on, or -1 while it has not. */ + private volatile int locked = -1; + /** The phone's addresses for the subnet hint; null: the real ones. For tests. */ + List localForHint; final Window window = new Window(); private final Fragment fragment = new Fragment(); private final byte[] ack = new byte[ACK_CAPACITY]; + private final DatagramPacket out = new DatagramPacket(ack, 0); private final byte[][] jpegOut = new byte[1][]; private byte[] header; /** Quality and size the header was made for. */ @@ -523,6 +744,7 @@ static final class Client implements Closeable { private volatile boolean receiving; private volatile boolean playing; private volatile String network = "default"; + private volatile String hint; private volatile int probes; private volatile int acks; private volatile int packets; @@ -536,8 +758,10 @@ static final class Client implements Closeable { private volatile String lastService; private volatile String firstHeader; private volatile String size; - private volatile int quality; - private volatile boolean gl = true; + private volatile String dqtUsed; + private volatile int mainCamera = -1; + private volatile int flowCamera = -1; + private int cameraChanges; private volatile float fragmentsPerS; private volatile float framesPerS; /** Any fragment from the drone, usable or not: it is still there. */ @@ -547,43 +771,101 @@ static final class Client implements Closeable { this.tag = tag; this.url = url; this.setup = setup; + ports = url.ports(); + sockets = new DatagramSocket[ports.length]; + window.qpara = url.qpara; } @Override public void close() { closed = true; - DatagramSocket s = socket; - if (s != null) s.close(); + synchronized (sockets) { + for (DatagramSocket s : sockets) { + if (s != null) s.close(); + } + } + } + + /** BL618 (port 8801) probes every 200 ms, BL608 and others every 100 ms. */ + static long probeInterval(int port) { + return port == BL618_PORT ? PROBE_MS_BL618 : PROBE_MS; + } + + /** BL618 acknowledges every 50 ms and only with a record; BL608 every 25 ms. */ + static long ackInterval(int port) { + return port == BL618_PORT ? ACK_MS_BL618 : ACK_MS; + } + + static boolean ackNeedsRecord(int port) { + return port == BL618_PORT; + } + + static String engine(int port) { + return port == DEFAULT_PORT ? "BL608" : port == BL618_PORT ? "BL618" : "port " + port; + } + + /** The port the drone answers on, or -1. */ + int lockedPort() { + int i = locked; + return i >= 0 ? ports[i] : -1; } String probingText(boolean again) { - return (again ? "The toy went quiet: probing " : "Waiting for the toy: probing ") - + url.endpoint() + (again ? " again" : ""); + String t = (again ? "The toy went quiet: probing " : "Waiting for the toy: probing ") + + url.probed() + (again ? " again" : ""); + String h = hint; + return h != null ? t + ". " + h : t; } - private DatagramSocket open(InetAddress target) throws IOException { + private void open(int i, InetAddress target) throws IOException { DatagramSocket s = new DatagramSocket(); try { s.setReceiveBufferSize(1024 * 1024); } catch (IOException ignored) {} if (setup != null) network = setup.prepare(s, target); - socket = s; - if (closed) s.close(); - return s; + synchronized (sockets) { + if (sockets[i] != null) sockets[i].close(); + sockets[i] = s; + if (closed) s.close(); + } + } + + private void closeSocket(int i) { + synchronized (sockets) { + if (sockets[i] != null) sockets[i].close(); + sockets[i] = null; + } + } + + private DatagramSocket socket(int i) { + synchronized (sockets) { + return sockets[i]; + } + } + + private String localPorts() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < ports.length; i++) { + DatagramSocket s = socket(i); + if (s == null) continue; + if (sb.length() > 0) sb.append(", "); + sb.append(s.getLocalPort()).append(" for ").append(ports[i]); + } + return sb.toString(); } void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException { InetAddress address = InetAddress.getByName(url.host); - InetSocketAddress target = new InetSocketAddress(address, url.port); - DatagramSocket s = open(address); - Log.i(TAG, tag + ": FLOW-UFO: probing " + url.endpoint() + " from local port " - + s.getLocalPort() + " (network: " + network + ")"); + hint = subnetHint(address, localForHint != null ? localForHint : localAddresses()); + if (hint != null) Log.w(TAG, tag + ": FLOW-UFO: " + hint); + for (int i = 0; i < ports.length; i++) open(i, address); + Log.i(TAG, tag + ": FLOW-UFO: probing " + url.probed() + " from local port " + + localPorts() + " (network: " + network + "), qpara " + + hex(url.qpara, 0, 4) + ", dqt " + DQT_NAMES[url.dqt]); listener.onState(false, probingText(false)); byte[] rx = new byte[2048]; DatagramPacket in = new DatagramPacket(rx, rx.length); - DatagramPacket probe = new DatagramPacket(PROBE, PROBE.length, target); - DatagramPacket out = new DatagramPacket(ack, 0, target); - long nextProbe = 0; + long[] nextProbe = new long[ports.length]; long nextAck = 0; long rateMark = now(); int rateFragments = 0; @@ -605,88 +887,149 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException if (receiving && t - lastFragmentAt > resetMs) { Log.w(TAG, tag + ": FLOW-UFO: no fragment for " + resetMs + " ms, probing again (" + summary() + ")"); - probeAgain(listener, t); - nextProbe = t; + probeAgain(listener, address); + Arrays.fill(nextProbe, t); } if (receiving && t >= nextSummary) { nextSummary = t + 30_000; Log.i(TAG, tag + ": FLOW-UFO: " + summary()); } boolean sendFailed = false; - if (!receiving && t >= nextProbe) { - nextProbe = t + PROBE_MS; - sendFailed = !send(s, probe); - if (!sendFailed) probes++; - } else if (receiving && t >= nextAck) { + int lk = locked; + if (!receiving) { + for (int i = 0; i < ports.length; i++) { + DatagramSocket s = socket(i); + if (s == null || t < nextProbe[i]) continue; + nextProbe[i] = t + probeInterval(ports[i]); + if (send(s, PROBE, PROBE.length, address, ports[i])) { + probes++; + } else { + sendFailed = true; + } + } + } else if (lk >= 0 && t >= nextAck) { // Behind schedule (a slow thread) is caught up, not sent in a burst - nextAck = Math.max(nextAck + ACK_MS, t); + int port = ports[lk]; + nextAck = Math.max(nextAck + ackInterval(port), t); int len = window.buildAck(ack, t); - out.setLength(len); - sendFailed = !send(s, out); - if (!sendFailed) acks++; + if (len > ACK_HEADER || !ackNeedsRecord(port)) { + if (send(socket(lk), ack, len, address, port)) { + acks++; + } else { + sendFailed = true; + } + } } if (sendFailed) { - // Off the toy's Wi-Fi, or its network went away: a new socket after + // Off the toy's Wi-Fi, or its network went away: new sockets after // 3 s of failures, bound again to whatever network reaches it now. The - // drone sends to the port of the probe: the new port probes at once + // drone sends to the port of the probe: the new ports probe at once // (Qodo on #188) if (failingSince == 0) failingSince = t; if (t - failingSince >= 3000) { failingSince = 0; - s.close(); - s = open(address); - Log.w(TAG, tag + ": FLOW-UFO: new socket after send errors, local port " - + s.getLocalPort() + " (network: " + network + ")"); - if (receiving) probeAgain(listener, t); - nextProbe = t; + if (receiving) probeAgain(listener, address); + locked = -1; + for (int i = 0; i < ports.length; i++) open(i, address); + Log.w(TAG, tag + ": FLOW-UFO: new sockets after send errors, local" + + " port " + localPorts() + " (network: " + network + ")"); + Arrays.fill(nextProbe, t); } } else { failingSince = 0; } - long wake = receiving ? nextAck : nextProbe; + long wake = Long.MAX_VALUE; + if (receiving) { + wake = nextAck; + } else { + for (int i = 0; i < ports.length; i++) { + if (socket(i) != null) wake = Math.min(wake, nextProbe[i]); + } + } int timeout = (int) Math.max(1, Math.min(100, wake - now())); - try { - s.setSoTimeout(timeout); - in.setLength(rx.length); - s.receive(in); - } catch (SocketTimeoutException e) { - continue; + int got = -1; + lk = locked; + if (lk >= 0) { + if (receive(socket(lk), in, rx, timeout)) got = lk; + } else { + // Not locked yet: each socket in turn; what is not read waits in its + // buffer for the next turn + int each = Math.max(1, timeout / ports.length); + for (int i = 0; i < ports.length && got < 0; i++) { + DatagramSocket s = socket(i); + if (s != null && receive(s, in, rx, each)) got = i; + } } - if (!in.getAddress().equals(address)) { + if (got < 0) continue; + // By source address only: the drone sends from port 1234, not the probed one + if (!address.equals(in.getAddress())) { if (++foreign <= 3) { Log.w(TAG, tag + ": FLOW-UFO: packet from " + in.getAddress() + ", not the toy: ignored"); } continue; } + if (locked < 0 && packetType(rx, 0, in.getLength()) >= 0) lock(got, in); handle(rx, in.getLength(), listener); } } catch (IOException e) { if (closed) return; throw e; } finally { - s.close(); + close(); } } - /** Back to probing: the stream is gone. */ - private void probeAgain(Listener listener, long t) { + private boolean receive(DatagramSocket s, DatagramPacket in, byte[] rx, int timeout) + throws IOException { + if (s == null) return false; + try { + s.setSoTimeout(timeout); + in.setData(rx); + s.receive(in); + return true; + } catch (SocketTimeoutException e) { + return false; + } + } + + /** The drone answered on ports[i]: acks go there, the other probes stop. */ + private void lock(int i, DatagramPacket from) { + locked = i; + for (int j = 0; j < ports.length; j++) { + if (j != i) closeSocket(j); + } + Log.i(TAG, tag + ": FLOW-UFO: the toy answers the probe to port " + ports[i] + " (" + + engine(ports[i]) + "), from port " + from.getPort() + + (ports.length > 1 ? "; the other port is not probed any more" : "")); + } + + /** Back to probing on every port: the stream is gone. */ + private void probeAgain(Listener listener, InetAddress address) throws IOException { resets++; receiving = false; playing = false; window.reset(); + locked = -1; + for (int i = 0; i < ports.length; i++) { + if (socket(i) == null) open(i, address); + } listener.onState(false, probingText(true)); } - private boolean send(DatagramSocket s, DatagramPacket p) { + private boolean send(DatagramSocket s, byte[] data, int len, InetAddress a, int port) { + if (s == null) return false; try { - s.send(p); + out.setData(data, 0, len); + out.setAddress(a); + out.setPort(port); + s.send(out); return true; } catch (IOException e) { sendErrors++; lastError = e.getMessage(); if (sendErrors <= 3 || sendErrors % 100 == 0) { - Log.w(TAG, tag + ": FLOW-UFO: cannot send to " + url.endpoint() + " (" + Log.w(TAG, tag + ": FLOW-UFO: cannot send to " + url.endpoint(port) + " (" + sendErrors + "): " + e.getMessage() + "; is this device on the toy's Wi-Fi?"); } @@ -723,7 +1066,8 @@ private void handle(byte[] b, int len, Listener listener) { if (firstHeader == null) firstHeader = hex(b, 0, FRAGMENT_HEADER); Log.i(TAG, tag + ": FLOW-UFO: first fragment after " + probes + " probes: " + hex(b, 0, FRAGMENT_HEADER)); - listener.onState(true, "Receiving from the toy at " + url.endpoint() + "…"); + listener.onState(true, "Receiving from the toy at " + + url.endpoint(lockedPort() > 0 ? lockedPort() : url.port) + "…"); } int n = window.fragments; if (n < 5) { @@ -732,24 +1076,28 @@ private void handle(byte[] b, int len, Listener listener) { + fragment.bodyLength + ", " + fragment.width + "x" + fragment.height + " Q" + fragment.quality + ", bytes 0x08-0x17: " + hex(b, 0x08, 16)); } - if (!fragment.gl && gl) { - gl = false; - Log.w(TAG, tag + ": FLOW-UFO: bytes 0x34/0x35 are " + hex(b, 0x34, 2) - + ", not 01 01: another camera family, frames may not decode"); + if (fragment.mainCamera != mainCamera || fragment.flowCamera != flowCamera) { + boolean first = mainCamera < 0; + mainCamera = fragment.mainCamera; + flowCamera = fragment.flowCamera; + if (++cameraChanges <= 5) { + Log.i(TAG, tag + ": FLOW-UFO: camera status (0x34/0x35): main " + mainCamera + + ", flow " + flowCamera + (first ? "" : " (changed)")); + } } Slot done = window.add(fragment, t); if (done == null) return; if (header == null || done.quality != headerQuality || done.width != headerWidth || done.height != headerHeight) { - header = jpegHeader(done.quality, done.width, done.height); + header = jpegHeader(done.quality, done.width, done.height, url.dqt); headerQuality = done.quality; headerWidth = done.width; headerHeight = done.height; - quality = done.quality; size = done.width + "x" + done.height; + dqtUsed = dqtLabel(url.dqt, done.quality); Log.i(TAG, tag + ": FLOW-UFO: frames " + size + " Q" + done.quality + (done.quality >= 1 && done.quality <= 100 ? "" - : " (not a quality, Q" + DEFAULT_QUALITY + " tables)")); + : " (not a quality)") + ", tables " + dqtUsed); } int length = jpeg(header, done.body, done.bodyLength, jpegOut); if (!playing) { @@ -764,8 +1112,8 @@ private void handle(byte[] b, int len, Listener listener) { private String summary() { Window w = window; return String.format(Locale.ROOT, "%.1f fps, %.0f fragments/s, %d frames, %d lost," - + " %d late, %d whole requests, %d next requests, %d acks, %d bad", - framesPerS, fragmentsPerS, w.delivered, w.lost, w.late, w.wholeRequests, + + " %d late, %d lagging left out, %d next requests, %d acks, %d bad", + framesPerS, fragmentsPerS, w.delivered, w.lost, w.late, w.lagging, w.nextRequests, acks, badPackets); } @@ -774,13 +1122,26 @@ Map stats() { Window w = window; Map m = new LinkedHashMap<>(); m.put("state", playing ? "playing" : receiving ? "receiving" : "probing"); - m.put("target", url.endpoint()); + m.put("target", url.probed()); + int lp = lockedPort(); + if (lp > 0) { + m.put("port", lp); + m.put("engine", engine(lp)); + } m.put("network", network); + if (hint != null) m.put("hint", hint); + m.put("qpara", hex(url.qpara, 0, 4)); + m.put("dqt", dqtUsed != null ? dqtUsed : DQT_NAMES[url.dqt]); m.put("frames_per_s", Math.round(framesPerS * 10) / 10f); m.put("fragments_per_s", Math.round(fragmentsPerS)); if (size != null) m.put("size", size); - if (size != null) m.put("quality", quality); - m.put("gl", gl); + if (size != null) m.put("quality", headerQuality); + if (mainCamera >= 0) { + Map cam = new LinkedHashMap<>(); + cam.put("main", mainCamera); + cam.put("flow", flowCamera); + m.put("camera_status", cam); + } m.put("probes", probes); m.put("acks", acks); m.put("packets", packets); @@ -795,7 +1156,7 @@ Map stats() { m.put("frames_complete", w.completed); m.put("frames_lost", w.lost); m.put("frames_late", w.late); - m.put("whole_requests", w.wholeRequests); + m.put("lagging_left_out", w.lagging); m.put("next_requests", w.nextRequests); m.put("resets", resets); m.put("seq_restarts", w.restarts); diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index 5c3f3699..d9e71e8b 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -431,6 +431,9 @@ static boolean isKeyFrameNal(int type, boolean h265) { /** FLOW-UFO statistics of a flowufo:// cell (#187), else null. */ java.util.Map flowUfoStats() { return session.flowUfoStats(); } + /** The cooingdv heartbeat of an RTSP cell once it used one (#187), else null. */ + java.util.Map heartbeatStats() { return session.heartbeatStats(); } + /** Transport in use, with the mode after a fallback (#112). */ String transportLabel() { return Transport.label(session.transportInUse(), transport); } diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java index 84a46e42..c40e424f 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java @@ -89,6 +89,8 @@ void configure(String host, String userAgent, Transport.Mode transport) { long getBitrateMarkNs() { return session.getBitrateMarkNs(); } /** FLOW-UFO statistics while a flowufo:// slot plays (#187), else null. */ java.util.Map flowUfoStats() { return session.flowUfoStats(); } + /** The cooingdv heartbeat once this slot used one (#187), else null. */ + java.util.Map heartbeatStats() { return session.heartbeatStats(); } void start() { nalAssembler.reset(); diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index 4174bd03..bad7a2c9 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -107,6 +107,12 @@ default void onDetail(String text) {} private volatile boolean listening; /** Client of a flowufo:// slot (#187), else null: it probes on its own when silent. */ private volatile FlowUfo.Client ufo; + /** Heartbeat of the RTSP connection now (#187), else null; counts of the earlier ones. */ + private volatile UdpHeartbeat heartbeat; + private volatile int heartbeatsBefore; + private volatile int heartbeatErrorsBefore; + private volatile boolean heartbeatUsed; + private volatile boolean heartbeatLogged; private Thread networkThread; private Thread watchdogThread; @@ -210,6 +216,7 @@ void stop() { private void closeSocketLocked() { if (listener != null) listener.close(); if (ufo != null) ufo.close(); + if (heartbeat != null) heartbeat.close(); if (udp != null) { udp.close(); if (udpPlaying) return; @@ -311,9 +318,11 @@ private boolean connect(int gen) throws IOException { } Conn c = null; UdpReceiver rx = null; + UdpHeartbeat hb = null; String session = null; try { s.connect(new InetSocketAddress(u.host, u.port), CONNECT_TIMEOUT_MS); + if (u.heartbeatPort > 0) hb = startHeartbeat(gen, s, u.heartbeatPort); s.setTcpNoDelay(true); s.setSoTimeout(HANDSHAKE_TIMEOUT_MS); BufferedInputStream in = new BufferedInputStream(s.getInputStream(), 65536); @@ -493,6 +502,7 @@ private boolean connect(int gen) throws IOException { Keepalive k = keepalive; if (k != null && k.conn == c) keepalive = null; } + stopHeartbeat(hb); if (rx != null) { rx.close(); if (session != null && !s.isClosed()) { @@ -599,6 +609,51 @@ private boolean httpMjpeg(int gen, HttpMjpeg.Url u) throws IOException { throw new IOException("401 Unauthorized"); } + /** + * The cooingdv heartbeat (#187): 01 01 to the camera's UDP port every second, from the + * address of the RTSP connection, while the connection lasts. Logged at the first start. + */ + private UdpHeartbeat startHeartbeat(int gen, Socket s, int port) { + UdpHeartbeat h = new UdpHeartbeat(tag, s.getLocalAddress(), + new InetSocketAddress(s.getInetAddress(), port)); + synchronized (lock) { + if (gen != generation) return null; + stopHeartbeat(heartbeat); + heartbeat = h; + heartbeatUsed = true; + } + try { + h.start(!heartbeatLogged); + heartbeatLogged = true; + } catch (IOException e) { + Log.w(TAG, tag + ": no heartbeat socket: " + e.getMessage()); + } + return h; + } + + /** + * Ends the heartbeat {@code h} of a connection; when it is the current one its counts + * go to the totals of the stats. A newer connection's heartbeat is left alone. + */ + private void stopHeartbeat(UdpHeartbeat h) { + if (h == null) return; + h.close(); + synchronized (lock) { + if (heartbeat != h) return; + heartbeatsBefore += h.sent(); + heartbeatErrorsBefore += h.errors(); + heartbeat = null; + } + } + + /** Heartbeat statistics for /stats.json once an RTSP slot used one (#187), else null. */ + Map heartbeatStats() { + if (!heartbeatUsed) return null; + synchronized (lock) { + return UdpHeartbeat.stats(heartbeat, heartbeatsBefore, heartbeatErrorsBefore); + } + } + /** FLOW-UFO statistics for /stats.json while a flowufo:// slot runs (#187), else null. */ Map flowUfoStats() { FlowUfo.Client c = ufo; @@ -1191,6 +1246,12 @@ static final class Url { String userInfo; /** URL without credentials, used in requests. */ String requestUrl; + /** + * UDP port for the cooingdv heartbeat (#187), or 0 for none: 7099 for + * rtsp://{host}:7070/webcam, else ?heartbeat={port}; ?heartbeat=off turns it off. The + * option is ours: it is taken out of the URL sent to the camera. + */ + int heartbeatPort; static Url parse(String url) throws IOException { if (url == null) throw new IOException("Camera slot not configured"); @@ -1237,10 +1298,51 @@ static Url parse(String url) throws IOException { u.host = hostPort; } if (u.host.isEmpty()) throw new IOException("Invalid RTSP URL: host is missing"); + rest = takeHeartbeat(u, rest); u.requestUrl = scheme + "://" + authority + rest; return u; } + /** Sets heartbeatPort from path and query; returns them without our option. */ + private static String takeHeartbeat(Url u, String rest) throws IOException { + int q = rest.indexOf('?'); + String path = q >= 0 ? rest.substring(0, q) : rest; + if (u.port == 7070 && (path.equalsIgnoreCase("/webcam") + || path.equalsIgnoreCase("/webcam/"))) { + u.heartbeatPort = UdpHeartbeat.DEFAULT_PORT; + } + if (q < 0) return rest; + StringBuilder kept = new StringBuilder(); + boolean found = false; + for (String param : rest.substring(q + 1).split("&", -1)) { + int eq = param.indexOf('='); + String name = (eq < 0 ? param : param.substring(0, eq)).toLowerCase(Locale.ROOT); + if (!name.equals("heartbeat")) { + if (kept.length() > 0) kept.append('&'); + kept.append(param); + continue; + } + found = true; + String v = eq < 0 ? "" : param.substring(eq + 1).trim().toLowerCase(Locale.ROOT); + if (v.equals("off") || v.equals("0")) { + u.heartbeatPort = 0; + } else { + try { + u.heartbeatPort = Integer.parseInt(v); + } catch (NumberFormatException e) { + u.heartbeatPort = -1; + } + if (u.heartbeatPort <= 0 || u.heartbeatPort > 65535) { + throw new IOException("Invalid heartbeat port: " + v + + " (a UDP port, e.g. heartbeat=7099, or heartbeat=off)"); + } + } + } + // Untouched unless our option was there + if (!found) return rest; + return kept.length() > 0 ? path + "?" + kept : path; + } + private static int parsePort(String s, String url) throws IOException { if (s.isEmpty()) return 554; try { diff --git a/android/app/src/main/java/com/openipc/decoder/UdpHeartbeat.java b/android/app/src/main/java/com/openipc/decoder/UdpHeartbeat.java new file mode 100644 index 00000000..72fe420f --- /dev/null +++ b/android/app/src/main/java/com/openipc/decoder/UdpHeartbeat.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * UdpHeartbeat.java — the "01 01" heartbeat that cooingdv RTSP toys expect on UDP 7099 (#187) + * + */ + +package com.openipc.decoder; + +import android.util.Log; + +import java.io.Closeable; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Toys of the cooingdv apps (KY UFO, RC UFO, ...; Wi-Fi "FLY-UFO-…" at 192.168.1.1) stream + * RTSP from rtsp://192.168.1.1:7070/webcam, and their apps send {@code 01 01} to UDP port + * 7099 once a second while they show it (described by turbodrone, docs/research/cooingdv.md). + * This sends exactly that heartbeat, from an ephemeral socket on the address the RTSP + * connection uses, and nothing else: no flight command and no "08 01" either. One thread per + * heartbeat, until close(). + */ +final class UdpHeartbeat implements Closeable { + private static final String TAG = "OpenIPCDecoder"; + + static final byte[] PAYLOAD = {0x01, 0x01}; + static final int DEFAULT_PORT = 7099; + /** Not final so unit tests can shorten it. */ + static volatile long intervalMs = 1000; + + private final String tag; + private final InetAddress local; + private final InetSocketAddress target; + private volatile boolean closed; + private volatile DatagramSocket socket; + private Thread thread; + private volatile int sent; + private volatile int errors; + private volatile String lastError; + + /** @param local address to send from (the RTSP connection's), or null for any. */ + UdpHeartbeat(String tag, InetAddress local, InetSocketAddress target) { + this.tag = tag; + this.local = local; + this.target = target; + } + + /** Opens the socket and starts sending, the first heartbeat at once. */ + synchronized void start(boolean log) throws IOException { + if (closed || thread != null) return; + DatagramSocket s = local != null + ? new DatagramSocket(new InetSocketAddress(local, 0)) : new DatagramSocket(); + String network = NetInfo.bindToNetworkOf(s, target.getAddress()); + socket = s; + if (log) { + Log.i(TAG, tag + ": heartbeat 01 01 every " + intervalMs + " ms to " + + target.getAddress().getHostAddress() + ":" + target.getPort() + + " from local port " + s.getLocalPort() + " (network: " + network + ")"); + } + thread = new Thread(() -> loop(s), tag + "-hb"); + thread.setDaemon(true); + thread.start(); + } + + private void loop(DatagramSocket s) { + DatagramPacket p = new DatagramPacket(PAYLOAD, PAYLOAD.length, target); + long next = System.nanoTime() / 1_000_000; + while (!closed) { + try { + s.send(p); + sent++; + } catch (IOException e) { + if (closed) break; + errors++; + lastError = e.getMessage(); + if (errors <= 3 || errors % 100 == 0) { + Log.w(TAG, tag + ": heartbeat failed (" + errors + "): " + e.getMessage()); + } + } + // A fixed schedule: a slow send does not shift the following ones + next += intervalMs; + long wait = next - System.nanoTime() / 1_000_000; + if (wait < 0) { + next = System.nanoTime() / 1_000_000; + wait = 0; + } + try { + Thread.sleep(wait); + } catch (InterruptedException e) { + break; + } + } + s.close(); + } + + int sent() { return sent; } + + int errors() { return errors; } + + String lastError() { return lastError; } + + InetSocketAddress target() { return target; } + + @Override + public synchronized void close() { + closed = true; + DatagramSocket s = socket; + if (s != null) s.close(); + if (thread != null) thread.interrupt(); + } + + /** Running or not: true until close(). */ + boolean running() { return !closed && thread != null; } + + static Map stats(UdpHeartbeat h, int sentBefore, int errorsBefore) { + Map m = new LinkedHashMap<>(); + m.put("payload", "01 01"); + if (h != null) { + m.put("target", h.target.getAddress().getHostAddress() + ":" + h.target.getPort()); + m.put("running", h.running()); + if (h.lastError != null) m.put("last_error", h.lastError); + } else { + m.put("running", false); + } + m.put("sent", sentBefore + (h != null ? h.sent : 0)); + int e = errorsBefore + (h != null ? h.errors : 0); + if (e > 0) m.put("errors", e); + return m; + } +} diff --git a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java index 84ac2841..4ec2f8da 100644 --- a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java +++ b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java @@ -196,7 +196,14 @@ public void fragmentFields() { assertEquals(640, f.width); assertEquals(360, f.height); assertEquals(75, f.quality); - assertTrue(f.gl); + // Camera status bytes: main and flow camera + assertEquals(1, f.mainCamera); + assertEquals(1, f.flowCamera); + p[0x34] = 0; + p[0x35] = 2; + f = parsed(p); + assertEquals(0, f.mainCamera); + assertEquals(2, f.flowCamera); assertEquals(FlowUfo.FRAGMENT_HEADER, f.dataOffset); assertEquals(1024, f.dataLength); assertSame(p, f.buf); @@ -239,7 +246,7 @@ public void ackBytesExact() { String header = "EF 02 %s 02 02 00 01 %s 00 00 00 00 00 00" + " 00000000000000000000000000000000 00000000000000000000000000000000" + " 00000000000000000000000000000000 00000000000000000000000000000000" - + " 19 32 23 4B 00 00"; + + " 32 4B 14 2D 00 00"; byte[] expected = bytes(String.format(header, "6C 00", "01 00 00 00") + " 01 00 00 00 00 00 00 00 01 00 00 00 14 00 00 00 FF FF FF FF"); assertArrayEquals(expected, ack(w, 1015)); @@ -278,7 +285,7 @@ public void maskOfManyFragments() { } @Test - public void lateSlotAsksForItsNextFrameWhole() { + public void lateSlotIsLeftOut() { FlowUfo.Window w = new FlowUfo.Window(); byte[] body = new byte[100]; // Frames 1..4 and 6, each one fragment; frame 5 (slot of frame 1) never comes @@ -286,17 +293,34 @@ public void lateSlotAsksForItsNextFrameWhole() { assertNotNull(w.add(parsed(piece(seq, body, 0)), 0)); } List r = records(ack(w, 5)); - // Slot 0 (frame 1) is 5 behind: frame 5 whole; 3 and 4 complete; 6 is arriving - assertEquals(3, r.size()); - assertArrayEquals(new long[] {5, FlowUfo.STATUS_WHOLE, 16}, Arrays.copyOf(r.get(0), 3)); - assertArrayEquals(new long[] {3, FlowUfo.STATUS_COMPLETE, 20}, Arrays.copyOf(r.get(1), 3)); - assertArrayEquals(new long[] {4, FlowUfo.STATUS_COMPLETE, 20}, Arrays.copyOf(r.get(2), 3)); - assertEquals(1, w.wholeRequests); + // Slot 0 (frame 1) is 5 behind: no record (as in turbodrone); 3 and 4 complete; 6 (in + // the slot of 2) is arriving + assertEquals(2, r.size()); + assertArrayEquals(new long[] {3, FlowUfo.STATUS_COMPLETE, 20}, Arrays.copyOf(r.get(0), 3)); + assertArrayEquals(new long[] {4, FlowUfo.STATUS_COMPLETE, 20}, Arrays.copyOf(r.get(1), 3)); + assertEquals(1, w.lagging); + for (long[] x : r) assertFalse("no seq+4 request", x[1] == FlowUfo.STATUS_WHOLE); // Frame 9 takes slot 0: frame 1 was complete, frame 5 never came w.add(parsed(piece(9, body, 0)), 10); assertEquals(1, w.lost); } + @Test + public void incompleteLateSlotDoesNotHoldUpTheNextRequest() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + byte[] one = Arrays.copyOf(body, 10); + // Frame 1 misses its second fragment; frames 2, 3, 4 and 6 complete + w.add(parsed(piece(1, body, 0)), 0); + for (long seq : new long[] {2, 3, 4, 6}) assertNotNull(w.add(parsed(piece(seq, one, 0)), 0)); + List r = records(ack(w, 100)); + // Quiet: 6, 3, 4 complete, then frame 7 asked for; frame 1 (5 behind) left out + long[] last = r.get(r.size() - 1); + assertArrayEquals(new long[] {7, FlowUfo.STATUS_WHOLE, 16}, Arrays.copyOf(last, 3)); + for (long[] x : r) assertFalse("frame 1 left out", x[0] == 1); + assertEquals(1, w.nextRequests); + } + @Test public void baseHidesOldFrames() { FlowUfo.Window w = new FlowUfo.Window(); @@ -877,4 +901,255 @@ public void sameSubnet() throws Exception { assertFalse(NetInfo.sameSubnet(InetAddress.getByName("fe80::1"), 64, toy)); assertFalse(NetInfo.sameSubnet(phone, 33, toy)); } + + // ====================== Options, ports, hints (#187, after turbodrone) ====================== + + @Test + public void urlOptions() throws IOException { + FlowUfo.Url u = FlowUfo.Url.parse("flowufo://"); + assertArrayEquals(bytes("32 4B 14 2D"), u.qpara); + assertEquals(FlowUfo.DQT_ZIGZAG, u.dqt); + u = FlowUfo.Url.parse("flowufo://192.168.169.1:8800?qpara=19322348&dqt=raster"); + assertEquals("192.168.169.1", u.host); + assertEquals(8800, u.port); + assertArrayEquals(bytes("19 32 23 48"), u.qpara); + assertEquals(FlowUfo.DQT_RASTER, u.dqt); + u = FlowUfo.Url.parse("flowufo://192.168.169.1:8800?q=19322348"); + assertArrayEquals(bytes("19 32 23 48"), u.qpara); + u = FlowUfo.Url.parse("flowufo://?DQT=Q50&other=1#x"); + assertEquals(FlowUfo.DEFAULT_HOST, u.host); + assertEquals(FlowUfo.DQT_Q50, u.dqt); + u = FlowUfo.Url.parse("flowufo://10.0.0.5/?dqt=zigzag&qpara=aabbccdd"); + assertEquals("10.0.0.5", u.host); + assertEquals(FlowUfo.DQT_ZIGZAG, u.dqt); + assertArrayEquals(bytes("AA BB CC DD"), u.qpara); + for (String bad : new String[] {"flowufo://?qpara=1932234", "flowufo://?qpara=1932234G", + "flowufo://?qpara=193223480", "flowufo://?dqt=natural", "flowufo://?dqt="}) { + try { + FlowUfo.Url.parse(bad); + fail(bad); + } catch (IOException expected) { + // ok + } + } + } + + @Test + public void portsProbed() throws IOException { + // 8800, given or not, probes the BL618 port 8801 too; any other port only itself + assertArrayEquals(new int[] {8800, 8801}, FlowUfo.Url.parse("flowufo://").ports()); + assertArrayEquals(new int[] {8800, 8801}, + FlowUfo.Url.parse("flowufo://192.168.169.1:8800").ports()); + assertArrayEquals(new int[] {8801}, + FlowUfo.Url.parse("flowufo://192.168.169.1:8801").ports()); + assertArrayEquals(new int[] {9000}, FlowUfo.Url.parse("flowufo://:9000").ports()); + assertEquals("192.168.169.1:8800 and :8801", FlowUfo.Url.parse("flowufo://").probed()); + assertEquals("192.168.169.1:8801", FlowUfo.Url.parse("flowufo://:8801").probed()); + } + + @Test + public void bl618Timing() { + assertEquals(100, FlowUfo.Client.probeInterval(8800)); + assertEquals(200, FlowUfo.Client.probeInterval(8801)); + assertEquals(25, FlowUfo.Client.ackInterval(8800)); + assertEquals(50, FlowUfo.Client.ackInterval(8801)); + assertEquals(25, FlowUfo.Client.ackInterval(9000)); + assertFalse(FlowUfo.Client.ackNeedsRecord(8800)); + assertTrue(FlowUfo.Client.ackNeedsRecord(8801)); + assertEquals("BL608", FlowUfo.Client.engine(8800)); + assertEquals("BL618", FlowUfo.Client.engine(8801)); + assertEquals("port 9000", FlowUfo.Client.engine(9000)); + } + + @Test + public void ackCarriesTheQualityParametersAndNoCommand() throws IOException { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] a = ack(w, 0); + assertEquals(FlowUfo.ACK_HEADER, a.length); + assertArrayEquals(bytes("32 4B 14 2D 00 00"), Arrays.copyOfRange(a, 0x52, 0x58)); + // Command counter, command length and the command itself: all zero + for (int i = 0x0C; i < 0x52; i++) assertEquals("byte " + i, 0, a[i]); + w.qpara = FlowUfo.Url.parse("flowufo://?qpara=19322348").qpara; + assertArrayEquals(bytes("19 32 23 48 00 00"), Arrays.copyOfRange(ack(w, 0), 0x52, 0x58)); + } + + @Test + public void dqtOrders() { + byte[] raster = FlowUfo.dqtTables(FlowUfo.DQT_RASTER, 75); + // Annex K row by row: luma 16 11 10 16 24 40 51 61 / 12 12 14 19 26 58 60 55 + assertArrayEquals(bytes("10 0B 0A 10 18 28 33 3D 0C 0C 0E 13 1A 3A 3C 37"), + Arrays.copyOf(raster, 16)); + // Chroma 17 18 24 47 99 99 99 99 / 18 21 26 66 99 99 99 99 + assertArrayEquals(bytes("11 12 18 2F 63 63 63 63 12 15 1A 42 63 63 63 63"), + Arrays.copyOfRange(raster, 64, 80)); + // Unscaled whatever the quality byte says + assertArrayEquals(raster, FlowUfo.dqtTables(FlowUfo.DQT_RASTER, 20)); + byte[] zz = JpegAssembler.ijgTables(50); + for (int k = 0; k < 64; k++) { + assertEquals(zz[k], raster[NATURAL[k]]); + assertEquals(zz[64 + k], raster[64 + NATURAL[k]]); + } + assertArrayEquals(zz, FlowUfo.dqtTables(FlowUfo.DQT_Q50, 90)); + assertArrayEquals(JpegAssembler.ijgTables(90), FlowUfo.dqtTables(FlowUfo.DQT_ZIGZAG, 90)); + assertArrayEquals(zz, FlowUfo.dqtTables(FlowUfo.DQT_ZIGZAG, 0)); + // In the header: luma after "FF DB 00 43 00", chroma after "FF DB 00 43 01" + byte[] h = FlowUfo.jpegHeader(75, 640, 360, FlowUfo.DQT_RASTER); + assertEquals(605, h.length); + assertArrayEquals(Arrays.copyOf(raster, 64), Arrays.copyOfRange(h, 7, 71)); + assertArrayEquals(Arrays.copyOfRange(raster, 64, 128), Arrays.copyOfRange(h, 76, 140)); + assertArrayEquals(FlowUfo.jpegHeader(75, 640, 360), + FlowUfo.jpegHeader(75, 640, 360, FlowUfo.DQT_ZIGZAG)); + assertEquals("zigzag Q75", FlowUfo.dqtLabel(FlowUfo.DQT_ZIGZAG, 75)); + assertEquals("zigzag Q50", FlowUfo.dqtLabel(FlowUfo.DQT_ZIGZAG, 0)); + assertEquals("raster (unscaled)", FlowUfo.dqtLabel(FlowUfo.DQT_RASTER, 75)); + assertEquals("q50 (unscaled)", FlowUfo.dqtLabel(FlowUfo.DQT_Q50, 75)); + } + + private static InetAddress ip(String s) throws IOException { + return InetAddress.getByName(s); + } + + @Test + public void subnetHints() throws Exception { + String ky = "This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam"; + // #187: the K200 slot pointed at the phone itself, on a 192.168.1.x network + List on1 = Collections.singletonList(ip("192.168.1.100")); + assertEquals("192.168.1.100 is this phone; the toy is usually 192.168.1.1. " + ky, + FlowUfo.subnetHint(ip("192.168.1.100"), on1)); + assertEquals(ky, FlowUfo.subnetHint(ip("192.168.169.1"), on1)); + assertNull(FlowUfo.subnetHint(ip("192.168.1.1"), on1)); + assertNull(FlowUfo.subnetHint(ip("10.0.0.5"), on1)); + List on169 = Collections.singletonList(ip("192.168.169.2")); + assertNull(FlowUfo.subnetHint(ip("192.168.169.1"), on169)); + assertEquals("192.168.169.2 is this phone; the toy is usually 192.168.169.1", + FlowUfo.subnetHint(ip("192.168.169.2"), on169)); + assertEquals("This phone is on 192.168.169.x: the toy is usually flowufo://192.168.169.1", + FlowUfo.subnetHint(ip("192.168.1.1"), on169)); + assertEquals("Toys on 192.168.80.x (HF_ Wi-Fi) use another protocol, not supported yet", + FlowUfo.subnetHint(ip("192.168.169.1"), + Collections.singletonList(ip("192.168.80.3")))); + // On both networks: no guess + assertNull(FlowUfo.subnetHint(ip("192.168.169.1"), + Arrays.asList(ip("192.168.1.5"), ip("192.168.169.2")))); + assertNull(FlowUfo.subnetHint(ip("192.168.169.1"), Collections.emptyList())); + assertNull(FlowUfo.subnetHint(ip("::1"), on1)); + } + + @Test + public void hintInTheStatusText() throws Exception { + try (DatagramSocket drone = new DatagramSocket(0, InetAddress.getLoopbackAddress())) { + FlowUfo.Url u = FlowUfo.Url.parse("flowufo://127.0.0.1:" + drone.getLocalPort()); + FlowUfo.Client client = new FlowUfo.Client("test", u, null); + client.localForHint = Collections.singletonList(InetAddress.getByName("127.0.0.1")); + List states = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch first = new CountDownLatch(1); + Thread t = new Thread(() -> { + try { + client.run(() -> true, new FlowUfo.Listener() { + @Override + public void onState(boolean receiving, String text) { + states.add(text); + first.countDown(); + } + + @Override + public void onFrame(byte[] buf, int length) {} + }); + } catch (IOException ignored) { + // closed + } + }); + t.start(); + try { + assertTrue(first.await(3, TimeUnit.SECONDS)); + assertEquals("Waiting for the toy: probing 127.0.0.1:" + drone.getLocalPort() + + ". 127.0.0.1 is this phone; the toy is usually 127.0.0.1", states.get(0)); + assertNotNull(client.stats().get("hint")); + } finally { + client.close(); + t.join(2000); + } + } + } + + /** The next probe arriving at {@code drone}; returns where it came from. */ + private static SocketAddress probeAt(DatagramSocket drone) throws IOException { + byte[] buf = new byte[64]; + long deadline = System.currentTimeMillis() + 3000; + while (System.currentTimeMillis() < deadline) { + DatagramPacket p = new DatagramPacket(buf, buf.length); + try { + drone.receive(p); + } catch (SocketTimeoutException e) { + continue; + } + if (Arrays.equals(FlowUfo.PROBE, Arrays.copyOf(buf, p.getLength()))) { + return p.getSocketAddress(); + } + } + throw new AssertionError("no probe"); + } + + @Test + public void probesBothPortsAndLocksOntoTheOneThatAnswers() throws Exception { + InetAddress lo = InetAddress.getLoopbackAddress(); + try (DatagramSocket a = new DatagramSocket(0, lo); + DatagramSocket b = new DatagramSocket(0, lo); + DatagramSocket from1234 = new DatagramSocket(0, lo)) { + a.setSoTimeout(200); + b.setSoTimeout(200); + // a stands for 8800, b for 8801 + FlowUfo.Url u = FlowUfo.Url.parse("flowufo://127.0.0.1:" + a.getLocalPort()); + u.alternatePort = b.getLocalPort(); + assertArrayEquals(new int[] {a.getLocalPort(), b.getLocalPort()}, u.ports()); + FlowUfo.Client client = new FlowUfo.Client("test", u, null); + client.localForHint = Collections.emptyList(); + Thread t = new Thread(() -> { + try { + client.run(() -> true, new FlowUfo.Listener() { + @Override + public void onState(boolean receiving, String text) {} + + @Override + public void onFrame(byte[] buf, int length) {} + }); + } catch (IOException ignored) { + // closed + } + }); + t.start(); + try { + // Both ports are probed, each from a socket of its own + SocketAddress phoneA = probeAt(a); + SocketAddress phoneB = probeAt(b); + assertFalse(phoneA.equals(phoneB)); + assertEquals(-1, client.lockedPort()); + // The engine behind b answers, from another source port (the drone's 1234) + byte[] f = piece(1, new byte[2000], 0); + from1234.send(new DatagramPacket(f, f.length, phoneB)); + // Acknowledgements go to b, to the port that was probed + nextAck(b, System.currentTimeMillis() + 3000); + assertEquals(b.getLocalPort(), client.lockedPort()); + assertEquals(b.getLocalPort(), client.stats().get("port")); + // a gets nothing any more once what was sent before the lock is read + a.setSoTimeout(50); + try { + while (true) a.receive(new DatagramPacket(new byte[64], 64)); + } catch (SocketTimeoutException expected) { + // drained + } + a.setSoTimeout(400); + try { + a.receive(new DatagramPacket(new byte[64], 64)); + fail("probe to the other port after the lock"); + } catch (SocketTimeoutException expected) { + // ok + } + } finally { + client.close(); + t.join(2000); + } + assertFalse(t.isAlive()); + } + } } diff --git a/android/app/src/test/java/com/openipc/decoder/RtspSessionTest.java b/android/app/src/test/java/com/openipc/decoder/RtspSessionTest.java index 1c1e6799..3e282f4b 100644 --- a/android/app/src/test/java/com/openipc/decoder/RtspSessionTest.java +++ b/android/app/src/test/java/com/openipc/decoder/RtspSessionTest.java @@ -428,6 +428,51 @@ public void handshakeWithAudio() throws Exception { assertEquals(1, rec.audio.get()); } + @Test + public void heartbeatWhilePlayingStopsWithTheSlot() throws Exception { + // #187: cooingdv toys get 01 01 on UDP while their RTSP stream plays + UdpHeartbeat.intervalMs = 50; + try (DatagramSocket toy = new DatagramSocket(0, InetAddress.getLoopbackAddress())) { + toy.setSoTimeout(3000); + runUntilVideo(server.url(null) + "?heartbeat=" + toy.getLocalPort(), false); + byte[] buf = new byte[16]; + for (int i = 0; i < 2; i++) { + DatagramPacket p = new DatagramPacket(buf, buf.length); + toy.receive(p); + assertEquals(2, p.getLength()); + assertEquals(1, buf[0]); + assertEquals(1, buf[1]); + } + // The option is ours: the camera never sees it + assertEquals("DESCRIBE " + base() + " RTSP/1.0", server.methods().get(0)); + java.util.Map m = session.heartbeatStats(); + assertTrue((Integer) m.get("sent") >= 2); + assertEquals(true, m.get("running")); + session.stop(); + toy.setSoTimeout(100); + try { + while (true) toy.receive(new DatagramPacket(buf, buf.length)); + } catch (java.net.SocketTimeoutException expected) { + // drained + } + toy.setSoTimeout(300); + try { + toy.receive(new DatagramPacket(buf, buf.length)); + throw new AssertionError("heartbeat after stop"); + } catch (java.net.SocketTimeoutException expected) { + // ok + } + } finally { + UdpHeartbeat.intervalMs = 1000; + } + } + + @Test + public void noHeartbeatForOrdinaryCameras() throws Exception { + runUntilVideo(server.url(null), false); + assertEquals(null, session.heartbeatStats()); + } + @Test public void noAudioTrackMeansNoAudioSetup() throws Exception { // #18 diff --git a/android/app/src/test/java/com/openipc/decoder/UdpHeartbeatTest.java b/android/app/src/test/java/com/openipc/decoder/UdpHeartbeatTest.java new file mode 100644 index 00000000..5004a161 --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/UdpHeartbeatTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * UdpHeartbeatTest.java — the cooingdv heartbeat: when RTSP slots send it, and its schedule + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketTimeoutException; +import java.util.Arrays; +import java.util.Map; + +public class UdpHeartbeatTest { + @After + public void restoreInterval() { + UdpHeartbeat.intervalMs = 1000; + } + + @Test + public void onForTheCooingdvWebcamPath() throws IOException { + // Any host: the toy is usually 192.168.1.1, but not always + RtspSession.Url u = RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam"); + assertEquals(7099, u.heartbeatPort); + assertEquals("rtsp://192.168.1.1:7070/webcam", u.requestUrl); + assertEquals(7099, RtspSession.Url.parse("rtsp://10.0.0.9:7070/webcam").heartbeatPort); + assertEquals(7099, RtspSession.Url.parse("rtsp://10.0.0.9:7070/webcam/").heartbeatPort); + // Not the path or not the port: off + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1:554/webcam").heartbeatPort); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1:7070/live").heartbeatPort); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1/webcam").heartbeatPort); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.10:554/stream=0").heartbeatPort); + } + + @Test + public void optionTurnsItOnOrOffAndIsNotSentToTheCamera() throws IOException { + RtspSession.Url u = RtspSession.Url.parse("rtsp://192.168.1.1:554/live?heartbeat=7099"); + assertEquals(7099, u.heartbeatPort); + assertEquals("rtsp://192.168.1.1:554/live", u.requestUrl); + u = RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam?heartbeat=off"); + assertEquals(0, u.heartbeatPort); + assertEquals("rtsp://192.168.1.1:7070/webcam", u.requestUrl); + u = RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam?HeartBeat=0"); + assertEquals(0, u.heartbeatPort); + // Other parameters stay, in their order + u = RtspSession.Url.parse("rtsp://u:p@cam/x?a=1&heartbeat=7000&b=2#frag"); + assertEquals(7000, u.heartbeatPort); + assertEquals("rtsp://cam/x?a=1&b=2", u.requestUrl); + u = RtspSession.Url.parse("rtsp://cam/x?a=1"); + assertEquals(0, u.heartbeatPort); + assertEquals("rtsp://cam/x?a=1", u.requestUrl); + for (String bad : new String[] {"rtsp://cam/x?heartbeat=abc", "rtsp://cam/x?heartbeat=", + "rtsp://cam/x?heartbeat=70000", "rtsp://cam/x?heartbeat=-5"}) { + try { + RtspSession.Url.parse(bad); + fail(bad); + } catch (IOException expected) { + // ok + } + } + } + + @Test + public void sendsOneOneEverySecondUntilClosed() throws Exception { + UdpHeartbeat.intervalMs = 100; + InetAddress lo = InetAddress.getLoopbackAddress(); + try (DatagramSocket toy = new DatagramSocket(0, lo)) { + toy.setSoTimeout(2000); + UdpHeartbeat h = new UdpHeartbeat("test", lo, + new InetSocketAddress(lo, toy.getLocalPort())); + h.start(true); + assertTrue(h.running()); + long[] at = new long[4]; + byte[] buf = new byte[16]; + int port = -1; + for (int i = 0; i < at.length; i++) { + DatagramPacket p = new DatagramPacket(buf, buf.length); + toy.receive(p); + at[i] = System.nanoTime() / 1_000_000; + assertArrayEquals(UdpHeartbeat.PAYLOAD, Arrays.copyOf(buf, p.getLength())); + // Always from the same ephemeral port + if (port < 0) port = p.getPort(); + assertEquals(port, p.getPort()); + } + // About one interval apart: the scheduler neither bursts nor stalls + long span = at[at.length - 1] - at[0]; + assertTrue("span " + span, span >= 3 * 100 - 40 && span < 3 * 100 + 400); + assertTrue(h.sent() >= 4); + h.close(); + assertFalse(h.running()); + // What was on its way is read; then nothing more + toy.setSoTimeout(50); + try { + while (true) toy.receive(new DatagramPacket(buf, buf.length)); + } catch (SocketTimeoutException expected) { + // drained + } + toy.setSoTimeout(400); + try { + toy.receive(new DatagramPacket(buf, buf.length)); + fail("heartbeat after close"); + } catch (SocketTimeoutException expected) { + // ok + } + Map m = UdpHeartbeat.stats(h, 3, 0); + assertEquals(3 + h.sent(), m.get("sent")); + assertEquals(false, m.get("running")); + assertEquals("01 01", m.get("payload")); + assertEquals(5, UdpHeartbeat.stats(null, 5, 0).get("sent")); + } + } +} diff --git a/docs/debug.md b/docs/debug.md index e31c2f48..0ea44458 100644 --- a/docs/debug.md +++ b/docs/debug.md @@ -168,7 +168,7 @@ Single view and U view: every cell of the grid as the debug page shows it (3×3 | `codec`, `size` | when found | | `fps`, `kbps`, `key_frames` | measured during the check | -The FLOW-UFO test build ([toys.md](toys.md#flow-ufo)) adds a `flowufo` object per cell and in `single`; it is not in the releases. +The FLOW-UFO test build ([toys.md](toys.md#flow-ufo)) adds a `flowufo` object per cell and in `single`, and a `heartbeat` object for RTSP slots that send the KY UFO heartbeat ([toys.md](toys.md#ky-ufo-rtsp-toys)); it is not in the releases. ## Log diff --git a/docs/toys.md b/docs/toys.md index e9155cbb..ace5f21c 100644 --- a/docs/toys.md +++ b/docs/toys.md @@ -2,7 +2,10 @@ - [How such toys stream](#how-such-toys-stream) - [Connecting](#connecting) +- [Toy families by Wi‑Fi name](#toy-families-by-wifi-name) - [FLOW-UFO](#flow-ufo) +- [KY UFO RTSP toys](#ky-ufo-rtsp-toys) +- [Credits](#credits) - [An unknown toy](#an-unknown-toy) - [What to send in a report](#what-to-send-in-a-report) @@ -31,28 +34,66 @@ Whether a toy can be played here depends on its protocol. Toys that do offer RTS 4. Close the vendor app: the toy usually streams to one client only. 5. While the phone is only on the toy's network, open the [debug pages](debug.md) on the phone itself: `http://127.0.0.1:8085/stats.json` and `http://127.0.0.1:8085/log`. +## Toy families by Wi‑Fi name + +The Wi‑Fi name and the address the phone gets usually tell the family. The toy is almost always the `.1` address of that network, **not** the phone's own address. + +| Wi‑Fi name | Phone gets | Family | Slot URL | Status | +|---|---|---|---|---| +| `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF | — | **not supported yet** | + +When a `flowufo://` slot points at the phone itself, or at the network of another family, its status says so (for example *192.168.1.100 is this phone; the toy is usually 192.168.1.1* or *This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam*). + ## FLOW-UFO | | | |---|---| -| Wi‑Fi name | `FLOW-UFO-xxxxxx` | -| Camera | BL60x Wi‑Fi camera module | -| Toy address | `192.168.169.1`, UDP port 8800 | +| Wi‑Fi name | `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | +| Camera | BL60x Wi‑Fi camera module (BL608 or BL618 engine) | +| Toy address | `192.168.169.1`, UDP port 8800 (BL608) or 8801 (BL618); the toy answers from port 1234 | | Slot URL | `flowufo://192.168.169.1:8800` (or `flowufo://` alone) | | Status | **in testing**, not in the releases: issue [#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188) | | Test build | [flowufo-187-test1](https://github.com/OpenIPC/decoder/releases/tag/flowufo-187-test1) (2.10 with the FLOW-UFO transport; installs over the release and keeps the settings) | What the toy does, as far as is known: -- It sends nothing until a client sends it a short **probe** on UDP port 8800. The app repeats the probe until video arrives, and starts probing again when the toy goes quiet for 3 s. +- It sends nothing until a client sends it a short **probe**. The vendor library starts two engines, BL608 on UDP port 8800 and BL618 on 8801, and keeps the one that delivers. The app does the same: a slot on 8800 (or without a port) probes **both 8800 and 8801** at once, each from a socket of its own, and locks onto the port whose packets arrive first; the acknowledgements then go there. A slot with another port, `8801` included, probes only that port. The probes repeat until video arrives (every 100 ms; 200 ms on 8801), and start again on both ports when the toy goes quiet for 3 s. - The video is **MJPEG**: each frame is cut into fragments of about 1 KB. The fragments carry the compressed picture without JPEG headers; the app adds standard headers and shows the frames through its normal MJPEG path. -- The client **acknowledges** regularly which fragments it has, and can ask for a frame again. The app does this; whether the toy really needs it is one of the open questions of the test. -- Video only, no sound. No flight command is ever sent. +- The client **acknowledges** regularly which fragments of the last frames it has (every 25 ms; on 8801 every 50 ms and only when there is something to acknowledge), and asks for the next frame when the stream pauses. Frames 5 or more behind the newest are left out of the acknowledgement. +- Bytes 0x34/0x35 of each fragment are the main and flow camera status; the stats show them as `camera_status`. +- Video only, no sound. **No flight command is ever sent**: the vendor app and turbodrone put a (neutral) stick command into every acknowledgement; this app leaves that field empty. -The implementation was written from a description of the protocol, not tested on a real toy yet. Status texts while it connects: *Waiting for the toy: probing 192.168.169.1:8800*, *Receiving from the toy at …*, *The toy went quiet: probing … again*. The test build adds a `flowufo` object to `/stats.json` (state, network, frame and fragment counters, losses, resets, errors). +URL options, for comparing with what the toy expects (as with `udp://`, separated by `?` and `&`): + +| Option | Values | Default | What it does | +|---|---|---|---| +| `qpara` (or `q`) | 8 hex digits | `324B142D` | the four quality parameters in each acknowledgement; `324B142D` is what turbodrone sends to working drones, `19322348` an older reading of the vendor app | +| `dqt` | `zigzag`, `raster`, `q50` | `zigzag` | the quantization tables in the JPEG header the app puts in front of each frame: `zigzag` = standard tables scaled to the quality byte of the fragment, in zig-zag order as the JPEG standard has it; `raster` = unscaled standard tables in raster order, quality ignored (what turbodrone writes, and it shows real drones' frames); `q50` = unscaled, zig-zag. If the picture is there but blocky, too dark/bright or noisy, try the others | + +Examples: `flowufo://192.168.169.1:8800`, `flowufo://192.168.169.1:8800?dqt=raster`, `flowufo://192.168.169.1:8800?q=19322348`, `flowufo://192.168.169.1:8801?dqt=q50`. + +The implementation was written from a description of the protocol, not tested on a real toy yet. Status texts while it connects: *Waiting for the toy: probing 192.168.169.1:8800*, *Receiving from the toy at …*, *The toy went quiet: probing … again*. The test build adds a `flowufo` object to `/stats.json`: state, the ports probed (`target`), the port and engine that answered (`port`, `engine`), network, `qpara`, `dqt` (e.g. `zigzag Q50` or `raster (unscaled)`), `camera_status`, frame and fragment counters, losses, `lagging_left_out`, resets, errors, and a `hint` when the address looks wrong. To test: install the test build, join the `FLOW-UFO-…` Wi‑Fi, [stay connected](#connecting), put `flowufo://192.168.169.1:8800` into a slot (**Settings**, or share the text to the app). If there is no picture, or it looks wrong, wait about 30 s and [send a report](#what-to-send-in-a-report) in [#187](https://github.com/OpenIPC/decoder/issues/187). +## KY UFO RTSP toys + +Toys of the cooingdv apps (*KY UFO*, *RC UFO* and others), often with a `FLY-UFO-…` Wi‑Fi, give the phone an address in `192.168.1.x` and stream plain RTSP: + +``` +rtsp://192.168.1.1:7070/webcam +``` + +Their apps send a heartbeat, the two bytes `01 01`, to UDP port 7099 of the toy every second while the video plays. For an RTSP slot with port 7070 and path `/webcam` (any host) the app does the same: from the address of the RTSP connection, once a second, until the slot stops. Nothing else is sent to port 7099 (no flight command, no "stop" command). Other RTSP slots send no heartbeat unless asked with `?heartbeat=`, e.g. `rtsp://192.168.1.1:554/live?heartbeat=7099`; `?heartbeat=off` turns it off for a `:7070/webcam` slot. The option is the app's own and is not sent to the camera. The log has one line when it starts (*heartbeat 01 01 every 1000 ms to …*), and `/stats.json` a `heartbeat` object (`target`, `running`, `sent`) for such a slot. + +Some of these toys use the FLOW-UFO protocol instead (the KY UFO app has both); if RTSP shows nothing, try `flowufo://192.168.1.1`. + +## Credits + +What the app does for FLOW-UFO and KY UFO toys was checked against [turbodrone](https://github.com/marshallrichards/turbodrone) (Apache-2.0), which flies these drones from a computer: its `wifi_uav` backend is the FLOW-UFO protocol and its `cooingdv` backend the KY UFO RTSP toys. The default quality parameters, the second port 8801 and its timing, the camera status bytes, leaving lagging frames out of the acknowledgement, the raster tables of `dqt=raster` and the 7099 heartbeat come from its code and research notes; no code of it is used. More open-source work on such toys is collected in [#196](https://github.com/OpenIPC/decoder/issues/196). + ## An unknown toy First find the toy's address: on Android it is usually the gateway of the Wi‑Fi (*Wi‑Fi → the network → Details*; often `192.168.0.1`, `192.168.1.1`, `192.168.4.1`, `192.168.169.1`, `172.16.10.1` or `192.168.25.1`). Then try, in a slot: diff --git a/docs/transports.md b/docs/transports.md index 5bc7c690..8d5634d2 100644 --- a/docs/transports.md +++ b/docs/transports.md @@ -100,7 +100,9 @@ For example Majestic's `http://user:password@/mjpeg`: a `multipart/x-mix ## Toys: FLOW-UFO -`flowufo://192.168.169.1:8800` (or `flowufo://` alone) plays the video of toy drones with a `FLOW-UFO-xxxxxx` Wi‑Fi access point. It exists only in a test build so far ([#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188)), not in the releases. Setup and status: [toys.md](toys.md#flow-ufo). +`flowufo://192.168.169.1:8800` (or `flowufo://` alone) plays the video of toy drones with a `FLOW-UFO-xxxxxx` Wi‑Fi access point. It exists only in a test build so far ([#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188)), not in the releases. Port 8800 (the default) probes 8801 as well and keeps the port that answers. Options: `?qpara=<8 hex digits>` and `?dqt=zigzag|raster|q50`. Setup, options and status: [toys.md](toys.md#flow-ufo). + +RTSP slots on `rtsp://:7070/webcam` (KY UFO and other cooingdv toys) also send the toy's UDP heartbeat `01 01` to port 7099 once a second; `?heartbeat=` turns it on for other RTSP URLs, `?heartbeat=off` off. See [toys.md](toys.md#ky-ufo-rtsp-toys). ## Codecs From dd56ae7a1f0458ea00667ee39146d67616bbf09b Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 18:38:11 +0000 Subject: [PATCH 06/21] feat: Jieli HF toy drones over CTP (#187) jieli://192.168.80.1 for toys with an HF_xxxxxxxx Wi-Fi (Jieli AC79, vendor app DVRunning 2), after the PCAPdroid capture in #196 and turbodrone's research notes (Apache-2.0, behaviour reference only, no code taken over): - CTP on TCP 3333 (CTP: + u16 topic length + topic + u32 JSON length + JSON), a reader that reassembles split and joined messages and skips what is not CTP - APP_ACCESS, DATE_TIME, CTP_KEEP_ALIVE every min(timeout / 4, 7.5 s), OPEN_RT_STREAM with the size of VIDEO_PARAM (else 640x480 at 25), CLOSE_RT_STREAM on stop - video expected on the phone's UDP 6666 (unverified): PT 26 into the RTP/JPEG path, PT 96 into the H.264 path, whole JPEG datagrams too; first packets logged with sender, PT and bytes; no video for 5 s: said in the status, UDP 1234 and 2224 opened for the log only, OPEN_RT_STREAM again every 10 s - options w, h, fps, rtp, format; sockets bound to the toy's network (TCP from Android 6) - jieli object in /stats.json; subnet hints both ways between flowufo:// and jieli:// - never FLIGHT_CONTROL, nothing to UDP 3456 - docs/toys.md: HF / Jieli section, credits; unit tests with a fake toy on loopback --- .../java/com/openipc/decoder/Decoder.java | 5 + .../java/com/openipc/decoder/FlowUfo.java | 8 +- .../main/java/com/openipc/decoder/Jieli.java | 1118 +++++++++++++++++ .../java/com/openipc/decoder/NetInfo.java | 66 +- .../java/com/openipc/decoder/QuadCell.java | 3 + .../java/com/openipc/decoder/RtspClient.java | 2 + .../java/com/openipc/decoder/RtspSession.java | 104 +- .../com/openipc/decoder/StreamProbes.java | 4 +- .../java/com/openipc/decoder/FlowUfoTest.java | 3 +- .../java/com/openipc/decoder/JieliTest.java | 580 +++++++++ docs/debug.md | 2 +- docs/toys.md | 42 +- docs/transports.md | 2 + 13 files changed, 1912 insertions(+), 27 deletions(-) create mode 100644 android/app/src/main/java/com/openipc/decoder/Jieli.java create mode 100644 android/app/src/test/java/com/openipc/decoder/JieliTest.java diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index d00efd27..b592638c 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -1558,6 +1558,8 @@ private Map debugState() { // The toy drone's protocol counters (#187) Map ufo = rtspClient != null ? rtspClient.flowUfoStats() : null; if (ufo != null) single.put("flowufo", ufo); + Map jl = rtspClient != null ? rtspClient.jieliStats() : null; + if (jl != null) single.put("jieli", jl); Map hb = rtspClient != null ? rtspClient.heartbeatStats() : null; if (hb != null) single.put("heartbeat", hb); m.put("single", single); @@ -1600,6 +1602,8 @@ private Map debugState() { c.put("audio", i == quadAudio); Map ufo = cell.flowUfoStats(); if (ufo != null) c.put("flowufo", ufo); + Map jl = cell.jieliStats(); + if (jl != null) c.put("jieli", jl); Map hb = cell.heartbeatStats(); if (hb != null) c.put("heartbeat", hb); } @@ -1670,6 +1674,7 @@ private String slotTransport(int i, String url) { if (RtpListener.Url.matches(url)) return "UDP listen"; if (HttpMjpeg.matches(url)) return "HTTP"; if (FlowUfo.Url.matches(url)) return "FLOW-UFO UDP"; + if (Jieli.Url.matches(url)) return "Jieli RTP"; Transport.Mode m = StreamProbe.probeMode(mTransports[i]); return "check over " + (m == Transport.Mode.MULTICAST ? "multicast" : m.name()); } diff --git a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java index f5a2869f..0e0f1336 100644 --- a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java +++ b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java @@ -635,19 +635,19 @@ static String subnetHint(InetAddress target, List local) { + " flowufo://192.168.169.1"); } if (on80 && !on169 && !on1) { - sentence(parts, "Toys on 192.168.80.x (HF_ Wi-Fi) use another protocol, not supported" - + " yet"); + sentence(parts, "This phone is on 192.168.80.x (HF_ Wi-Fi), a Jieli toy: try" + + " jieli://192.168.80.1"); } return parts.length() > 0 ? parts.toString() : null; } - private static void sentence(StringBuilder sb, String s) { + static void sentence(StringBuilder sb, String s) { if (sb.length() > 0) sb.append(". "); sb.append(s); } /** {@code a} is in 192.168.{@code third}.0/24. */ - private static boolean in24(InetAddress a, int third) { + static boolean in24(InetAddress a, int third) { if (!(a instanceof Inet4Address)) return false; byte[] b = a.getAddress(); return (b[0] & 0xFF) == 192 && (b[1] & 0xFF) == 168 && (b[2] & 0xFF) == third; diff --git a/android/app/src/main/java/com/openipc/decoder/Jieli.java b/android/app/src/main/java/com/openipc/decoder/Jieli.java new file mode 100644 index 00000000..53488394 --- /dev/null +++ b/android/app/src/main/java/com/openipc/decoder/Jieli.java @@ -0,0 +1,1118 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * Jieli.java — Jieli toy drones ("HF_xxxxxxxx" Wi-Fi): CTP over TCP, RTP to UDP 6666 (#187) + * + */ + +package com.openipc.decoder; + +import android.util.Log; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Toy drones with an "HF_xxxxxxxx" access point (Jieli AC79 camera, vendor app "DVRunning 2") + * at 192.168.80.1 (#187, #196). The app talks CTP to TCP port 3333: "CTP:", the topic length + * (u16 LE), the topic, the JSON length (u32 LE) and the JSON, which may be empty. The session + * as a PCAPdroid capture of the vendor app shows it: APP_ACCESS; the toy announces + * KEEP_ALIVE_INTERVAL, VIDEO_PARAM and more; DATE_TIME; CTP_KEEP_ALIVE every 7.5 s; + * OPEN_RT_STREAM; CLOSE_RT_STREAM at the end. The video itself was not in the capture: after + * turbodrone's research on the same SDK (marshallrichards/turbodrone, Apache-2.0, used as a + * description only) it is RTP/JPEG (payload type 26) or H.264 (96) to the phone's UDP port + * 6666. The packets go into the app's normal RTP paths. Unverified, so the log shows what + * really arrives on 6666, and on the audio and stream ports 1234 and 2224 too when 6666 stays + * silent. Video only: FLIGHT_CONTROL is never sent, nothing goes to the toy's UDP port 3456. + * Pure Java apart from the log. + */ +final class Jieli { + private static final String TAG = "OpenIPCDecoder"; + + static final String SCHEME = "jieli://"; + static final String DEFAULT_HOST = "192.168.80.1"; + static final int CTP_PORT = 3333; + static final int RTP_PORT = 6666; + static final int DEFAULT_WIDTH = 640; + static final int DEFAULT_HEIGHT = 480; + static final int DEFAULT_FPS = 25; + static final int FORMAT_JPEG = 0; + static final int FORMAT_H264 = 1; + static final int PT_JPEG = 26; + static final int PT_H264 = 96; + + static final byte[] MAGIC = {'C', 'T', 'P', ':'}; + /** Longer topics or JSON than this are not CTP: the reader looks for the next "CTP:". */ + static final int MAX_TOPIC = 256; + static final int MAX_JSON = 256 * 1024; + + static final String APP_ACCESS = "APP_ACCESS"; + static final String DATE_TIME = "DATE_TIME"; + static final String KEEP_ALIVE = "CTP_KEEP_ALIVE"; + static final String KEEP_ALIVE_INTERVAL = "KEEP_ALIVE_INTERVAL"; + static final String VIDEO_PARAM = "VIDEO_PARAM"; + static final String OPEN_RT_STREAM = "OPEN_RT_STREAM"; + static final String CLOSE_RT_STREAM = "CLOSE_RT_STREAM"; + + private static final int CONNECT_TIMEOUT_MS = 5000; + /** Read timeout of the CTP socket: how often the timers are looked at. */ + private static final int TICK_MS = 100; + /** The keepalive of the vendor app; a shorter announced timeout makes it timeout / 4. */ + static final long KEEP_ALIVE_MAX_MS = 7500; + private static final long KEEP_ALIVE_MIN_MS = 250; + // Not final so unit tests can shorten them + /** How long APP_ACCESS waits for VIDEO_PARAM before the stream is asked for anyway. */ + static volatile long paramWaitMs = 1500; + /** No video this long after OPEN_RT_STREAM: said in the log and status, diagnostics on. */ + static volatile long noVideoMs = 5000; + /** No video this long (after a request or the last packet): OPEN_RT_STREAM again. */ + static volatile long reopenMs = 10_000; + /** Video that stops for this long is "quiet". */ + static volatile long quietMs = 3000; + /** The toy says nothing on CTP for this long (at least): the connection is given up. */ + static volatile long ctpSilenceMs = 20_000; + /** + * Ports opened only to log what arrives when 6666 stays silent: 1234 (audio in the + * vendor SDK) and 2224 (its RT stream port). Nothing received there is played. + */ + static volatile int[] diagnosticPorts = {1234, 2224}; + + private Jieli() {} + + // ====================== URL ====================== + + /** + * jieli://[host][:port][/...][?option=value&...]: the toy's CTP address, 192.168.80.1:3333 + * when left out. Options, as in udp:// URLs (unknown ones are ignored, bad values + * refused): w, h, fps (the stream asked for; default what the toy announces in + * VIDEO_PARAM, else 640x480 at 25), rtp (the local UDP port for the video, 6666), + * format (0 JPEG, the default, or 1 H.264). + */ + static final class Url { + String host = DEFAULT_HOST; + int port = CTP_PORT; + int rtpPort = RTP_PORT; + /** 0: from VIDEO_PARAM, else the default. */ + int width; + int height; + int fps; + int format = FORMAT_JPEG; + + static boolean matches(String url) { + return url != null && url.trim().toLowerCase(Locale.ROOT).startsWith(SCHEME); + } + + static Url parse(String url) throws IOException { + if (!matches(url)) throw new IOException("Not a jieli:// URL"); + String rest = url.trim().substring(SCHEME.length()); + int end = rest.length(); + for (int i = 0; i < rest.length(); i++) { + char ch = rest.charAt(i); + if (ch == '/' || ch == '?' || ch == '#') { + end = i; + break; + } + } + String authority = rest.substring(0, end).trim(); + String query = ""; + int q = rest.indexOf('?'); + int hash = rest.indexOf('#'); + if (q >= 0 && (hash < 0 || q < hash)) { + query = rest.substring(q + 1, hash >= 0 ? hash : rest.length()); + } + Url u = new Url(); + String host = authority; + int colon = authority.lastIndexOf(':'); + if (authority.startsWith("[")) { + int close = authority.indexOf(']'); + if (close < 0) throw new IOException("Bad host in URL"); + host = authority.substring(1, close); + colon = authority.indexOf(':', close); + } else if (colon >= 0) { + host = authority.substring(0, colon); + } + if (colon >= 0) u.port = number(authority.substring(colon + 1), 1, 65535, "port"); + if (!host.isEmpty()) u.host = host; + for (int i = 0; i < u.host.length(); i++) { + char ch = u.host.charAt(i); + if (ch <= 0x20 || ch == 0x7F || ch == '@') throw new IOException("Bad host in URL"); + } + for (String param : query.split("&")) { + int eq = param.indexOf('='); + if (eq < 0) continue; + String name = param.substring(0, eq).trim().toLowerCase(Locale.ROOT); + String value = param.substring(eq + 1).trim(); + switch (name) { + case "w": + u.width = number(value, 16, 4096, "w"); + break; + case "h": + u.height = number(value, 16, 4096, "h"); + break; + case "fps": + u.fps = number(value, 1, 120, "fps"); + break; + case "rtp": + u.rtpPort = number(value, 1, 65535, "rtp"); + break; + case "format": + u.format = number(value, 0, 1, "format (0 JPEG, 1 H.264)"); + break; + default: + break; + } + } + return u; + } + + private static int number(String s, int min, int max, String what) throws IOException { + int v; + try { + v = Integer.parseInt(s.trim()); + } catch (NumberFormatException e) { + throw new IOException("Bad " + what + " in URL: " + s); + } + if (v < min || v > max) throw new IOException("Bad " + what + " in URL: " + s); + return v; + } + + String endpoint() { + return (host.indexOf(':') >= 0 ? "[" + host + "]" : host) + ":" + port; + } + } + + // ====================== CTP messages ====================== + + /** One CTP message: a topic and its JSON ("" when there is none). */ + static final class Message { + final String topic; + final String json; + + Message(String topic, String json) { + this.topic = topic; + this.json = json; + } + + @Override + public String toString() { + return json.isEmpty() ? topic : topic + " " + json; + } + } + + /** "CTP:" + topic length (u16 LE) + topic + JSON length (u32 LE) + JSON. */ + static byte[] encode(String topic, String json) { + byte[] t = topic.getBytes(StandardCharsets.UTF_8); + byte[] j = json != null ? json.getBytes(StandardCharsets.UTF_8) : new byte[0]; + byte[] out = new byte[MAGIC.length + 2 + t.length + 4 + j.length]; + System.arraycopy(MAGIC, 0, out, 0, MAGIC.length); + int pos = MAGIC.length; + out[pos] = (byte) t.length; + out[pos + 1] = (byte) (t.length >> 8); + pos += 2; + System.arraycopy(t, 0, out, pos, t.length); + pos += t.length; + for (int i = 0; i < 4; i++) out[pos + i] = (byte) (j.length >> (8 * i)); + pos += 4; + System.arraycopy(j, 0, out, pos, j.length); + return out; + } + + /** + * Cuts the TCP byte stream into messages: a message may arrive in pieces or several in + * one read. Bytes that are not a message (no "CTP:", a length out of range) are skipped + * up to the next "CTP:" and counted. + */ + static final class Reader { + private byte[] buf = new byte[4096]; + private int start; + private int end; + long skipped; + + void feed(byte[] b, int off, int len) { + if (end + len > buf.length) { + int have = end - start; + if (have + len > buf.length) { + byte[] n = new byte[Math.max(buf.length * 2, have + len)]; + System.arraycopy(buf, start, n, 0, have); + buf = n; + } else { + System.arraycopy(buf, start, buf, 0, have); + } + start = 0; + end = have; + } + System.arraycopy(b, off, buf, end, len); + end += len; + } + + /** The next whole message, or null until more bytes arrive. */ + Message next() { + while (true) { + int have = end - start; + if (have < MAGIC.length) return null; + if (!startsWithMagic(start)) { + skipToMagic(); + continue; + } + if (have < MAGIC.length + 2) return null; + int topicLen = (buf[start + 4] & 0xFF) | (buf[start + 5] & 0xFF) << 8; + if (topicLen > MAX_TOPIC) { + skipOne(); + continue; + } + int jsonAt = start + MAGIC.length + 2 + topicLen; + if (end < jsonAt + 4) return null; + long jsonLen = (buf[jsonAt] & 0xFFL) | (buf[jsonAt + 1] & 0xFFL) << 8 + | (buf[jsonAt + 2] & 0xFFL) << 16 | (buf[jsonAt + 3] & 0xFFL) << 24; + if (jsonLen > MAX_JSON) { + skipOne(); + continue; + } + int total = MAGIC.length + 2 + topicLen + 4 + (int) jsonLen; + if (have < total) return null; + String topic = new String(buf, start + 6, topicLen, StandardCharsets.UTF_8); + String json = new String(buf, jsonAt + 4, (int) jsonLen, StandardCharsets.UTF_8); + start += total; + if (start == end) start = end = 0; + return new Message(topic, json); + } + } + + private boolean startsWithMagic(int at) { + for (int i = 0; i < MAGIC.length; i++) { + if (buf[at + i] != MAGIC[i]) return false; + } + return true; + } + + /** Drops the first byte (a "CTP:" with bad lengths) and looks for the next one. */ + private void skipOne() { + start++; + skipped++; + skipToMagic(); + } + + private void skipToMagic() { + int i = start; + while (i + MAGIC.length <= end && !startsWithMagic(i)) i++; + if (i + MAGIC.length > end) { + // Keep a tail that may be the start of "CTP:" + i = Math.max(start, end - (MAGIC.length - 1)); + } + skipped += i - start; + start = i; + } + } + + // ====================== JSON ====================== + + /** {"op":"PUT"}, or with "param" from name/value pairs, in their order. */ + static String put(String... nameValues) { + StringBuilder sb = new StringBuilder("{\"op\":\"PUT\""); + if (nameValues.length > 0) { + sb.append(",\"param\":{"); + for (int i = 0; i + 1 < nameValues.length; i += 2) { + if (i > 0) sb.append(','); + quote(sb, nameValues[i]); + sb.append(':'); + quote(sb, nameValues[i + 1]); + } + sb.append('}'); + } + return sb.append('}').toString(); + } + + private static void quote(StringBuilder sb, String s) { + sb.append('"'); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '"' || c == '\\') { + sb.append('\\').append(c); + } else if (c < 0x20) { + sb.append(String.format(Locale.ROOT, "\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + sb.append('"'); + } + + /** + * A small JSON reader (org.json is a stub in unit tests): objects become maps, arrays + * lists, strings strings, and numbers, true, false and null their text. Null when the + * text is not one JSON object. + */ + static Map parseJson(String s) { + if (s == null) return null; + try { + Json j = new Json(s); + j.space(); + if (j.peek() != '{') return null; + @SuppressWarnings("unchecked") + Map m = (Map) j.value(0); + j.space(); + return j.pos == s.length() ? m : null; + } catch (RuntimeException e) { + return null; + } + } + + private static final class Json { + final String s; + int pos; + + Json(String s) { + this.s = s; + } + + char peek() { + if (pos >= s.length()) throw new IllegalArgumentException("end"); + return s.charAt(pos); + } + + void space() { + while (pos < s.length() && Character.isWhitespace(s.charAt(pos))) pos++; + } + + void expect(char c) { + space(); + if (peek() != c) throw new IllegalArgumentException("expected " + c); + pos++; + } + + Object value(int depth) { + if (depth > 16) throw new IllegalArgumentException("too deep"); + space(); + char c = peek(); + if (c == '{') { + pos++; + Map m = new LinkedHashMap<>(); + space(); + if (peek() == '}') { + pos++; + return m; + } + while (true) { + space(); + String key = string(); + expect(':'); + m.put(key, value(depth + 1)); + space(); + if (peek() == ',') { + pos++; + continue; + } + expect('}'); + return m; + } + } + if (c == '[') { + pos++; + List l = new ArrayList<>(); + space(); + if (peek() == ']') { + pos++; + return l; + } + while (true) { + l.add(value(depth + 1)); + space(); + if (peek() == ',') { + pos++; + continue; + } + expect(']'); + return l; + } + } + if (c == '"') return string(); + int from = pos; + while (pos < s.length() && ",}] \t\r\n".indexOf(s.charAt(pos)) < 0) pos++; + if (pos == from) throw new IllegalArgumentException("value"); + return s.substring(from, pos); + } + + String string() { + if (peek() != '"') throw new IllegalArgumentException("string"); + pos++; + StringBuilder sb = new StringBuilder(); + while (true) { + char c = peek(); + pos++; + if (c == '"') return sb.toString(); + if (c != '\\') { + sb.append(c); + continue; + } + char e = peek(); + pos++; + switch (e) { + case 'n': sb.append('\n'); break; + case 't': sb.append('\t'); break; + case 'r': sb.append('\r'); break; + case 'b': sb.append('\b'); break; + case 'f': sb.append('\f'); break; + case 'u': + sb.append((char) Integer.parseInt(s.substring(pos, pos + 4), 16)); + pos += 4; + break; + default: sb.append(e); break; + } + } + } + } + + /** {@code name} in the message's "param" object, else at the top, as text; or null. */ + static String param(Map json, String name) { + if (json == null) return null; + Object p = json.get("param"); + if (p instanceof Map) { + Object v = ((Map) p).get(name); + if (v instanceof String) return (String) v; + } + Object v = json.get(name); + return v instanceof String ? (String) v : null; + } + + /** The integer {@code name} of a message, or -1. */ + static int intParam(Map json, String name) { + String v = param(json, name); + if (v == null) return -1; + try { + return Integer.parseInt(v.trim()); + } catch (NumberFormatException e) { + return -1; + } + } + + /** min(timeout / 4, 7.5 s), not below 250 ms; 7.5 s when the toy announced none. */ + static long keepAliveMs(long timeoutMs) { + if (timeoutMs <= 0) return KEEP_ALIVE_MAX_MS; + return Math.max(KEEP_ALIVE_MIN_MS, Math.min(timeoutMs / 4, KEEP_ALIVE_MAX_MS)); + } + + /** The request for the stream, in the field order of the vendor app. */ + static String openStream(int format, int width, int height, int fps) { + return put("format", String.valueOf(format), "h", String.valueOf(height), + "w", String.valueOf(width), "fps", String.valueOf(fps)); + } + + static String truncate(String s, int max) { + return s.length() <= max ? s : s.substring(0, max) + "…"; + } + + // ====================== Subnet hint ====================== + + /** + * A hint when a jieli:// slot cannot be right for the phone's networks, or null: the + * target is the phone itself, or the phone is on another toy family's network. + */ + static String subnetHint(InetAddress target, List local) { + if (!(target instanceof Inet4Address) || local == null) return null; + boolean own = local.contains(target); + boolean on1 = false; + boolean on169 = false; + boolean on80 = false; + for (InetAddress a : local) { + on1 |= FlowUfo.in24(a, 1); + on169 |= FlowUfo.in24(a, 169); + on80 |= FlowUfo.in24(a, 80); + } + StringBuilder parts = new StringBuilder(); + byte[] b = target.getAddress(); + if (own) { + FlowUfo.sentence(parts, target.getHostAddress() + " is this phone; the toy is usually " + + (b[0] & 0xFF) + "." + (b[1] & 0xFF) + "." + (b[2] & 0xFF) + ".1"); + } + if (on80 && !FlowUfo.in24(target, 80)) { + FlowUfo.sentence(parts, "This phone is on 192.168.80.x: the toy is usually" + + " jieli://192.168.80.1"); + } + if (!on80 && on169) { + FlowUfo.sentence(parts, "This network looks like a FLOW-UFO toy: try" + + " flowufo://192.168.169.1"); + } + if (!on80 && on1) { + FlowUfo.sentence(parts, "This network looks like a KY UFO RTSP toy: try" + + " rtsp://192.168.1.1:7070/webcam"); + } + return parts.length() > 0 ? parts.toString() : null; + } + + // ====================== Client ====================== + + interface Listener { + /** Waiting (no video) or receiving; text for the status. */ + void onState(boolean receiving, String text); + + /** One RTP packet with payload type 26 (JPEG) or 96 (H.264), valid during the call. */ + void onRtp(byte[] buf, int offset, int length); + + /** One whole JPEG in a datagram without RTP, valid during the call. */ + default void onJpeg(byte[] buf, int offset, int length) {} + } + + /** Binds the CTP socket to the toy's Wi-Fi before it connects; returns what was done. */ + interface TcpSetup { + String prepare(Socket socket, InetAddress target); + } + + /** + * One CTP session on the calling thread (connect, the commands, keepalives, the timers), + * and the video socket on a thread of its own, until {@code keepGoing} fails or close(). + * The stream is closed with CLOSE_RT_STREAM on the way out. A lost connection ends run() + * with an exception: the session starts again. + */ + static final class Client implements Closeable { + private final String tag; + private final Url url; + private final TcpSetup tcpSetup; + private final FlowUfo.SocketSetup udpSetup; + private volatile boolean closed; + /** The phone's addresses for the subnet hint; null: the real ones. For tests. */ + List localForHint; + + private final Object lock = new Object(); + private Socket tcp; + private boolean tcpConnected; + private final List udpSockets = new ArrayList<>(); + private volatile InetAddress address; + + // For the stats + private volatile String state = "connecting"; + private volatile String hint; + private volatile String networkTcp = "default"; + private volatile String networkUdp = "default"; + private volatile int ctpSent; + private volatile int ctpReceived; + private volatile int keepAlives; + private volatile int keepAliveReplies; + private volatile long keepAliveIntervalMs = KEEP_ALIVE_MAX_MS; + private volatile long announcedTimeoutMs = -1; + private volatile int openRequests; + private volatile String lastError; + private volatile String videoParam; + private volatile String requested; + private final ArrayDeque lastNotifies = new ArrayDeque<>(); + private final Map topics = new LinkedHashMap<>(); + private final Reader reader = new Reader(); + private int ctpLogged; + + // Video socket; written by its thread + private volatile long lastVideoAt; + private volatile int packets; + private volatile long bytes; + private volatile int rtpPackets; + private volatile int rtpFrames; + private volatile int otherRtp; + private volatile int nonRtp; + private volatile int rawJpegs; + private volatile int foreign; + private volatile String firstPacket; + private volatile String lastSource; + private volatile boolean h264Logged; + private final Map payloadTypes = new LinkedHashMap<>(); + private final Map sources = new LinkedHashMap<>(); + private final Map diagnostic = new LinkedHashMap<>(); + private final Map diagnosticFirst = new LinkedHashMap<>(); + private int logged; + + Client(String tag, Url url, TcpSetup tcpSetup, FlowUfo.SocketSetup udpSetup) { + this.tag = tag; + this.url = url; + this.tcpSetup = tcpSetup; + this.udpSetup = udpSetup; + } + + /** + * Ends run() from any thread. The video sockets close at once; a connected CTP socket + * is left to the session thread, which sends CLOSE_RT_STREAM first (within TICK_MS). + */ + @Override + public void close() { + closed = true; + synchronized (lock) { + for (DatagramSocket s : udpSockets) s.close(); + if (tcp != null && !tcpConnected) { + try { + tcp.close(); + } catch (IOException ignored) {} + } + } + } + + private String withHint(String text) { + String h = hint; + return h != null ? text + ". " + h : text; + } + + String connectingText() { + return withHint("Connecting to the toy (CTP " + url.endpoint() + ")"); + } + + String requestedText() { + return withHint("Stream requested, waiting for video on UDP " + url.rtpPort); + } + + String noVideoText() { + return withHint("No video on UDP " + url.rtpPort + " yet: the toy accepted the" + + " request; see /log"); + } + + String quietText() { + return withHint("The toy's video stopped: waiting on UDP " + url.rtpPort); + } + + void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException { + InetAddress a = InetAddress.getByName(url.host); + address = a; + hint = subnetHint(a, localForHint != null ? localForHint : FlowUfo.localAddresses()); + if (hint != null) Log.w(TAG, tag + ": Jieli: " + hint); + OutputStream out = null; + boolean requestedStream = false; + Socket s = new Socket(); + try { + // The video port first: nothing the toy sends after OPEN_RT_STREAM is lost + DatagramSocket video = openUdp(url.rtpPort, a); + startThread(video, true, listener); + state = "connecting"; + listener.onState(false, connectingText()); + if (tcpSetup != null) networkTcp = tcpSetup.prepare(s, a); + synchronized (lock) { + tcp = s; + if (closed) return; + } + Log.i(TAG, tag + ": Jieli: connecting to CTP " + url.endpoint() + " (network: " + + networkTcp + "), video on UDP " + url.rtpPort + " (network: " + + networkUdp + ")"); + try { + s.connect(new InetSocketAddress(a, url.port), CONNECT_TIMEOUT_MS); + } catch (IOException e) { + if (closed) return; + throw new IOException("Cannot reach the toy's CTP port " + url.endpoint() + + ": " + e.getMessage()); + } + synchronized (lock) { + tcpConnected = true; + if (closed) return; + } + s.setTcpNoDelay(true); + s.setSoTimeout(TICK_MS); + out = s.getOutputStream(); + InputStream in = s.getInputStream(); + state = "ctp"; + Log.i(TAG, tag + ": Jieli: CTP connected to " + url.endpoint()); + send(out, APP_ACCESS, put("ver", "25", "type", "0")); + long t0 = now(); + long lastRx = t0; + long nextKeepAlive = 0; + long openAt = 0; + boolean diagnostics = false; + boolean receiving = false; + int reopens = 0; + byte[] buf = new byte[8192]; + while (!closed && keepGoing.holds()) { + long t = now(); + if (!requestedStream && (videoParam != null || t - t0 >= paramWaitMs)) { + send(out, DATE_TIME, put("date", new SimpleDateFormat("yyyyMMddHHmmss", + Locale.ROOT).format(new Date()))); + keepAlive(out); + nextKeepAlive = t + keepAliveIntervalMs; + requestStream(out); + requestedStream = true; + openAt = t; + state = "requested"; + listener.onState(false, requestedText()); + } + if (requestedStream && t >= nextKeepAlive) { + keepAlive(out); + nextKeepAlive = t + keepAliveIntervalMs; + } + if (requestedStream) { + long last = lastVideoAt; + boolean flowing = last > 0 && t - last < quietMs; + if (flowing && !receiving) { + receiving = true; + state = "playing"; + Log.i(TAG, tag + ": Jieli: video arrives on UDP " + url.rtpPort + + " from " + lastSource); + listener.onState(true, "Receiving from the toy on UDP " + + url.rtpPort + "…"); + } else if (!flowing && receiving) { + receiving = false; + state = "quiet"; + Log.w(TAG, tag + ": Jieli: no video for " + quietMs + " ms (" + + summary() + ")"); + listener.onState(false, quietText()); + } + if (!diagnostics && last == 0 && t - openAt >= noVideoMs) { + diagnostics = true; + Log.w(TAG, tag + ": Jieli: no video on UDP " + url.rtpPort + " " + + noVideoMs + " ms after OPEN_RT_STREAM (" + summary() + + "); listening on " + ports(diagnosticPorts) + + " too, for the log only"); + for (int p : diagnosticPorts) { + if (p == url.rtpPort) continue; + try { + startThread(openUdp(p, a), false, listener); + } catch (IOException e) { + Log.w(TAG, tag + ": Jieli: cannot listen on UDP " + p + ": " + + e.getMessage()); + } + } + listener.onState(false, noVideoText()); + } + if (!receiving && t - Math.max(openAt, last) >= reopenMs) { + reopens++; + if (reopens <= 3 || reopens % 10 == 0) { + Log.w(TAG, tag + ": Jieli: still no video, OPEN_RT_STREAM again (" + + reopens + ")"); + } + requestStream(out); + openAt = t; + } + } + if (t - lastRx > Math.max(ctpSilenceMs, announcedTimeoutMs)) { + throw new IOException("The toy stopped answering on CTP " + + url.endpoint()); + } + int n; + try { + n = in.read(buf); + } catch (SocketTimeoutException e) { + continue; + } + if (n < 0) throw new IOException("The toy closed the CTP connection"); + lastRx = now(); + reader.feed(buf, 0, n); + Message m; + while ((m = reader.next()) != null) handle(m); + } + } catch (IOException e) { + lastError = e.getMessage(); + if (closed) return; + throw e; + } finally { + if (out != null && requestedStream && !s.isClosed()) { + try { + send(out, CLOSE_RT_STREAM, put("status", "1")); + } catch (IOException ignored) {} + } + try { + s.close(); + } catch (IOException ignored) {} + synchronized (lock) { + for (DatagramSocket d : udpSockets) d.close(); + udpSockets.clear(); + tcp = null; + tcpConnected = false; + } + Log.i(TAG, tag + ": Jieli: session ended (" + summary() + ")"); + } + } + + private static String ports(int[] p) { + StringBuilder sb = new StringBuilder("UDP"); + for (int i = 0; i < p.length; i++) sb.append(i == 0 ? " " : ", ").append(p[i]); + return sb.toString(); + } + + private DatagramSocket openUdp(int port, InetAddress target) throws IOException { + DatagramSocket d = new DatagramSocket(null); + try { + d.bind(new InetSocketAddress(port)); + } catch (IOException e) { + d.close(); + throw new IOException("UDP port " + port + " is in use (another jieli:// slot" + + " or app?): " + e.getMessage()); + } + try { + d.setReceiveBufferSize(1024 * 1024); + } catch (IOException ignored) {} + String n = udpSetup != null ? udpSetup.prepare(d, target) : "default"; + if (port == url.rtpPort) networkUdp = n; + synchronized (lock) { + udpSockets.add(d); + if (closed) d.close(); + } + return d; + } + + private void startThread(DatagramSocket d, boolean video, Listener listener) { + int port = d.getLocalPort(); + Thread th = new Thread(() -> { + byte[] b = new byte[65536]; + DatagramPacket p = new DatagramPacket(b, b.length); + try { + while (!closed && !d.isClosed()) { + p.setLength(b.length); + d.receive(p); + if (video) { + video(p, listener); + } else { + diagnostic(port, p); + } + } + } catch (IOException e) { + // Closed + } + }, tag + "-jieli-" + port); + th.setDaemon(true); + th.start(); + } + + private void send(OutputStream out, String topic, String json) throws IOException { + out.write(encode(topic, json)); + out.flush(); + ctpSent++; + if (!KEEP_ALIVE.equals(topic) || keepAlives < 2) logCtp("→", topic, json); + } + + private void keepAlive(OutputStream out) throws IOException { + send(out, KEEP_ALIVE, put()); + keepAlives++; + } + + private void requestStream(OutputStream out) throws IOException { + int[] p = videoParams(); + int w = url.width > 0 ? url.width : p[0] > 0 ? p[0] : DEFAULT_WIDTH; + int h = url.height > 0 ? url.height : p[1] > 0 ? p[1] : DEFAULT_HEIGHT; + int fps = url.fps > 0 ? url.fps : p[2] > 0 ? p[2] : DEFAULT_FPS; + requested = w + "x" + h + " at " + fps + " fps, format " + url.format + + (url.format == FORMAT_H264 ? " (H.264)" : " (JPEG)"); + send(out, OPEN_RT_STREAM, openStream(url.format, w, h, fps)); + openRequests++; + } + + /** Width, height and fps of VIDEO_PARAM, -1 each when not received. */ + private int[] videoParams() { + Map j = parseJson(videoParamJson); + return new int[] {intParam(j, "w"), intParam(j, "h"), intParam(j, "fps")}; + } + + private volatile String videoParamJson; + + private void logCtp(String dir, String topic, String json) { + int n = ++ctpLogged; + if (n <= 60 || n % 100 == 0) { + Log.i(TAG, tag + ": Jieli: CTP " + dir + " " + topic + + (json.isEmpty() ? "" : " " + truncate(json, 200))); + } + } + + private void handle(Message m) { + ctpReceived++; + synchronized (topics) { + Integer c = topics.get(m.topic); + topics.put(m.topic, c == null ? 1 : c + 1); + if (!KEEP_ALIVE.equals(m.topic)) { + lastNotifies.addLast(truncate(m.toString(), 200)); + while (lastNotifies.size() > 8) lastNotifies.removeFirst(); + } + } + if (KEEP_ALIVE.equals(m.topic)) { + keepAliveReplies++; + if (keepAliveReplies <= 2) logCtp("←", m.topic, m.json); + return; + } + logCtp("←", m.topic, m.json); + Map j = parseJson(m.json); + if (KEEP_ALIVE_INTERVAL.equals(m.topic)) { + int timeout = intParam(j, "timeout"); + if (timeout > 0) { + announcedTimeoutMs = timeout; + keepAliveIntervalMs = keepAliveMs(timeout); + Log.i(TAG, tag + ": Jieli: keepalive every " + keepAliveIntervalMs + + " ms (timeout " + timeout + " ms)"); + } + } else if (VIDEO_PARAM.equals(m.topic)) { + int w = intParam(j, "w"); + int h = intParam(j, "h"); + if (w > 0 && h > 0) { + videoParamJson = m.json; + videoParam = w + "x" + h + " at " + intParam(j, "fps") + " fps, format " + + intParam(j, "format"); + } + } + } + + private void video(DatagramPacket p, Listener listener) { + byte[] b = p.getData(); + int len = p.getLength(); + packets++; + bytes += len; + String from = p.getAddress().getHostAddress() + ":" + p.getPort(); + lastSource = from; + synchronized (sources) { + Integer c = sources.get(from); + if (c != null || sources.size() < 8) { + sources.put(from, c == null ? 1 : c + 1); + } else { + Integer o = sources.get("other"); + sources.put("other", o == null ? 1 : o + 1); + } + } + boolean rtp = len >= 12 && (b[0] & 0xC0) == 0x80; + int pt = rtp ? b[1] & 0x7F : -1; + boolean toy = p.getAddress().equals(address); + if (firstPacket == null) { + firstPacket = from + ", " + len + " bytes: " + FlowUfo.hex(b, 0, Math.min(len, 32)); + } + int n = ++logged; + if (n <= 8 || n % 1000 == 0) { + Log.i(TAG, tag + ": Jieli: UDP " + url.rtpPort + " packet " + n + " from " + from + + (toy ? "" : " (not the toy: ignored)") + ", " + len + " bytes, " + + (rtp ? "RTP PT " + pt + " seq " + ((b[2] & 0xFF) << 8 | (b[3] & 0xFF)) + + ((b[1] & 0x80) != 0 ? " marker" : "") : "not RTP") + + ": " + FlowUfo.hex(b, 0, Math.min(len, 32))); + } + if (!toy) { + foreign++; + return; + } + if (rtp) { + synchronized (payloadTypes) { + Integer c = payloadTypes.get(pt); + payloadTypes.put(pt, c == null ? 1 : c + 1); + } + } + if (rtp && (pt == PT_JPEG || pt == PT_H264)) { + rtpPackets++; + if ((b[1] & 0x80) != 0) rtpFrames++; + if (pt == PT_H264 && !h264Logged) { + h264Logged = true; + Log.i(TAG, tag + ": Jieli: RTP payload type 96: played as H.264"); + } + lastVideoAt = now(); + listener.onRtp(b, 0, len); + } else if (!rtp && len >= 4 && (b[0] & 0xFF) == 0xFF && (b[1] & 0xFF) == 0xD8 + && (b[len - 2] & 0xFF) == 0xFF && (b[len - 1] & 0xFF) == 0xD9) { + // A whole JPEG without RTP: turbodrone takes those too + rawJpegs++; + lastVideoAt = now(); + listener.onJpeg(b, 0, len); + } else if (rtp) { + if (++otherRtp <= 3) { + Log.w(TAG, tag + ": Jieli: RTP payload type " + pt + " on UDP " + + url.rtpPort + ": not video we know, ignored"); + } + } else { + nonRtp++; + } + } + + private void diagnostic(int port, DatagramPacket p) { + int count; + synchronized (diagnostic) { + Integer c = diagnostic.get(port); + count = c == null ? 1 : c + 1; + diagnostic.put(port, count); + if (count == 1) { + diagnosticFirst.put(port, p.getAddress().getHostAddress() + ":" + p.getPort() + + ", " + p.getLength() + " bytes: " + + FlowUfo.hex(p.getData(), 0, Math.min(p.getLength(), 32))); + } + } + if (count <= 5 || count % 1000 == 0) { + byte[] b = p.getData(); + int len = p.getLength(); + boolean rtp = len >= 12 && (b[0] & 0xC0) == 0x80; + Log.i(TAG, tag + ": Jieli: diagnostic UDP " + port + " packet " + count + " from " + + p.getAddress().getHostAddress() + ":" + p.getPort() + ", " + len + + " bytes" + (rtp ? ", RTP PT " + (b[1] & 0x7F) : "") + ": " + + FlowUfo.hex(b, 0, Math.min(len, 32)) + " (not played)"); + } + } + + private String summary() { + return String.format(Locale.ROOT, "CTP %d sent, %d received, %d keepalives;" + + " UDP %d: %d packets, %d RTP video, %d frames, %d not RTP," + + " %d from others", ctpSent, ctpReceived, keepAlives, url.rtpPort, + packets, rtpPackets, rtpFrames, nonRtp, foreign); + } + + /** For /stats.json. */ + Map stats() { + Map m = new LinkedHashMap<>(); + m.put("state", state); + m.put("target", url.endpoint()); + m.put("network", networkTcp); + if (hint != null) m.put("hint", hint); + Map ctp = new LinkedHashMap<>(); + ctp.put("sent", ctpSent); + ctp.put("received", ctpReceived); + ctp.put("keepalives", keepAlives); + ctp.put("keepalive_replies", keepAliveReplies); + ctp.put("keepalive_ms", keepAliveIntervalMs); + if (announcedTimeoutMs > 0) ctp.put("announced_timeout_ms", announcedTimeoutMs); + ctp.put("open_requests", openRequests); + if (reader.skipped > 0) ctp.put("skipped_bytes", reader.skipped); + synchronized (topics) { + ctp.put("topics", new LinkedHashMap<>(topics)); + ctp.put("last_notifies", new ArrayList<>(lastNotifies)); + } + m.put("ctp", ctp); + if (videoParam != null) m.put("video_param", videoParam); + if (requested != null) m.put("requested", requested); + Map rtp = new LinkedHashMap<>(); + rtp.put("port", url.rtpPort); + rtp.put("network", networkUdp); + rtp.put("packets", packets); + rtp.put("bytes", bytes); + rtp.put("video_packets", rtpPackets); + rtp.put("frames", rtpFrames); + if (rawJpegs > 0) rtp.put("raw_jpegs", rawJpegs); + rtp.put("not_rtp", nonRtp); + if (otherRtp > 0) rtp.put("other_rtp", otherRtp); + if (foreign > 0) rtp.put("from_others", foreign); + synchronized (payloadTypes) { + Map pts = new LinkedHashMap<>(); + for (Map.Entry e : payloadTypes.entrySet()) { + pts.put(String.valueOf(e.getKey()), e.getValue()); + } + rtp.put("payload_types", pts); + } + if (firstPacket != null) rtp.put("first_packet", firstPacket); + m.put("rtp", rtp); + synchronized (sources) { + m.put("sources", new LinkedHashMap<>(sources)); + } + synchronized (diagnostic) { + if (!diagnostic.isEmpty()) { + Map d = new LinkedHashMap<>(); + for (Map.Entry e : diagnostic.entrySet()) { + Map one = new LinkedHashMap<>(); + one.put("packets", e.getValue()); + one.put("first", diagnosticFirst.get(e.getKey())); + d.put(String.valueOf(e.getKey()), one); + } + m.put("diagnostic_ports", d); + } + } + if (lastError != null) m.put("last_error", lastError); + return m; + } + } + + private static long now() { + return System.nanoTime() / 1_000_000; + } +} diff --git a/android/app/src/main/java/com/openipc/decoder/NetInfo.java b/android/app/src/main/java/com/openipc/decoder/NetInfo.java index c155e2e6..2ce650f9 100644 --- a/android/app/src/main/java/com/openipc/decoder/NetInfo.java +++ b/android/app/src/main/java/com/openipc/decoder/NetInfo.java @@ -20,6 +20,7 @@ import java.net.Inet6Address; import java.net.InetAddress; import java.net.NetworkInterface; +import java.net.Socket; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -73,27 +74,62 @@ static boolean isLocalClient(InetAddress a) { * network is found (or before Android 5.1), else the interface bound to. */ static String bindToNetworkOf(DatagramSocket socket, InetAddress target) { - Context ctx = App.context; - if (ctx == null || Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP_MR1) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP_MR1) return "default"; + try { + Found f = networkOf(target); + if (f == null) return "default"; + f.network.bindSocket(socket); + return "bound to " + f.name; + } catch (Exception e) { + Log.w(TAG, "No network bound for " + target.getHostAddress() + ": " + e.getMessage()); return "default"; } + } + + /** + * The same for a TCP socket that is not connected yet (the CTP connection of a jieli:// + * slot, #187); binding one needs Android 6. + */ + static String bindToNetworkOf(Socket socket, InetAddress target) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) return "default"; try { - ConnectivityManager cm = - (ConnectivityManager) ctx.getSystemService(Context.CONNECTIVITY_SERVICE); - if (cm == null) return "default"; - for (Network n : cm.getAllNetworks()) { - LinkProperties lp = cm.getLinkProperties(n); - if (lp == null) continue; - for (LinkAddress la : lp.getLinkAddresses()) { - if (!sameSubnet(la.getAddress(), la.getPrefixLength(), target)) continue; - n.bindSocket(socket); - return "bound to " + lp.getInterfaceName(); - } - } + Found f = networkOf(target); + if (f == null) return "default"; + f.network.bindSocket(socket); + return "bound to " + f.name; } catch (Exception e) { Log.w(TAG, "No network bound for " + target.getHostAddress() + ": " + e.getMessage()); + return "default"; + } + } + + private static final class Found { + final Network network; + final String name; + + Found(Network network, String name) { + this.network = network; + this.name = name; + } + } + + /** The network whose own subnet holds {@code target}, or null. */ + private static Found networkOf(InetAddress target) { + Context ctx = App.context; + if (ctx == null) return null; + ConnectivityManager cm = + (ConnectivityManager) ctx.getSystemService(Context.CONNECTIVITY_SERVICE); + if (cm == null) return null; + for (Network n : cm.getAllNetworks()) { + LinkProperties lp = cm.getLinkProperties(n); + if (lp == null) continue; + for (LinkAddress la : lp.getLinkAddresses()) { + if (sameSubnet(la.getAddress(), la.getPrefixLength(), target)) { + return new Found(n, lp.getInterfaceName()); + } + } } - return "default"; + return null; } /** {@code a} and {@code b} share the first {@code prefix} bits (and the address family). */ diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index d9e71e8b..d1c49a2f 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -431,6 +431,9 @@ static boolean isKeyFrameNal(int type, boolean h265) { /** FLOW-UFO statistics of a flowufo:// cell (#187), else null. */ java.util.Map flowUfoStats() { return session.flowUfoStats(); } + /** Jieli statistics of a jieli:// cell (#187), else null. */ + java.util.Map jieliStats() { return session.jieliStats(); } + /** The cooingdv heartbeat of an RTSP cell once it used one (#187), else null. */ java.util.Map heartbeatStats() { return session.heartbeatStats(); } diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java index c40e424f..940a6010 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java @@ -89,6 +89,8 @@ void configure(String host, String userAgent, Transport.Mode transport) { long getBitrateMarkNs() { return session.getBitrateMarkNs(); } /** FLOW-UFO statistics while a flowufo:// slot plays (#187), else null. */ java.util.Map flowUfoStats() { return session.flowUfoStats(); } + /** Jieli statistics while a jieli:// slot plays (#187), else null. */ + java.util.Map jieliStats() { return session.jieliStats(); } /** The cooingdv heartbeat once this slot used one (#187), else null. */ java.util.Map heartbeatStats() { return session.heartbeatStats(); } diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index bad7a2c9..05b590b0 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -24,6 +24,7 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; /** @@ -107,6 +108,8 @@ default void onDetail(String text) {} private volatile boolean listening; /** Client of a flowufo:// slot (#187), else null: it probes on its own when silent. */ private volatile FlowUfo.Client ufo; + /** Client of a jieli:// slot (#187), else null: it asks for the stream again on its own. */ + private volatile Jieli.Client jieli; /** Heartbeat of the RTSP connection now (#187), else null; counts of the earlier ones. */ private volatile UdpHeartbeat heartbeat; private volatile int heartbeatsBefore; @@ -216,6 +219,7 @@ void stop() { private void closeSocketLocked() { if (listener != null) listener.close(); if (ufo != null) ufo.close(); + if (jieli != null) jieli.close(); if (heartbeat != null) heartbeat.close(); if (udp != null) { udp.close(); @@ -289,7 +293,7 @@ private void watchdogLoop(int gen) { } } } - if (active && !listening && ufo == null + if (active && !listening && ufo == null && jieli == null && System.nanoTime() / 1_000_000 - lastPacketMs > watchdogMs) { Log.w(TAG, tag + ": stream is inactive, reconnecting"); active = false; @@ -310,6 +314,7 @@ private boolean connect(int gen) throws IOException { if (RtpListener.Url.matches(url)) return listen(gen, RtpListener.Url.parse(url)); if (HttpMjpeg.matches(url)) return httpMjpeg(gen, HttpMjpeg.Url.parse(url)); if (FlowUfo.Url.matches(url)) return flowUfo(gen, FlowUfo.Url.parse(url)); + if (Jieli.Url.matches(url)) return jieli(gen, Jieli.Url.parse(url)); Url u = Url.parse(url); Socket s = new Socket(); synchronized (lock) { @@ -724,6 +729,103 @@ public void onFrame(byte[] buf, int length) { } } + /** Jieli statistics for /stats.json while a jieli:// slot runs (#187), else null. */ + Map jieliStats() { + Jieli.Client c = jieli; + return c != null ? c.stats() : null; + } + + /** + * A jieli:// slot (#187): CTP over TCP asks the toy for the stream, which comes as RTP to + * a fixed local UDP port: payload type 26 goes to the RTP/JPEG path, 96 to the H.264 one, + * as on a udp:// slot. The client asks again when no video comes, so the watchdog leaves + * it alone. Returns true once it ran, like a PLAY that succeeded. + */ + private boolean jieli(int gen, Jieli.Url u) throws IOException { + Jieli.Client c = new Jieli.Client(tag, u, NetInfo::bindToNetworkOf, + NetInfo::bindToNetworkOf); + synchronized (lock) { + if (gen != generation) return false; + jieli = c; + keepalive = null; + lastPacketMs = System.nanoTime() / 1_000_000; + streamBytes = 0; + bitrateMarkNs = System.nanoTime(); + active = true; + } + try { + final int[] pt = {u.format == Jieli.FORMAT_H264 ? Jieli.PT_H264 : Jieli.PT_JPEG}; + callback.onSdp(jieliSdp(pt[0])); + // onState comes from the CTP thread, the video from the UDP one + final AtomicBoolean first = new AtomicBoolean(true); + final AtomicBoolean received = new AtomicBoolean(false); + c.run(() -> active && gen == generation, new Jieli.Listener() { + @Override + public void onState(boolean receiving, String text) { + if (gen != generation) return; + if (receiving) { + inUse = "Jieli RTP"; + received.set(true); + } else { + inUse = null; + // Video that stopped says so; the next frame reports "connected" again + status(gen, received.get() ? "disconnected" : "buffering"); + first.set(true); + } + callback.onDetail(text); + } + + @Override + public void onRtp(byte[] buf, int offset, int length) { + if (gen != generation) return; + int type = buf[offset + 1] & 0x7F; + if (type != pt[0]) { + // The toy sends the other codec than asked for: follow it + pt[0] = type; + Log.i(TAG, tag + ": Jieli: video is " + + (type == Jieli.PT_H264 ? "H.264" : "JPEG") + " (RTP PT " + type + + ")"); + callback.onSdp(jieliSdp(type)); + } + streamBytes += length; + if (callback.onPacket(CHANNEL_VIDEO, buf, offset, length)) usable(); + } + + @Override + public void onJpeg(byte[] buf, int offset, int length) { + if (gen != generation) return; + if (pt[0] != Jieli.PT_JPEG) { + pt[0] = Jieli.PT_JPEG; + callback.onSdp(jieliSdp(Jieli.PT_JPEG)); + } + streamBytes += length; + if (callback.onJpegFrame(buf, offset, length)) usable(); + } + + private void usable() { + lastPacketMs = System.nanoTime() / 1_000_000; + if (first.compareAndSet(true, false)) status(gen, "connected"); + } + }); + return true; + } finally { + synchronized (lock) { + if (jieli == c) { + jieli = null; + inUse = null; + } + } + c.close(); + } + } + + private static Sdp jieliSdp(int pt) { + Sdp sdp = new Sdp(); + sdp.videoPt = pt; + sdp.videoJpeg = pt == Jieli.PT_JPEG; + return sdp; + } + private boolean listen(int gen, RtpListener.Url u) throws IOException { RtpListener l = new RtpListener(tag, u); synchronized (lock) { diff --git a/android/app/src/main/java/com/openipc/decoder/StreamProbes.java b/android/app/src/main/java/com/openipc/decoder/StreamProbes.java index 4c1ddfd6..7a3c08a6 100644 --- a/android/app/src/main/java/com/openipc/decoder/StreamProbes.java +++ b/android/app/src/main/java/com/openipc/decoder/StreamProbes.java @@ -50,7 +50,9 @@ static String skipReason(String url, String defaultUrl, String screenUrl) { // A listening slot would take the port; the camera pushes to it, nothing to ask if (RtpListener.Url.matches(url)) return "UDP listen slot: not checked in single view"; // A toy drone sends to one client; a check would take the stream from the screen - if (FlowUfo.Url.matches(url)) return "toy drone slot: not checked in single view"; + if (FlowUfo.Url.matches(url) || Jieli.Url.matches(url)) { + return "toy drone slot: not checked in single view"; + } String e = endpoint(url); if (e == null) return "invalid URL"; if (e.equals(endpoint(screenUrl))) return "same camera as on screen"; diff --git a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java index 4ec2f8da..87d164c2 100644 --- a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java +++ b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java @@ -1025,7 +1025,8 @@ public void subnetHints() throws Exception { FlowUfo.subnetHint(ip("192.168.169.2"), on169)); assertEquals("This phone is on 192.168.169.x: the toy is usually flowufo://192.168.169.1", FlowUfo.subnetHint(ip("192.168.1.1"), on169)); - assertEquals("Toys on 192.168.80.x (HF_ Wi-Fi) use another protocol, not supported yet", + assertEquals("This phone is on 192.168.80.x (HF_ Wi-Fi), a Jieli toy: try" + + " jieli://192.168.80.1", FlowUfo.subnetHint(ip("192.168.169.1"), Collections.singletonList(ip("192.168.80.3")))); // On both networks: no guess diff --git a/android/app/src/test/java/com/openipc/decoder/JieliTest.java b/android/app/src/test/java/com/openipc/decoder/JieliTest.java new file mode 100644 index 00000000..6f86b41c --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/JieliTest.java @@ -0,0 +1,580 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * JieliTest.java — jieli:// slots: CTP framing, the session against a fake toy, RTP/JPEG on UDP + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class JieliTest { + private static final InetAddress LOOPBACK = InetAddress.getLoopbackAddress(); + + @Before + public void shortTimers() { + Jieli.paramWaitMs = 300; + Jieli.diagnosticPorts = new int[0]; + } + + @After + public void restoreTimers() { + Jieli.paramWaitMs = 1500; + Jieli.noVideoMs = 5000; + Jieli.reopenMs = 10_000; + Jieli.quietMs = 3000; + Jieli.ctpSilenceMs = 20_000; + Jieli.diagnosticPorts = new int[] {1234, 2224}; + } + + // ====================== Helpers ====================== + + private static byte[] bytes(String hex) { + String h = hex.replaceAll("\\s", ""); + byte[] b = new byte[h.length() / 2]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) Integer.parseInt(h.substring(2 * i, 2 * i + 2), 16); + } + return b; + } + + private static byte[] concat(byte[]... parts) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (byte[] p : parts) out.write(p, 0, p.length); + return out.toByteArray(); + } + + /** RTP/JPEG packets of a 64x32 test picture (see JpegAssemblerTest). */ + private static List jpegPackets() throws IOException { + ByteArrayOutputStream all = new ByteArrayOutputStream(); + try (InputStream in = JieliTest.class.getResourceAsStream("/rtp/jpeg_q50_64x32.rtp")) { + assertNotNull(in); + byte[] b = new byte[4096]; + int n; + while ((n = in.read(b)) > 0) all.write(b, 0, n); + } + byte[] d = all.toByteArray(); + List out = new ArrayList<>(); + for (int i = 0; i < d.length; ) { + int len = ((d[i] & 0xFF) << 8) | (d[i + 1] & 0xFF); + out.add(Arrays.copyOfRange(d, i + 2, i + 2 + len)); + i += 2 + len; + } + return out; + } + + private static byte[] resource(String name) throws IOException { + ByteArrayOutputStream all = new ByteArrayOutputStream(); + try (InputStream in = JieliTest.class.getResourceAsStream("/rtp/" + name)) { + assertNotNull(in); + byte[] b = new byte[4096]; + int n; + while ((n = in.read(b)) > 0) all.write(b, 0, n); + } + return all.toByteArray(); + } + + private static int freeUdpPort() throws IOException { + try (DatagramSocket s = new DatagramSocket(0)) { + return s.getLocalPort(); + } + } + + private static long now() { + return System.nanoTime() / 1_000_000; + } + + // ====================== Framing ====================== + + @Test + public void encodesAsTheVendorApp() { + // CTP_KEEP_ALIVE {"op":"PUT"} as in the capture of #196 + assertArrayEquals(concat("CTP:".getBytes(StandardCharsets.US_ASCII), bytes("0E 00"), + "CTP_KEEP_ALIVE".getBytes(StandardCharsets.US_ASCII), bytes("0C 00 00 00"), + "{\"op\":\"PUT\"}".getBytes(StandardCharsets.US_ASCII)), + Jieli.encode(Jieli.KEEP_ALIVE, Jieli.put())); + // An empty JSON has length 0 + assertArrayEquals(concat("CTP:".getBytes(StandardCharsets.US_ASCII), bytes("02 00"), + "AB".getBytes(StandardCharsets.US_ASCII), bytes("00 00 00 00")), + Jieli.encode("AB", "")); + } + + @Test + public void commandsAsTheVendorApp() { + assertEquals("{\"op\":\"PUT\",\"param\":{\"ver\":\"25\",\"type\":\"0\"}}", + Jieli.put("ver", "25", "type", "0")); + assertEquals("{\"op\":\"PUT\",\"param\":{\"format\":\"0\",\"h\":\"480\",\"w\":\"640\"," + + "\"fps\":\"25\"}}", Jieli.openStream(0, 640, 480, 25)); + assertEquals("{\"op\":\"PUT\",\"param\":{\"status\":\"1\"}}", Jieli.put("status", "1")); + assertEquals("{\"op\":\"PUT\",\"param\":{\"a\\\"b\":\"c\\\\\"}}", Jieli.put("a\"b", "c\\")); + } + + @Test + public void readsMessagesInPiecesAndTogether() { + byte[] a = Jieli.encode("VIDEO_PARAM", + "{\"op\":\"NOTIFY\",\"param\":{\"w\":\"640\",\"h\":\"480\"}}"); + byte[] b = Jieli.encode(Jieli.KEEP_ALIVE, ""); + byte[] c = Jieli.encode("KEEP_ALIVE_INTERVAL", "{\"param\":{\"timeout\":\"60000\"}}"); + byte[] all = concat(a, b, c); + // Byte by byte + Jieli.Reader r = new Jieli.Reader(); + List got = new ArrayList<>(); + for (int i = 0; i < all.length; i++) { + r.feed(all, i, 1); + Jieli.Message m; + while ((m = r.next()) != null) got.add(m); + } + assertEquals(3, got.size()); + assertEquals("VIDEO_PARAM", got.get(0).topic); + assertEquals("{\"op\":\"NOTIFY\",\"param\":{\"w\":\"640\",\"h\":\"480\"}}", + got.get(0).json); + assertEquals(Jieli.KEEP_ALIVE, got.get(1).topic); + assertEquals("", got.get(1).json); + assertEquals("KEEP_ALIVE_INTERVAL", got.get(2).topic); + assertEquals(0, r.skipped); + // All in one read, split across a length field + r = new Jieli.Reader(); + r.feed(all, 0, a.length + 5); + assertEquals("VIDEO_PARAM", r.next().topic); + assertNull(r.next()); + r.feed(all, a.length + 5, all.length - a.length - 5); + assertEquals(Jieli.KEEP_ALIVE, r.next().topic); + assertEquals("KEEP_ALIVE_INTERVAL", r.next().topic); + assertNull(r.next()); + } + + @Test + public void skipsWhatIsNotCtp() { + byte[] good = Jieli.encode("SD_STATUS", "{\"online\":\"1\"}"); + // Garbage, then a "CTP:" with a topic length out of range, then a real message + byte[] bad = concat("xyCT".getBytes(StandardCharsets.US_ASCII), + "CTP:".getBytes(StandardCharsets.US_ASCII), bytes("FF FF"), good); + Jieli.Reader r = new Jieli.Reader(); + r.feed(bad, 0, bad.length); + Jieli.Message m = r.next(); + assertNotNull(m); + assertEquals("SD_STATUS", m.topic); + assertEquals(10, r.skipped); + assertNull(r.next()); + // A large message grows the buffer + StringBuilder big = new StringBuilder("{\"x\":\""); + for (int i = 0; i < 20000; i++) big.append('a'); + big.append("\"}"); + byte[] large = Jieli.encode("BIG", big.toString()); + r.feed(large, 0, large.length); + assertEquals(big.toString(), r.next().json); + } + + @Test + public void readsJson() { + Map j = Jieli.parseJson("{\"errno\":0,\"op\":\"NOTIFY\",\"param\":" + + "{\"w\":\"640\",\"h\":\"480\",\"format\":\"0\",\"fps\":\"25\",\"l\":[1,\"x\"]}}"); + assertNotNull(j); + assertEquals("NOTIFY", j.get("op")); + assertEquals("0", j.get("errno")); + assertEquals("640", Jieli.param(j, "w")); + assertEquals(25, Jieli.intParam(j, "fps")); + assertEquals("NOTIFY", Jieli.param(j, "op")); + assertEquals(-1, Jieli.intParam(j, "missing")); + assertEquals("a\"b\u00e9", Jieli.parseJson("{\"k\":\"a\\\"b\\u00e9\"}").get("k")); + assertNull(Jieli.parseJson("")); + assertNull(Jieli.parseJson("{\"a\":")); + assertNull(Jieli.parseJson("[1]")); + assertNull(Jieli.parseJson("{} x")); + } + + @Test + public void keepAliveInterval() { + assertEquals(7500, Jieli.keepAliveMs(60000)); + assertEquals(7500, Jieli.keepAliveMs(-1)); + assertEquals(2500, Jieli.keepAliveMs(10000)); + assertEquals(250, Jieli.keepAliveMs(100)); + } + + // ====================== URL ====================== + + @Test + public void urls() throws Exception { + Jieli.Url u = Jieli.Url.parse("jieli://"); + assertEquals("192.168.80.1", u.host); + assertEquals(3333, u.port); + assertEquals(6666, u.rtpPort); + assertEquals(0, u.width); + assertEquals(Jieli.FORMAT_JPEG, u.format); + u = Jieli.Url.parse(" JIELI://192.168.80.1:3334/?w=1280&h=720&fps=30&rtp=7000&format=1 "); + assertEquals(3334, u.port); + assertEquals(1280, u.width); + assertEquals(720, u.height); + assertEquals(30, u.fps); + assertEquals(7000, u.rtpPort); + assertEquals(Jieli.FORMAT_H264, u.format); + assertEquals("192.168.80.1:3334", u.endpoint()); + assertTrue(Jieli.Url.matches("jieli://x")); + assertTrue(!Jieli.Url.matches("flowufo://")); + for (String bad : new String[] {"jieli://h:0", "jieli://h:x", "jieli://?w=8", + "jieli://?fps=0", "jieli://?rtp=70000", "jieli://?format=2", "jieli://a b", + "jieli://[::1"}) { + try { + Jieli.Url.parse(bad); + fail(bad); + } catch (IOException expected) { + // refused + } + } + assertEquals("toy drone slot: not checked in single view", + StreamProbes.skipReason("jieli://", "rtsp://default", null)); + } + + @Test + public void subnetHints() throws Exception { + InetAddress toy = InetAddress.getByName("192.168.80.1"); + List on80 = Collections.singletonList(InetAddress.getByName("192.168.80.2")); + assertNull(Jieli.subnetHint(toy, on80)); + assertEquals("192.168.80.2 is this phone; the toy is usually 192.168.80.1", + Jieli.subnetHint(InetAddress.getByName("192.168.80.2"), on80)); + assertEquals("This phone is on 192.168.80.x: the toy is usually jieli://192.168.80.1", + Jieli.subnetHint(InetAddress.getByName("192.168.169.1"), on80)); + assertEquals("This network looks like a FLOW-UFO toy: try flowufo://192.168.169.1", + Jieli.subnetHint(toy, + Collections.singletonList(InetAddress.getByName("192.168.169.3")))); + assertEquals("This network looks like a KY UFO RTSP toy: try" + + " rtsp://192.168.1.1:7070/webcam", + Jieli.subnetHint(toy, + Collections.singletonList(InetAddress.getByName("192.168.1.100")))); + assertNull(Jieli.subnetHint(toy, Collections.emptyList())); + } + + // ====================== A fake toy on loopback ====================== + + /** One message the fake toy received, with the time. */ + private static final class Got { + final String topic; + final String json; + final long at; + + Got(Jieli.Message m) { + topic = m.topic; + json = m.json; + at = now(); + } + } + + /** + * Answers like the toy of #196: KEEP_ALIVE_INTERVAL and VIDEO_PARAM after APP_ACCESS, + * empty keepalive replies, an OPEN_RT_STREAM notify; then the video to {@code rtpPort}. + */ + private static final class FakeToy implements AutoCloseable { + final ServerSocket server; + final List got = new CopyOnWriteArrayList<>(); + final CountDownLatch closeStream = new CountDownLatch(1); + final CountDownLatch opened = new CountDownLatch(1); + final int timeoutMs; + final boolean sendVideoParam; + volatile int rtpPort; + volatile boolean sendVideo = true; + private final Thread thread; + + FakeToy(int timeoutMs, boolean sendVideoParam) throws IOException { + this.timeoutMs = timeoutMs; + this.sendVideoParam = sendVideoParam; + server = new ServerSocket(0, 1, LOOPBACK); + thread = new Thread(this::serve, "fake-jieli"); + thread.setDaemon(true); + thread.start(); + } + + int port() { + return server.getLocalPort(); + } + + private void serve() { + try (Socket s = server.accept()) { + InputStream in = s.getInputStream(); + OutputStream out = s.getOutputStream(); + Jieli.Reader r = new Jieli.Reader(); + byte[] buf = new byte[4096]; + int n; + while ((n = in.read(buf)) > 0) { + r.feed(buf, 0, n); + Jieli.Message m; + while ((m = r.next()) != null) { + got.add(new Got(m)); + switch (m.topic) { + case Jieli.APP_ACCESS: + out.write(Jieli.encode(Jieli.KEEP_ALIVE_INTERVAL, + "{\"op\":\"NOTIFY\",\"param\":{\"timeout\":\"" + + timeoutMs + "\"}}")); + out.write(Jieli.encode("BAT_STATUS", + "{\"op\":\"NOTIFY\",\"param\":{\"level\":\"4\"}}")); + if (sendVideoParam) { + out.write(Jieli.encode(Jieli.VIDEO_PARAM, + "{\"op\":\"NOTIFY\",\"param\":{\"w\":\"320\"," + + "\"h\":\"240\",\"format\":\"0\"," + + "\"fps\":\"15\"}}")); + } + break; + case Jieli.KEEP_ALIVE: + out.write(Jieli.encode(Jieli.KEEP_ALIVE, "")); + break; + case Jieli.OPEN_RT_STREAM: + out.write(Jieli.encode(Jieli.OPEN_RT_STREAM, + "{\"op\":\"NOTIFY\",\"param\":{\"status\":\"1\"}}")); + if (opened.getCount() > 0 && sendVideo) sendVideo(); + opened.countDown(); + break; + case Jieli.CLOSE_RT_STREAM: + closeStream.countDown(); + break; + default: + break; + } + out.flush(); + } + } + } catch (IOException e) { + // Closed + } + } + + private void sendVideo() throws IOException { + try (DatagramSocket u = new DatagramSocket(0, LOOPBACK)) { + InetSocketAddress to = new InetSocketAddress(LOOPBACK, rtpPort); + // Something that is not RTP first: counted and logged, not played + byte[] junk = {1, 2, 3}; + u.send(new DatagramPacket(junk, junk.length, to)); + for (byte[] p : jpegPackets()) u.send(new DatagramPacket(p, p.length, to)); + } + } + + List topics() { + List t = new ArrayList<>(); + for (Got g : got) t.add(g.topic); + return t; + } + + @Override + public void close() throws IOException { + server.close(); + } + } + + /** Runs the client on a thread; frames assembled from the RTP it delivers. */ + private static final class Running { + final Jieli.Client client; + final AtomicBoolean keepGoing = new AtomicBoolean(true); + final List frames = new CopyOnWriteArrayList<>(); + final List states = new CopyOnWriteArrayList<>(); + final CountDownLatch frame = new CountDownLatch(1); + final CountDownLatch ended = new CountDownLatch(1); + volatile Exception failure; + final Thread thread; + + Running(String url) throws IOException { + client = new Jieli.Client("test", Jieli.Url.parse(url), null, null); + client.localForHint = Collections.emptyList(); + JpegAssembler assembler = new JpegAssembler(new FramePool(4)); + thread = new Thread(() -> { + try { + client.run(keepGoing::get, new Jieli.Listener() { + @Override + public void onState(boolean receiving, String text) { + states.add(text); + } + + @Override + public void onRtp(byte[] buf, int offset, int length) { + assembler.assemble(buf, offset, length, f -> { + frames.add(Arrays.copyOf(f.data(), f.length())); + frame.countDown(); + }); + } + }); + } catch (Exception e) { + failure = e; + } finally { + ended.countDown(); + } + }, "jieli-client"); + thread.start(); + } + } + + @Test + public void sessionAsTheVendorApp() throws Exception { + int rtp = freeUdpPort(); + try (FakeToy toy = new FakeToy(2000, true)) { + toy.rtpPort = rtp; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp); + assertTrue("no frame", r.frame.await(5, TimeUnit.SECONDS)); + // A few keepalives at 2000 / 4 = 500 ms + Thread.sleep(1700); + r.keepGoing.set(false); + assertTrue("no CLOSE_RT_STREAM", toy.closeStream.await(3, TimeUnit.SECONDS)); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + assertNull(r.failure); + + List t = toy.topics(); + assertEquals(Arrays.asList(Jieli.APP_ACCESS, Jieli.DATE_TIME, Jieli.KEEP_ALIVE, + Jieli.OPEN_RT_STREAM), t.subList(0, 4)); + assertEquals(Jieli.CLOSE_RT_STREAM, t.get(t.size() - 1)); + assertEquals("{\"op\":\"PUT\",\"param\":{\"ver\":\"25\",\"type\":\"0\"}}", + toy.got.get(0).json); + assertTrue(toy.got.get(1).json, + toy.got.get(1).json.matches("\\{\"op\":\"PUT\",\"param\":\\{\"date\":" + + "\"\\d{14}\"}}")); + // The size the toy announced in VIDEO_PARAM + assertEquals(Jieli.openStream(0, 320, 240, 15), toy.got.get(3).json); + assertEquals("{\"op\":\"PUT\",\"param\":{\"status\":\"1\"}}", + toy.got.get(t.size() - 1).json); + assertEquals(1, Collections.frequency(t, Jieli.OPEN_RT_STREAM)); + + List keepAlives = new ArrayList<>(); + for (Got g : toy.got) { + if (g.topic.equals(Jieli.KEEP_ALIVE)) keepAlives.add(g.at); + } + assertTrue("keepalives: " + keepAlives.size(), keepAlives.size() >= 3); + for (int i = 1; i < keepAlives.size(); i++) { + long gap = keepAlives.get(i) - keepAlives.get(i - 1); + assertTrue("gap " + gap, gap >= 400 && gap <= 900); + } + + // The JPEG the RTP/JPEG path makes of the packets + assertArrayEquals(resource("jpeg_q50_64x32.jpg"), r.frames.get(0)); + Map st = r.client.stats(); + @SuppressWarnings("unchecked") + Map ctp = (Map) st.get("ctp"); + assertEquals(500L, ctp.get("keepalive_ms")); + assertEquals("320x240 at 15 fps, format 0", st.get("video_param")); + @SuppressWarnings("unchecked") + Map rtpStats = (Map) st.get("rtp"); + assertEquals(1, rtpStats.get("not_rtp")); + assertTrue((Integer) rtpStats.get("frames") >= 1); + @SuppressWarnings("unchecked") + Map sources = (Map) st.get("sources"); + assertEquals(1, sources.size()); + assertTrue(r.states.get(0), r.states.get(0).startsWith("Connecting to the toy (CTP")); + assertTrue(r.states.contains("Stream requested, waiting for video on UDP " + rtp)); + } + } + + @Test + public void defaultSizeAndCloseOnStop() throws Exception { + int rtp = freeUdpPort(); + try (FakeToy toy = new FakeToy(60000, false)) { + toy.rtpPort = rtp; + toy.sendVideo = false; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp); + assertTrue(toy.opened.await(5, TimeUnit.SECONDS)); + // No VIDEO_PARAM: 640x480 at 25 after the wait + Got open = null; + for (Got g : toy.got) if (g.topic.equals(Jieli.OPEN_RT_STREAM)) open = g; + assertNotNull(open); + assertEquals(Jieli.openStream(0, 640, 480, 25), open.json); + assertTrue(open.at - toy.got.get(0).at >= 250); + // close() as RtspSession.stop() does it: CLOSE_RT_STREAM still goes out + r.client.close(); + assertTrue("no CLOSE_RT_STREAM", toy.closeStream.await(3, TimeUnit.SECONDS)); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + assertNull(r.failure); + @SuppressWarnings("unchecked") + Map ctp = (Map) r.client.stats().get("ctp"); + assertEquals(7500L, ctp.get("keepalive_ms")); + } + } + + @Test + public void noVideoListensElsewhereAndAsksAgain() throws Exception { + int rtp = freeUdpPort(); + int diag = freeUdpPort(); + Jieli.noVideoMs = 300; + Jieli.reopenMs = 700; + Jieli.diagnosticPorts = new int[] {diag}; + try (FakeToy toy = new FakeToy(60000, true)) { + toy.rtpPort = rtp; + toy.sendVideo = false; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp); + assertTrue(toy.opened.await(5, TimeUnit.SECONDS)); + long until = now() + 3000; + while (now() < until && !r.states.contains(r.client.noVideoText())) Thread.sleep(20); + assertTrue(r.states.toString(), r.states.contains(r.client.noVideoText())); + Thread.sleep(100); + try (DatagramSocket u = new DatagramSocket(0, LOOPBACK)) { + byte[] p = {(byte) 0x80, 97, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 9}; + u.send(new DatagramPacket(p, p.length, new InetSocketAddress(LOOPBACK, diag))); + } + until = now() + 3000; + while (now() < until && (Collections.frequency(toy.topics(), Jieli.OPEN_RT_STREAM) < 2 + || r.client.stats().get("diagnostic_ports") == null)) { + Thread.sleep(20); + } + assertTrue(Collections.frequency(toy.topics(), Jieli.OPEN_RT_STREAM) >= 2); + Map st = r.client.stats(); + @SuppressWarnings("unchecked") + Map d = (Map) st.get("diagnostic_ports"); + assertNotNull(d); + @SuppressWarnings("unchecked") + Map one = (Map) d.get(String.valueOf(diag)); + assertEquals(1, one.get("packets")); + r.keepGoing.set(false); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + assertNull(r.failure); + // Nothing from a diagnostic port reaches the video path + assertTrue(r.frames.isEmpty()); + } + } + + @Test + public void unreachableToyFails() throws Exception { + int port; + try (ServerSocket s = new ServerSocket(0, 1, LOOPBACK)) { + port = s.getLocalPort(); + } + Jieli.Client c = new Jieli.Client("test", + Jieli.Url.parse("jieli://127.0.0.1:" + port + "?rtp=" + freeUdpPort()), null, null); + c.localForHint = Collections.emptyList(); + try { + c.run(() -> true, new Jieli.Listener() { + @Override + public void onState(boolean receiving, String text) {} + + @Override + public void onRtp(byte[] buf, int offset, int length) {} + }); + fail("connected"); + } catch (IOException e) { + assertTrue(e.getMessage(), + e.getMessage().startsWith("Cannot reach the toy's CTP port")); + } + } +} diff --git a/docs/debug.md b/docs/debug.md index 0ea44458..27fb7b78 100644 --- a/docs/debug.md +++ b/docs/debug.md @@ -168,7 +168,7 @@ Single view and U view: every cell of the grid as the debug page shows it (3×3 | `codec`, `size` | when found | | `fps`, `kbps`, `key_frames` | measured during the check | -The FLOW-UFO test build ([toys.md](toys.md#flow-ufo)) adds a `flowufo` object per cell and in `single`, and a `heartbeat` object for RTSP slots that send the KY UFO heartbeat ([toys.md](toys.md#ky-ufo-rtsp-toys)); it is not in the releases. +The FLOW-UFO test build ([toys.md](toys.md#flow-ufo)) adds a `flowufo` object per cell and in `single` (`jieli` for `jieli://` slots, [toys.md](toys.md#hf--jieli-toys)), and a `heartbeat` object for RTSP slots that send the KY UFO heartbeat ([toys.md](toys.md#ky-ufo-rtsp-toys)); it is not in the releases. ## Log diff --git a/docs/toys.md b/docs/toys.md index ace5f21c..8ea89c50 100644 --- a/docs/toys.md +++ b/docs/toys.md @@ -5,6 +5,7 @@ - [Toy families by Wi‑Fi name](#toy-families-by-wifi-name) - [FLOW-UFO](#flow-ufo) - [KY UFO RTSP toys](#ky-ufo-rtsp-toys) +- [HF / Jieli toys](#hf--jieli-toys) - [Credits](#credits) - [An unknown toy](#an-unknown-toy) - [What to send in a report](#what-to-send-in-a-report) @@ -24,7 +25,7 @@ Cheap drones, cars and other toys with a Wi‑Fi camera usually work the same wa - The video is often **not a standard stream**: no RTSP, no plain RTP. Many toys send JPEG frames over UDP in their own framing and start only after the client has sent a request; some also expect regular replies while they stream. A standard player then sees nothing, and this app stays in *Buffering*. - Flight or drive commands go over the same network. **This app never sends any**: it only asks for and receives video. -Whether a toy can be played here depends on its protocol. Toys that do offer RTSP or HTTP MJPEG play like any camera ([transports.md](transports.md)); others need a transport of their own, like [FLOW-UFO](#flow-ufo). +Whether a toy can be played here depends on its protocol. Toys that do offer RTSP or HTTP MJPEG play like any camera ([transports.md](transports.md)); others need a transport of their own, like [FLOW-UFO](#flow-ufo) or [HF / Jieli](#hf--jieli-toys). ## Connecting @@ -42,9 +43,9 @@ The Wi‑Fi name and the address the phone gets usually tell the family. The toy |---|---|---|---|---| | `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | | `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | -| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF | — | **not supported yet** | +| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 6666) | test build 3, video path unverified ([#187](https://github.com/OpenIPC/decoder/issues/187)) | -When a `flowufo://` slot points at the phone itself, or at the network of another family, its status says so (for example *192.168.1.100 is this phone; the toy is usually 192.168.1.1* or *This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam*). +When a `flowufo://` or `jieli://` slot points at the phone itself, or at the network of another family, its status says so (for example *192.168.1.100 is this phone; the toy is usually 192.168.1.1*, *This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam* or, on a `192.168.80.x` network, *try jieli://192.168.80.1*). ## FLOW-UFO @@ -90,9 +91,42 @@ Their apps send a heartbeat, the two bytes `01 01`, to UDP port 7099 of the toy Some of these toys use the FLOW-UFO protocol instead (the KY UFO app has both); if RTSP shows nothing, try `flowufo://192.168.1.1`. +## HF / Jieli toys + +| | | +|---|---| +| Wi‑Fi name | `HF_xxxxxxxx` | +| Camera | Jieli AC79 (`AC79X_wifi_car_camera`); vendor app *DVRunning 2* (also *HF UFO*) | +| Toy address | `192.168.80.1`: control (CTP) on TCP port 3333; the video goes to UDP port 6666 **of the phone** | +| Slot URL | `jieli://192.168.80.1` (or `jieli://` alone) | +| Status | **test build 3**, not in the releases: the control part follows a capture of the vendor app, the video part is **unverified**; issue [#187](https://github.com/OpenIPC/decoder/issues/187) | + +What the app does, as a capture of the vendor app shows it ([#196](https://github.com/OpenIPC/decoder/issues/196)): + +- It connects to TCP port 3333 and talks **CTP**: each message is `CTP:`, the length of a topic name, the topic (e.g. `OPEN_RT_STREAM`), the length of a JSON text and the JSON. +- `APP_ACCESS` first; the toy answers with `KEEP_ALIVE_INTERVAL` (a timeout, 60 s), its battery and SD card state and `VIDEO_PARAM` (the size, e.g. 640×480 at 25 fps). Then `DATE_TIME` (the phone's time), a `CTP_KEEP_ALIVE` every 7.5 s (a quarter of a shorter timeout), and `OPEN_RT_STREAM` with the size from `VIDEO_PARAM` (640×480 at 25 if the toy sent none). When the slot stops: `CLOSE_RT_STREAM`. +- The video was **not** in the capture (PCAPdroid without root cannot see it). After turbodrone's research on the same Jieli SDK it is **RTP/JPEG** (payload type 26) or H.264 (96) to the phone's **UDP port 6666**. The app listens there and plays payload type 26 through its RTP/JPEG path and 96 through its H.264 path; a whole JPEG in a datagram without RTP is shown too. +- If nothing arrives on 6666 for 5 s after `OPEN_RT_STREAM`, the status says so and the app also listens on UDP 1234 (audio in the vendor SDK) and 2224 (its stream port), **only to log** what arrives there; it asks for the stream again every 10 s without video. +- **No flight command is ever sent**: no `FLIGHT_CONTROL` over CTP, nothing to the toy's UDP port 3456 (where the vendor app sends the sticks). +- Port 6666 can be used by one slot at a time: one `jieli://` slot in a grid. + +URL options (as with `udp://`, separated by `?` and `&`): + +| Option | Values | Default | What it does | +|---|---|---|---| +| `w`, `h`, `fps` | numbers | from `VIDEO_PARAM`, else 640, 480, 25 | the stream asked for in `OPEN_RT_STREAM` | +| `rtp` | a UDP port | `6666` | the local port the video is expected on | +| `format` | `0`, `1` | `0` | `0` asks for JPEG (as the vendor app does), `1` for H.264 | + +Examples: `jieli://192.168.80.1`, `jieli://192.168.80.1?w=1280&h=720`, `jieli://192.168.80.1?format=1`. + +Status texts while it connects: *Connecting to the toy (CTP 192.168.80.1:3333)*, *Stream requested, waiting for video on UDP 6666*, *No video on UDP 6666 yet: the toy accepted the request; see /log*, *Receiving from the toy on UDP 6666…*, *The toy's video stopped: waiting on UDP 6666*. The test build adds a `jieli` object to `/stats.json`: `state` (`connecting`, `ctp`, `requested`, `playing`, `quiet`), `target`, `network`, `ctp` (messages sent and received, keepalives and replies, the keepalive interval, the topics seen, the last 8 messages from the toy), `video_param`, `requested`, `rtp` (port, packets, video packets, frames, payload types, packets that are not RTP, the first packet in hex), `sources` (sender address:port → packets) and `diagnostic_ports`. The log has the CTP messages (JSON shortened) and the first packets on each port with sender, payload type and first bytes: exactly what a report needs. + +To test: install the test build, join the `HF_…` Wi‑Fi, [stay connected](#connecting), close the vendor app, put `jieli://192.168.80.1` into a slot. Wait about 30 s and [send a report](#what-to-send-in-a-report) in [#187](https://github.com/OpenIPC/decoder/issues/187), with or without a picture: `/stats.json` and `/log` tell what the toy really sends. + ## Credits -What the app does for FLOW-UFO and KY UFO toys was checked against [turbodrone](https://github.com/marshallrichards/turbodrone) (Apache-2.0), which flies these drones from a computer: its `wifi_uav` backend is the FLOW-UFO protocol and its `cooingdv` backend the KY UFO RTSP toys. The default quality parameters, the second port 8801 and its timing, the camera status bytes, leaving lagging frames out of the acknowledgement, the raster tables of `dqt=raster` and the 7099 heartbeat come from its code and research notes; no code of it is used. More open-source work on such toys is collected in [#196](https://github.com/OpenIPC/decoder/issues/196). +What the app does for FLOW-UFO and KY UFO toys was checked against [turbodrone](https://github.com/marshallrichards/turbodrone) (Apache-2.0), which flies these drones from a computer: its `wifi_uav` backend is the FLOW-UFO protocol and its `cooingdv` backend the KY UFO RTSP toys. The default quality parameters, the second port 8801 and its timing, the camera status bytes, leaving lagging frames out of the acknowledgement, the raster tables of `dqt=raster` and the 7099 heartbeat come from its code and research notes; no code of it is used. For the HF / Jieli toys its research notes on the same Jieli SDK (`rxdrone`, `cooingdv_jieli`: RTP/JPEG to UDP 6666, audio on 1234, the stream ports) describe the video part; the control part follows a PCAPdroid capture of the vendor app posted in [#196](https://github.com/OpenIPC/decoder/issues/196). More open-source work on such toys is collected in [#196](https://github.com/OpenIPC/decoder/issues/196). ## An unknown toy diff --git a/docs/transports.md b/docs/transports.md index 8d5634d2..375d8576 100644 --- a/docs/transports.md +++ b/docs/transports.md @@ -102,6 +102,8 @@ For example Majestic's `http://user:password@/mjpeg`: a `multipart/x-mix `flowufo://192.168.169.1:8800` (or `flowufo://` alone) plays the video of toy drones with a `FLOW-UFO-xxxxxx` Wi‑Fi access point. It exists only in a test build so far ([#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188)), not in the releases. Port 8800 (the default) probes 8801 as well and keeps the port that answers. Options: `?qpara=<8 hex digits>` and `?dqt=zigzag|raster|q50`. Setup, options and status: [toys.md](toys.md#flow-ufo). +`jieli://192.168.80.1` (or `jieli://` alone) plays toy drones with an `HF_xxxxxxxx` Wi‑Fi (Jieli AC79 camera): CTP commands on TCP 3333 ask for the stream, which is expected as RTP/JPEG on the phone's UDP port 6666 (test build only, video path unverified). Options: `?w=`, `?h=`, `?fps=`, `?rtp=`, `?format=0|1`. See [toys.md](toys.md#hf--jieli-toys). + RTSP slots on `rtsp://:7070/webcam` (KY UFO and other cooingdv toys) also send the toy's UDP heartbeat `01 01` to port 7099 once a second; `?heartbeat=` turns it on for other RTSP URLs, `?heartbeat=off` off. See [toys.md](toys.md#ky-ufo-rtsp-toys). ## Codecs From 4a9245a018e3e672f43f9ecc1256d66f5c734562 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 18:42:56 +0000 Subject: [PATCH 07/21] fix: lock onto a port only by a frame fragment, new sockets on every silence reset, network binding from Android 6 (Qodo on #188) --- .../java/com/openipc/decoder/FlowUfo.java | 24 ++++++++++++------- .../java/com/openipc/decoder/NetInfo.java | 5 ++-- .../java/com/openipc/decoder/FlowUfoTest.java | 20 +++++++++++++++- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java index f5a2869f..594580e9 100644 --- a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java +++ b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java @@ -928,9 +928,12 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException if (failingSince == 0) failingSince = t; if (t - failingSince >= 3000) { failingSince = 0; - if (receiving) probeAgain(listener, address); - locked = -1; - for (int i = 0; i < ports.length; i++) open(i, address); + if (receiving) { + probeAgain(listener, address); + } else { + locked = -1; + for (int i = 0; i < ports.length; i++) open(i, address); + } Log.w(TAG, tag + ": FLOW-UFO: new sockets after send errors, local" + " port " + localPorts() + " (network: " + network + ")"); Arrays.fill(nextProbe, t); @@ -969,7 +972,11 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException } continue; } - if (locked < 0 && packetType(rx, 0, in.getLength()) >= 0) lock(got, in); + // Only a frame fragment locks: a service message may come from the engine + // that sends no video (Qodo on #188) + if (locked < 0 && packetType(rx, 0, in.getLength()) == TYPE_FRAGMENT) { + lock(got, in); + } handle(rx, in.getLength(), listener); } } catch (IOException e) { @@ -1004,16 +1011,17 @@ private void lock(int i, DatagramPacket from) { + (ports.length > 1 ? "; the other port is not probed any more" : "")); } - /** Back to probing on every port: the stream is gone. */ + /** + * Back to probing on every port: the stream is gone. Every socket is new, bound again + * to whatever network reaches the toy now (Qodo on #188). + */ private void probeAgain(Listener listener, InetAddress address) throws IOException { resets++; receiving = false; playing = false; window.reset(); locked = -1; - for (int i = 0; i < ports.length; i++) { - if (socket(i) == null) open(i, address); - } + for (int i = 0; i < ports.length; i++) open(i, address); listener.onState(false, probingText(true)); } diff --git a/android/app/src/main/java/com/openipc/decoder/NetInfo.java b/android/app/src/main/java/com/openipc/decoder/NetInfo.java index c155e2e6..5c7f0790 100644 --- a/android/app/src/main/java/com/openipc/decoder/NetInfo.java +++ b/android/app/src/main/java/com/openipc/decoder/NetInfo.java @@ -70,11 +70,12 @@ static boolean isLocalClient(InetAddress a) { * Binds {@code socket} to the network whose own subnet holds {@code target} (#187). A toy's * Wi-Fi has no internet: with mobile data on, Android may send even on-link traffic over the * default network instead. Returns what was done, for the log: "default" when no such - * network is found (or before Android 5.1), else the interface bound to. + * network is found (or before Android 6: Network.bindSocket(DatagramSocket) is API 23, + * Qodo on #188), else the interface bound to. */ static String bindToNetworkOf(DatagramSocket socket, InetAddress target) { Context ctx = App.context; - if (ctx == null || Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP_MR1) { + if (ctx == null || Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { return "default"; } try { diff --git a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java index 4ec2f8da..25d26fd5 100644 --- a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java +++ b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java @@ -781,7 +781,12 @@ public void fakeDroneOnLoopback() throws Exception { try (DatagramSocket drone = new DatagramSocket(0, InetAddress.getLoopbackAddress())) { drone.setSoTimeout(200); FlowUfo.Url u = FlowUfo.Url.parse("flowufo://127.0.0.1:" + drone.getLocalPort()); - FlowUfo.Client client = new FlowUfo.Client("test", u, null); + java.util.concurrent.atomic.AtomicInteger setups = + new java.util.concurrent.atomic.AtomicInteger(); + FlowUfo.Client client = new FlowUfo.Client("test", u, (s, target) -> { + setups.incrementAndGet(); + return "test"; + }); List frames = Collections.synchronizedList(new ArrayList<>()); List states = Collections.synchronizedList(new ArrayList<>()); CountDownLatch frame = new CountDownLatch(1); @@ -882,6 +887,8 @@ public void onFrame(byte[] buf, int length) { } assertTrue(probed); assertEquals("probing", client.stats().get("state")); + // A new socket, set up for the network again (Qodo on #188) + assertEquals(2, setups.get()); } finally { client.close(); t.join(2000); @@ -1124,6 +1131,17 @@ public void onFrame(byte[] buf, int length) {} SocketAddress phoneB = probeAt(b); assertFalse(phoneA.equals(phoneB)); assertEquals(-1, client.lockedPort()); + // A service message on a's socket does not lock it (Qodo on #188) + byte[] service = bytes("93 04 0E 00 01 00 00 00 02 00 00 00 55 5A"); + from1234.send(new DatagramPacket(service, service.length, phoneA)); + long until = System.currentTimeMillis() + 3000; + while (client.stats().get("service_messages").equals(0) + && System.currentTimeMillis() < until) { + Thread.sleep(10); + } + assertEquals(1, client.stats().get("service_messages")); + assertEquals(-1, client.lockedPort()); + probeAt(b); // The engine behind b answers, from another source port (the drone's 1234) byte[] f = piece(1, new byte[2000], 0); from1234.send(new DatagramPacket(f, f.length, phoneB)); From 2eacdfada6e84b23dffc3e839f74211b08e2db46 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 18:51:05 +0000 Subject: [PATCH 08/21] fix: keepalive from the last one sent, quiet status in order with the video (Qodo on #198) - a KEEP_ALIVE_INTERVAL announced after the first keepalive applies at once: the next one is due interval ms after the last one sent, not on the old schedule; test with the announcement after the first keepalive - the quiet status from the CTP thread and "connected" from the UDP thread are serialised, and a quiet notice is dropped when video arrived meanwhile --- .../main/java/com/openipc/decoder/Jieli.java | 16 ++++++-- .../java/com/openipc/decoder/RtspSession.java | 28 ++++++++----- .../java/com/openipc/decoder/JieliTest.java | 40 +++++++++++++++++-- 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/Jieli.java b/android/app/src/main/java/com/openipc/decoder/Jieli.java index 53488394..4fa128ba 100644 --- a/android/app/src/main/java/com/openipc/decoder/Jieli.java +++ b/android/app/src/main/java/com/openipc/decoder/Jieli.java @@ -664,6 +664,12 @@ private String withHint(String text) { return h != null ? text + ". " + h : text; } + /** Milliseconds since the last video packet from the toy, or Long.MAX_VALUE. */ + long videoAgeMs() { + long last = lastVideoAt; + return last > 0 ? now() - last : Long.MAX_VALUE; + } + String connectingText() { return withHint("Connecting to the toy (CTP " + url.endpoint() + ")"); } @@ -723,7 +729,9 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException send(out, APP_ACCESS, put("ver", "25", "type", "0")); long t0 = now(); long lastRx = t0; - long nextKeepAlive = 0; + // From the last one sent: an interval announced later applies at once (Qodo + // on #198) + long lastKeepAlive = 0; long openAt = 0; boolean diagnostics = false; boolean receiving = false; @@ -735,16 +743,16 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException send(out, DATE_TIME, put("date", new SimpleDateFormat("yyyyMMddHHmmss", Locale.ROOT).format(new Date()))); keepAlive(out); - nextKeepAlive = t + keepAliveIntervalMs; + lastKeepAlive = t; requestStream(out); requestedStream = true; openAt = t; state = "requested"; listener.onState(false, requestedText()); } - if (requestedStream && t >= nextKeepAlive) { + if (requestedStream && t - lastKeepAlive >= keepAliveIntervalMs) { keepAlive(out); - nextKeepAlive = t + keepAliveIntervalMs; + lastKeepAlive = t; } if (requestedStream) { long last = lastVideoAt; diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index 05b590b0..fa7c55e6 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -759,20 +759,26 @@ private boolean jieli(int gen, Jieli.Url u) throws IOException { // onState comes from the CTP thread, the video from the UDP one final AtomicBoolean first = new AtomicBoolean(true); final AtomicBoolean received = new AtomicBoolean(false); + final Object order = new Object(); c.run(() -> active && gen == generation, new Jieli.Listener() { @Override public void onState(boolean receiving, String text) { if (gen != generation) return; - if (receiving) { - inUse = "Jieli RTP"; - received.set(true); - } else { - inUse = null; - // Video that stopped says so; the next frame reports "connected" again - status(gen, received.get() ? "disconnected" : "buffering"); - first.set(true); + // In order with usable() on the UDP thread: a packet that came in since + // the CTP thread saw the video stop wins (Qodo on #198) + synchronized (order) { + if (receiving) { + inUse = "Jieli RTP"; + received.set(true); + } else { + if (c.videoAgeMs() < Jieli.quietMs) return; + inUse = null; + // Video that stopped says so; the next frame reports "connected" + status(gen, received.get() ? "disconnected" : "buffering"); + first.set(true); + } + callback.onDetail(text); } - callback.onDetail(text); } @Override @@ -804,7 +810,9 @@ public void onJpeg(byte[] buf, int offset, int length) { private void usable() { lastPacketMs = System.nanoTime() / 1_000_000; - if (first.compareAndSet(true, false)) status(gen, "connected"); + synchronized (order) { + if (first.compareAndSet(true, false)) status(gen, "connected"); + } } }); return true; diff --git a/android/app/src/test/java/com/openipc/decoder/JieliTest.java b/android/app/src/test/java/com/openipc/decoder/JieliTest.java index 6f86b41c..0474451a 100644 --- a/android/app/src/test/java/com/openipc/decoder/JieliTest.java +++ b/android/app/src/test/java/com/openipc/decoder/JieliTest.java @@ -303,6 +303,9 @@ private static final class FakeToy implements AutoCloseable { final boolean sendVideoParam; volatile int rtpPort; volatile boolean sendVideo = true; + /** KEEP_ALIVE_INTERVAL only after the first keepalive, not after APP_ACCESS. */ + volatile boolean lateInterval; + private boolean intervalSent; private final Thread thread; FakeToy(int timeoutMs, boolean sendVideoParam) throws IOException { @@ -332,9 +335,7 @@ private void serve() { got.add(new Got(m)); switch (m.topic) { case Jieli.APP_ACCESS: - out.write(Jieli.encode(Jieli.KEEP_ALIVE_INTERVAL, - "{\"op\":\"NOTIFY\",\"param\":{\"timeout\":\"" - + timeoutMs + "\"}}")); + if (!lateInterval) interval(out); out.write(Jieli.encode("BAT_STATUS", "{\"op\":\"NOTIFY\",\"param\":{\"level\":\"4\"}}")); if (sendVideoParam) { @@ -346,6 +347,7 @@ private void serve() { break; case Jieli.KEEP_ALIVE: out.write(Jieli.encode(Jieli.KEEP_ALIVE, "")); + if (lateInterval && !intervalSent) interval(out); break; case Jieli.OPEN_RT_STREAM: out.write(Jieli.encode(Jieli.OPEN_RT_STREAM, @@ -367,6 +369,12 @@ private void serve() { } } + private void interval(OutputStream out) throws IOException { + intervalSent = true; + out.write(Jieli.encode(Jieli.KEEP_ALIVE_INTERVAL, + "{\"op\":\"NOTIFY\",\"param\":{\"timeout\":\"" + timeoutMs + "\"}}")); + } + private void sendVideo() throws IOException { try (DatagramSocket u = new DatagramSocket(0, LOOPBACK)) { InetSocketAddress to = new InetSocketAddress(LOOPBACK, rtpPort); @@ -513,6 +521,32 @@ public void defaultSizeAndCloseOnStop() throws Exception { } } + @Test + public void lateIntervalAppliesAtOnce() throws Exception { + int rtp = freeUdpPort(); + try (FakeToy toy = new FakeToy(2000, true)) { + toy.rtpPort = rtp; + toy.sendVideo = false; + toy.lateInterval = true; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp); + assertTrue(toy.opened.await(5, TimeUnit.SECONDS)); + long until = now() + 3000; + while (now() < until && Collections.frequency(toy.topics(), Jieli.KEEP_ALIVE) < 2) { + Thread.sleep(20); + } + r.keepGoing.set(false); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + List keepAlives = new ArrayList<>(); + for (Got g : toy.got) { + if (g.topic.equals(Jieli.KEEP_ALIVE)) keepAlives.add(g.at); + } + // The second one 500 ms after the first, not 7.5 s + assertTrue("keepalives: " + keepAlives.size(), keepAlives.size() >= 2); + long gap = keepAlives.get(1) - keepAlives.get(0); + assertTrue("gap " + gap, gap >= 400 && gap <= 900); + } + } + @Test public void noVideoListensElsewhereAndAsksAgain() throws Exception { int rtp = freeUdpPort(); From 3a04551726f0792f03ab9ccd53aa2556647c8a58 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 19:32:13 +0000 Subject: [PATCH 09/21] fix: /log keeps the app's lines and only Info and up from system libraries, so chatty graphics logs no longer push them out (#187) --- .../app/src/main/java/com/openipc/decoder/LogBuffer.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/LogBuffer.java b/android/app/src/main/java/com/openipc/decoder/LogBuffer.java index 89a08c5f..775acbf0 100644 --- a/android/app/src/main/java/com/openipc/decoder/LogBuffer.java +++ b/android/app/src/main/java/com/openipc/decoder/LogBuffer.java @@ -51,8 +51,12 @@ void startReader() { while (true) { Process p = null; try { - // No -d: dump what logcat still has, then keep following - p = new ProcessBuilder("logcat", "-v", "threadtime") + // No -d: dump what logcat still has, then keep following. Every line of the + // app's own tag, but only Info and up from the rest of the process: graphics + // libraries on some phones (mali_gralloc) log dozens of Debug lines a second + // and would push the app's lines out of the ring within seconds (#187) + p = new ProcessBuilder("logcat", "-v", "threadtime", + "OpenIPCDecoder:V", "*:I") .redirectErrorStream(true).start(); try (BufferedReader r = new BufferedReader( new InputStreamReader(p.getInputStream(), StandardCharsets.UTF_8))) { From b93fd4713cf71cbd759fa8980afd0538249ea4b1 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 19:55:53 +0000 Subject: [PATCH 10/21] =?UTF-8?q?feat:=20KY=20UFO=20toys=20(rtsp://?= =?UTF-8?q?=E2=80=A6:7070/webcam)=20show=20MJPEG=20turned=2090=C2=B0=20cou?= =?UTF-8?q?nter-clockwise,=20=3Frotate=3D=20to=20override=20(#200)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The K200's camera is mounted sideways. For :7070/webcam slots the software MJPEG path draws the frames turned 90° CCW and reports the turned size, so the view fits a landscape picture; ?rotate=0|90|180|270 overrides it for any RTSP MJPEG slot and is not sent to the camera. H.264/H.265 are not turned (#201). /stats.json gets rotate; screenshot and debug snapshot copy the turned surface. --- .../java/com/openipc/decoder/Decoder.java | 9 +- .../com/openipc/decoder/JpegRenderer.java | 60 ++++++++-- .../java/com/openipc/decoder/QuadCell.java | 5 +- .../java/com/openipc/decoder/RtspClient.java | 3 + .../java/com/openipc/decoder/RtspSession.java | 72 +++++++++++- .../java/com/openipc/decoder/RotateTest.java | 104 ++++++++++++++++++ docs/debug.md | 2 +- docs/toys.md | 2 + docs/transports.md | 2 +- 9 files changed, 243 insertions(+), 16 deletions(-) create mode 100644 android/app/src/test/java/com/openipc/decoder/RotateTest.java diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index b592638c..8fedbadc 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -1562,6 +1562,9 @@ private Map debugState() { if (jl != null) single.put("jieli", jl); Map hb = rtspClient != null ? rtspClient.heartbeatStats() : null; if (hb != null) single.put("heartbeat", hb); + // MJPEG turned counter-clockwise, "size" is the turned one (#200) + int rotate = rtspClient != null && rtspClient.isJpeg() ? rtspClient.rotate() : 0; + if (rotate != 0) single.put("rotate", rotate); m.put("single", single); } QuadCell[] cells = quadCells; @@ -1606,6 +1609,7 @@ private Map debugState() { if (jl != null) c.put("jieli", jl); Map hb = cell.heartbeatStats(); if (hb != null) c.put("heartbeat", hb); + if (cell.rotate() != 0) c.put("rotate", cell.rotate()); } list.add(c); } @@ -1975,7 +1979,10 @@ private int dp(float dp) { */ private boolean drawJpeg(Frame frame, int gen) { Surface s = mVideoSurface; - int[] size = jpegRenderer.render(frame, s, () -> gen == listenerGen && s == mVideoSurface); + RtspClient client = rtspClient; + int rotate = client != null ? client.rotate() : 0; + int[] size = jpegRenderer.render(frame, s, rotate, + () -> gen == listenerGen && s == mVideoSurface); if (size != null) jpegShown(size[0], size[1]); return size != null; } diff --git a/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java b/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java index 999c02c0..0149da83 100644 --- a/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java +++ b/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java @@ -13,6 +13,7 @@ import android.graphics.Color; import android.graphics.Paint; import android.graphics.Rect; +import android.graphics.RectF; import android.util.Log; import android.view.Surface; @@ -41,13 +42,16 @@ final class JpegRenderer { private Bitmap bitmap; /** - * Shows one frame on {@code s}. Returns its {width, height} when it is on screen, null when - * not (yet). {@code current} is false once the stream or the surface has been replaced. + * Shows one frame on {@code s}, turned {@code rotate} degrees counter-clockwise (0, 90, + * 180 or 270, #200). Returns its {width, height} as shown (turned) when it is on screen, + * null when not (yet). {@code current} is false once the stream or the surface has been + * replaced. */ - int[] render(Frame frame, Surface s, JpegCodec.Current current) { + int[] render(Frame frame, Surface s, int rotate, JpegCodec.Current current) { if (s == null || !s.isValid() || !current.holds()) return null; - // The hardware JPEG decoder first, where there is one (#132) - if (!hwFailed && !drawn && JpegCodec.candidate() != null) { + // The hardware JPEG decoder first, where there is one (#132); a turned frame is drawn + // by the app, which turns it while drawing (#200) + if (rotate == 0 && !hwFailed && !drawn && JpegCodec.candidate() != null) { int[] size = JpegAssembler.size(frame.data(), frame.length()); if (size != null) { try { @@ -83,7 +87,16 @@ int[] render(Frame frame, Surface s, JpegCodec.Current current) { Canvas c = s.lockCanvas(null); try { c.drawColor(Color.BLACK); - c.drawBitmap(b, null, new Rect(0, 0, c.getWidth(), c.getHeight()), paint); + if (rotate == 0) { + c.drawBitmap(b, null, new Rect(0, 0, c.getWidth(), c.getHeight()), paint); + } else { + float[] t = turn(rotate, c.getWidth(), c.getHeight()); + c.save(); + c.translate(t[0], t[1]); + c.rotate(t[2]); + c.drawBitmap(b, null, new RectF(t[3], t[4], t[5], t[6]), paint); + c.restore(); + } } finally { s.unlockCanvasAndPost(c); } @@ -92,7 +105,40 @@ int[] render(Frame frame, Surface s, JpegCodec.Current current) { return null; } drawn = true; - return new int[] {b.getWidth(), b.getHeight()}; + return shownSize(b.getWidth(), b.getHeight(), rotate); + } + + /** {width, height} of a {@code w}×{@code h} frame turned {@code rotate} degrees (#200). */ + static int[] shownSize(int w, int h, int rotate) { + return rotate == 90 || rotate == 270 ? new int[] {h, w} : new int[] {w, h}; + } + + /** + * How a frame turned {@code rotate} degrees counter-clockwise fills a {@code cw}×{@code ch} + * canvas (#200): translate by {t[0], t[1]} (the centre), rotate by t[2] degrees (Canvas + * turns clockwise, so negative), then draw the frame into the rectangle {t[3], t[4], t[5], + * t[6]} (left, top, right, bottom), which is the canvas with width and height swapped for + * a quarter turn. + */ + static float[] turn(int rotate, int cw, int ch) { + boolean quarter = rotate == 90 || rotate == 270; + float hw = (quarter ? ch : cw) / 2f; + float hh = (quarter ? cw : ch) / 2f; + return new float[] {cw / 2f, ch / 2f, -rotate, -hw, -hh, hw, hh}; + } + + /** + * Where the point ({@code u}, {@code v}) of the frame, as fractions of its width and height + * from the top left, lands on the canvas with {@link #turn}: the same transform as + * Canvas.translate() and rotate(), for tests without a bitmap. + */ + static float[] map(float[] t, float u, float v) { + float x = t[3] + u * (t[5] - t[3]); + float y = t[4] + v * (t[6] - t[4]); + double a = Math.toRadians(t[2]); + double cos = Math.cos(a); + double sin = Math.sin(a); + return new float[] {(float) (t[0] + x * cos - y * sin), (float) (t[1] + x * sin + y * cos)}; } /** diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index d1c49a2f..aa8b4ef7 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -437,6 +437,9 @@ static boolean isKeyFrameNal(int type, boolean h265) { /** The cooingdv heartbeat of an RTSP cell once it used one (#187), else null. */ java.util.Map heartbeatStats() { return session.heartbeatStats(); } + /** Degrees counter-clockwise the MJPEG frames are turned (#200), 0 for none. */ + int rotate() { return jpeg ? session.rotate() : 0; } + /** Transport in use, with the mode after a fallback (#112). */ String transportLabel() { return Transport.label(session.transportInUse(), transport); } @@ -730,7 +733,7 @@ private void decodeJpeg(Frame frame) { return; } Surface s = surface; - int[] size = jpegRenderer.render(frame, s, + int[] size = jpegRenderer.render(frame, s, session.rotate(), () -> running && jpeg && s == surface); if (size == null) return; lastJpegDecodeMs = now; diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java index 940a6010..50eb8153 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java @@ -94,6 +94,9 @@ void configure(String host, String userAgent, Transport.Mode transport) { /** The cooingdv heartbeat once this slot used one (#187), else null. */ java.util.Map heartbeatStats() { return session.heartbeatStats(); } + /** Degrees counter-clockwise the MJPEG frames are turned (#200), 0 for none. */ + int rotate() { return session.rotate(); } + void start() { nalAssembler.reset(); jpegAssembler.reset(); diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index fa7c55e6..c816ccf9 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -116,6 +116,10 @@ default void onDetail(String text) {} private volatile int heartbeatErrorsBefore; private volatile boolean heartbeatUsed; private volatile boolean heartbeatLogged; + /** Degrees counter-clockwise the MJPEG frames of this connection are turned (#200). */ + private volatile int rotate; + private volatile boolean rotateLogged; + private volatile boolean rotateIgnoredLogged; private Thread networkThread; private Thread watchdogThread; @@ -137,6 +141,11 @@ void configure(String url, String userAgent, boolean withAudio) { } void configure(String url, String userAgent, boolean withAudio, Transport.Mode mode) { + if (!java.util.Objects.equals(url, this.url)) { + // Another camera: its rotation is logged again (#200) + rotateLogged = false; + rotateIgnoredLogged = false; + } this.url = url; this.userAgent = userAgent != null ? userAgent : ""; this.withAudio = withAudio; @@ -311,6 +320,8 @@ private void watchdogLoop(int gen) { /** One session from connect to disconnect. Returns true if PLAY succeeded. */ private boolean connect(int gen) throws IOException { + // Only an RTSP MJPEG stream is turned (#200); the other transports draw it as it comes + rotate = 0; if (RtpListener.Url.matches(url)) return listen(gen, RtpListener.Url.parse(url)); if (HttpMjpeg.matches(url)) return httpMjpeg(gen, HttpMjpeg.Url.parse(url)); if (FlowUfo.Url.matches(url)) return flowUfo(gen, FlowUfo.Url.parse(url)); @@ -468,6 +479,7 @@ private boolean connect(int gen) throws IOException { } if (gen != generation) return false; + rotate = rotation(u.rotate, sdp.videoJpeg); callback.onSdp(sdp); // PLAY on the request URL, as before: accepted by the servers seen so far @@ -651,6 +663,30 @@ private void stopHeartbeat(UdpHeartbeat h) { } } + /** + * The turn for frames of this connection: {@code wanted} for MJPEG, 0 for H.264/H.265, + * which have no rotation yet (#201). Logged once each. + */ + private int rotation(int wanted, boolean jpeg) { + if (wanted == 0) return 0; + if (!jpeg) { + if (!rotateIgnoredLogged) { + rotateIgnoredLogged = true; + Log.i(TAG, tag + ": rotate= only applies to MJPEG streams for now (#201)"); + } + return 0; + } + if (!rotateLogged) { + rotateLogged = true; + Log.i(TAG, tag + ": MJPEG frames turned " + wanted + "\u00b0 counter-clockwise" + + " (#200; ?rotate=0 turns it off)"); + } + return wanted; + } + + /** Degrees counter-clockwise the MJPEG frames are turned now (#200), 0 for none. */ + int rotate() { return rotate; } + /** Heartbeat statistics for /stats.json once an RTSP slot used one (#187), else null. */ Map heartbeatStats() { if (!heartbeatUsed) return null; @@ -1362,6 +1398,12 @@ static final class Url { * option is ours: it is taken out of the URL sent to the camera. */ int heartbeatPort; + /** + * Degrees counter-clockwise to turn MJPEG frames when drawing them (#200): 90 for + * rtsp://{host}:7070/webcam, whose toy camera is mounted sideways, else 0; + * ?rotate=0|90|180|270 overrides it. Ours as well: not sent to the camera. + */ + int rotate; static Url parse(String url) throws IOException { if (url == null) throw new IOException("Camera slot not configured"); @@ -1408,18 +1450,23 @@ static Url parse(String url) throws IOException { u.host = hostPort; } if (u.host.isEmpty()) throw new IOException("Invalid RTSP URL: host is missing"); - rest = takeHeartbeat(u, rest); + rest = takeOptions(u, rest); u.requestUrl = scheme + "://" + authority + rest; return u; } - /** Sets heartbeatPort from path and query; returns them without our option. */ - private static String takeHeartbeat(Url u, String rest) throws IOException { + /** + * Sets heartbeatPort and rotate from path and query; returns them without our + * options. + */ + private static String takeOptions(Url u, String rest) throws IOException { int q = rest.indexOf('?'); String path = q >= 0 ? rest.substring(0, q) : rest; if (u.port == 7070 && (path.equalsIgnoreCase("/webcam") || path.equalsIgnoreCase("/webcam/"))) { + // The cooingdv toys (#187): heartbeat, and the K200's sideways camera (#200) u.heartbeatPort = UdpHeartbeat.DEFAULT_PORT; + u.rotate = 90; } if (q < 0) return rest; StringBuilder kept = new StringBuilder(); @@ -1427,14 +1474,16 @@ private static String takeHeartbeat(Url u, String rest) throws IOException { for (String param : rest.substring(q + 1).split("&", -1)) { int eq = param.indexOf('='); String name = (eq < 0 ? param : param.substring(0, eq)).toLowerCase(Locale.ROOT); - if (!name.equals("heartbeat")) { + if (!name.equals("heartbeat") && !name.equals("rotate")) { if (kept.length() > 0) kept.append('&'); kept.append(param); continue; } found = true; String v = eq < 0 ? "" : param.substring(eq + 1).trim().toLowerCase(Locale.ROOT); - if (v.equals("off") || v.equals("0")) { + if (name.equals("rotate")) { + u.rotate = parseRotate(v); + } else if (v.equals("off") || v.equals("0")) { u.heartbeatPort = 0; } else { try { @@ -1453,6 +1502,19 @@ private static String takeHeartbeat(Url u, String rest) throws IOException { return kept.length() > 0 ? path + "?" + kept : path; } + /** ?rotate= in degrees counter-clockwise: 0, 90, 180 or 270 (#200). */ + static int parseRotate(String v) throws IOException { + switch (v) { + case "0": return 0; + case "90": return 90; + case "180": return 180; + case "270": return 270; + default: + throw new IOException("Invalid rotate: " + v + + " (degrees counter-clockwise: rotate=0, 90, 180 or 270)"); + } + } + private static int parsePort(String s, String url) throws IOException { if (s.isEmpty()) return 554; try { diff --git a/android/app/src/test/java/com/openipc/decoder/RotateTest.java b/android/app/src/test/java/com/openipc/decoder/RotateTest.java new file mode 100644 index 00000000..9db4d6c7 --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/RotateTest.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * RotateTest.java — MJPEG frames turned for the sideways KY UFO camera (#200) + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import org.junit.Test; + +import java.io.IOException; + +public class RotateTest { + private static final float EPS = 0.01f; + + @Test + public void ninetyForTheCooingdvWebcamPathOnly() throws IOException { + assertEquals(90, RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam").rotate); + assertEquals(90, RtspSession.Url.parse("rtsp://10.0.0.9:7070/WEBCAM/").rotate); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1:554/webcam").rotate); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1:7070/live").rotate); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.10:554/stream=0").rotate); + } + + @Test + public void optionOverridesAndIsNotSentToTheCamera() throws IOException { + RtspSession.Url u = RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam?rotate=0"); + assertEquals(0, u.rotate); + assertEquals(7099, u.heartbeatPort); + assertEquals("rtsp://192.168.1.1:7070/webcam", u.requestUrl); + u = RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam?Rotate=270&heartbeat=off"); + assertEquals(270, u.rotate); + assertEquals(0, u.heartbeatPort); + assertEquals("rtsp://192.168.1.1:7070/webcam", u.requestUrl); + // Any camera, other parameters kept in their order + u = RtspSession.Url.parse("rtsp://u:p@cam/x?a=1&rotate=180&b=2"); + assertEquals(180, u.rotate); + assertEquals("rtsp://cam/x?a=1&b=2", u.requestUrl); + u = RtspSession.Url.parse("rtsp://cam/x?rotate=90"); + assertEquals(90, u.rotate); + assertEquals("rtsp://cam/x", u.requestUrl); + for (String bad : new String[] {"rtsp://cam/x?rotate=45", "rtsp://cam/x?rotate=", + "rtsp://cam/x?rotate", "rtsp://cam/x?rotate=-90", "rtsp://cam/x?rotate=360", + "rtsp://cam/x?rotate=left"}) { + try { + RtspSession.Url.parse(bad); + fail(bad); + } catch (IOException expected) { + // ok + } + } + } + + @Test + public void quarterTurnSwapsTheShownSize() { + assertArrayEquals(new int[] {640, 480}, JpegRenderer.shownSize(480, 640, 90)); + assertArrayEquals(new int[] {640, 480}, JpegRenderer.shownSize(480, 640, 270)); + assertArrayEquals(new int[] {480, 640}, JpegRenderer.shownSize(480, 640, 180)); + assertArrayEquals(new int[] {480, 640}, JpegRenderer.shownSize(480, 640, 0)); + } + + @Test + public void turnedPortraitSourceFillsALandscapeScreen() { + // The K200's 480x640 on a 1600x720 phone: 960x720, not a 540x720 strip + int[] shown = JpegRenderer.shownSize(480, 640, 90); + assertArrayEquals(new int[] {960, 720}, Decoder.fitInside(shown[0], shown[1], 1600, 720)); + assertArrayEquals(new int[] {540, 720}, Decoder.fitInside(480, 640, 1600, 720)); + } + + @Test + public void ninetyCounterClockwiseMovesTheCorners() { + // A 960x720 surface; the frame's top left goes to the bottom left, its top right to + // the top left: the top edge becomes the left one + float[] t = JpegRenderer.turn(90, 960, 720); + assertPoint(0, 720, JpegRenderer.map(t, 0, 0)); + assertPoint(0, 0, JpegRenderer.map(t, 1, 0)); + assertPoint(960, 720, JpegRenderer.map(t, 0, 1)); + assertPoint(960, 0, JpegRenderer.map(t, 1, 1)); + assertPoint(480, 360, JpegRenderer.map(t, 0.5f, 0.5f)); + } + + @Test + public void otherTurnsFillTheSurfaceToo() { + float[] t = JpegRenderer.turn(270, 960, 720); + assertPoint(960, 0, JpegRenderer.map(t, 0, 0)); + assertPoint(0, 720, JpegRenderer.map(t, 1, 1)); + t = JpegRenderer.turn(180, 960, 720); + assertPoint(960, 720, JpegRenderer.map(t, 0, 0)); + assertPoint(0, 0, JpegRenderer.map(t, 1, 1)); + t = JpegRenderer.turn(0, 960, 720); + assertPoint(0, 0, JpegRenderer.map(t, 0, 0)); + assertPoint(960, 720, JpegRenderer.map(t, 1, 1)); + } + + private static void assertPoint(float x, float y, float[] p) { + assertEquals(x, p[0], EPS); + assertEquals(y, p[1], EPS); + } +} diff --git a/docs/debug.md b/docs/debug.md index 27fb7b78..703edc4d 100644 --- a/docs/debug.md +++ b/docs/debug.md @@ -168,7 +168,7 @@ Single view and U view: every cell of the grid as the debug page shows it (3×3 | `codec`, `size` | when found | | `fps`, `kbps`, `key_frames` | measured during the check | -The FLOW-UFO test build ([toys.md](toys.md#flow-ufo)) adds a `flowufo` object per cell and in `single` (`jieli` for `jieli://` slots, [toys.md](toys.md#hf--jieli-toys)), and a `heartbeat` object for RTSP slots that send the KY UFO heartbeat ([toys.md](toys.md#ky-ufo-rtsp-toys)); it is not in the releases. +The FLOW-UFO test build ([toys.md](toys.md#flow-ufo)) adds a `flowufo` object per cell and in `single` (`jieli` for `jieli://` slots, [toys.md](toys.md#hf--jieli-toys)), and a `heartbeat` object for RTSP slots that send the KY UFO heartbeat, and `rotate` (degrees counter-clockwise) where MJPEG frames are turned ([toys.md](toys.md#ky-ufo-rtsp-toys)); it is not in the releases. ## Log diff --git a/docs/toys.md b/docs/toys.md index 8ea89c50..c7b278aa 100644 --- a/docs/toys.md +++ b/docs/toys.md @@ -89,6 +89,8 @@ rtsp://192.168.1.1:7070/webcam Their apps send a heartbeat, the two bytes `01 01`, to UDP port 7099 of the toy every second while the video plays. For an RTSP slot with port 7070 and path `/webcam` (any host) the app does the same: from the address of the RTSP connection, once a second, until the slot stops. Nothing else is sent to port 7099 (no flight command, no "stop" command). Other RTSP slots send no heartbeat unless asked with `?heartbeat=`, e.g. `rtsp://192.168.1.1:554/live?heartbeat=7099`; `?heartbeat=off` turns it off for a `:7070/webcam` slot. The option is the app's own and is not sent to the camera. The log has one line when it starts (*heartbeat 01 01 every 1000 ms to …*), and `/stats.json` a `heartbeat` object (`target`, `running`, `sent`) for such a slot. +The camera of some of these toys (the K200) is mounted sideways: its frames come turned, a mug lies on its side as a portrait strip in a landscape screen. For a `:7070/webcam` slot the app therefore turns the MJPEG frames 90° counter-clockwise when drawing them ([#200](https://github.com/OpenIPC/decoder/issues/200)), and fits the turned frame (a 480×640 frame shows as 640×480 and fills a landscape screen). `?rotate=0|90|180|270` (degrees counter-clockwise) overrides it, e.g. `rtsp://192.168.1.1:7070/webcam?rotate=0` for a unit mounted upright, and also turns other RTSP MJPEG slots; any other value is an error. The option is the app's own and is not sent to the camera. It turns MJPEG only: an H.264/H.265 stream is shown as it comes and the log says *rotate= only applies to MJPEG streams for now (#201)*. The turn is in the picture itself, so zoom, the long-press screenshot and the debug snapshot show it turned too. The log has one line when it is active (*MJPEG frames turned 90° counter-clockwise*), and `/stats.json` a `rotate` (degrees) for such a slot; its `size` is the turned one. + Some of these toys use the FLOW-UFO protocol instead (the KY UFO app has both); if RTSP shows nothing, try `flowufo://192.168.1.1`. ## HF / Jieli toys diff --git a/docs/transports.md b/docs/transports.md index 375d8576..5187e26a 100644 --- a/docs/transports.md +++ b/docs/transports.md @@ -104,7 +104,7 @@ For example Majestic's `http://user:password@/mjpeg`: a `multipart/x-mix `jieli://192.168.80.1` (or `jieli://` alone) plays toy drones with an `HF_xxxxxxxx` Wi‑Fi (Jieli AC79 camera): CTP commands on TCP 3333 ask for the stream, which is expected as RTP/JPEG on the phone's UDP port 6666 (test build only, video path unverified). Options: `?w=`, `?h=`, `?fps=`, `?rtp=`, `?format=0|1`. See [toys.md](toys.md#hf--jieli-toys). -RTSP slots on `rtsp://:7070/webcam` (KY UFO and other cooingdv toys) also send the toy's UDP heartbeat `01 01` to port 7099 once a second; `?heartbeat=` turns it on for other RTSP URLs, `?heartbeat=off` off. See [toys.md](toys.md#ky-ufo-rtsp-toys). +RTSP slots on `rtsp://:7070/webcam` (KY UFO and other cooingdv toys) also send the toy's UDP heartbeat `01 01` to port 7099 once a second; `?heartbeat=` turns it on for other RTSP URLs, `?heartbeat=off` off. Their MJPEG frames are drawn turned 90° counter-clockwise (the K200's camera is mounted sideways, [#200](https://github.com/OpenIPC/decoder/issues/200)); `?rotate=0|90|180|270` (degrees counter-clockwise, any RTSP MJPEG slot) overrides it and is not sent to the camera either. See [toys.md](toys.md#ky-ufo-rtsp-toys). ## Codecs From 05f309367cc0a70bb5498bb3c4a12db8d4722b93 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Thu, 15 Oct 2026 20:36:54 +0000 Subject: [PATCH 11/21] fix: a reconfigured RTSP session drops the previous camera's heartbeat stats (Qodo on #188, #200) --- .../main/java/com/openipc/decoder/RtspSession.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index c816ccf9..dff7c5ea 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -142,9 +142,18 @@ void configure(String url, String userAgent, boolean withAudio) { void configure(String url, String userAgent, boolean withAudio, Transport.Mode mode) { if (!java.util.Objects.equals(url, this.url)) { - // Another camera: its rotation is logged again (#200) + // Another camera: its rotation is logged again (#200), and the heartbeat counts + // of the previous one leave /stats.json (Qodo on #188). A heartbeat still running + // is closed by its connection, without adding to the new totals. rotateLogged = false; rotateIgnoredLogged = false; + synchronized (lock) { + heartbeat = null; + heartbeatUsed = false; + heartbeatsBefore = 0; + heartbeatErrorsBefore = 0; + heartbeatLogged = false; + } } this.url = url; this.userAgent = userAgent != null ? userAgent : ""; From 4e4c483784c8b24c28ec31cf5e602e94f40f0d12 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 13:07:43 +0000 Subject: [PATCH 12/21] =?UTF-8?q?fix:=20KY=20UFO=20toys=20turned=20270?= =?UTF-8?q?=C2=B0=20by=20default;=20MJPEG=20shows=20the=20newest=20frame?= =?UTF-8?q?=20when=20drawing=20falls=20behind=20(#187,=20#200)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/src/main/java/com/openipc/decoder/Decoder.java | 8 ++++++++ .../src/main/java/com/openipc/decoder/RtspSession.java | 4 ++-- .../app/src/test/java/com/openipc/decoder/RotateTest.java | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index 8fedbadc..ef92916b 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -2045,6 +2045,14 @@ private void startListener() { try { Frame buffer = nalQueue.poll(5, TimeUnit.MILLISECONDS); if (buffer != null && rtspClient.isJpeg()) { + // Every JPEG frame is whole: when drawing falls behind (a turned frame + // is drawn in software, #187), show the newest and drop the older ones + // instead of letting the delay grow + Frame newer; + while ((newer = nalQueue.poll()) != null) { + framePool.recycle(buffer); + buffer = newer; + } // S: MJPEG has no key frames, one frame per second instead (#153); the // second counts from a frame really shown (Qodo on #155) long now = System.nanoTime() / 1_000_000; diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index dff7c5ea..b5793dd2 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -1408,7 +1408,7 @@ static final class Url { */ int heartbeatPort; /** - * Degrees counter-clockwise to turn MJPEG frames when drawing them (#200): 90 for + * Degrees counter-clockwise to turn MJPEG frames when drawing them (#200): 270 for * rtsp://{host}:7070/webcam, whose toy camera is mounted sideways, else 0; * ?rotate=0|90|180|270 overrides it. Ours as well: not sent to the camera. */ @@ -1475,7 +1475,7 @@ private static String takeOptions(Url u, String rest) throws IOException { || path.equalsIgnoreCase("/webcam/"))) { // The cooingdv toys (#187): heartbeat, and the K200's sideways camera (#200) u.heartbeatPort = UdpHeartbeat.DEFAULT_PORT; - u.rotate = 90; + u.rotate = 270; } if (q < 0) return rest; StringBuilder kept = new StringBuilder(); diff --git a/android/app/src/test/java/com/openipc/decoder/RotateTest.java b/android/app/src/test/java/com/openipc/decoder/RotateTest.java index 9db4d6c7..b84d9998 100644 --- a/android/app/src/test/java/com/openipc/decoder/RotateTest.java +++ b/android/app/src/test/java/com/openipc/decoder/RotateTest.java @@ -20,8 +20,8 @@ public class RotateTest { @Test public void ninetyForTheCooingdvWebcamPathOnly() throws IOException { - assertEquals(90, RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam").rotate); - assertEquals(90, RtspSession.Url.parse("rtsp://10.0.0.9:7070/WEBCAM/").rotate); + assertEquals(270, RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam").rotate); + assertEquals(270, RtspSession.Url.parse("rtsp://10.0.0.9:7070/WEBCAM/").rotate); assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1:554/webcam").rotate); assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1:7070/live").rotate); assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.10:554/stream=0").rotate); From 3696475c7c826b2f092a03d0615212d0ad5fa958 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 13:07:43 +0000 Subject: [PATCH 13/21] feat: Jieli toys play the JPEG stream on UDP 2224 (YL3, #187) --- .../main/java/com/openipc/decoder/Jieli.java | 201 ++++++++++++++++-- .../java/com/openipc/decoder/JieliTest.java | 129 ++++++++++- docs/toys.md | 13 +- docs/transports.md | 4 +- 4 files changed, 318 insertions(+), 29 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/Jieli.java b/android/app/src/main/java/com/openipc/decoder/Jieli.java index 4fa128ba..b7fe40d2 100644 --- a/android/app/src/main/java/com/openipc/decoder/Jieli.java +++ b/android/app/src/main/java/com/openipc/decoder/Jieli.java @@ -39,9 +39,10 @@ * OPEN_RT_STREAM; CLOSE_RT_STREAM at the end. The video itself was not in the capture: after * turbodrone's research on the same SDK (marshallrichards/turbodrone, Apache-2.0, used as a * description only) it is RTP/JPEG (payload type 26) or H.264 (96) to the phone's UDP port - * 6666. The packets go into the app's normal RTP paths. Unverified, so the log shows what - * really arrives on 6666, and on the audio and stream ports 1234 and 2224 too when 6666 stays - * silent. Video only: FLIGHT_CONTROL is never sent, nothing goes to the toy's UDP port 3456. + * 6666. The packets go into the app's normal RTP paths. A YL3 sent nothing there but its own + * stream to UDP 2224 instead (JPEG behind a 20-byte header, see {@link RtStream}): both ports + * are played. When neither has video, the log shows what arrives on the audio port 1234. + * Video only: FLIGHT_CONTROL is never sent, nothing goes to the toy's UDP port 3456. * Pure Java apart from the log. */ final class Jieli { @@ -51,6 +52,10 @@ final class Jieli { static final String DEFAULT_HOST = "192.168.80.1"; static final int CTP_PORT = 3333; static final int RTP_PORT = 6666; + /** The toy's own stream (#187): a YL3 sends its JPEG frames here, not as RTP to 6666. */ + static final int STREAM_PORT = 2224; + /** The port listened on for that stream; STREAM_PORT but for the tests. */ + static volatile int streamPort = STREAM_PORT; static final int DEFAULT_WIDTH = 640; static final int DEFAULT_HEIGHT = 480; static final int DEFAULT_FPS = 25; @@ -90,10 +95,10 @@ final class Jieli { /** The toy says nothing on CTP for this long (at least): the connection is given up. */ static volatile long ctpSilenceMs = 20_000; /** - * Ports opened only to log what arrives when 6666 stays silent: 1234 (audio in the - * vendor SDK) and 2224 (its RT stream port). Nothing received there is played. + * Ports opened only to log what arrives when the video ports stay silent: 1234 (audio in + * the vendor SDK). Nothing received there is played. */ - static volatile int[] diagnosticPorts = {1234, 2224}; + static volatile int[] diagnosticPorts = {1234}; private Jieli() {} @@ -557,6 +562,86 @@ static String subnetHint(InetAddress target, List local) { return parts.length() > 0 ? parts.toString() : null; } + // ====================== RT stream (UDP 2224) ====================== + + /** + * The toy's own video stream on UDP 2224 (#187), as a YL3 sent it: a 20-byte header, + * then (a piece of) a JPEG. Header as far as seen, little-endian: byte 0 the type (2 for + * JPEG), bytes 4–7 a packet counter, 8–11 the length of the JPEG, 12–15 the offset of + * this piece in it. A 640×480 frame fitted one datagram (4600 bytes, 20 more in the + * datagram); longer ones are put together by offset. Pure Java, for the unit tests. + */ + static final class RtStream { + static final int HEADER = 20; + static final int TYPE_JPEG = 2; + static final int MAX_FRAME = 1 << 20; + + private byte[] frame = new byte[64 * 1024]; + private int size = -1; + private int filled; + int frames; + int dropped; + int otherTypes; + + /** + * Takes one datagram; returns the length of a JPEG now complete in {@link #frame()} + * (from 0, FFD8 to its last FFD9), or 0. + */ + int feed(byte[] b, int off, int len) { + if (len <= HEADER) return 0; + if ((b[off] & 0xFF) != TYPE_JPEG) { + otherTypes++; + return 0; + } + int total = le32(b, off + 8); + int at = le32(b, off + 12); + if (total < 4 || total > MAX_FRAME || at < 0 || at >= total) { + dropped++; + return 0; + } + if (at == 0) { + // A new frame; one still missing pieces is lost + if (size > 0) dropped++; + size = total; + filled = 0; + if (frame.length < total) frame = new byte[total]; + } else if (size != total) { + return 0; // a piece of a frame whose start was lost + } + int piece = Math.min(len - HEADER, total - at); + System.arraycopy(b, off + HEADER, frame, at, piece); + filled += piece; + if (filled < size) return 0; + int n = size; + size = -1; + int end = jpegEnd(frame, n); + if (end == 0) { + dropped++; + return 0; + } + frames++; + return end; + } + + byte[] frame() { + return frame; + } + + /** Length up to the last FFD9 when {@code b} starts with FFD8, else 0. */ + static int jpegEnd(byte[] b, int n) { + if (n < 4 || (b[0] & 0xFF) != 0xFF || (b[1] & 0xFF) != 0xD8) return 0; + for (int end = n; end >= 4; end--) { + if ((b[end - 2] & 0xFF) == 0xFF && (b[end - 1] & 0xFF) == 0xD9) return end; + } + return 0; + } + + static int le32(byte[] b, int at) { + return (b[at] & 0xFF) | (b[at + 1] & 0xFF) << 8 | (b[at + 2] & 0xFF) << 16 + | (b[at + 3] & 0xFF) << 24; + } + } + // ====================== Client ====================== interface Listener { @@ -627,6 +712,13 @@ static final class Client implements Closeable { private volatile int rawJpegs; private volatile int foreign; private volatile String firstPacket; + /** UDP port the video came from last: url.rtpPort or streamPort. */ + private volatile int videoPort; + private final RtStream rtStream = new RtStream(); + private volatile int streamPackets; + private volatile int streamForeign; + private volatile String streamFirst; + private int droppedLogged; private volatile String lastSource; private volatile boolean h264Logged; private final Map payloadTypes = new LinkedHashMap<>(); @@ -675,16 +767,21 @@ String connectingText() { } String requestedText() { - return withHint("Stream requested, waiting for video on UDP " + url.rtpPort); + return withHint("Stream requested, waiting for video on UDP " + videoPorts()); } String noVideoText() { - return withHint("No video on UDP " + url.rtpPort + " yet: the toy accepted the" + return withHint("No video on UDP " + videoPorts() + " yet: the toy accepted the" + " request; see /log"); } String quietText() { - return withHint("The toy's video stopped: waiting on UDP " + url.rtpPort); + return withHint("The toy's video stopped: waiting on UDP " + videoPorts()); + } + + private String videoPorts() { + return url.rtpPort == streamPort ? String.valueOf(streamPort) + : url.rtpPort + " and " + streamPort; } void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException { @@ -698,7 +795,15 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException try { // The video port first: nothing the toy sends after OPEN_RT_STREAM is lost DatagramSocket video = openUdp(url.rtpPort, a); - startThread(video, true, listener); + startThread(video, PORT_RTP, listener); + if (url.rtpPort != streamPort) { + try { + startThread(openUdp(streamPort, a), PORT_STREAM, listener); + } catch (IOException e) { + Log.w(TAG, tag + ": Jieli: cannot listen on UDP " + streamPort + ": " + + e.getMessage()); + } + } state = "connecting"; listener.onState(false, connectingText()); if (tcpSetup != null) networkTcp = tcpSetup.prepare(s, a); @@ -707,7 +812,7 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException if (closed) return; } Log.i(TAG, tag + ": Jieli: connecting to CTP " + url.endpoint() + " (network: " - + networkTcp + "), video on UDP " + url.rtpPort + " (network: " + + networkTcp + "), video on UDP " + videoPorts() + " (network: " + networkUdp + ")"); try { s.connect(new InetSocketAddress(a, url.port), CONNECT_TIMEOUT_MS); @@ -760,10 +865,10 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException if (flowing && !receiving) { receiving = true; state = "playing"; - Log.i(TAG, tag + ": Jieli: video arrives on UDP " + url.rtpPort + Log.i(TAG, tag + ": Jieli: video arrives on UDP " + videoPort + " from " + lastSource); listener.onState(true, "Receiving from the toy on UDP " - + url.rtpPort + "…"); + + videoPort + "…"); } else if (!flowing && receiving) { receiving = false; state = "quiet"; @@ -773,14 +878,14 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException } if (!diagnostics && last == 0 && t - openAt >= noVideoMs) { diagnostics = true; - Log.w(TAG, tag + ": Jieli: no video on UDP " + url.rtpPort + " " + Log.w(TAG, tag + ": Jieli: no video on UDP " + videoPorts() + " " + noVideoMs + " ms after OPEN_RT_STREAM (" + summary() + "); listening on " + ports(diagnosticPorts) + " too, for the log only"); for (int p : diagnosticPorts) { - if (p == url.rtpPort) continue; + if (p == url.rtpPort || p == streamPort) continue; try { - startThread(openUdp(p, a), false, listener); + startThread(openUdp(p, a), PORT_DIAGNOSTIC, listener); } catch (IOException e) { Log.w(TAG, tag + ": Jieli: cannot listen on UDP " + p + ": " + e.getMessage()); @@ -864,7 +969,11 @@ private DatagramSocket openUdp(int port, InetAddress target) throws IOException return d; } - private void startThread(DatagramSocket d, boolean video, Listener listener) { + private static final int PORT_DIAGNOSTIC = 0; + private static final int PORT_RTP = 1; + private static final int PORT_STREAM = 2; + + private void startThread(DatagramSocket d, int kind, Listener listener) { int port = d.getLocalPort(); Thread th = new Thread(() -> { byte[] b = new byte[65536]; @@ -873,8 +982,10 @@ private void startThread(DatagramSocket d, boolean video, Listener listener) { while (!closed && !d.isClosed()) { p.setLength(b.length); d.receive(p); - if (video) { + if (kind == PORT_RTP) { video(p, listener); + } else if (kind == PORT_STREAM) { + stream(p, listener); } else { diagnostic(port, p); } @@ -1009,12 +1120,14 @@ private void video(DatagramPacket p, Listener listener) { h264Logged = true; Log.i(TAG, tag + ": Jieli: RTP payload type 96: played as H.264"); } + videoPort = url.rtpPort; lastVideoAt = now(); listener.onRtp(b, 0, len); } else if (!rtp && len >= 4 && (b[0] & 0xFF) == 0xFF && (b[1] & 0xFF) == 0xD8 && (b[len - 2] & 0xFF) == 0xFF && (b[len - 1] & 0xFF) == 0xD9) { // A whole JPEG without RTP: turbodrone takes those too rawJpegs++; + videoPort = url.rtpPort; lastVideoAt = now(); listener.onJpeg(b, 0, len); } else if (rtp) { @@ -1027,6 +1140,42 @@ private void video(DatagramPacket p, Listener listener) { } } + /** One datagram of the toy's own stream on UDP 2224 (see {@link RtStream}). */ + private void stream(DatagramPacket p, Listener listener) { + byte[] b = p.getData(); + int len = p.getLength(); + int n = ++streamPackets; + String from = p.getAddress().getHostAddress() + ":" + p.getPort(); + boolean toy = p.getAddress().equals(address); + if (n <= 5 || n % 1000 == 0) { + // The whole header and the end: the format is only partly known (#187) + String head = FlowUfo.hex(b, 0, Math.min(len, RtStream.HEADER + 4)); + String tail = FlowUfo.hex(b, Math.max(0, len - 8), Math.min(len, 8)); + Log.i(TAG, tag + ": Jieli: UDP " + streamPort + " packet " + n + " from " + from + + (toy ? "" : " (not the toy: ignored)") + ", " + len + " bytes: " + head + + " … " + tail + (len > RtStream.HEADER + 12 ? " (type " + (b[0] & 0xFF) + + ", counter " + RtStream.le32(b, 4) + ", length " + RtStream.le32(b, 8) + + ", offset " + RtStream.le32(b, 12) + ")" : "")); + } + if (streamFirst == null) streamFirst = from + ", " + len + " bytes: " + + FlowUfo.hex(b, 0, Math.min(len, 32)); + if (!toy) { + streamForeign++; + return; + } + int jpeg = rtStream.feed(b, 0, len); + if (jpeg > 0) { + lastSource = from; + videoPort = streamPort; + lastVideoAt = now(); + listener.onJpeg(rtStream.frame(), 0, jpeg); + } else if (rtStream.dropped != droppedLogged && rtStream.dropped <= 3) { + droppedLogged = rtStream.dropped; + Log.w(TAG, tag + ": Jieli: UDP " + streamPort + ": " + droppedLogged + + " frame(s) not put together (length or JPEG markers do not fit)"); + } + } + private void diagnostic(int port, DatagramPacket p) { int count; synchronized (diagnostic) { @@ -1053,8 +1202,10 @@ private void diagnostic(int port, DatagramPacket p) { private String summary() { return String.format(Locale.ROOT, "CTP %d sent, %d received, %d keepalives;" + " UDP %d: %d packets, %d RTP video, %d frames, %d not RTP," - + " %d from others", ctpSent, ctpReceived, keepAlives, url.rtpPort, - packets, rtpPackets, rtpFrames, nonRtp, foreign); + + " %d from others; UDP %d: %d packets, %d frames, %d dropped", + ctpSent, ctpReceived, keepAlives, url.rtpPort, packets, rtpPackets, + rtpFrames, nonRtp, foreign, streamPort, streamPackets, rtStream.frames, + rtStream.dropped); } /** For /stats.json. */ @@ -1100,6 +1251,16 @@ Map stats() { } if (firstPacket != null) rtp.put("first_packet", firstPacket); m.put("rtp", rtp); + Map stream = new LinkedHashMap<>(); + stream.put("port", streamPort); + stream.put("packets", streamPackets); + stream.put("frames", rtStream.frames); + stream.put("dropped", rtStream.dropped); + if (rtStream.otherTypes > 0) stream.put("other_types", rtStream.otherTypes); + if (streamForeign > 0) stream.put("from_others", streamForeign); + if (streamFirst != null) stream.put("first_packet", streamFirst); + m.put("stream", stream); + if (videoPort > 0) m.put("video_port", videoPort); synchronized (sources) { m.put("sources", new LinkedHashMap<>(sources)); } diff --git a/android/app/src/test/java/com/openipc/decoder/JieliTest.java b/android/app/src/test/java/com/openipc/decoder/JieliTest.java index 0474451a..2806c6e4 100644 --- a/android/app/src/test/java/com/openipc/decoder/JieliTest.java +++ b/android/app/src/test/java/com/openipc/decoder/JieliTest.java @@ -46,6 +46,7 @@ public class JieliTest { public void shortTimers() { Jieli.paramWaitMs = 300; Jieli.diagnosticPorts = new int[0]; + Jieli.streamPort = freeUdpPortOrZero(); } @After @@ -55,7 +56,100 @@ public void restoreTimers() { Jieli.reopenMs = 10_000; Jieli.quietMs = 3000; Jieli.ctpSilenceMs = 20_000; - Jieli.diagnosticPorts = new int[] {1234, 2224}; + Jieli.diagnosticPorts = new int[] {1234}; + Jieli.streamPort = Jieli.STREAM_PORT; + } + + private static int freeUdpPortOrZero() { + try (DatagramSocket d = new DatagramSocket(0, LOOPBACK)) { + return d.getLocalPort(); + } catch (IOException e) { + return 0; + } + } + + /** A datagram of the toy's stream on UDP 2224 as a YL3 sent it (#187). */ + private static byte[] streamPacket(int counter, int total, int offset, byte[] piece, int tail) { + byte[] b = new byte[Jieli.RtStream.HEADER + piece.length + tail]; + b[0] = 2; + b[2] = (byte) 0x94; + b[3] = 0x0B; + le32(b, 4, counter); + le32(b, 8, total); + le32(b, 12, offset); + System.arraycopy(piece, 0, b, Jieli.RtStream.HEADER, piece.length); + return b; + } + + private static void le32(byte[] b, int at, int v) { + for (int i = 0; i < 4; i++) b[at + i] = (byte) (v >>> (8 * i)); + } + + private static byte[] jpeg(int length) { + byte[] j = new byte[length]; + j[0] = (byte) 0xFF; + j[1] = (byte) 0xD8; + for (int i = 2; i < length - 2; i++) j[i] = (byte) (i * 7); + j[length - 2] = (byte) 0xFF; + j[length - 1] = (byte) 0xD9; + return j; + } + + // ====================== RT stream ====================== + + @Test + public void rtStreamOneDatagramFrame() { + // As seen from a YL3: 4600 bytes of JPEG, 20 more bytes after it in the datagram + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(4600); + byte[] p = streamPacket(42, 4600, 0, j, 20); + assertEquals(4640, p.length); + int n = st.feed(p, 0, p.length); + assertEquals(4600, n); + assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); + assertEquals(1, st.frames); + assertEquals(0, st.dropped); + } + + @Test + public void rtStreamFrameInPieces() { + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(10000); + assertEquals(0, st.feed(streamPacket(1, 10000, 0, Arrays.copyOfRange(j, 0, 4000), 0), + 0, Jieli.RtStream.HEADER + 4000)); + assertEquals(0, st.feed(streamPacket(2, 10000, 4000, Arrays.copyOfRange(j, 4000, 8000), 0), + 0, Jieli.RtStream.HEADER + 4000)); + byte[] last = streamPacket(3, 10000, 8000, Arrays.copyOfRange(j, 8000, 10000), 0); + int n = st.feed(last, 0, last.length); + assertEquals(10000, n); + assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); + } + + @Test + public void rtStreamDropsBrokenFrames() { + Jieli.RtStream st = new Jieli.RtStream(); + // Start lost: a middle piece alone gives nothing + byte[] mid = streamPacket(5, 9000, 4000, new byte[4000], 0); + assertEquals(0, st.feed(mid, 0, mid.length)); + // Not a JPEG + byte[] junk = streamPacket(6, 100, 0, new byte[100], 0); + assertEquals(0, st.feed(junk, 0, junk.length)); + assertEquals(1, st.dropped); + // A frame cut short by the next one + byte[] j = jpeg(8000); + byte[] first = streamPacket(7, 8000, 0, Arrays.copyOfRange(j, 0, 4000), 0); + assertEquals(0, st.feed(first, 0, first.length)); + byte[] next = streamPacket(8, 4600, 0, jpeg(4600), 0); + assertEquals(4600, st.feed(next, 0, next.length)); + assertEquals(2, st.dropped); + // Another type (audio?) and a bad length are ignored + byte[] other = streamPacket(9, 4600, 0, jpeg(4600), 0); + other[0] = 1; + assertEquals(0, st.feed(other, 0, other.length)); + assertEquals(1, st.otherTypes); + byte[] bad = streamPacket(10, -5, 0, jpeg(100), 0); + assertEquals(0, st.feed(bad, 0, bad.length)); + assertEquals(1, st.frames); } // ====================== Helpers ====================== @@ -402,6 +496,7 @@ private static final class Running { final Jieli.Client client; final AtomicBoolean keepGoing = new AtomicBoolean(true); final List frames = new CopyOnWriteArrayList<>(); + final List jpegs = new CopyOnWriteArrayList<>(); final List states = new CopyOnWriteArrayList<>(); final CountDownLatch frame = new CountDownLatch(1); final CountDownLatch ended = new CountDownLatch(1); @@ -427,6 +522,12 @@ public void onRtp(byte[] buf, int offset, int length) { frame.countDown(); }); } + + @Override + public void onJpeg(byte[] buf, int offset, int length) { + jpegs.add(Arrays.copyOfRange(buf, offset, offset + length)); + frame.countDown(); + } }); } catch (Exception e) { failure = e; @@ -588,6 +689,32 @@ public void noVideoListensElsewhereAndAsksAgain() throws Exception { } } + @Test + public void streamPortVideoIsPlayed() throws Exception { + int rtp = freeUdpPort(); + try (FakeToy toy = new FakeToy(60000, true)) { + toy.rtpPort = rtp; + toy.sendVideo = false; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp); + assertTrue(toy.opened.await(5, TimeUnit.SECONDS)); + byte[] j = jpeg(4600); + try (DatagramSocket u = new DatagramSocket(0, LOOPBACK)) { + byte[] p = streamPacket(1, 4600, 0, j, 20); + u.send(new DatagramPacket(p, p.length, + new InetSocketAddress(LOOPBACK, Jieli.streamPort))); + } + assertTrue(r.frame.await(3, TimeUnit.SECONDS)); + assertArrayEquals(j, r.jpegs.get(0)); + assertTrue(r.frames.isEmpty()); + long until = now() + 3000; + while (now() < until && !r.client.stats().containsKey("video_port")) Thread.sleep(20); + assertEquals(Jieli.streamPort, r.client.stats().get("video_port")); + r.keepGoing.set(false); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + assertNull(r.failure); + } + } + @Test public void unreachableToyFails() throws Exception { int port; diff --git a/docs/toys.md b/docs/toys.md index c7b278aa..69ba1891 100644 --- a/docs/toys.md +++ b/docs/toys.md @@ -43,7 +43,7 @@ The Wi‑Fi name and the address the phone gets usually tell the family. The toy |---|---|---|---|---| | `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | | `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | -| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 6666) | test build 3, video path unverified ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 2224 or 6666) | test build 3: video on UDP 2224 from a YL3, playing it is new ([#187](https://github.com/OpenIPC/decoder/issues/187)) | When a `flowufo://` or `jieli://` slot points at the phone itself, or at the network of another family, its status says so (for example *192.168.1.100 is this phone; the toy is usually 192.168.1.1*, *This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam* or, on a `192.168.80.x` network, *try jieli://192.168.80.1*). @@ -89,7 +89,7 @@ rtsp://192.168.1.1:7070/webcam Their apps send a heartbeat, the two bytes `01 01`, to UDP port 7099 of the toy every second while the video plays. For an RTSP slot with port 7070 and path `/webcam` (any host) the app does the same: from the address of the RTSP connection, once a second, until the slot stops. Nothing else is sent to port 7099 (no flight command, no "stop" command). Other RTSP slots send no heartbeat unless asked with `?heartbeat=`, e.g. `rtsp://192.168.1.1:554/live?heartbeat=7099`; `?heartbeat=off` turns it off for a `:7070/webcam` slot. The option is the app's own and is not sent to the camera. The log has one line when it starts (*heartbeat 01 01 every 1000 ms to …*), and `/stats.json` a `heartbeat` object (`target`, `running`, `sent`) for such a slot. -The camera of some of these toys (the K200) is mounted sideways: its frames come turned, a mug lies on its side as a portrait strip in a landscape screen. For a `:7070/webcam` slot the app therefore turns the MJPEG frames 90° counter-clockwise when drawing them ([#200](https://github.com/OpenIPC/decoder/issues/200)), and fits the turned frame (a 480×640 frame shows as 640×480 and fills a landscape screen). `?rotate=0|90|180|270` (degrees counter-clockwise) overrides it, e.g. `rtsp://192.168.1.1:7070/webcam?rotate=0` for a unit mounted upright, and also turns other RTSP MJPEG slots; any other value is an error. The option is the app's own and is not sent to the camera. It turns MJPEG only: an H.264/H.265 stream is shown as it comes and the log says *rotate= only applies to MJPEG streams for now (#201)*. The turn is in the picture itself, so zoom, the long-press screenshot and the debug snapshot show it turned too. The log has one line when it is active (*MJPEG frames turned 90° counter-clockwise*), and `/stats.json` a `rotate` (degrees) for such a slot; its `size` is the turned one. +The camera of some of these toys (the K200) is mounted sideways: its frames come turned, a mug lies on its side as a portrait strip in a landscape screen. For a `:7070/webcam` slot the app therefore turns the MJPEG frames 270° counter-clockwise (90° clockwise) when drawing them ([#200](https://github.com/OpenIPC/decoder/issues/200)), and fits the turned frame (a 480×640 frame shows as 640×480 and fills a landscape screen). `?rotate=0|90|180|270` (degrees counter-clockwise) overrides it, e.g. `rtsp://192.168.1.1:7070/webcam?rotate=0` for a unit mounted upright, and also turns other RTSP MJPEG slots; any other value is an error. The option is the app's own and is not sent to the camera. It turns MJPEG only: an H.264/H.265 stream is shown as it comes and the log says *rotate= only applies to MJPEG streams for now (#201)*. The turn is in the picture itself, so zoom, the long-press screenshot and the debug snapshot show it turned too. The log has one line when it is active (*MJPEG frames turned 270° counter-clockwise*), and `/stats.json` a `rotate` (degrees) for such a slot; its `size` is the turned one. Some of these toys use the FLOW-UFO protocol instead (the KY UFO app has both); if RTSP shows nothing, try `flowufo://192.168.1.1`. @@ -99,7 +99,7 @@ Some of these toys use the FLOW-UFO protocol instead (the KY UFO app has both); |---|---| | Wi‑Fi name | `HF_xxxxxxxx` | | Camera | Jieli AC79 (`AC79X_wifi_car_camera`); vendor app *DVRunning 2* (also *HF UFO*) | -| Toy address | `192.168.80.1`: control (CTP) on TCP port 3333; the video goes to UDP port 6666 **of the phone** | +| Toy address | `192.168.80.1`: control (CTP) on TCP port 3333; the video goes to UDP port 2224 (or 6666) **of the phone** | | Slot URL | `jieli://192.168.80.1` (or `jieli://` alone) | | Status | **test build 3**, not in the releases: the control part follows a capture of the vendor app, the video part is **unverified**; issue [#187](https://github.com/OpenIPC/decoder/issues/187) | @@ -108,9 +108,10 @@ What the app does, as a capture of the vendor app shows it ([#196](https://githu - It connects to TCP port 3333 and talks **CTP**: each message is `CTP:`, the length of a topic name, the topic (e.g. `OPEN_RT_STREAM`), the length of a JSON text and the JSON. - `APP_ACCESS` first; the toy answers with `KEEP_ALIVE_INTERVAL` (a timeout, 60 s), its battery and SD card state and `VIDEO_PARAM` (the size, e.g. 640×480 at 25 fps). Then `DATE_TIME` (the phone's time), a `CTP_KEEP_ALIVE` every 7.5 s (a quarter of a shorter timeout), and `OPEN_RT_STREAM` with the size from `VIDEO_PARAM` (640×480 at 25 if the toy sent none). When the slot stops: `CLOSE_RT_STREAM`. - The video was **not** in the capture (PCAPdroid without root cannot see it). After turbodrone's research on the same Jieli SDK it is **RTP/JPEG** (payload type 26) or H.264 (96) to the phone's **UDP port 6666**. The app listens there and plays payload type 26 through its RTP/JPEG path and 96 through its H.264 path; a whole JPEG in a datagram without RTP is shown too. -- If nothing arrives on 6666 for 5 s after `OPEN_RT_STREAM`, the status says so and the app also listens on UDP 1234 (audio in the vendor SDK) and 2224 (its stream port), **only to log** what arrives there; it asks for the stream again every 10 s without video. +- A YL3 sent nothing to 6666 but its own stream to the phone's **UDP port 2224**: per datagram a 20-byte header, then the JPEG (seen: byte 0 = 2, a packet counter in bytes 4–7, the JPEG's length in 8–11, an offset in 12–15, little-endian; a 640×480 frame of 4600 bytes in one 4640-byte datagram). The app listens on 2224 as well and shows those JPEGs; longer frames are put together by the offset. The first five datagrams go to the log with the whole header, `/stats.json` has a `stream` object (`packets`, `frames`, `dropped`) and `video_port`. +- If nothing arrives on 6666 or 2224 for 5 s after `OPEN_RT_STREAM`, the status says so and the app also listens on UDP 1234 (audio in the vendor SDK), **only to log** what arrives there; it asks for the stream again every 10 s without video. - **No flight command is ever sent**: no `FLIGHT_CONTROL` over CTP, nothing to the toy's UDP port 3456 (where the vendor app sends the sticks). -- Port 6666 can be used by one slot at a time: one `jieli://` slot in a grid. +- Ports 6666 and 2224 can be used by one slot at a time: one `jieli://` slot in a grid. URL options (as with `udp://`, separated by `?` and `&`): @@ -122,7 +123,7 @@ URL options (as with `udp://`, separated by `?` and `&`): Examples: `jieli://192.168.80.1`, `jieli://192.168.80.1?w=1280&h=720`, `jieli://192.168.80.1?format=1`. -Status texts while it connects: *Connecting to the toy (CTP 192.168.80.1:3333)*, *Stream requested, waiting for video on UDP 6666*, *No video on UDP 6666 yet: the toy accepted the request; see /log*, *Receiving from the toy on UDP 6666…*, *The toy's video stopped: waiting on UDP 6666*. The test build adds a `jieli` object to `/stats.json`: `state` (`connecting`, `ctp`, `requested`, `playing`, `quiet`), `target`, `network`, `ctp` (messages sent and received, keepalives and replies, the keepalive interval, the topics seen, the last 8 messages from the toy), `video_param`, `requested`, `rtp` (port, packets, video packets, frames, payload types, packets that are not RTP, the first packet in hex), `sources` (sender address:port → packets) and `diagnostic_ports`. The log has the CTP messages (JSON shortened) and the first packets on each port with sender, payload type and first bytes: exactly what a report needs. +Status texts while it connects: *Connecting to the toy (CTP 192.168.80.1:3333)*, *Stream requested, waiting for video on UDP 6666 and 2224*, *No video on UDP 6666 and 2224 yet: the toy accepted the request; see /log*, *Receiving from the toy on UDP 2224…* (the port the video came on), *The toy's video stopped: waiting on UDP 6666 and 2224*. The test build adds a `jieli` object to `/stats.json`: `state` (`connecting`, `ctp`, `requested`, `playing`, `quiet`), `target`, `network`, `ctp` (messages sent and received, keepalives and replies, the keepalive interval, the topics seen, the last 8 messages from the toy), `video_param`, `requested`, `rtp` (port, packets, video packets, frames, payload types, packets that are not RTP, the first packet in hex), `sources` (sender address:port → packets) and `diagnostic_ports`. The log has the CTP messages (JSON shortened) and the first packets on each port with sender, payload type and first bytes: exactly what a report needs. To test: install the test build, join the `HF_…` Wi‑Fi, [stay connected](#connecting), close the vendor app, put `jieli://192.168.80.1` into a slot. Wait about 30 s and [send a report](#what-to-send-in-a-report) in [#187](https://github.com/OpenIPC/decoder/issues/187), with or without a picture: `/stats.json` and `/log` tell what the toy really sends. diff --git a/docs/transports.md b/docs/transports.md index 5187e26a..85737fc0 100644 --- a/docs/transports.md +++ b/docs/transports.md @@ -102,9 +102,9 @@ For example Majestic's `http://user:password@/mjpeg`: a `multipart/x-mix `flowufo://192.168.169.1:8800` (or `flowufo://` alone) plays the video of toy drones with a `FLOW-UFO-xxxxxx` Wi‑Fi access point. It exists only in a test build so far ([#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188)), not in the releases. Port 8800 (the default) probes 8801 as well and keeps the port that answers. Options: `?qpara=<8 hex digits>` and `?dqt=zigzag|raster|q50`. Setup, options and status: [toys.md](toys.md#flow-ufo). -`jieli://192.168.80.1` (or `jieli://` alone) plays toy drones with an `HF_xxxxxxxx` Wi‑Fi (Jieli AC79 camera): CTP commands on TCP 3333 ask for the stream, which is expected as RTP/JPEG on the phone's UDP port 6666 (test build only, video path unverified). Options: `?w=`, `?h=`, `?fps=`, `?rtp=`, `?format=0|1`. See [toys.md](toys.md#hf--jieli-toys). +`jieli://192.168.80.1` (or `jieli://` alone) plays toy drones with an `HF_xxxxxxxx` Wi‑Fi (Jieli AC79 camera): CTP commands on TCP 3333 ask for the stream, which arrives as JPEG frames behind a 20-byte header on the phone's UDP port 2224 (seen from a YL3) or as RTP/JPEG on UDP 6666 (test build only). Options: `?w=`, `?h=`, `?fps=`, `?rtp=`, `?format=0|1`. See [toys.md](toys.md#hf--jieli-toys). -RTSP slots on `rtsp://:7070/webcam` (KY UFO and other cooingdv toys) also send the toy's UDP heartbeat `01 01` to port 7099 once a second; `?heartbeat=` turns it on for other RTSP URLs, `?heartbeat=off` off. Their MJPEG frames are drawn turned 90° counter-clockwise (the K200's camera is mounted sideways, [#200](https://github.com/OpenIPC/decoder/issues/200)); `?rotate=0|90|180|270` (degrees counter-clockwise, any RTSP MJPEG slot) overrides it and is not sent to the camera either. See [toys.md](toys.md#ky-ufo-rtsp-toys). +RTSP slots on `rtsp://:7070/webcam` (KY UFO and other cooingdv toys) also send the toy's UDP heartbeat `01 01` to port 7099 once a second; `?heartbeat=` turns it on for other RTSP URLs, `?heartbeat=off` off. Their MJPEG frames are drawn turned 270° counter-clockwise (90° clockwise) (the K200's camera is mounted sideways, [#200](https://github.com/OpenIPC/decoder/issues/200)); `?rotate=0|90|180|270` (degrees counter-clockwise, any RTSP MJPEG slot) overrides it and is not sent to the camera either. See [toys.md](toys.md#ky-ufo-rtsp-toys). ## Codecs From 51b4ab111864adc0b0a4ff87b75df2e38e297503 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 13:11:34 +0000 Subject: [PATCH 14/21] test: Jieli status names both video ports (#187) --- android/app/src/test/java/com/openipc/decoder/JieliTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/android/app/src/test/java/com/openipc/decoder/JieliTest.java b/android/app/src/test/java/com/openipc/decoder/JieliTest.java index 2806c6e4..10b87cbf 100644 --- a/android/app/src/test/java/com/openipc/decoder/JieliTest.java +++ b/android/app/src/test/java/com/openipc/decoder/JieliTest.java @@ -593,7 +593,9 @@ public void sessionAsTheVendorApp() throws Exception { Map sources = (Map) st.get("sources"); assertEquals(1, sources.size()); assertTrue(r.states.get(0), r.states.get(0).startsWith("Connecting to the toy (CTP")); - assertTrue(r.states.contains("Stream requested, waiting for video on UDP " + rtp)); + assertTrue(r.states.toString(), r.states.contains( + "Stream requested, waiting for video on UDP " + rtp + " and " + Jieli.streamPort)); + assertEquals(rtp, st.get("video_port")); } } From 168012d6d1e82ae9c506c8bcd8659b82c166f314 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 13:15:31 +0000 Subject: [PATCH 15/21] chore: version 2.15 (2.14 went to #220) --- android/app/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/app/build.gradle b/android/app/build.gradle index 8fadf408..53d4226c 100755 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -23,8 +23,8 @@ android { applicationId "com.openipc.decoder" minSdk 21 targetSdk 36 - versionCode 38 - versionName "2.13" + versionCode 40 + versionName "2.15" buildConfigField "String", "GIT_HASH", "\"${getGitHash()}\"" } From 916c88d2b853a4b03e52e856b698f21e4d7e5f9b Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 14:00:05 +0000 Subject: [PATCH 16/21] docs: align Markdown tables for reading as plain text (toys, transports, debug) --- docs/debug.md | 196 ++++++++++++++++++++++----------------------- docs/toys.md | 82 +++++++++---------- docs/transports.md | 68 ++++++++-------- 3 files changed, 173 insertions(+), 173 deletions(-) diff --git a/docs/debug.md b/docs/debug.md index d0f27da9..7a71bafa 100644 --- a/docs/debug.md +++ b/docs/debug.md @@ -14,13 +14,13 @@ While the app runs it serves its debug information over HTTP on port **8085**, for a browser or `curl` on the same network. It is always on and read-only. -| Path | Content | -|---|---| -| `/` (or `/index.html`) | the [page](#page): snapshot with the text of each cell | -| `/stats.json` | the state as JSON, pretty-printed, see [fields](#statsjson-fields) | -| `/snapshot.jpg` | a picture of the screen, see [Snapshot](#snapshot); `?labels=0`: frames only | -| `/log` | the last 500 lines of the app's log, see [Log](#log) | -| `/report.txt` | a header, `/stats.json` and `/log` in one file, downloaded as `report-YYYYMMDD-HHMMSS.txt`, see [Save report](#save-report) | +| Path | Content | +|------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| `/` (or `/index.html`) | the [page](#page): snapshot with the text of each cell | +| `/stats.json` | the state as JSON, pretty-printed, see [fields](#statsjson-fields) | +| `/snapshot.jpg` | a picture of the screen, see [Snapshot](#snapshot); `?labels=0`: frames only | +| `/log` | the last 500 lines of the app's log, see [Log](#log) | +| `/report.txt` | a header, `/stats.json` and `/log` in one file, downloaded as `report-YYYYMMDD-HHMMSS.txt`, see [Save report](#save-report) | - Only clients on the local network are answered; others get `403`. The rule and what is hidden: [security.md](security.md#debug-server). - `GET` only (`405` otherwise), one request per connection, `Cache-Control: no-store`. @@ -46,13 +46,13 @@ The device's addresses are in `addresses` of `/stats.json`, and the [U cell](usa - **Snapshot 3 s / 5 s / 10 s**: how often the picture is refreshed (default 5 s). `/stats.json` is fetched every 1.5 s. Both only while the tab is visible. - On a screen narrower than 720 px the snapshot takes the full width and the cells are listed below it. -| Status dot / header note | Meaning | -|---|---| -| green | the app answers | -| orange, `UI not responding within 1000 ms` | the UI thread is stuck (an ANR); only the device facts are reported | -| orange, `no activity` / `App in background` | no activity behind the server, or the app is paused | +| Status dot / header note | Meaning | +|----------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------| +| green | the app answers | +| orange, `UI not responding within 1000 ms` | the UI thread is stuck (an ANR); only the device facts are reported | +| orange, `no activity` / `App in background` | no activity behind the server, or the app is paused | | red, `video decoder not responding — restart the device` | `decoder_stuck`: frames arrive in several cells, none is decoded for 20 s ([#145](https://github.com/OpenIPC/decoder/issues/145)) | -| red, `offline` | the page cannot reach the app | +| red, `offline` | the page cannot reach the app | ## Snapshot @@ -76,99 +76,99 @@ In the single view (and the U view) the page and the snapshot still show the who ### Top level -| Field | Type | Meaning | -|---|---|---| -| `app` | string | `OpenIPC Decoder` | -| `version` | string | versionName, e.g. `2.10` | -| `git` | string | short commit hash of the build | -| `time` | string | device time, `yyyy-MM-dd HH:mm:ss` | -| `uptime_s` | number | seconds since the device booted | -| `device` | string | manufacturer, model, SoC (Android 12+; else `Build.HARDWARE`) and board | -| `android` | string | release and SDK level, e.g. `10 (SDK 29)` | -| `screen` | string | display size in pixels, `WxH` | -| `addresses` | list | `http://:8085/` for every address of the device | -| `server` | string | `listening`, or why port 8085 could not be opened | -| `decoders` | list or `"loading"` | the device's video decoders, see [decoders](#decoders) | -| `ui_thread` | string | `ok`, or `not responding within 1000 ms` (then only the fields above are present) | -| `mode` | string | `single`, `multi-view`, `slides` (multi-view with Slides on), `udp` (U cell full-screen), `paused` (app in background); `no activity` when no activity is attached | -| `active_slot` | number | active slot, 1–8 | -| `decoder` | object | multi-view only: `capacity` (macroblocks/s), `learnt` (bool), `failed_load` (macroblocks/s, 0: none), see [architecture.md](architecture.md#decode-planner) | -| `decoder_stuck` | bool | multi-view only: frames arrive in at least two cells and none is decoded for 20 s | -| `udp` | object | U view only: `port` (5600) and `addresses` (the device's IPs to send to) | -| `single` | object | single view only, see [single](#single) | -| `cells` | list | multi-view and U view, see [cells](#cells) | -| `slots` | list | single view and U view: every grid cell with the stream checks, see [slots](#slots) | +| Field | Type | Meaning | +|-----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `app` | string | `OpenIPC Decoder` | +| `version` | string | versionName, e.g. `2.10` | +| `git` | string | short commit hash of the build | +| `time` | string | device time, `yyyy-MM-dd HH:mm:ss` | +| `uptime_s` | number | seconds since the device booted | +| `device` | string | manufacturer, model, SoC (Android 12+; else `Build.HARDWARE`) and board | +| `android` | string | release and SDK level, e.g. `10 (SDK 29)` | +| `screen` | string | display size in pixels, `WxH` | +| `addresses` | list | `http://:8085/` for every address of the device | +| `server` | string | `listening`, or why port 8085 could not be opened | +| `decoders` | list or `"loading"` | the device's video decoders, see [decoders](#decoders) | +| `ui_thread` | string | `ok`, or `not responding within 1000 ms` (then only the fields above are present) | +| `mode` | string | `single`, `multi-view`, `slides` (multi-view with Slides on), `udp` (U cell full-screen), `paused` (app in background); `no activity` when no activity is attached | +| `active_slot` | number | active slot, 1–8 | +| `decoder` | object | multi-view only: `capacity` (macroblocks/s), `learnt` (bool), `failed_load` (macroblocks/s, 0: none), see [architecture.md](architecture.md#decode-planner) | +| `decoder_stuck` | bool | multi-view only: frames arrive in at least two cells and none is decoded for 20 s | +| `udp` | object | U view only: `port` (5600) and `addresses` (the device's IPs to send to) | +| `single` | object | single view only, see [single](#single) | +| `cells` | list | multi-view and U view, see [cells](#cells) | +| `slots` | list | single view and U view: every grid cell with the stream checks, see [slots](#slots) | ### decoders One entry per video decoder and MIME type (encoders are left out). -| Field | Meaning | -|---|---| -| `name` | codec name as Android lists it | -| `type` | MIME type, e.g. `video/hevc` | -| `hardware` | hardware-accelerated (Android 10+) | -| `adaptive` | supports adaptive playback | -| `max_instances` | instances the codec declares (Android 6+) | -| `fps_1080p` | highest declared frame rate at 1920×1080, when 1080p is supported | +| Field | Meaning | +|-----------------|-------------------------------------------------------------------| +| `name` | codec name as Android lists it | +| `type` | MIME type, e.g. `video/hevc` | +| `hardware` | hardware-accelerated (Android 10+) | +| `adaptive` | supports adaptive playback | +| `max_instances` | instances the codec declares (Android 6+) | +| `fps_1080p` | highest declared frame rate at 1920×1080, when 1080p is supported | ### single -| Field | Meaning | -|---|---| -| `url` | the slot URL, redacted | -| `transport` | in use, as in [cells](#cells) | -| `transport_mode` | the slot's setting: `TCP`, `UDP`, `AUTO`, `MULTICAST` | -| `video` | a frame was shown within the last 2 s | -| `status` | the status text on screen, or the session state | -| `size` | `WxH`, once known | -| `codec` | `H.264`, `H.265` or `MJPEG` | -| `audio` | audio codec: `NONE`, `AAC`, `LATM`, `OPUS`, `PCMU`, `PCMA`, `PCM` | -| `out_fps` | frames shown in the last second | -| `decode` | `full`, or `key frames` with Slides | +| Field | Meaning | +|------------------|-------------------------------------------------------------------| +| `url` | the slot URL, redacted | +| `transport` | in use, as in [cells](#cells) | +| `transport_mode` | the slot's setting: `TCP`, `UDP`, `AUTO`, `MULTICAST` | +| `video` | a frame was shown within the last 2 s | +| `status` | the status text on screen, or the session state | +| `size` | `WxH`, once known | +| `codec` | `H.264`, `H.265` or `MJPEG` | +| `audio` | audio codec: `NONE`, `AAC`, `LATM`, `OPUS`, `PCMU`, `PCMA`, `PCM` | +| `out_fps` | frames shown in the last second | +| `decode` | `full`, or `key frames` with Slides | ### cells One entry per grid cell; `slot` is 1–8 or `U`. An empty cell has only `slot`, `video: false` and `text: "not configured"`. -| Field | Meaning | -|---|---| -| `slot` | 1–8, or `U` | -| `video` | the cell showed a frame within the last 2 s | -| `text` | the corner text of the cell, see [below](#corner-text-in-multi-view) | -| `url` | the slot URL, redacted (`udp://@:5600` for U) | -| `transport` | in use: `TCP`, `UDP`, `multicast`, `HTTP`, `UDP push`, `multicast push`; after a fallback `TCP (auto)` or `TCP (multicast)`; the mode alone before anything plays | -| `transport_mode` | the slot's setting (`UDP` for U) | -| `out_fps` | frames decoded per second | -| `in_fps` | frames received per second | -| `lost_fps` | frames per second the decoder lost (queue full, no input buffer); network losses do not count | -| `nal_dropped` | NAL units dropped because one of their packets was lost | -| `nal_damaged` | slices passed on cut short at a lost piece; absent while damaged decoding is off | -| `decode` | `full` or `key frames` | -| `load` | macroblocks/s the cell costs in its current mode, once size and rate are known | -| `paused` | paused by the decode planner (`Paused: decoder overloaded`) | -| `audio` | this cell's sound plays | +| Field | Meaning | +|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `slot` | 1–8, or `U` | +| `video` | the cell showed a frame within the last 2 s | +| `text` | the corner text of the cell, see [below](#corner-text-in-multi-view) | +| `url` | the slot URL, redacted (`udp://@:5600` for U) | +| `transport` | in use: `TCP`, `UDP`, `multicast`, `HTTP`, `UDP push`, `multicast push`; after a fallback `TCP (auto)` or `TCP (multicast)`; the mode alone before anything plays | +| `transport_mode` | the slot's setting (`UDP` for U) | +| `out_fps` | frames decoded per second | +| `in_fps` | frames received per second | +| `lost_fps` | frames per second the decoder lost (queue full, no input buffer); network losses do not count | +| `nal_dropped` | NAL units dropped because one of their packets was lost | +| `nal_damaged` | slices passed on cut short at a lost piece; absent while damaged decoding is off | +| `decode` | `full` or `key frames` | +| `load` | macroblocks/s the cell costs in its current mode, once size and rate are known | +| `paused` | paused by the decode planner (`Paused: decoder overloaded`) | +| `audio` | this cell's sound plays | ### slots Single view and U view: every cell of the grid as the debug page shows it (3×3 in the U view or when the active slot is 5–8). -| Field | Meaning | -|---|---| -| `slot` | 1–8, or `U` | -| `on_screen` | this is the camera on screen | -| `url`, `transport_mode` | as in [cells](#cells), for configured slots only | -| `transport` | on screen: in use; otherwise what a check goes over (`check over TCP` for TCP and Auto slots, `check over UDP`, `check over multicast`, `UDP listen`, `HTTP`, `UDP push` for U) | -| `video`, `text` | on screen: as it plays; otherwise `false` and the check result or why there is none | -| `probe` | not on screen: the check, see below | - -| `probe` field | Meaning | -|---|---| -| `state` | `ok`, `error`, `checking`, `waiting`, `not checked` | -| `checked_s_ago` | seconds since the last check began | -| `summary` | result text | -| `codec`, `size` | when found | -| `fps`, `kbps`, `key_frames` | measured during the check | +| Field | Meaning | +|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `slot` | 1–8, or `U` | +| `on_screen` | this is the camera on screen | +| `url`, `transport_mode` | as in [cells](#cells), for configured slots only | +| `transport` | on screen: in use; otherwise what a check goes over (`check over TCP` for TCP and Auto slots, `check over UDP`, `check over multicast`, `UDP listen`, `HTTP`, `UDP push` for U) | +| `video`, `text` | on screen: as it plays; otherwise `false` and the check result or why there is none | +| `probe` | not on screen: the check, see below | + +| `probe` field | Meaning | +|-----------------------------|-----------------------------------------------------| +| `state` | `ok`, `error`, `checking`, `waiting`, `not checked` | +| `checked_s_ago` | seconds since the last check began | +| `summary` | result text | +| `codec`, `size` | when found | +| `fps`, `kbps`, `key_frames` | measured during the check | The FLOW-UFO test build ([toys.md](toys.md#flow-ufo)) adds a `flowufo` object per cell and in `single` (`jieli` for `jieli://` slots, [toys.md](toys.md#hf--jieli-toys)), and a `heartbeat` object for RTSP slots that send the KY UFO heartbeat, and `rotate` (degrees counter-clockwise) where MJPEG frames are turned ([toys.md](toys.md#ky-ufo-rtsp-toys)); it is not in the releases. @@ -206,16 +206,16 @@ Uptime: 1h 2m 3s Examples: `1920×1080 H.265 25/25 fps` at full rate, `1920×1080 H.265 key frames · 0.5 fps` on key frames only. -| Part | Meaning | -|---|---| -| `1920×1080 H.265` | stream size and codec (`H.264`, `H.265`, `MJPEG`) | -| `25/25 fps` | full rate: decoded / received frames per second | -| `key frames · 0.5 fps` | key frames only, with the frames shown per second (MJPEG: `reduced`, one frame per second) | -| `· software` | key frames on a software decoder, beyond the hardware instance limit (MJPEG: BitmapFactory) | -| `· damaged` | frames with lost pieces are decoded with the damage concealed | -| `Connecting…`, `Buffering…`, `Disconnected` | the session state before video arrives | -| `Decoder limit: N × H.265` | no decoder instance left, not even a software one | -| `Paused: decoder overloaded` | paused by the decode planner, tried again later | +| Part | Meaning | +|---------------------------------------------|---------------------------------------------------------------------------------------------| +| `1920×1080 H.265` | stream size and codec (`H.264`, `H.265`, `MJPEG`) | +| `25/25 fps` | full rate: decoded / received frames per second | +| `key frames · 0.5 fps` | key frames only, with the frames shown per second (MJPEG: `reduced`, one frame per second) | +| `· software` | key frames on a software decoder, beyond the hardware instance limit (MJPEG: BitmapFactory) | +| `· damaged` | frames with lost pieces are decoded with the damage concealed | +| `Connecting…`, `Buffering…`, `Disconnected` | the session state before video arrives | +| `Decoder limit: N × H.265` | no decoder instance left, not even a software one | +| `Paused: decoder overloaded` | paused by the decode planner, tried again later | ## Bug reports diff --git a/docs/toys.md b/docs/toys.md index ebf405e1..662829f6 100644 --- a/docs/toys.md +++ b/docs/toys.md @@ -40,24 +40,24 @@ Whether a toy can be played here depends on its protocol. Toys that do offer RTS The Wi‑Fi name and the address the phone gets usually tell the family. The toy is almost always the `.1` address of that network, **not** the phone's own address. -| Wi‑Fi name | Phone gets | Family | Slot URL | Status | -|---|---|---|---|---| -| `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | -| `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | -| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 2224 or 6666) | test build 3: video on UDP 2224 from a YL3, playing it is new ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| Wi‑Fi name | Phone gets | Family | Slot URL | Status | +|----------------------------------------------------------------|-----------------|----------------------------------------------------------|---------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 2224 or 6666) | test build 3: video on UDP 2224 from a YL3, playing it is new ([#187](https://github.com/OpenIPC/decoder/issues/187)) | When a `flowufo://` or `jieli://` slot points at the phone itself, or at the network of another family, its status says so (for example *192.168.1.100 is this phone; the toy is usually 192.168.1.1*, *This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam* or, on a `192.168.80.x` network, *try jieli://192.168.80.1*). ## FLOW-UFO -| | | -|---|---| -| Wi‑Fi name | `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | -| Camera | BL60x Wi‑Fi camera module (BL608 or BL618 engine) | -| Toy address | `192.168.169.1`, UDP port 8800 (BL608) or 8801 (BL618); the toy answers from port 1234 | -| Slot URL | `flowufo://192.168.169.1:8800` (or `flowufo://` alone) | -| Status | **in testing**, not in the releases: issue [#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188) | -| Test build | [toys-187-test2](https://github.com/OpenIPC/decoder/releases/tag/toys-187-test2) (2.11 with the FLOW-UFO, Jieli and KY UFO changes; installs over the release and keeps the settings) | +| | | +|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Wi‑Fi name | `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | +| Camera | BL60x Wi‑Fi camera module (BL608 or BL618 engine) | +| Toy address | `192.168.169.1`, UDP port 8800 (BL608) or 8801 (BL618); the toy answers from port 1234 | +| Slot URL | `flowufo://192.168.169.1:8800` (or `flowufo://` alone) | +| Status | **in testing**, not in the releases: issue [#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188) | +| Test build | [toys-187-test2](https://github.com/OpenIPC/decoder/releases/tag/toys-187-test2) (2.11 with the FLOW-UFO, Jieli and KY UFO changes; installs over the release and keeps the settings) | What the toy does, as far as is known: @@ -69,10 +69,10 @@ What the toy does, as far as is known: URL options, for comparing with what the toy expects (as with `udp://`, separated by `?` and `&`): -| Option | Values | Default | What it does | -|---|---|---|---| -| `qpara` (or `q`) | 8 hex digits | `324B142D` | the four quality parameters in each acknowledgement; `324B142D` is what turbodrone sends to working drones, `19322348` an older reading of the vendor app | -| `dqt` | `zigzag`, `raster`, `q50` | `zigzag` | the quantization tables in the JPEG header the app puts in front of each frame: `zigzag` = standard tables scaled to the quality byte of the fragment, in zig-zag order as the JPEG standard has it; `raster` = unscaled standard tables in raster order, quality ignored (what turbodrone writes, and it shows real drones' frames); `q50` = unscaled, zig-zag. If the picture is there but blocky, too dark/bright or noisy, try the others | +| Option | Values | Default | What it does | +|------------------|---------------------------|------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `qpara` (or `q`) | 8 hex digits | `324B142D` | the four quality parameters in each acknowledgement; `324B142D` is what turbodrone sends to working drones, `19322348` an older reading of the vendor app | +| `dqt` | `zigzag`, `raster`, `q50` | `zigzag` | the quantization tables in the JPEG header the app puts in front of each frame: `zigzag` = standard tables scaled to the quality byte of the fragment, in zig-zag order as the JPEG standard has it; `raster` = unscaled standard tables in raster order, quality ignored (what turbodrone writes, and it shows real drones' frames); `q50` = unscaled, zig-zag. If the picture is there but blocky, too dark/bright or noisy, try the others | Examples: `flowufo://192.168.169.1:8800`, `flowufo://192.168.169.1:8800?dqt=raster`, `flowufo://192.168.169.1:8800?q=19322348`, `flowufo://192.168.169.1:8801?dqt=q50`. @@ -96,13 +96,13 @@ Some of these toys use the FLOW-UFO protocol instead (the KY UFO app has both); ## HF / Jieli toys -| | | -|---|---| -| Wi‑Fi name | `HF_xxxxxxxx` | -| Camera | Jieli AC79 (`AC79X_wifi_car_camera`); vendor app *DVRunning 2* (also *HF UFO*) | -| Toy address | `192.168.80.1`: control (CTP) on TCP port 3333; the video goes to UDP port 2224 (or 6666) **of the phone** | -| Slot URL | `jieli://192.168.80.1` (or `jieli://` alone) | -| Status | **test build 3**, not in the releases: the control part follows a capture of the vendor app, the video part is **unverified**; issue [#187](https://github.com/OpenIPC/decoder/issues/187) | +| | | +|-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Wi‑Fi name | `HF_xxxxxxxx` | +| Camera | Jieli AC79 (`AC79X_wifi_car_camera`); vendor app *DVRunning 2* (also *HF UFO*) | +| Toy address | `192.168.80.1`: control (CTP) on TCP port 3333; the video goes to UDP port 2224 (or 6666) **of the phone** | +| Slot URL | `jieli://192.168.80.1` (or `jieli://` alone) | +| Status | **test build 3**, not in the releases: the control part follows a capture of the vendor app, the video part is **unverified**; issue [#187](https://github.com/OpenIPC/decoder/issues/187) | What the app does, as a capture of the vendor app shows it ([#196](https://github.com/OpenIPC/decoder/issues/196)): @@ -116,11 +116,11 @@ What the app does, as a capture of the vendor app shows it ([#196](https://githu URL options (as with `udp://`, separated by `?` and `&`): -| Option | Values | Default | What it does | -|---|---|---|---| -| `w`, `h`, `fps` | numbers | from `VIDEO_PARAM`, else 640, 480, 25 | the stream asked for in `OPEN_RT_STREAM` | -| `rtp` | a UDP port | `6666` | the local port the video is expected on | -| `format` | `0`, `1` | `0` | `0` asks for JPEG (as the vendor app does), `1` for H.264 | +| Option | Values | Default | What it does | +|-----------------|------------|---------------------------------------|-----------------------------------------------------------| +| `w`, `h`, `fps` | numbers | from `VIDEO_PARAM`, else 640, 480, 25 | the stream asked for in `OPEN_RT_STREAM` | +| `rtp` | a UDP port | `6666` | the local port the video is expected on | +| `format` | `0`, `1` | `0` | `0` asks for JPEG (as the vendor app does), `1` for H.264 | Examples: `jieli://192.168.80.1`, `jieli://192.168.80.1?w=1280&h=720`, `jieli://192.168.80.1?format=1`. @@ -136,13 +136,13 @@ What the app does for FLOW-UFO and KY UFO toys was checked against [turbodrone]( First find the toy's address: on Android it is usually the gateway of the Wi‑Fi (*Wi‑Fi → the network → Details*; often `192.168.0.1`, `192.168.1.1`, `192.168.4.1`, `192.168.169.1`, `172.16.10.1` or `192.168.25.1`). Then try, in a slot: -| Try | Why | -|---|---| -| `rtsp://:554/` | some toys run a plain RTSP server; try the paths `/live`, `/stream`, `/11`, `/h264` too | -| `rtsp://:8554/` | another common RTSP port | -| `http://:8080/?action=stream` | MJPEG streamers (mjpg-streamer style) | -| `http:///` and `http://:81/stream` | MJPEG on the web port; open them in a browser first to see what answers | -| `udp://@:` | only if the toy pushes RTP by itself to a port of the phone, which is rare | +| Try | Why | +|----------------------------------------------|-----------------------------------------------------------------------------------------| +| `rtsp://:554/` | some toys run a plain RTSP server; try the paths `/live`, `/stream`, `/11`, `/h264` too | +| `rtsp://:8554/` | another common RTSP port | +| `http://:8080/?action=stream` | MJPEG streamers (mjpg-streamer style) | +| `http:///` and `http://:81/stream` | MJPEG on the web port; open them in a browser first to see what answers | +| `udp://@:` | only if the toy pushes RTP by itself to a port of the phone, which is rare | If nothing plays, the toy most likely uses a protocol of its own, which needs a [report](#what-to-send-in-a-report) and a transport written for it. @@ -163,9 +163,9 @@ Do not send the vendor app itself or parts of it. The capture contains only the The drones a tester checked in [#187](https://github.com/OpenIPC/decoder/issues/187): -| Toy | Wi‑Fi | Family | URL | Status | -|---|---|---|---|---| -| D16 | `FLOW_…` | FLOW-UFO, WIFI UAV app | `flowufo://192.168.169.1:8800` | video works | -| K200 | `FLY-UFO-…` | KY UFO, RTSP | `rtsp://192.168.1.1:7070/webcam` | video works; the picture comes sideways ([#200](https://github.com/OpenIPC/decoder/issues/200)) | -| Drone YL3 | `HF_…` | Jieli AC79, DVRunning 2 app | `jieli://192.168.80.1` (test build) | in testing | +| Toy | Wi‑Fi | Family | URL | Status | +|-----------|-------------|-----------------------------|-------------------------------------|-------------------------------------------------------------------------------------------------| +| D16 | `FLOW_…` | FLOW-UFO, WIFI UAV app | `flowufo://192.168.169.1:8800` | video works | +| K200 | `FLY-UFO-…` | KY UFO, RTSP | `rtsp://192.168.1.1:7070/webcam` | video works; the picture comes sideways ([#200](https://github.com/OpenIPC/decoder/issues/200)) | +| Drone YL3 | `HF_…` | Jieli AC79, DVRunning 2 app | `jieli://192.168.80.1` (test build) | in testing | diff --git a/docs/transports.md b/docs/transports.md index 78c306bd..ccc90941 100644 --- a/docs/transports.md +++ b/docs/transports.md @@ -14,12 +14,12 @@ A slot holds one URL. Its scheme picks the path the video takes; for `rtsp://` the slot's transport mode (chosen under the URL in **Settings**, or with the `transport` extra of a [shared link](usage.md#opening-a-camera-from-another-app)) picks how RTP travels. -| URL | Path | Single view | Multi-view | Audio | -|---|---|---|---|---| -| `rtsp://` | RTSP session, RTP over TCP, UDP or multicast | yes | yes | yes | -| `udp://`, `rtp://` | plain RTP pushed by the camera, no RTSP | yes | yes | no | -| `http://` | MJPEG, `multipart/x-mixed-replace` | yes | yes | no | -| `flowufo://` | toy drone protocol, **test build only** ([#187](https://github.com/OpenIPC/decoder/issues/187)) | yes | yes | no | +| URL | Path | Single view | Multi-view | Audio | +|--------------------|-------------------------------------------------------------------------------------------------|-------------|------------|-------| +| `rtsp://` | RTSP session, RTP over TCP, UDP or multicast | yes | yes | yes | +| `udp://`, `rtp://` | plain RTP pushed by the camera, no RTSP | yes | yes | no | +| `http://` | MJPEG, `multipart/x-mixed-replace` | yes | yes | no | +| `flowufo://` | toy drone protocol, **test build only** ([#187](https://github.com/OpenIPC/decoder/issues/187)) | yes | yes | no | The transport in use is shown when a stream starts in the single view (`Connected • UDP • H.265 …`), on the [debug page](debug.md) and in `/stats.json` (`transport`; the slot's setting is `transport_mode`). After a fallback it reads, for example, `TCP (auto)`. @@ -42,12 +42,12 @@ rtsp://[user:password@]host[:port]/path ## RTSP transport modes -| Mode | How the video arrives | Fallback | -|---|---|---| -| **TCP** | RTP interleaved in the RTSP connection | none needed; works through NAT and firewalls | -| **UDP** | RTP over UDP, a port pair per track | never falls back. Lost packets are skipped, not resent | -| **Auto** (default for a new slot) | UDP, else TCP | to TCP when the camera refuses UDP in `SETUP`, when no video arrives within 3 s, or when packets arrive but key frames keep losing a piece | -| **Multicast** | RTSP multicast (Android 8.0+) | to TCP in the same connection when the server refuses it or answers with unicast | +| Mode | How the video arrives | Fallback | +|-----------------------------------|----------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------| +| **TCP** | RTP interleaved in the RTSP connection | none needed; works through NAT and firewalls | +| **UDP** | RTP over UDP, a port pair per track | never falls back. Lost packets are skipped, not resent | +| **Auto** (default for a new slot) | UDP, else TCP | to TCP when the camera refuses UDP in `SETUP`, when no video arrives within 3 s, or when packets arrive but key frames keep losing a piece | +| **Multicast** | RTSP multicast (Android 8.0+) | to TCP in the same connection when the server refuses it or answers with unicast | - After a fallback the URL is remembered as failed for that mode for 10 minutes while the app runs: reconnects, a resume or a rebuilt grid go straight to TCP; after that UDP (or multicast) is tried again. - If the server answers a UDP request with interleaved TCP, the app takes what the server chose. @@ -62,17 +62,17 @@ A slot can receive RTP that the camera pushes by itself, without RTSP. Put a `ud udp://[source@][address]:port[?codec=h264|h265][&pt=N] ``` -| URL | Meaning | -|---|---| -| `udp://@:5600` | Listen on port 5600 | -| `udp://@239.1.2.3:5600` | Join the multicast group 239.1.2.3 on port 5600 | +| URL | Meaning | +|-------------------------------------|----------------------------------------------------| +| `udp://@:5600` | Listen on port 5600 | +| `udp://@239.1.2.3:5600` | Join the multicast group 239.1.2.3 on port 5600 | | `udp://192.168.1.10@239.1.2.3:5600` | The same, but accept packets from this sender only | -| `udp://@:5600?codec=h265&pt=97` | Force the codec or the video payload type | +| `udp://@:5600?codec=h265&pt=97` | Force the codec or the video payload type | -| Option | Values | Default | -|---|---|---| +| Option | Values | Default | +|---------|---------------------------------|-----------------------------------------------| | `codec` | `h264` / `avc`, `h265` / `hevc` | from the payload type: 96 → H.264, 97 → H.265 | -| `pt` | 0–127 | 96 or 97 | +| `pt` | 0–127 | 96 or 97 | - Addresses must be numeric (IPv4 or IPv6 in brackets); no name lookup. A port is required. - A codec change on the camera is followed while playing. Video only. @@ -108,12 +108,12 @@ RTSP slots on `rtsp://:7070/webcam` (KY UFO and other cooingdv toys) also ## Codecs -| Kind | Codecs | Decoding | -|---|---|---| -| Video | H.264, H.265 | MediaCodec (hardware); a software decoder for key-frame cells beyond the hardware instance limit | -| Video | MJPEG (RTP/JPEG over RTSP, HTTP MJPEG) | the device's JPEG decoder (`video/jpeg`) where it can be created, otherwise BitmapFactory | -| Audio | AAC (RFC 3640 `mpeg4-generic`, ADTS, MP4A-LATM with out-of-band config), Opus | MediaCodec | -| Audio | G.711 μ-law / A-law, L16 PCM (mono or stereo) | converted in the app | +| Kind | Codecs | Decoding | +|-------|-------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| +| Video | H.264, H.265 | MediaCodec (hardware); a software decoder for key-frame cells beyond the hardware instance limit | +| Video | MJPEG (RTP/JPEG over RTSP, HTTP MJPEG) | the device's JPEG decoder (`video/jpeg`) where it can be created, otherwise BitmapFactory | +| Audio | AAC (RFC 3640 `mpeg4-generic`, ADTS, MP4A-LATM with out-of-band config), Opus | MediaCodec | +| Audio | G.711 μ-law / A-law, L16 PCM (mono or stereo) | converted in the app | Not supported: G.726, MP4A-LATM with in-band configuration (`cpresent=1`) ([#14](https://github.com/OpenIPC/decoder/issues/14)). An unsupported audio track is not set up; the video plays without sound. @@ -127,13 +127,13 @@ Not supported: G.726, MP4A-LATM with in-band configuration (`cpresent=1`) ([#14] Measured on the lab cameras. -| Feature | OpenIPC Majestic | XM (`H264DVR 1.0`) | -|---|---|---| -| RTP over TCP (interleaved) | yes | yes | -| RTP over UDP | yes | yes (reply without `/UDP`) | -| RTSP multicast | no: `400` to a multicast request, silent unicast when the request also has `client_port` | no: `451`, silent unicast likewise | -| Plain RTP push (`outgoing`), unicast or multicast | yes | not known | -| HTTP MJPEG | yes (`/mjpeg`) | not tested | -| Audio | Opus, AAC, L16 (G.711 not testable on the lab build) | none (video H.264 only) | +| Feature | OpenIPC Majestic | XM (`H264DVR 1.0`) | +|---------------------------------------------------|------------------------------------------------------------------------------------------|------------------------------------| +| RTP over TCP (interleaved) | yes | yes | +| RTP over UDP | yes | yes (reply without `/UDP`) | +| RTSP multicast | no: `400` to a multicast request, silent unicast when the request also has `client_port` | no: `451`, silent unicast likewise | +| Plain RTP push (`outgoing`), unicast or multicast | yes | not known | +| HTTP MJPEG | yes (`/mjpeg`) | not tested | +| Audio | Opus, AAC, L16 (G.711 not testable on the lab build) | none (video H.264 only) | With a Multicast slot both play over TCP in the same connection. Multicast from OpenIPC is available only as plain RTP via `outgoing`. A server that does RTSP multicast (for example mediamtx) is needed to use the Multicast mode ([#69](https://github.com/OpenIPC/decoder/issues/69)). From e37c2d07a3c570664255c5e653571aff8bfe2a45 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 14:09:54 +0000 Subject: [PATCH 17/21] =?UTF-8?q?feat:=20jieli://=20turns=20frames=20270?= =?UTF-8?q?=C2=B0=20by=20default=20(=3Frotate=3D),=20why=20frames=20are=20?= =?UTF-8?q?dropped,=20raw=20frames=20on=20the=20debug=20page=20(#187)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/openipc/decoder/DebugServer.java | 17 +++++++ .../java/com/openipc/decoder/Decoder.java | 4 ++ .../main/java/com/openipc/decoder/Jieli.java | 46 ++++++++++++++++--- .../java/com/openipc/decoder/RtspClient.java | 1 + .../java/com/openipc/decoder/RtspSession.java | 7 +++ .../java/com/openipc/decoder/JieliTest.java | 7 ++- docs/debug.md | 15 +++--- docs/toys.md | 23 +++++----- 8 files changed, 95 insertions(+), 25 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/DebugServer.java b/android/app/src/main/java/com/openipc/decoder/DebugServer.java index ab8c9a0b..b80c72ff 100644 --- a/android/app/src/main/java/com/openipc/decoder/DebugServer.java +++ b/android/app/src/main/java/com/openipc/decoder/DebugServer.java @@ -63,6 +63,9 @@ interface Source { * labels: only the frames, for the page that draws the text over them. */ byte[] snapshotJpeg(boolean labels); + + /** Raw bytes of the single view's jieli:// slot (#187), or null. */ + default byte[] jieliRaw(String which) { return null; } } private static DebugServer instance; @@ -210,6 +213,20 @@ void serve(OutputStream out, String path, String query) throws IOException { case "/log": reply(out, 200, "text/plain; charset=utf-8", log.text()); break; + case "/jieli-frame.jpg": + case "/jieli-dropped.bin": + // The toy's frames as they came (#187), to see what the picture is made from + Source src = source; + byte[] raw = src != null ? src.jieliRaw(path.startsWith("/jieli-frame") + ? "frame" : "dropped") : null; + if (raw != null) { + reply(out, 200, path.endsWith(".jpg") ? "image/jpeg" + : "application/octet-stream", raw); + } else { + reply(out, 404, "text/plain", "Nothing yet (a jieli:// slot in the single" + + " view)\n"); + } + break; case "/report.txt": // Downloaded by the browser, not shown (#207) Date now = new Date(); diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index 3020744c..d7e47748 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -1388,6 +1388,10 @@ private void removeQuadOverlay(int index) { private final DebugServer.Source debugSource = new DebugServer.Source() { @Override public Map snapshot() { return debugSnapshot(); } @Override public byte[] snapshotJpeg(boolean labels) { return debugSnapshotJpeg(labels); } + @Override public byte[] jieliRaw(String which) { + RtspClient c = rtspClient; + return c != null ? c.jieliRaw(which) : null; + } }; /** A cell counts as playing in a snapshot when it showed a frame this recently (#98). */ private static final long SNAPSHOT_LIVE_MS = 2000; diff --git a/android/app/src/main/java/com/openipc/decoder/Jieli.java b/android/app/src/main/java/com/openipc/decoder/Jieli.java index b7fe40d2..3672cc66 100644 --- a/android/app/src/main/java/com/openipc/decoder/Jieli.java +++ b/android/app/src/main/java/com/openipc/decoder/Jieli.java @@ -24,6 +24,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; @@ -61,6 +62,8 @@ final class Jieli { static final int DEFAULT_FPS = 25; static final int FORMAT_JPEG = 0; static final int FORMAT_H264 = 1; + /** The YL3's frames come turned (portrait 240×320), as the K200's do (#187, #200). */ + static final int DEFAULT_ROTATE = 270; static final int PT_JPEG = 26; static final int PT_H264 = 96; @@ -109,7 +112,8 @@ private Jieli() {} * when left out. Options, as in udp:// URLs (unknown ones are ignored, bad values * refused): w, h, fps (the stream asked for; default what the toy announces in * VIDEO_PARAM, else 640x480 at 25), rtp (the local UDP port for the video, 6666), - * format (0 JPEG, the default, or 1 H.264). + * format (0 JPEG, the default, or 1 H.264), rotate (degrees counter-clockwise for JPEG, + * 0, 90, 180 or 270; default 270: the YL3's camera is mounted sideways like the K200's). */ static final class Url { String host = DEFAULT_HOST; @@ -120,6 +124,7 @@ static final class Url { int height; int fps; int format = FORMAT_JPEG; + int rotate = DEFAULT_ROTATE; static boolean matches(String url) { return url != null && url.trim().toLowerCase(Locale.ROOT).startsWith(SCHEME); @@ -181,6 +186,9 @@ static Url parse(String url) throws IOException { case "format": u.format = number(value, 0, 1, "format (0 JPEG, 1 H.264)"); break; + case "rotate": + u.rotate = RtspSession.Url.parseRotate(value); + break; default: break; } @@ -582,6 +590,15 @@ static final class RtStream { int frames; int dropped; int otherTypes; + /** Why the last frame was dropped, and its bytes as put together (for the debug page). */ + volatile String lastDropReason; + volatile byte[] lastDropped; + + private void drop(String reason, int n) { + dropped++; + lastDropReason = reason; + if (n > 0) lastDropped = Arrays.copyOf(frame, n); + } /** * Takes one datagram; returns the length of a JPEG now complete in {@link #frame()} @@ -596,12 +613,12 @@ int feed(byte[] b, int off, int len) { int total = le32(b, off + 8); int at = le32(b, off + 12); if (total < 4 || total > MAX_FRAME || at < 0 || at >= total) { - dropped++; + drop("bad header: length " + total + ", offset " + at, 0); return 0; } if (at == 0) { // A new frame; one still missing pieces is lost - if (size > 0) dropped++; + if (size > 0) drop("incomplete: " + filled + " of " + size + " bytes", filled); size = total; filled = 0; if (frame.length < total) frame = new byte[total]; @@ -616,7 +633,9 @@ int feed(byte[] b, int off, int len) { size = -1; int end = jpegEnd(frame, n); if (end == 0) { - dropped++; + drop(n + " bytes, starts " + FlowUfo.hex(frame, 0, Math.min(n, 4)) + ", ends " + + FlowUfo.hex(frame, Math.max(0, n - 8), Math.min(n, 8)) + + ": no FFD8 at the start or no FFD9", n); return 0; } frames++; @@ -719,6 +738,8 @@ static final class Client implements Closeable { private volatile int streamForeign; private volatile String streamFirst; private int droppedLogged; + /** The last JPEG put together from UDP 2224, as received (for the debug page). */ + private volatile byte[] lastStreamFrame; private volatile String lastSource; private volatile boolean h264Logged; private final Map payloadTypes = new LinkedHashMap<>(); @@ -1165,14 +1186,15 @@ private void stream(DatagramPacket p, Listener listener) { } int jpeg = rtStream.feed(b, 0, len); if (jpeg > 0) { + lastStreamFrame = Arrays.copyOf(rtStream.frame(), jpeg); lastSource = from; videoPort = streamPort; lastVideoAt = now(); listener.onJpeg(rtStream.frame(), 0, jpeg); } else if (rtStream.dropped != droppedLogged && rtStream.dropped <= 3) { droppedLogged = rtStream.dropped; - Log.w(TAG, tag + ": Jieli: UDP " + streamPort + ": " + droppedLogged - + " frame(s) not put together (length or JPEG markers do not fit)"); + Log.w(TAG, tag + ": Jieli: UDP " + streamPort + ": frame " + droppedLogged + + " not put together: " + rtStream.lastDropReason); } } @@ -1208,6 +1230,15 @@ private String summary() { rtStream.dropped); } + /** + * For the debug page (#187): "frame" the last JPEG put together from UDP 2224, "dropped" + * the last one dropped, as the bytes came; null when there is none. + */ + byte[] raw(String which) { + return "frame".equals(which) ? lastStreamFrame + : "dropped".equals(which) ? rtStream.lastDropped : null; + } + /** For /stats.json. */ Map stats() { Map m = new LinkedHashMap<>(); @@ -1257,6 +1288,9 @@ Map stats() { stream.put("frames", rtStream.frames); stream.put("dropped", rtStream.dropped); if (rtStream.otherTypes > 0) stream.put("other_types", rtStream.otherTypes); + if (rtStream.lastDropReason != null) { + stream.put("last_drop", rtStream.lastDropReason); + } if (streamForeign > 0) stream.put("from_others", streamForeign); if (streamFirst != null) stream.put("first_packet", streamFirst); m.put("stream", stream); diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java index 50eb8153..d0f550cb 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java @@ -91,6 +91,7 @@ void configure(String host, String userAgent, Transport.Mode transport) { java.util.Map flowUfoStats() { return session.flowUfoStats(); } /** Jieli statistics while a jieli:// slot plays (#187), else null. */ java.util.Map jieliStats() { return session.jieliStats(); } + byte[] jieliRaw(String which) { return session.jieliRaw(which); } /** The cooingdv heartbeat once this slot used one (#187), else null. */ java.util.Map heartbeatStats() { return session.heartbeatStats(); } diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index b5793dd2..3a6bc66c 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -780,6 +780,12 @@ Map jieliStats() { return c != null ? c.stats() : null; } + /** Raw bytes of a jieli:// slot for the debug page (#187), see Jieli.Client.raw(). */ + byte[] jieliRaw(String which) { + Jieli.Client c = jieli; + return c != null ? c.raw(which) : null; + } + /** * A jieli:// slot (#187): CTP over TCP asks the toy for the stream, which comes as RTP to * a fixed local UDP port: payload type 26 goes to the RTP/JPEG path, 96 to the H.264 one, @@ -800,6 +806,7 @@ private boolean jieli(int gen, Jieli.Url u) throws IOException { } try { final int[] pt = {u.format == Jieli.FORMAT_H264 ? Jieli.PT_H264 : Jieli.PT_JPEG}; + rotate = rotation(u.rotate, true); callback.onSdp(jieliSdp(pt[0])); // onState comes from the CTP thread, the video from the UDP one final AtomicBoolean first = new AtomicBoolean(true); diff --git a/android/app/src/test/java/com/openipc/decoder/JieliTest.java b/android/app/src/test/java/com/openipc/decoder/JieliTest.java index 10b87cbf..4cbbbb60 100644 --- a/android/app/src/test/java/com/openipc/decoder/JieliTest.java +++ b/android/app/src/test/java/com/openipc/decoder/JieliTest.java @@ -135,6 +135,8 @@ public void rtStreamDropsBrokenFrames() { byte[] junk = streamPacket(6, 100, 0, new byte[100], 0); assertEquals(0, st.feed(junk, 0, junk.length)); assertEquals(1, st.dropped); + assertTrue(st.lastDropReason, st.lastDropReason.contains("no FFD8")); + assertEquals(100, st.lastDropped.length); // A frame cut short by the next one byte[] j = jpeg(8000); byte[] first = streamPacket(7, 8000, 0, Arrays.copyOfRange(j, 0, 4000), 0); @@ -324,6 +326,9 @@ public void urls() throws Exception { assertEquals("192.168.80.1", u.host); assertEquals(3333, u.port); assertEquals(6666, u.rtpPort); + assertEquals(270, u.rotate); + assertEquals(0, Jieli.Url.parse("jieli://?rotate=0").rotate); + assertEquals(90, Jieli.Url.parse("jieli://?rotate=90").rotate); assertEquals(0, u.width); assertEquals(Jieli.FORMAT_JPEG, u.format); u = Jieli.Url.parse(" JIELI://192.168.80.1:3334/?w=1280&h=720&fps=30&rtp=7000&format=1 "); @@ -337,7 +342,7 @@ public void urls() throws Exception { assertTrue(Jieli.Url.matches("jieli://x")); assertTrue(!Jieli.Url.matches("flowufo://")); for (String bad : new String[] {"jieli://h:0", "jieli://h:x", "jieli://?w=8", - "jieli://?fps=0", "jieli://?rtp=70000", "jieli://?format=2", "jieli://a b", + "jieli://?fps=0", "jieli://?rtp=70000", "jieli://?format=2", "jieli://?rotate=45", "jieli://a b", "jieli://[::1"}) { try { Jieli.Url.parse(bad); diff --git a/docs/debug.md b/docs/debug.md index 7a71bafa..b1e7d65e 100644 --- a/docs/debug.md +++ b/docs/debug.md @@ -14,13 +14,14 @@ While the app runs it serves its debug information over HTTP on port **8085**, for a browser or `curl` on the same network. It is always on and read-only. -| Path | Content | -|------------------------|-----------------------------------------------------------------------------------------------------------------------------| -| `/` (or `/index.html`) | the [page](#page): snapshot with the text of each cell | -| `/stats.json` | the state as JSON, pretty-printed, see [fields](#statsjson-fields) | -| `/snapshot.jpg` | a picture of the screen, see [Snapshot](#snapshot); `?labels=0`: frames only | -| `/log` | the last 500 lines of the app's log, see [Log](#log) | -| `/report.txt` | a header, `/stats.json` and `/log` in one file, downloaded as `report-YYYYMMDD-HHMMSS.txt`, see [Save report](#save-report) | +| Path | Content | +|------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `/` (or `/index.html`) | the [page](#page): snapshot with the text of each cell | +| `/stats.json` | the state as JSON, pretty-printed, see [fields](#statsjson-fields) | +| `/snapshot.jpg` | a picture of the screen, see [Snapshot](#snapshot); `?labels=0`: frames only | +| `/log` | the last 500 lines of the app's log, see [Log](#log) | +| `/report.txt` | a header, `/stats.json` and `/log` in one file, downloaded as `report-YYYYMMDD-HHMMSS.txt`, see [Save report](#save-report) | +| `/jieli-frame.jpg`, `/jieli-dropped.bin` | test build only: the last JPEG of a `jieli://` slot in the single view as the toy sent it, and the last frame that could not be put together ([toys.md](toys.md#hf--jieli-toys)) | - Only clients on the local network are answered; others get `403`. The rule and what is hidden: [security.md](security.md#debug-server). - `GET` only (`405` otherwise), one request per connection, `Cache-Control: no-store`. diff --git a/docs/toys.md b/docs/toys.md index 662829f6..17a6db8d 100644 --- a/docs/toys.md +++ b/docs/toys.md @@ -40,11 +40,11 @@ Whether a toy can be played here depends on its protocol. Toys that do offer RTS The Wi‑Fi name and the address the phone gets usually tell the family. The toy is almost always the `.1` address of that network, **not** the phone's own address. -| Wi‑Fi name | Phone gets | Family | Slot URL | Status | -|----------------------------------------------------------------|-----------------|----------------------------------------------------------|---------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| -| `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | -| `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | -| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 2224 or 6666) | test build 3: video on UDP 2224 from a YL3, playing it is new ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| Wi‑Fi name | Phone gets | Family | Slot URL | Status | +|----------------------------------------------------------------|-----------------|----------------------------------------------------------|---------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------| +| `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 2224 or 6666) | test build 4: YL3 video on UDP 2224 plays, lower half broken in test build 3 ([#187](https://github.com/OpenIPC/decoder/issues/187)) | When a `flowufo://` or `jieli://` slot points at the phone itself, or at the network of another family, its status says so (for example *192.168.1.100 is this phone; the toy is usually 192.168.1.1*, *This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam* or, on a `192.168.80.x` network, *try jieli://192.168.80.1*). @@ -109,18 +109,19 @@ What the app does, as a capture of the vendor app shows it ([#196](https://githu - It connects to TCP port 3333 and talks **CTP**: each message is `CTP:`, the length of a topic name, the topic (e.g. `OPEN_RT_STREAM`), the length of a JSON text and the JSON. - `APP_ACCESS` first; the toy answers with `KEEP_ALIVE_INTERVAL` (a timeout, 60 s), its battery and SD card state and `VIDEO_PARAM` (the size, e.g. 640×480 at 25 fps). Then `DATE_TIME` (the phone's time), a `CTP_KEEP_ALIVE` every 7.5 s (a quarter of a shorter timeout), and `OPEN_RT_STREAM` with the size from `VIDEO_PARAM` (640×480 at 25 if the toy sent none). When the slot stops: `CLOSE_RT_STREAM`. - The video was **not** in the capture (PCAPdroid without root cannot see it). After turbodrone's research on the same Jieli SDK it is **RTP/JPEG** (payload type 26) or H.264 (96) to the phone's **UDP port 6666**. The app listens there and plays payload type 26 through its RTP/JPEG path and 96 through its H.264 path; a whole JPEG in a datagram without RTP is shown too. -- A YL3 sent nothing to 6666 but its own stream to the phone's **UDP port 2224**: per datagram a 20-byte header, then the JPEG (seen: byte 0 = 2, a packet counter in bytes 4–7, the JPEG's length in 8–11, an offset in 12–15, little-endian; a 640×480 frame of 4600 bytes in one 4640-byte datagram). The app listens on 2224 as well and shows those JPEGs; longer frames are put together by the offset. The first five datagrams go to the log with the whole header, `/stats.json` has a `stream` object (`packets`, `frames`, `dropped`) and `video_port`. +- A YL3 sent nothing to 6666 but its own stream to the phone's **UDP port 2224**: per datagram a 20-byte header, then the JPEG (seen: byte 0 = 2, a packet counter in bytes 4–7, the JPEG's length in 8–11, an offset in 12–15, little-endian; frames of 4–8 KB in one datagram, 40–60 zero bytes after the JPEG). Its frames are portrait (240×320 when 640×480 was asked for): the camera is mounted sideways, so `jieli://` turns them 270° counter-clockwise like the K200's, `?rotate=` overrides it. The app listens on 2224 as well and shows those JPEGs; longer frames are put together by the offset. The first five datagrams go to the log with the whole header, `/stats.json` has a `stream` object (`packets`, `frames`, `dropped`, `last_drop`) and `video_port`; why a frame was dropped is in the log for the first three. The debug page serves the last frame as it came, `/jieli-frame.jpg`, and the last dropped one, `/jieli-dropped.bin`. - If nothing arrives on 6666 or 2224 for 5 s after `OPEN_RT_STREAM`, the status says so and the app also listens on UDP 1234 (audio in the vendor SDK), **only to log** what arrives there; it asks for the stream again every 10 s without video. - **No flight command is ever sent**: no `FLIGHT_CONTROL` over CTP, nothing to the toy's UDP port 3456 (where the vendor app sends the sticks). - Ports 6666 and 2224 can be used by one slot at a time: one `jieli://` slot in a grid. URL options (as with `udp://`, separated by `?` and `&`): -| Option | Values | Default | What it does | -|-----------------|------------|---------------------------------------|-----------------------------------------------------------| -| `w`, `h`, `fps` | numbers | from `VIDEO_PARAM`, else 640, 480, 25 | the stream asked for in `OPEN_RT_STREAM` | -| `rtp` | a UDP port | `6666` | the local port the video is expected on | -| `format` | `0`, `1` | `0` | `0` asks for JPEG (as the vendor app does), `1` for H.264 | +| Option | Values | Default | What it does | +|-----------------|-------------------------|---------------------------------------|---------------------------------------------------------------------------------------------| +| `w`, `h`, `fps` | numbers | from `VIDEO_PARAM`, else 640, 480, 25 | the stream asked for in `OPEN_RT_STREAM` | +| `rtp` | a UDP port | `6666` | the local port the video is expected on | +| `format` | `0`, `1` | `0` | `0` asks for JPEG (as the vendor app does), `1` for H.264 | +| `rotate` | `0`, `90`, `180`, `270` | `270` | degrees counter-clockwise the JPEG frames are turned (the YL3's camera is mounted sideways) | Examples: `jieli://192.168.80.1`, `jieli://192.168.80.1?w=1280&h=720`, `jieli://192.168.80.1?format=1`. From 3f4ae3e3f0530f5358eaae854214749bc7214e3a Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 14:36:55 +0000 Subject: [PATCH 18/21] =?UTF-8?q?fix:=20open=20Qodo=20findings=20on=20#188?= =?UTF-8?q?=20and=20#198=20=E2=80=94=20FLOW-UFO=20sockets=20closed=20when?= =?UTF-8?q?=20startup=20fails;=20Jieli:=20only=20usable=20video=20counts?= =?UTF-8?q?=20as=20receiving,=20a=20stopped=20slot=20cannot=20hang=20on=20?= =?UTF-8?q?a=20blocked=20CTP=20write,=20diagnostic=20ports=20freed=20after?= =?UTF-8?q?=2030=20s,=20topic=20stats=20bounded,=20a=20dropout=20after=20t?= =?UTF-8?q?he=20stream=20request=20reconnects=20without=20back-off?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/openipc/decoder/FlowUfo.java | 16 ++- .../main/java/com/openipc/decoder/Jieli.java | 94 +++++++++++++--- .../java/com/openipc/decoder/RtspSession.java | 106 ++++++++++-------- .../java/com/openipc/decoder/JieliTest.java | 56 ++++++++- docs/toys.md | 2 +- 5 files changed, 205 insertions(+), 69 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java index 62e67eda..c708b48c 100644 --- a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java +++ b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java @@ -822,7 +822,13 @@ private void open(int i, InetAddress target) throws IOException { try { s.setReceiveBufferSize(1024 * 1024); } catch (IOException ignored) {} - if (setup != null) network = setup.prepare(s, target); + try { + if (setup != null) network = setup.prepare(s, target); + } catch (RuntimeException e) { + // Not registered yet: close() would not see it (Qodo on #188) + s.close(); + throw e; + } synchronized (sockets) { if (sockets[i] != null) sockets[i].close(); sockets[i] = s; @@ -858,7 +864,13 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException InetAddress address = InetAddress.getByName(url.host); hint = subnetHint(address, localForHint != null ? localForHint : localAddresses()); if (hint != null) Log.w(TAG, tag + ": FLOW-UFO: " + hint); - for (int i = 0; i < ports.length; i++) open(i, address); + try { + for (int i = 0; i < ports.length; i++) open(i, address); + } catch (IOException | RuntimeException e) { + // The sockets opened before the one that failed (Qodo on #188) + close(); + throw e; + } Log.i(TAG, tag + ": FLOW-UFO: probing " + url.probed() + " from local port " + localPorts() + " (network: " + network + "), qpara " + hex(url.qpara, 0, 4) + ", dqt " + DQT_NAMES[url.dqt]); diff --git a/android/app/src/main/java/com/openipc/decoder/Jieli.java b/android/app/src/main/java/com/openipc/decoder/Jieli.java index 3672cc66..e776265d 100644 --- a/android/app/src/main/java/com/openipc/decoder/Jieli.java +++ b/android/app/src/main/java/com/openipc/decoder/Jieli.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.CountDownLatch; /** * Toy drones with an "HF_xxxxxxxx" access point (Jieli AC79 camera, vendor app "DVRunning 2") @@ -71,6 +72,13 @@ final class Jieli { /** Longer topics or JSON than this are not CTP: the reader looks for the next "CTP:". */ static final int MAX_TOPIC = 256; static final int MAX_JSON = 256 * 1024; + /** Distinct CTP topics counted one by one for the stats; the rest together. */ + static final int MAX_TOPICS = 64; + /** How long close() lets the session send CLOSE_RT_STREAM before it closes the TCP. */ + static volatile long closeGraceMs = 1000; + /** Diagnostic ports listen this long, or for this many packets, then are freed. */ + static volatile long diagnosticMs = 30_000; + static final int DIAGNOSTIC_PACKETS = 20; static final String APP_ACCESS = "APP_ACCESS"; static final String DATE_TIME = "DATE_TIME"; @@ -667,11 +675,15 @@ interface Listener { /** Waiting (no video) or receiving; text for the status. */ void onState(boolean receiving, String text); - /** One RTP packet with payload type 26 (JPEG) or 96 (H.264), valid during the call. */ - void onRtp(byte[] buf, int offset, int length); + /** + * One RTP packet with payload type 26 (JPEG) or 96 (H.264), valid during the call. + * Returns whether it was usable: only usable video keeps the toy "receiving" (Qodo on + * #198), so a toy sending broken packets is still asked for the stream again. + */ + boolean onRtp(byte[] buf, int offset, int length); - /** One whole JPEG in a datagram without RTP, valid during the call. */ - default void onJpeg(byte[] buf, int offset, int length) {} + /** One whole JPEG without RTP, valid during the call; returns whether it was shown. */ + default boolean onJpeg(byte[] buf, int offset, int length) { return false; } } /** Binds the CTP socket to the toy's Wi-Fi before it connects; returns what was done. */ @@ -717,6 +729,12 @@ static final class Client implements Closeable { private volatile String requested; private final ArrayDeque lastNotifies = new ArrayDeque<>(); private final Map topics = new LinkedHashMap<>(); + /** Messages with a topic beyond the first MAX_TOPICS distinct ones. */ + private int otherTopics; + /** OPEN_RT_STREAM was sent: the session got as far as a PLAY would (Qodo on #198). */ + volatile boolean streamRequested; + /** Counted down when run() has cleaned up. */ + private final CountDownLatch ended = new CountDownLatch(1); private final Reader reader = new Reader(); private int ctpLogged; @@ -762,14 +780,31 @@ static final class Client implements Closeable { @Override public void close() { closed = true; + Socket connected = null; synchronized (lock) { for (DatagramSocket s : udpSockets) s.close(); if (tcp != null && !tcpConnected) { try { tcp.close(); } catch (IOException ignored) {} + } else if (tcp != null) { + connected = tcp; } } + if (connected == null) return; + // The session sends CLOSE_RT_STREAM and closes the TCP itself; a write blocked on a + // toy that stopped reading would keep it from that, so after a grace period the + // socket is closed from here (Qodo on #188) + final Socket s = connected; + Thread t = new Thread(() -> { + try { + if (!ended.await(closeGraceMs, java.util.concurrent.TimeUnit.MILLISECONDS)) { + s.close(); + } + } catch (InterruptedException | IOException ignored) {} + }, tag + "-jieli-close"); + t.setDaemon(true); + t.start(); } private String withHint(String text) { @@ -960,6 +995,7 @@ void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException tcpConnected = false; } Log.i(TAG, tag + ": Jieli: session ended (" + summary() + ")"); + ended.countDown(); } } @@ -999,21 +1035,37 @@ private void startThread(DatagramSocket d, int kind, Listener listener) { Thread th = new Thread(() -> { byte[] b = new byte[65536]; DatagramPacket p = new DatagramPacket(b, b.length); + // A diagnostic port is only borrowed: freed after a while or a few packets, so + // another slot can use it (Qodo on #198) + long until = kind == PORT_DIAGNOSTIC ? now() + diagnosticMs : Long.MAX_VALUE; + int seen = 0; try { - while (!closed && !d.isClosed()) { + if (kind == PORT_DIAGNOSTIC) d.setSoTimeout(TICK_MS * 10); + while (!closed && !d.isClosed() && now() < until) { p.setLength(b.length); - d.receive(p); + try { + d.receive(p); + } catch (SocketTimeoutException e) { + continue; + } if (kind == PORT_RTP) { video(p, listener); } else if (kind == PORT_STREAM) { stream(p, listener); } else { diagnostic(port, p); + if (++seen >= DIAGNOSTIC_PACKETS) break; } } } catch (IOException e) { // Closed } + if (kind == PORT_DIAGNOSTIC) { + synchronized (lock) { + udpSockets.remove(d); + } + d.close(); + } }, tag + "-jieli-" + port); th.setDaemon(true); th.start(); @@ -1032,6 +1084,7 @@ private void keepAlive(OutputStream out) throws IOException { } private void requestStream(OutputStream out) throws IOException { + streamRequested = true; int[] p = videoParams(); int w = url.width > 0 ? url.width : p[0] > 0 ? p[0] : DEFAULT_WIDTH; int h = url.height > 0 ? url.height : p[1] > 0 ? p[1] : DEFAULT_HEIGHT; @@ -1062,7 +1115,12 @@ private void handle(Message m) { ctpReceived++; synchronized (topics) { Integer c = topics.get(m.topic); - topics.put(m.topic, c == null ? 1 : c + 1); + // Bounded: a toy inventing topic names cannot grow it (Qodo on #198) + if (c != null || topics.size() < MAX_TOPICS) { + topics.put(m.topic, c == null ? 1 : c + 1); + } else { + otherTopics++; + } if (!KEEP_ALIVE.equals(m.topic)) { lastNotifies.addLast(truncate(m.toString(), 200)); while (lastNotifies.size() > 8) lastNotifies.removeFirst(); @@ -1141,16 +1199,18 @@ private void video(DatagramPacket p, Listener listener) { h264Logged = true; Log.i(TAG, tag + ": Jieli: RTP payload type 96: played as H.264"); } - videoPort = url.rtpPort; - lastVideoAt = now(); - listener.onRtp(b, 0, len); + if (listener.onRtp(b, 0, len)) { + videoPort = url.rtpPort; + lastVideoAt = now(); + } } else if (!rtp && len >= 4 && (b[0] & 0xFF) == 0xFF && (b[1] & 0xFF) == 0xD8 && (b[len - 2] & 0xFF) == 0xFF && (b[len - 1] & 0xFF) == 0xD9) { // A whole JPEG without RTP: turbodrone takes those too rawJpegs++; - videoPort = url.rtpPort; - lastVideoAt = now(); - listener.onJpeg(b, 0, len); + if (listener.onJpeg(b, 0, len)) { + videoPort = url.rtpPort; + lastVideoAt = now(); + } } else if (rtp) { if (++otherRtp <= 3) { Log.w(TAG, tag + ": Jieli: RTP payload type " + pt + " on UDP " @@ -1188,9 +1248,10 @@ private void stream(DatagramPacket p, Listener listener) { if (jpeg > 0) { lastStreamFrame = Arrays.copyOf(rtStream.frame(), jpeg); lastSource = from; - videoPort = streamPort; - lastVideoAt = now(); - listener.onJpeg(rtStream.frame(), 0, jpeg); + if (listener.onJpeg(rtStream.frame(), 0, jpeg)) { + videoPort = streamPort; + lastVideoAt = now(); + } } else if (rtStream.dropped != droppedLogged && rtStream.dropped <= 3) { droppedLogged = rtStream.dropped; Log.w(TAG, tag + ": Jieli: UDP " + streamPort + ": frame " + droppedLogged @@ -1257,6 +1318,7 @@ Map stats() { if (reader.skipped > 0) ctp.put("skipped_bytes", reader.skipped); synchronized (topics) { ctp.put("topics", new LinkedHashMap<>(topics)); + if (otherTopics > 0) ctp.put("other_topics", otherTopics); ctp.put("last_notifies", new ArrayList<>(lastNotifies)); } m.put("ctp", ctp); diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index 3a6bc66c..9e1abc3c 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -812,61 +812,73 @@ private boolean jieli(int gen, Jieli.Url u) throws IOException { final AtomicBoolean first = new AtomicBoolean(true); final AtomicBoolean received = new AtomicBoolean(false); final Object order = new Object(); - c.run(() -> active && gen == generation, new Jieli.Listener() { - @Override - public void onState(boolean receiving, String text) { - if (gen != generation) return; - // In order with usable() on the UDP thread: a packet that came in since - // the CTP thread saw the video stop wins (Qodo on #198) - synchronized (order) { - if (receiving) { - inUse = "Jieli RTP"; - received.set(true); - } else { - if (c.videoAgeMs() < Jieli.quietMs) return; - inUse = null; - // Video that stopped says so; the next frame reports "connected" - status(gen, received.get() ? "disconnected" : "buffering"); - first.set(true); + try { + c.run(() -> active && gen == generation, new Jieli.Listener() { + @Override + public void onState(boolean receiving, String text) { + if (gen != generation) return; + // In order with usable() on the UDP thread: a packet that came in since + // the CTP thread saw the video stop wins (Qodo on #198) + synchronized (order) { + if (receiving) { + inUse = "Jieli RTP"; + received.set(true); + } else { + if (c.videoAgeMs() < Jieli.quietMs) return; + inUse = null; + // Video that stopped says so; the next frame reports "connected" + status(gen, received.get() ? "disconnected" : "buffering"); + first.set(true); + } + callback.onDetail(text); } - callback.onDetail(text); } - } - @Override - public void onRtp(byte[] buf, int offset, int length) { - if (gen != generation) return; - int type = buf[offset + 1] & 0x7F; - if (type != pt[0]) { - // The toy sends the other codec than asked for: follow it - pt[0] = type; - Log.i(TAG, tag + ": Jieli: video is " - + (type == Jieli.PT_H264 ? "H.264" : "JPEG") + " (RTP PT " + type - + ")"); - callback.onSdp(jieliSdp(type)); + @Override + public boolean onRtp(byte[] buf, int offset, int length) { + if (gen != generation) return false; + int type = buf[offset + 1] & 0x7F; + if (type != pt[0]) { + // The toy sends the other codec than asked for: follow it + pt[0] = type; + Log.i(TAG, tag + ": Jieli: video is " + + (type == Jieli.PT_H264 ? "H.264" : "JPEG") + + " (RTP PT " + type + ")"); + callback.onSdp(jieliSdp(type)); + } + streamBytes += length; + if (!callback.onPacket(CHANNEL_VIDEO, buf, offset, length)) return false; + usable(); + return true; } - streamBytes += length; - if (callback.onPacket(CHANNEL_VIDEO, buf, offset, length)) usable(); - } - @Override - public void onJpeg(byte[] buf, int offset, int length) { - if (gen != generation) return; - if (pt[0] != Jieli.PT_JPEG) { - pt[0] = Jieli.PT_JPEG; - callback.onSdp(jieliSdp(Jieli.PT_JPEG)); + @Override + public boolean onJpeg(byte[] buf, int offset, int length) { + if (gen != generation) return false; + if (pt[0] != Jieli.PT_JPEG) { + pt[0] = Jieli.PT_JPEG; + callback.onSdp(jieliSdp(Jieli.PT_JPEG)); + } + streamBytes += length; + if (!callback.onJpegFrame(buf, offset, length)) return false; + usable(); + return true; } - streamBytes += length; - if (callback.onJpegFrame(buf, offset, length)) usable(); - } - private void usable() { - lastPacketMs = System.nanoTime() / 1_000_000; - synchronized (order) { - if (first.compareAndSet(true, false)) status(gen, "connected"); + private void usable() { + lastPacketMs = System.nanoTime() / 1_000_000; + synchronized (order) { + if (first.compareAndSet(true, false)) status(gen, "connected"); + } } - } - }); + }); + } catch (IOException e) { + // A toy that dropped after the stream was asked for ends like a PLAY that + // worked: the next attempt comes soon, not after a growing back-off (Qodo on #198) + if (!c.streamRequested || gen != generation) throw e; + Log.w(TAG, tag + ": Jieli: " + e.getMessage() + "; connecting again"); + callback.onFailure(e.getMessage()); + } return true; } finally { synchronized (lock) { diff --git a/android/app/src/test/java/com/openipc/decoder/JieliTest.java b/android/app/src/test/java/com/openipc/decoder/JieliTest.java index 4cbbbb60..5e0fb433 100644 --- a/android/app/src/test/java/com/openipc/decoder/JieliTest.java +++ b/android/app/src/test/java/com/openipc/decoder/JieliTest.java @@ -9,6 +9,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -57,6 +58,7 @@ public void restoreTimers() { Jieli.quietMs = 3000; Jieli.ctpSilenceMs = 20_000; Jieli.diagnosticPorts = new int[] {1234}; + Jieli.diagnosticMs = 30_000; Jieli.streamPort = Jieli.STREAM_PORT; } @@ -508,7 +510,15 @@ private static final class Running { volatile Exception failure; final Thread thread; + /** False: the "decoder" takes nothing, as with broken packets. */ + final boolean usable; + Running(String url) throws IOException { + this(url, true); + } + + Running(String url, boolean usable) throws IOException { + this.usable = usable; client = new Jieli.Client("test", Jieli.Url.parse(url), null, null); client.localForHint = Collections.emptyList(); JpegAssembler assembler = new JpegAssembler(new FramePool(4)); @@ -521,17 +531,21 @@ public void onState(boolean receiving, String text) { } @Override - public void onRtp(byte[] buf, int offset, int length) { + public boolean onRtp(byte[] buf, int offset, int length) { + if (!usable) return false; assembler.assemble(buf, offset, length, f -> { frames.add(Arrays.copyOf(f.data(), f.length())); frame.countDown(); }); + return true; } @Override - public void onJpeg(byte[] buf, int offset, int length) { + public boolean onJpeg(byte[] buf, int offset, int length) { + if (!usable) return false; jpegs.add(Arrays.copyOfRange(buf, offset, offset + length)); frame.countDown(); + return true; } }); } catch (Exception e) { @@ -557,6 +571,7 @@ public void sessionAsTheVendorApp() throws Exception { assertTrue("no CLOSE_RT_STREAM", toy.closeStream.await(3, TimeUnit.SECONDS)); assertTrue(r.ended.await(3, TimeUnit.SECONDS)); assertNull(r.failure); + assertTrue(r.client.streamRequested); List t = toy.topics(); assertEquals(Arrays.asList(Jieli.APP_ACCESS, Jieli.DATE_TIME, Jieli.KEEP_ALIVE, @@ -662,6 +677,7 @@ public void noVideoListensElsewhereAndAsksAgain() throws Exception { Jieli.noVideoMs = 300; Jieli.reopenMs = 700; Jieli.diagnosticPorts = new int[] {diag}; + Jieli.diagnosticMs = 500; try (FakeToy toy = new FakeToy(60000, true)) { toy.rtpPort = rtp; toy.sendVideo = false; @@ -688,6 +704,17 @@ public void noVideoListensElsewhereAndAsksAgain() throws Exception { @SuppressWarnings("unchecked") Map one = (Map) d.get(String.valueOf(diag)); assertEquals(1, one.get("packets")); + // The diagnostic port is freed after diagnosticMs while the session goes on + until = now() + 3000; + boolean freed = false; + while (now() < until && !freed) { + try (DatagramSocket again = new DatagramSocket(diag, LOOPBACK)) { + freed = true; + } catch (IOException e) { + Thread.sleep(50); + } + } + assertTrue("diagnostic port still taken", freed); r.keepGoing.set(false); assertTrue(r.ended.await(3, TimeUnit.SECONDS)); assertNull(r.failure); @@ -696,6 +723,27 @@ public void noVideoListensElsewhereAndAsksAgain() throws Exception { } } + @Test + public void unusableVideoIsNotReceiving() throws Exception { + int rtp = freeUdpPort(); + Jieli.reopenMs = 700; + try (FakeToy toy = new FakeToy(60000, true)) { + toy.rtpPort = rtp; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp, false); + assertTrue(toy.opened.await(5, TimeUnit.SECONDS)); + // Packets arrive but the decoder takes none: the toy is asked again (Qodo on #198) + long until = now() + 3000; + while (now() < until + && Collections.frequency(toy.topics(), Jieli.OPEN_RT_STREAM) < 2) { + Thread.sleep(20); + } + assertTrue(Collections.frequency(toy.topics(), Jieli.OPEN_RT_STREAM) >= 2); + for (String st : r.states) assertFalse(st, st.startsWith("Receiving")); + r.keepGoing.set(false); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + } + } + @Test public void streamPortVideoIsPlayed() throws Exception { int rtp = freeUdpPort(); @@ -737,7 +785,9 @@ public void unreachableToyFails() throws Exception { public void onState(boolean receiving, String text) {} @Override - public void onRtp(byte[] buf, int offset, int length) {} + public boolean onRtp(byte[] buf, int offset, int length) { + return false; + } }); fail("connected"); } catch (IOException e) { diff --git a/docs/toys.md b/docs/toys.md index 17a6db8d..023e73d4 100644 --- a/docs/toys.md +++ b/docs/toys.md @@ -110,7 +110,7 @@ What the app does, as a capture of the vendor app shows it ([#196](https://githu - `APP_ACCESS` first; the toy answers with `KEEP_ALIVE_INTERVAL` (a timeout, 60 s), its battery and SD card state and `VIDEO_PARAM` (the size, e.g. 640×480 at 25 fps). Then `DATE_TIME` (the phone's time), a `CTP_KEEP_ALIVE` every 7.5 s (a quarter of a shorter timeout), and `OPEN_RT_STREAM` with the size from `VIDEO_PARAM` (640×480 at 25 if the toy sent none). When the slot stops: `CLOSE_RT_STREAM`. - The video was **not** in the capture (PCAPdroid without root cannot see it). After turbodrone's research on the same Jieli SDK it is **RTP/JPEG** (payload type 26) or H.264 (96) to the phone's **UDP port 6666**. The app listens there and plays payload type 26 through its RTP/JPEG path and 96 through its H.264 path; a whole JPEG in a datagram without RTP is shown too. - A YL3 sent nothing to 6666 but its own stream to the phone's **UDP port 2224**: per datagram a 20-byte header, then the JPEG (seen: byte 0 = 2, a packet counter in bytes 4–7, the JPEG's length in 8–11, an offset in 12–15, little-endian; frames of 4–8 KB in one datagram, 40–60 zero bytes after the JPEG). Its frames are portrait (240×320 when 640×480 was asked for): the camera is mounted sideways, so `jieli://` turns them 270° counter-clockwise like the K200's, `?rotate=` overrides it. The app listens on 2224 as well and shows those JPEGs; longer frames are put together by the offset. The first five datagrams go to the log with the whole header, `/stats.json` has a `stream` object (`packets`, `frames`, `dropped`, `last_drop`) and `video_port`; why a frame was dropped is in the log for the first three. The debug page serves the last frame as it came, `/jieli-frame.jpg`, and the last dropped one, `/jieli-dropped.bin`. -- If nothing arrives on 6666 or 2224 for 5 s after `OPEN_RT_STREAM`, the status says so and the app also listens on UDP 1234 (audio in the vendor SDK), **only to log** what arrives there; it asks for the stream again every 10 s without video. +- If nothing arrives on 6666 or 2224 for 5 s after `OPEN_RT_STREAM`, the status says so and the app also listens on UDP 1234 (audio in the vendor SDK), **only to log** what arrives there, for 30 s or 20 packets, then frees the port; it asks for the stream again every 10 s without video. - **No flight command is ever sent**: no `FLIGHT_CONTROL` over CTP, nothing to the toy's UDP port 3456 (where the vendor app sends the sticks). - Ports 6666 and 2224 can be used by one slot at a time: one `jieli://` slot in a grid. From 100ac9bb81d26e4f38aea81dd8a7ceae99f87f34 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 15:36:01 +0000 Subject: [PATCH 19/21] =?UTF-8?q?fix:=20YL3=20frames=20put=20together=20fr?= =?UTF-8?q?om=20each=20piece's=20own=20length=20(bytes=202-3),=20not=20the?= =?UTF-8?q?=20whole=20datagram;=20jieli://=20turns=2090=C2=B0=20by=20defau?= =?UTF-8?q?lt;=20raw=20frames=20outlive=20the=20session=20(#187)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/openipc/decoder/DebugServer.java | 4 +- .../java/com/openipc/decoder/Decoder.java | 2 +- .../main/java/com/openipc/decoder/Jieli.java | 51 +++++++++++++++---- .../java/com/openipc/decoder/RtspSession.java | 3 +- .../java/com/openipc/decoder/JieliTest.java | 43 ++++++++++++++-- docs/toys.md | 6 +-- 6 files changed, 87 insertions(+), 22 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/DebugServer.java b/android/app/src/main/java/com/openipc/decoder/DebugServer.java index b80c72ff..07daace2 100644 --- a/android/app/src/main/java/com/openipc/decoder/DebugServer.java +++ b/android/app/src/main/java/com/openipc/decoder/DebugServer.java @@ -217,8 +217,8 @@ void serve(OutputStream out, String path, String query) throws IOException { case "/jieli-dropped.bin": // The toy's frames as they came (#187), to see what the picture is made from Source src = source; - byte[] raw = src != null ? src.jieliRaw(path.startsWith("/jieli-frame") - ? "frame" : "dropped") : null; + String which = path.startsWith("/jieli-frame") ? "frame" : "dropped"; + byte[] raw = src != null ? src.jieliRaw(which) : Jieli.Client.lastRaw(which); if (raw != null) { reply(out, 200, path.endsWith(".jpg") ? "image/jpeg" : "application/octet-stream", raw); diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index d7e47748..de6097e4 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -1390,7 +1390,7 @@ private void removeQuadOverlay(int index) { @Override public byte[] snapshotJpeg(boolean labels) { return debugSnapshotJpeg(labels); } @Override public byte[] jieliRaw(String which) { RtspClient c = rtspClient; - return c != null ? c.jieliRaw(which) : null; + return c != null ? c.jieliRaw(which) : Jieli.Client.lastRaw(which); } }; /** A cell counts as playing in a snapshot when it showed a frame this recently (#98). */ diff --git a/android/app/src/main/java/com/openipc/decoder/Jieli.java b/android/app/src/main/java/com/openipc/decoder/Jieli.java index e776265d..641afda0 100644 --- a/android/app/src/main/java/com/openipc/decoder/Jieli.java +++ b/android/app/src/main/java/com/openipc/decoder/Jieli.java @@ -56,6 +56,9 @@ final class Jieli { static final int RTP_PORT = 6666; /** The toy's own stream (#187): a YL3 sends its JPEG frames here, not as RTP to 6666. */ static final int STREAM_PORT = 2224; + /** The last frame of any session, and the last dropped one, for the debug page. */ + static volatile byte[] lastFrameAnySession; + static volatile byte[] lastDroppedAnySession; /** The port listened on for that stream; STREAM_PORT but for the tests. */ static volatile int streamPort = STREAM_PORT; static final int DEFAULT_WIDTH = 640; @@ -63,8 +66,8 @@ final class Jieli { static final int DEFAULT_FPS = 25; static final int FORMAT_JPEG = 0; static final int FORMAT_H264 = 1; - /** The YL3's frames come turned (portrait 240×320), as the K200's do (#187, #200). */ - static final int DEFAULT_ROTATE = 270; + /** The YL3's frames come turned (portrait 240×320); 270 showed them upside down (#187). */ + static final int DEFAULT_ROTATE = 90; static final int PT_JPEG = 26; static final int PT_H264 = 96; @@ -121,7 +124,7 @@ private Jieli() {} * refused): w, h, fps (the stream asked for; default what the toy announces in * VIDEO_PARAM, else 640x480 at 25), rtp (the local UDP port for the video, 6666), * format (0 JPEG, the default, or 1 H.264), rotate (degrees counter-clockwise for JPEG, - * 0, 90, 180 or 270; default 270: the YL3's camera is mounted sideways like the K200's). + * 0, 90, 180 or 270; default 90: the YL3's camera is mounted sideways). */ static final class Url { String host = DEFAULT_HOST; @@ -582,14 +585,17 @@ static String subnetHint(InetAddress target, List local) { /** * The toy's own video stream on UDP 2224 (#187), as a YL3 sent it: a 20-byte header, - * then (a piece of) a JPEG. Header as far as seen, little-endian: byte 0 the type (2 for - * JPEG), bytes 4–7 a packet counter, 8–11 the length of the JPEG, 12–15 the offset of - * this piece in it. A 640×480 frame fitted one datagram (4600 bytes, 20 more in the - * datagram); longer ones are put together by offset. Pure Java, for the unit tests. + * then a piece of a JPEG. Header, little-endian: byte 0 the type (2 for JPEG, bit 7 set + * on the last piece of a frame), bytes 2–3 the length of this piece, 4–7 a packet + * counter, 8–11 the length of the JPEG, 12–15 the offset of this piece in it. The + * datagram can be longer than the piece: what follows is left over from earlier frames + * (taking it gave a broken lower half), so only the piece is used. Pure Java, for the + * unit tests. */ static final class RtStream { static final int HEADER = 20; static final int TYPE_JPEG = 2; + static final int LAST_PIECE = 0x80; static final int MAX_FRAME = 1 << 20; private byte[] frame = new byte[64 * 1024]; @@ -605,7 +611,10 @@ static final class RtStream { private void drop(String reason, int n) { dropped++; lastDropReason = reason; - if (n > 0) lastDropped = Arrays.copyOf(frame, n); + if (n > 0) { + lastDropped = Arrays.copyOf(frame, n); + lastDroppedAnySession = lastDropped; + } } /** @@ -614,12 +623,15 @@ private void drop(String reason, int n) { */ int feed(byte[] b, int off, int len) { if (len <= HEADER) return 0; - if ((b[off] & 0xFF) != TYPE_JPEG) { + if ((b[off] & 0x7F) != TYPE_JPEG) { otherTypes++; return 0; } + boolean last = (b[off] & LAST_PIECE) != 0; int total = le32(b, off + 8); int at = le32(b, off + 12); + int length = (b[off + 2] & 0xFF) | (b[off + 3] & 0xFF) << 8; + if (length == 0 || length > len - HEADER) length = len - HEADER; if (total < 4 || total > MAX_FRAME || at < 0 || at >= total) { drop("bad header: length " + total + ", offset " + at, 0); return 0; @@ -633,10 +645,17 @@ int feed(byte[] b, int off, int len) { } else if (size != total) { return 0; // a piece of a frame whose start was lost } - int piece = Math.min(len - HEADER, total - at); + int piece = Math.min(length, total - at); System.arraycopy(b, off + HEADER, frame, at, piece); filled += piece; - if (filled < size) return 0; + if (filled < size) { + if (last) { + drop("incomplete: " + filled + " of " + size + " bytes at the last piece", + filled); + size = -1; + } + return 0; + } int n = size; size = -1; int end = jpegEnd(frame, n); @@ -1247,6 +1266,7 @@ private void stream(DatagramPacket p, Listener listener) { int jpeg = rtStream.feed(b, 0, len); if (jpeg > 0) { lastStreamFrame = Arrays.copyOf(rtStream.frame(), jpeg); + lastFrameAnySession = lastStreamFrame; lastSource = from; if (listener.onJpeg(rtStream.frame(), 0, jpeg)) { videoPort = streamPort; @@ -1300,6 +1320,15 @@ byte[] raw(String which) { : "dropped".equals(which) ? rtStream.lastDropped : null; } + /** + * As {@link #raw} after the session ended too: opening the browser on the same phone + * pauses the app and ends the session (#187), the frames must outlive it. + */ + static byte[] lastRaw(String which) { + return "frame".equals(which) ? lastFrameAnySession + : "dropped".equals(which) ? lastDroppedAnySession : null; + } + /** For /stats.json. */ Map stats() { Map m = new LinkedHashMap<>(); diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index 9e1abc3c..dd496e98 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -783,7 +783,8 @@ Map jieliStats() { /** Raw bytes of a jieli:// slot for the debug page (#187), see Jieli.Client.raw(). */ byte[] jieliRaw(String which) { Jieli.Client c = jieli; - return c != null ? c.raw(which) : null; + byte[] b = c != null ? c.raw(which) : null; + return b != null ? b : Jieli.Client.lastRaw(which); } /** diff --git a/android/app/src/test/java/com/openipc/decoder/JieliTest.java b/android/app/src/test/java/com/openipc/decoder/JieliTest.java index 5e0fb433..4aafe541 100644 --- a/android/app/src/test/java/com/openipc/decoder/JieliTest.java +++ b/android/app/src/test/java/com/openipc/decoder/JieliTest.java @@ -73,9 +73,10 @@ private static int freeUdpPortOrZero() { /** A datagram of the toy's stream on UDP 2224 as a YL3 sent it (#187). */ private static byte[] streamPacket(int counter, int total, int offset, byte[] piece, int tail) { byte[] b = new byte[Jieli.RtStream.HEADER + piece.length + tail]; - b[0] = 2; - b[2] = (byte) 0x94; - b[3] = 0x0B; + // Type 2, bit 7 on the last piece; bytes 2-3 the length of this piece + b[0] = (byte) (offset + piece.length >= total ? 0x82 : 0x02); + b[2] = (byte) piece.length; + b[3] = (byte) (piece.length >>> 8); le32(b, 4, counter); le32(b, 8, total); le32(b, 12, offset); @@ -127,6 +128,40 @@ public void rtStreamFrameInPieces() { assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); } + @Test + public void rtStreamUsesOnlyThePiece() { + // As test build 3 showed it: 2964 of 4600 bytes in a 4640-byte datagram whose rest is + // left over from an earlier frame (even with an FFD9); the rest comes at offset 2964 + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(4600); + byte[] first = streamPacket(1, 4600, 0, Arrays.copyOfRange(j, 0, 2964), 0); + byte[] dgram = Arrays.copyOf(first, 4640); + byte[] stale = jpeg(4600 - 2964 + 20); + System.arraycopy(stale, 0, dgram, first.length, stale.length); + assertEquals(0, st.feed(dgram, 0, dgram.length)); + byte[] second = streamPacket(2, 4600, 2964, Arrays.copyOfRange(j, 2964, 4600), 40); + int n = st.feed(second, 0, second.length); + assertEquals(4600, n); + assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); + assertEquals(0, st.dropped); + } + + @Test + public void rtStreamDropsAFrameWithAPieceMissing() { + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(6000); + byte[] first = streamPacket(1, 6000, 0, Arrays.copyOfRange(j, 0, 2000), 0); + assertEquals(0, st.feed(first, 0, first.length)); + // The middle piece is lost; the last one ends the frame + byte[] last = streamPacket(3, 6000, 4000, Arrays.copyOfRange(j, 4000, 6000), 0); + assertEquals(0, st.feed(last, 0, last.length)); + assertEquals(1, st.dropped); + assertTrue(st.lastDropReason, st.lastDropReason.startsWith("incomplete")); + // The next frame plays + byte[] next = streamPacket(4, 3000, 0, jpeg(3000), 0); + assertEquals(3000, st.feed(next, 0, next.length)); + } + @Test public void rtStreamDropsBrokenFrames() { Jieli.RtStream st = new Jieli.RtStream(); @@ -328,7 +363,7 @@ public void urls() throws Exception { assertEquals("192.168.80.1", u.host); assertEquals(3333, u.port); assertEquals(6666, u.rtpPort); - assertEquals(270, u.rotate); + assertEquals(90, u.rotate); assertEquals(0, Jieli.Url.parse("jieli://?rotate=0").rotate); assertEquals(90, Jieli.Url.parse("jieli://?rotate=90").rotate); assertEquals(0, u.width); diff --git a/docs/toys.md b/docs/toys.md index 023e73d4..091d032a 100644 --- a/docs/toys.md +++ b/docs/toys.md @@ -44,7 +44,7 @@ The Wi‑Fi name and the address the phone gets usually tell the family. The toy |----------------------------------------------------------------|-----------------|----------------------------------------------------------|---------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------| | `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | | `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | -| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 2224 or 6666) | test build 4: YL3 video on UDP 2224 plays, lower half broken in test build 3 ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 2224 or 6666) | test build 5: YL3 video on UDP 2224, pieces put together by their own length ([#187](https://github.com/OpenIPC/decoder/issues/187)) | When a `flowufo://` or `jieli://` slot points at the phone itself, or at the network of another family, its status says so (for example *192.168.1.100 is this phone; the toy is usually 192.168.1.1*, *This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam* or, on a `192.168.80.x` network, *try jieli://192.168.80.1*). @@ -109,7 +109,7 @@ What the app does, as a capture of the vendor app shows it ([#196](https://githu - It connects to TCP port 3333 and talks **CTP**: each message is `CTP:`, the length of a topic name, the topic (e.g. `OPEN_RT_STREAM`), the length of a JSON text and the JSON. - `APP_ACCESS` first; the toy answers with `KEEP_ALIVE_INTERVAL` (a timeout, 60 s), its battery and SD card state and `VIDEO_PARAM` (the size, e.g. 640×480 at 25 fps). Then `DATE_TIME` (the phone's time), a `CTP_KEEP_ALIVE` every 7.5 s (a quarter of a shorter timeout), and `OPEN_RT_STREAM` with the size from `VIDEO_PARAM` (640×480 at 25 if the toy sent none). When the slot stops: `CLOSE_RT_STREAM`. - The video was **not** in the capture (PCAPdroid without root cannot see it). After turbodrone's research on the same Jieli SDK it is **RTP/JPEG** (payload type 26) or H.264 (96) to the phone's **UDP port 6666**. The app listens there and plays payload type 26 through its RTP/JPEG path and 96 through its H.264 path; a whole JPEG in a datagram without RTP is shown too. -- A YL3 sent nothing to 6666 but its own stream to the phone's **UDP port 2224**: per datagram a 20-byte header, then the JPEG (seen: byte 0 = 2, a packet counter in bytes 4–7, the JPEG's length in 8–11, an offset in 12–15, little-endian; frames of 4–8 KB in one datagram, 40–60 zero bytes after the JPEG). Its frames are portrait (240×320 when 640×480 was asked for): the camera is mounted sideways, so `jieli://` turns them 270° counter-clockwise like the K200's, `?rotate=` overrides it. The app listens on 2224 as well and shows those JPEGs; longer frames are put together by the offset. The first five datagrams go to the log with the whole header, `/stats.json` has a `stream` object (`packets`, `frames`, `dropped`, `last_drop`) and `video_port`; why a frame was dropped is in the log for the first three. The debug page serves the last frame as it came, `/jieli-frame.jpg`, and the last dropped one, `/jieli-dropped.bin`. +- A YL3 sent nothing to 6666 but its own stream to the phone's **UDP port 2224**: per datagram a 20-byte header, then a piece of a JPEG. Header, little-endian: byte 0 the type (2 for JPEG, bit 7 set on the last piece of a frame), bytes 2–3 the length of this piece, 4–7 a packet counter, 8–11 the length of the JPEG, 12–15 the offset of the piece in it. The datagram can be longer than the piece; what follows is left over from earlier frames and is ignored (test build 3 took it, which broke the lower half of the picture). Its frames are portrait (240×320 or 320×240 rotated): the camera is mounted sideways, so `jieli://` turns them 90° counter-clockwise; 270° showed them upside down; `?rotate=` overrides it. The app listens on 2224 as well as 6666; the first five datagrams go to the log with the whole header, `/stats.json` has a `stream` object (`packets`, `frames`, `dropped`, `last_drop`) and `video_port`; why a frame was dropped is in the log for the first three. The debug page serves the last frame as it came, `/jieli-frame.jpg`, and the last dropped one, `/jieli-dropped.bin`, also after the session ended (the app pauses while the phone's browser is open). - If nothing arrives on 6666 or 2224 for 5 s after `OPEN_RT_STREAM`, the status says so and the app also listens on UDP 1234 (audio in the vendor SDK), **only to log** what arrives there, for 30 s or 20 packets, then frees the port; it asks for the stream again every 10 s without video. - **No flight command is ever sent**: no `FLIGHT_CONTROL` over CTP, nothing to the toy's UDP port 3456 (where the vendor app sends the sticks). - Ports 6666 and 2224 can be used by one slot at a time: one `jieli://` slot in a grid. @@ -121,7 +121,7 @@ URL options (as with `udp://`, separated by `?` and `&`): | `w`, `h`, `fps` | numbers | from `VIDEO_PARAM`, else 640, 480, 25 | the stream asked for in `OPEN_RT_STREAM` | | `rtp` | a UDP port | `6666` | the local port the video is expected on | | `format` | `0`, `1` | `0` | `0` asks for JPEG (as the vendor app does), `1` for H.264 | -| `rotate` | `0`, `90`, `180`, `270` | `270` | degrees counter-clockwise the JPEG frames are turned (the YL3's camera is mounted sideways) | +| `rotate` | `0`, `90`, `180`, `270` | `90` | degrees counter-clockwise the JPEG frames are turned (the YL3's camera is mounted sideways) | Examples: `jieli://192.168.80.1`, `jieli://192.168.80.1?w=1280&h=720`, `jieli://192.168.80.1?format=1`. From 776d32d2564faf61fe1a67a85112e77c7fb8a263 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 16:00:56 +0000 Subject: [PATCH 20/21] docs: toy drones are in the release (2.15): FLOW-UFO and KY UFO work, Jieli experimental (#187) --- README.md | 2 +- README.ru.md | 2 +- README.zh-CN.md | 2 +- docs/debug.md | 18 +++++++------- docs/toys.md | 59 +++++++++++++++++++++++----------------------- docs/transports.md | 16 ++++++------- 6 files changed, 49 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 56abfa69..0ec963e2 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Every item, the remote, sharing a URL from another app and **Reset**: [Usage](do | [Testing and releases](docs/testing.md) | branches, CI, unit tests, versions and releases, branch builds, `mapping.txt`, building locally | | [Security](docs/security.md) | credential store, debug server (LAN only), log redaction, no encryption on the network | | [Devices](docs/devices.md) | verified and not working devices, hardware decoder notes, device and camera reports | -| [Drones and Wi‑Fi toys](docs/toys.md) | how toys stream, FLOW-UFO (in testing), what to try with an unknown toy, what to send | +| [Drones and Wi‑Fi toys](docs/toys.md) | how toys stream, FLOW-UFO, KY UFO and Jieli drones, what to try with an unknown toy, what to send | The change history is the [merged pull requests](https://github.com/OpenIPC/decoder/pulls?q=is%3Apr+is%3Amerged+base%3Av2) and the [commits of v2](https://github.com/OpenIPC/decoder/commits/v2). diff --git a/README.ru.md b/README.ru.md index 6b47b53f..4040d840 100644 --- a/README.ru.md +++ b/README.ru.md @@ -112,7 +112,7 @@ | [Testing and releases](docs/testing.md) | ветки, CI, юнит-тесты, версии и релизы, сборки веток, `mapping.txt`, локальная сборка | | [Security](docs/security.md) | хранилище учётных данных, отладочный сервер (только локальная сеть), скрытие паролей в логах, отсутствие шифрования в сети | | [Devices](docs/devices.md) | проверенные и неработающие устройства, особенности аппаратных декодеров, отчёты об устройствах и камерах | -| [Drones and Wi‑Fi toys](docs/toys.md) | как передают видео игрушки, FLOW-UFO (в тестировании), что попробовать с неизвестной игрушкой, что прислать | +| [Drones and Wi‑Fi toys](docs/toys.md) | как передают видео игрушки, дроны FLOW-UFO, KY UFO и Jieli, что попробовать с неизвестной игрушкой, что прислать | История изменений — это [принятые pull request'ы](https://github.com/OpenIPC/decoder/pulls?q=is%3Apr+is%3Amerged+base%3Av2) и [коммиты ветки v2](https://github.com/OpenIPC/decoder/commits/v2). diff --git a/README.zh-CN.md b/README.zh-CN.md index 2b1c9b43..c3f08dec 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -112,7 +112,7 @@ | [Testing and releases](docs/testing.md) | 分支、CI、单元测试、版本与发布、分支构建、`mapping.txt`、本地构建 | | [Security](docs/security.md) | 凭据存储、调试服务器（仅限局域网）、日志脱敏、网络传输未加密 | | [Devices](docs/devices.md) | 已验证和无法使用的设备、硬件解码器说明、设备与摄像机报告 | -| [Drones and Wi‑Fi toys](docs/toys.md) | 玩具如何传输视频、FLOW-UFO（测试中）、遇到未知玩具时可以尝试什么、需要提交哪些信息 | +| [Drones and Wi‑Fi toys](docs/toys.md) | 玩具如何传输视频、FLOW-UFO、KY UFO 和 Jieli 无人机、遇到未知玩具时可以尝试什么、需要提交哪些信息 | 更新历史即[已合并的 pull request](https://github.com/OpenIPC/decoder/pulls?q=is%3Apr+is%3Amerged+base%3Av2) 和 [v2 分支的提交](https://github.com/OpenIPC/decoder/commits/v2)。 diff --git a/docs/debug.md b/docs/debug.md index b1e7d65e..ead0156c 100644 --- a/docs/debug.md +++ b/docs/debug.md @@ -14,14 +14,14 @@ While the app runs it serves its debug information over HTTP on port **8085**, for a browser or `curl` on the same network. It is always on and read-only. -| Path | Content | -|------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `/` (or `/index.html`) | the [page](#page): snapshot with the text of each cell | -| `/stats.json` | the state as JSON, pretty-printed, see [fields](#statsjson-fields) | -| `/snapshot.jpg` | a picture of the screen, see [Snapshot](#snapshot); `?labels=0`: frames only | -| `/log` | the last 500 lines of the app's log, see [Log](#log) | -| `/report.txt` | a header, `/stats.json` and `/log` in one file, downloaded as `report-YYYYMMDD-HHMMSS.txt`, see [Save report](#save-report) | -| `/jieli-frame.jpg`, `/jieli-dropped.bin` | test build only: the last JPEG of a `jieli://` slot in the single view as the toy sent it, and the last frame that could not be put together ([toys.md](toys.md#hf--jieli-toys)) | +| Path | Content | +|------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `/` (or `/index.html`) | the [page](#page): snapshot with the text of each cell | +| `/stats.json` | the state as JSON, pretty-printed, see [fields](#statsjson-fields) | +| `/snapshot.jpg` | a picture of the screen, see [Snapshot](#snapshot); `?labels=0`: frames only | +| `/log` | the last 500 lines of the app's log, see [Log](#log) | +| `/report.txt` | a header, `/stats.json` and `/log` in one file, downloaded as `report-YYYYMMDD-HHMMSS.txt`, see [Save report](#save-report) | +| `/jieli-frame.jpg`, `/jieli-dropped.bin` | the last JPEG of a `jieli://` slot in the single view as the toy sent it, and the last frame that could not be put together ([toys.md](toys.md#hf--jieli-toys)) | - Only clients on the local network are answered; others get `403`. The rule and what is hidden: [security.md](security.md#debug-server). - `GET` only (`405` otherwise), one request per connection, `Cache-Control: no-store`. @@ -171,7 +171,7 @@ Single view and U view: every cell of the grid as the debug page shows it (3×3 | `codec`, `size` | when found | | `fps`, `kbps`, `key_frames` | measured during the check | -The FLOW-UFO test build ([toys.md](toys.md#flow-ufo)) adds a `flowufo` object per cell and in `single` (`jieli` for `jieli://` slots, [toys.md](toys.md#hf--jieli-toys)), and a `heartbeat` object for RTSP slots that send the KY UFO heartbeat, and `rotate` (degrees counter-clockwise) where MJPEG frames are turned ([toys.md](toys.md#ky-ufo-rtsp-toys)); it is not in the releases. +Toy drone slots ([toys.md](toys.md#flow-ufo)) add a `flowufo` object per cell and in `single` (`jieli` for `jieli://` slots, [toys.md](toys.md#hf--jieli-toys)), and a `heartbeat` object for RTSP slots that send the KY UFO heartbeat, and `rotate` (degrees counter-clockwise) where MJPEG frames are turned ([toys.md](toys.md#ky-ufo-rtsp-toys)). ## Log diff --git a/docs/toys.md b/docs/toys.md index 091d032a..bc62e888 100644 --- a/docs/toys.md +++ b/docs/toys.md @@ -40,24 +40,23 @@ Whether a toy can be played here depends on its protocol. Toys that do offer RTS The Wi‑Fi name and the address the phone gets usually tell the family. The toy is almost always the `.1` address of that network, **not** the phone's own address. -| Wi‑Fi name | Phone gets | Family | Slot URL | Status | -|----------------------------------------------------------------|-----------------|----------------------------------------------------------|---------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------| -| `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | -| `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | in testing ([#187](https://github.com/OpenIPC/decoder/issues/187)) | -| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 2224 or 6666) | test build 5: YL3 video on UDP 2224, pieces put together by their own length ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| Wi‑Fi name | Phone gets | Family | Slot URL | Status | +|----------------------------------------------------------------|-----------------|----------------------------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | works since 2.15, checked with a D16 ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | works since 2.15, checked with a K200; the picture is turned upright ([#187](https://github.com/OpenIPC/decoder/issues/187), [#200](https://github.com/OpenIPC/decoder/issues/200)) | +| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 2224 or 6666) | **experimental** (2.15): YL3 video on UDP 2224 arrives and plays, picture quality still being checked ([#187](https://github.com/OpenIPC/decoder/issues/187)) | When a `flowufo://` or `jieli://` slot points at the phone itself, or at the network of another family, its status says so (for example *192.168.1.100 is this phone; the toy is usually 192.168.1.1*, *This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam* or, on a `192.168.80.x` network, *try jieli://192.168.80.1*). ## FLOW-UFO -| | | -|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Wi‑Fi name | `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | -| Camera | BL60x Wi‑Fi camera module (BL608 or BL618 engine) | -| Toy address | `192.168.169.1`, UDP port 8800 (BL608) or 8801 (BL618); the toy answers from port 1234 | -| Slot URL | `flowufo://192.168.169.1:8800` (or `flowufo://` alone) | -| Status | **in testing**, not in the releases: issue [#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188) | -| Test build | [toys-187-test2](https://github.com/OpenIPC/decoder/releases/tag/toys-187-test2) (2.11 with the FLOW-UFO, Jieli and KY UFO changes; installs over the release and keeps the settings) | +| | | +|-------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Wi‑Fi name | `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | +| Camera | BL60x Wi‑Fi camera module (BL608 or BL618 engine) | +| Toy address | `192.168.169.1`, UDP port 8800 (BL608) or 8801 (BL618); the toy answers from port 1234 | +| Slot URL | `flowufo://192.168.169.1:8800` (or `flowufo://` alone) | +| Status | in the releases since 2.15; checked with a D16 ([#187](https://github.com/OpenIPC/decoder/issues/187), PR [#215](https://github.com/OpenIPC/decoder/pull/215)) | What the toy does, as far as is known: @@ -76,9 +75,9 @@ URL options, for comparing with what the toy expects (as with `udp://`, separate Examples: `flowufo://192.168.169.1:8800`, `flowufo://192.168.169.1:8800?dqt=raster`, `flowufo://192.168.169.1:8800?q=19322348`, `flowufo://192.168.169.1:8801?dqt=q50`. -The implementation was written from a description of the protocol, not tested on a real toy yet. Status texts while it connects: *Waiting for the toy: probing 192.168.169.1:8800*, *Receiving from the toy at …*, *The toy went quiet: probing … again*. The test build adds a `flowufo` object to `/stats.json`: state, the ports probed (`target`), the port and engine that answered (`port`, `engine`), network, `qpara`, `dqt` (e.g. `zigzag Q50` or `raster (unscaled)`), `camera_status`, frame and fragment counters, losses, `lagging_left_out`, resets, errors, and a `hint` when the address looks wrong. +The implementation was written from a description of the protocol and checked with a D16. Status texts while it connects: *Waiting for the toy: probing 192.168.169.1:8800*, *Receiving from the toy at …*, *The toy went quiet: probing … again*. The app adds a `flowufo` object to `/stats.json`: state, the ports probed (`target`), the port and engine that answered (`port`, `engine`), network, `qpara`, `dqt` (e.g. `zigzag Q50` or `raster (unscaled)`), `camera_status`, frame and fragment counters, losses, `lagging_left_out`, resets, errors, and a `hint` when the address looks wrong. -To test: install the test build, join the `FLOW-UFO-…` Wi‑Fi, [stay connected](#connecting), put `flowufo://192.168.169.1:8800` into a slot (**Settings**, or share the text to the app). If there is no picture, or it looks wrong, wait about 30 s and [send a report](#what-to-send-in-a-report) in [#187](https://github.com/OpenIPC/decoder/issues/187). +To use it: join the `FLOW-UFO-…` Wi‑Fi, [stay connected](#connecting), put `flowufo://192.168.169.1:8800` into a slot (**Settings**, or share the text to the app). If there is no picture, or it looks wrong, wait about 30 s and [send a report](#what-to-send-in-a-report) in [#187](https://github.com/OpenIPC/decoder/issues/187). ## KY UFO RTSP toys @@ -96,20 +95,20 @@ Some of these toys use the FLOW-UFO protocol instead (the KY UFO app has both); ## HF / Jieli toys -| | | -|-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Wi‑Fi name | `HF_xxxxxxxx` | -| Camera | Jieli AC79 (`AC79X_wifi_car_camera`); vendor app *DVRunning 2* (also *HF UFO*) | -| Toy address | `192.168.80.1`: control (CTP) on TCP port 3333; the video goes to UDP port 2224 (or 6666) **of the phone** | -| Slot URL | `jieli://192.168.80.1` (or `jieli://` alone) | -| Status | **test build 3**, not in the releases: the control part follows a capture of the vendor app, the video part is **unverified**; issue [#187](https://github.com/OpenIPC/decoder/issues/187) | +| | | +|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Wi‑Fi name | `HF_xxxxxxxx` | +| Camera | Jieli AC79 (`AC79X_wifi_car_camera`); vendor app *DVRunning 2* (also *HF UFO*) | +| Toy address | `192.168.80.1`: control (CTP) on TCP port 3333; the video goes to UDP port 2224 (or 6666) **of the phone** | +| Slot URL | `jieli://192.168.80.1` (or `jieli://` alone) | +| Status | **experimental** since 2.15: the control part follows a capture of the vendor app; YL3 video arrives and plays, the picture quality is still being checked in [#187](https://github.com/OpenIPC/decoder/issues/187) | What the app does, as a capture of the vendor app shows it ([#196](https://github.com/OpenIPC/decoder/issues/196)): - It connects to TCP port 3333 and talks **CTP**: each message is `CTP:`, the length of a topic name, the topic (e.g. `OPEN_RT_STREAM`), the length of a JSON text and the JSON. - `APP_ACCESS` first; the toy answers with `KEEP_ALIVE_INTERVAL` (a timeout, 60 s), its battery and SD card state and `VIDEO_PARAM` (the size, e.g. 640×480 at 25 fps). Then `DATE_TIME` (the phone's time), a `CTP_KEEP_ALIVE` every 7.5 s (a quarter of a shorter timeout), and `OPEN_RT_STREAM` with the size from `VIDEO_PARAM` (640×480 at 25 if the toy sent none). When the slot stops: `CLOSE_RT_STREAM`. - The video was **not** in the capture (PCAPdroid without root cannot see it). After turbodrone's research on the same Jieli SDK it is **RTP/JPEG** (payload type 26) or H.264 (96) to the phone's **UDP port 6666**. The app listens there and plays payload type 26 through its RTP/JPEG path and 96 through its H.264 path; a whole JPEG in a datagram without RTP is shown too. -- A YL3 sent nothing to 6666 but its own stream to the phone's **UDP port 2224**: per datagram a 20-byte header, then a piece of a JPEG. Header, little-endian: byte 0 the type (2 for JPEG, bit 7 set on the last piece of a frame), bytes 2–3 the length of this piece, 4–7 a packet counter, 8–11 the length of the JPEG, 12–15 the offset of the piece in it. The datagram can be longer than the piece; what follows is left over from earlier frames and is ignored (test build 3 took it, which broke the lower half of the picture). Its frames are portrait (240×320 or 320×240 rotated): the camera is mounted sideways, so `jieli://` turns them 90° counter-clockwise; 270° showed them upside down; `?rotate=` overrides it. The app listens on 2224 as well as 6666; the first five datagrams go to the log with the whole header, `/stats.json` has a `stream` object (`packets`, `frames`, `dropped`, `last_drop`) and `video_port`; why a frame was dropped is in the log for the first three. The debug page serves the last frame as it came, `/jieli-frame.jpg`, and the last dropped one, `/jieli-dropped.bin`, also after the session ended (the app pauses while the phone's browser is open). +- A YL3 sent nothing to 6666 but its own stream to the phone's **UDP port 2224**: per datagram a 20-byte header, then a piece of a JPEG. Header, little-endian: byte 0 the type (2 for JPEG, bit 7 set on the last piece of a frame), bytes 2–3 the length of this piece, 4–7 a packet counter, 8–11 the length of the JPEG, 12–15 the offset of the piece in it. The datagram can be longer than the piece; what follows is left over from earlier frames and is ignored (an earlier test build took it, which broke the lower half of the picture). Its frames are portrait (240×320 or 320×240 rotated): the camera is mounted sideways, so `jieli://` turns them 90° counter-clockwise; 270° showed them upside down; `?rotate=` overrides it. The app listens on 2224 as well as 6666; the first five datagrams go to the log with the whole header, `/stats.json` has a `stream` object (`packets`, `frames`, `dropped`, `last_drop`) and `video_port`; why a frame was dropped is in the log for the first three. The debug page serves the last frame as it came, `/jieli-frame.jpg`, and the last dropped one, `/jieli-dropped.bin`, also after the session ended (the app pauses while the phone's browser is open). - If nothing arrives on 6666 or 2224 for 5 s after `OPEN_RT_STREAM`, the status says so and the app also listens on UDP 1234 (audio in the vendor SDK), **only to log** what arrives there, for 30 s or 20 packets, then frees the port; it asks for the stream again every 10 s without video. - **No flight command is ever sent**: no `FLIGHT_CONTROL` over CTP, nothing to the toy's UDP port 3456 (where the vendor app sends the sticks). - Ports 6666 and 2224 can be used by one slot at a time: one `jieli://` slot in a grid. @@ -125,9 +124,9 @@ URL options (as with `udp://`, separated by `?` and `&`): Examples: `jieli://192.168.80.1`, `jieli://192.168.80.1?w=1280&h=720`, `jieli://192.168.80.1?format=1`. -Status texts while it connects: *Connecting to the toy (CTP 192.168.80.1:3333)*, *Stream requested, waiting for video on UDP 6666 and 2224*, *No video on UDP 6666 and 2224 yet: the toy accepted the request; see /log*, *Receiving from the toy on UDP 2224…* (the port the video came on), *The toy's video stopped: waiting on UDP 6666 and 2224*. The test build adds a `jieli` object to `/stats.json`: `state` (`connecting`, `ctp`, `requested`, `playing`, `quiet`), `target`, `network`, `ctp` (messages sent and received, keepalives and replies, the keepalive interval, the topics seen, the last 8 messages from the toy), `video_param`, `requested`, `rtp` (port, packets, video packets, frames, payload types, packets that are not RTP, the first packet in hex), `sources` (sender address:port → packets) and `diagnostic_ports`. The log has the CTP messages (JSON shortened) and the first packets on each port with sender, payload type and first bytes: exactly what a report needs. +Status texts while it connects: *Connecting to the toy (CTP 192.168.80.1:3333)*, *Stream requested, waiting for video on UDP 6666 and 2224*, *No video on UDP 6666 and 2224 yet: the toy accepted the request; see /log*, *Receiving from the toy on UDP 2224…* (the port the video came on), *The toy's video stopped: waiting on UDP 6666 and 2224*. The app adds a `jieli` object to `/stats.json`: `state` (`connecting`, `ctp`, `requested`, `playing`, `quiet`), `target`, `network`, `ctp` (messages sent and received, keepalives and replies, the keepalive interval, the topics seen, the last 8 messages from the toy), `video_param`, `requested`, `rtp` (port, packets, video packets, frames, payload types, packets that are not RTP, the first packet in hex), `sources` (sender address:port → packets) and `diagnostic_ports`. The log has the CTP messages (JSON shortened) and the first packets on each port with sender, payload type and first bytes: exactly what a report needs. -To test: install the test build, join the `HF_…` Wi‑Fi, [stay connected](#connecting), close the vendor app, put `jieli://192.168.80.1` into a slot. Wait about 30 s and [send a report](#what-to-send-in-a-report) in [#187](https://github.com/OpenIPC/decoder/issues/187), with or without a picture: `/stats.json` and `/log` tell what the toy really sends. +To use it: join the `HF_…` Wi‑Fi, [stay connected](#connecting), close the vendor app, put `jieli://192.168.80.1` into a slot. Wait about 30 s and [send a report](#what-to-send-in-a-report) in [#187](https://github.com/OpenIPC/decoder/issues/187), with or without a picture: `/stats.json` and `/log` tell what the toy really sends. ## Credits @@ -164,9 +163,9 @@ Do not send the vendor app itself or parts of it. The capture contains only the The drones a tester checked in [#187](https://github.com/OpenIPC/decoder/issues/187): -| Toy | Wi‑Fi | Family | URL | Status | -|-----------|-------------|-----------------------------|-------------------------------------|-------------------------------------------------------------------------------------------------| -| D16 | `FLOW_…` | FLOW-UFO, WIFI UAV app | `flowufo://192.168.169.1:8800` | video works | -| K200 | `FLY-UFO-…` | KY UFO, RTSP | `rtsp://192.168.1.1:7070/webcam` | video works; the picture comes sideways ([#200](https://github.com/OpenIPC/decoder/issues/200)) | -| Drone YL3 | `HF_…` | Jieli AC79, DVRunning 2 app | `jieli://192.168.80.1` (test build) | in testing | +| Toy | Wi‑Fi | Family | URL | Status | +|-----------|-------------|-----------------------------|----------------------------------|-------------------------------------------------------------------------------------------------------------| +| D16 | `FLOW_…` | FLOW-UFO, WIFI UAV app | `flowufo://192.168.169.1:8800` | video works | +| K200 | `FLY-UFO-…` | KY UFO, RTSP | `rtsp://192.168.1.1:7070/webcam` | video works; the sideways picture is turned upright ([#200](https://github.com/OpenIPC/decoder/issues/200)) | +| Drone YL3 | `HF_…` | Jieli AC79, DVRunning 2 app | `jieli://192.168.80.1` | video plays, picture quality being checked | diff --git a/docs/transports.md b/docs/transports.md index ccc90941..5c61f329 100644 --- a/docs/transports.md +++ b/docs/transports.md @@ -14,12 +14,12 @@ A slot holds one URL. Its scheme picks the path the video takes; for `rtsp://` the slot's transport mode (chosen under the URL in **Settings**, or with the `transport` extra of a [shared link](usage.md#opening-a-camera-from-another-app)) picks how RTP travels. -| URL | Path | Single view | Multi-view | Audio | -|--------------------|-------------------------------------------------------------------------------------------------|-------------|------------|-------| -| `rtsp://` | RTSP session, RTP over TCP, UDP or multicast | yes | yes | yes | -| `udp://`, `rtp://` | plain RTP pushed by the camera, no RTSP | yes | yes | no | -| `http://` | MJPEG, `multipart/x-mixed-replace` | yes | yes | no | -| `flowufo://` | toy drone protocol, **test build only** ([#187](https://github.com/OpenIPC/decoder/issues/187)) | yes | yes | no | +| URL | Path | Single view | Multi-view | Audio | +|--------------------|----------------------------------------------------------------------------|-------------|------------|-------| +| `rtsp://` | RTSP session, RTP over TCP, UDP or multicast | yes | yes | yes | +| `udp://`, `rtp://` | plain RTP pushed by the camera, no RTSP | yes | yes | no | +| `http://` | MJPEG, `multipart/x-mixed-replace` | yes | yes | no | +| `flowufo://` | toy drone protocol ([#187](https://github.com/OpenIPC/decoder/issues/187)) | yes | yes | no | The transport in use is shown when a stream starts in the single view (`Connected • UDP • H.265 …`), on the [debug page](debug.md) and in `/stats.json` (`transport`; the slot's setting is `transport_mode`). After a fallback it reads, for example, `TCP (auto)`. @@ -100,9 +100,9 @@ For example Majestic's `http://user:password@/mjpeg`: a `multipart/x-mix ## Toys: FLOW-UFO -`flowufo://192.168.169.1:8800` (or `flowufo://` alone) plays the video of toy drones with a `FLOW-UFO-xxxxxx` Wi‑Fi access point. It exists only in a test build so far ([#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188)), not in the releases. Port 8800 (the default) probes 8801 as well and keeps the port that answers. Options: `?qpara=<8 hex digits>` and `?dqt=zigzag|raster|q50`. Setup, options and status: [toys.md](toys.md#flow-ufo). +`flowufo://192.168.169.1:8800` (or `flowufo://` alone) plays the video of toy drones with a `FLOW-UFO-xxxxxx` Wi‑Fi access point. Since 2.15 ([#187](https://github.com/OpenIPC/decoder/issues/187), PR [#215](https://github.com/OpenIPC/decoder/pull/215)). Port 8800 (the default) probes 8801 as well and keeps the port that answers. Options: `?qpara=<8 hex digits>` and `?dqt=zigzag|raster|q50`. Setup, options and status: [toys.md](toys.md#flow-ufo). -`jieli://192.168.80.1` (or `jieli://` alone) plays toy drones with an `HF_xxxxxxxx` Wi‑Fi (Jieli AC79 camera): CTP commands on TCP 3333 ask for the stream, which arrives as JPEG frames behind a 20-byte header on the phone's UDP port 2224 (seen from a YL3) or as RTP/JPEG on UDP 6666 (test build only). Options: `?w=`, `?h=`, `?fps=`, `?rtp=`, `?format=0|1`. See [toys.md](toys.md#hf--jieli-toys). +`jieli://192.168.80.1` (or `jieli://` alone) plays toy drones with an `HF_xxxxxxxx` Wi‑Fi (Jieli AC79 camera): CTP commands on TCP 3333 ask for the stream, which arrives as JPEG frames behind a 20-byte header on the phone's UDP port 2224 (seen from a YL3) or as RTP/JPEG on UDP 6666; experimental since 2.15. Options: `?w=`, `?h=`, `?fps=`, `?rtp=`, `?format=0|1`, `?rotate=`. See [toys.md](toys.md#hf--jieli-toys). RTSP slots on `rtsp://:7070/webcam` (KY UFO and other cooingdv toys) also send the toy's UDP heartbeat `01 01` to port 7099 once a second; `?heartbeat=` turns it on for other RTSP URLs, `?heartbeat=off` off. Their MJPEG frames are drawn turned 270° counter-clockwise (90° clockwise) (the K200's camera is mounted sideways, [#200](https://github.com/OpenIPC/decoder/issues/200)); `?rotate=0|90|180|270` (degrees counter-clockwise, any RTSP MJPEG slot) overrides it and is not sent to the camera either. See [toys.md](toys.md#ky-ufo-rtsp-toys). From 367b897fd5f9a15dd46aaa6a25a919dae3658aa9 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 16:09:54 +0000 Subject: [PATCH 21/21] fix: Jieli stream frames count each byte once and wait for pieces that come late (Qodo on #215) --- .../main/java/com/openipc/decoder/Jieli.java | 53 ++++++++++++------- .../java/com/openipc/decoder/JieliTest.java | 47 +++++++++++++--- 2 files changed, 73 insertions(+), 27 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/Jieli.java b/android/app/src/main/java/com/openipc/decoder/Jieli.java index 641afda0..6c3df414 100644 --- a/android/app/src/main/java/com/openipc/decoder/Jieli.java +++ b/android/app/src/main/java/com/openipc/decoder/Jieli.java @@ -25,6 +25,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; +import java.util.BitSet; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; @@ -586,8 +587,8 @@ static String subnetHint(InetAddress target, List local) { /** * The toy's own video stream on UDP 2224 (#187), as a YL3 sent it: a 20-byte header, * then a piece of a JPEG. Header, little-endian: byte 0 the type (2 for JPEG, bit 7 set - * on the last piece of a frame), bytes 2–3 the length of this piece, 4–7 a packet - * counter, 8–11 the length of the JPEG, 12–15 the offset of this piece in it. The + * on the last piece of a frame; not relied on), bytes 2–3 the length of this piece, 4–7 + * a packet counter, 8–11 the length of the JPEG, 12–15 the offset of this piece in it. The * datagram can be longer than the piece: what follows is left over from earlier frames * (taking it gave a broken lower half), so only the piece is used. Pure Java, for the * unit tests. @@ -600,6 +601,8 @@ static final class RtStream { private byte[] frame = new byte[64 * 1024]; private int size = -1; + /** Bytes of the frame received so far: duplicates and overlaps count once. */ + private final BitSet got = new BitSet(); private int filled; int frames; int dropped; @@ -627,7 +630,6 @@ int feed(byte[] b, int off, int len) { otherTypes++; return 0; } - boolean last = (b[off] & LAST_PIECE) != 0; int total = le32(b, off + 8); int at = le32(b, off + 12); int length = (b[off + 2] & 0xFF) | (b[off + 3] & 0xFF) << 8; @@ -636,28 +638,24 @@ int feed(byte[] b, int off, int len) { drop("bad header: length " + total + ", offset " + at, 0); return 0; } - if (at == 0) { - // A new frame; one still missing pieces is lost - if (size > 0) drop("incomplete: " + filled + " of " + size + " bytes", filled); - size = total; - filled = 0; - if (frame.length < total) frame = new byte[total]; + int piece = Math.min(length, total - at); + // Pieces may come twice or out of order (Qodo on #215): a piece belongs to the frame + // being put together if the length matches and it brings bytes not yet received; a + // repeat is ignored. Anything else starts the next frame, and only then is an + // unfinished one given up, even if its last piece came already. + if (size == total && got.get(at, at + piece).cardinality() == piece) { + if (same(b, off + HEADER, at, piece)) return 0; + startFrame(total); } else if (size != total) { - return 0; // a piece of a frame whose start was lost + startFrame(total); } - int piece = Math.min(length, total - at); System.arraycopy(b, off + HEADER, frame, at, piece); - filled += piece; - if (filled < size) { - if (last) { - drop("incomplete: " + filled + " of " + size + " bytes at the last piece", - filled); - size = -1; - } - return 0; - } + filled += piece - got.get(at, at + piece).cardinality(); + got.set(at, at + piece); + if (filled < size) return 0; int n = size; size = -1; + got.clear(); int end = jpegEnd(frame, n); if (end == 0) { drop(n + " bytes, starts " + FlowUfo.hex(frame, 0, Math.min(n, 4)) + ", ends " @@ -673,6 +671,21 @@ byte[] frame() { return frame; } + private void startFrame(int total) { + if (size > 0) drop("incomplete: " + filled + " of " + size + " bytes", size); + size = total; + filled = 0; + got.clear(); + if (frame.length < total) frame = new byte[total]; + } + + private boolean same(byte[] b, int from, int at, int n) { + for (int i = 0; i < n; i++) { + if (frame[at + i] != b[from + i]) return false; + } + return true; + } + /** Length up to the last FFD9 when {@code b} starts with FFD8, else 0. */ static int jpegEnd(byte[] b, int n) { if (n < 4 || (b[0] & 0xFF) != 0xFF || (b[1] & 0xFF) != 0xD8) return 0; diff --git a/android/app/src/test/java/com/openipc/decoder/JieliTest.java b/android/app/src/test/java/com/openipc/decoder/JieliTest.java index 4aafe541..9d99f5e0 100644 --- a/android/app/src/test/java/com/openipc/decoder/JieliTest.java +++ b/android/app/src/test/java/com/openipc/decoder/JieliTest.java @@ -152,14 +152,46 @@ public void rtStreamDropsAFrameWithAPieceMissing() { byte[] j = jpeg(6000); byte[] first = streamPacket(1, 6000, 0, Arrays.copyOfRange(j, 0, 2000), 0); assertEquals(0, st.feed(first, 0, first.length)); - // The middle piece is lost; the last one ends the frame + // The middle piece is lost; the last one alone does not finish the frame byte[] last = streamPacket(3, 6000, 4000, Arrays.copyOfRange(j, 4000, 6000), 0); assertEquals(0, st.feed(last, 0, last.length)); - assertEquals(1, st.dropped); - assertTrue(st.lastDropReason, st.lastDropReason.startsWith("incomplete")); - // The next frame plays + assertEquals(0, st.dropped); + // The next frame plays, the unfinished one is given up byte[] next = streamPacket(4, 3000, 0, jpeg(3000), 0); assertEquals(3000, st.feed(next, 0, next.length)); + assertEquals(1, st.dropped); + assertTrue(st.lastDropReason, st.lastDropReason.startsWith("incomplete")); + } + + @Test + public void rtStreamIgnoresRepeatedPieces() { + // A repeated piece must not count twice (Qodo on #215) + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(6000); + byte[] a = streamPacket(1, 6000, 0, Arrays.copyOfRange(j, 0, 3000), 0); + assertEquals(0, st.feed(a, 0, a.length)); + assertEquals(0, st.feed(a, 0, a.length)); + byte[] b = streamPacket(2, 6000, 3000, Arrays.copyOfRange(j, 3000, 6000), 0); + int n = st.feed(b, 0, b.length); + assertEquals(6000, n); + assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); + assertEquals(0, st.dropped); + } + + @Test + public void rtStreamTakesPiecesOutOfOrder() { + // The last piece before the middle one, the first one last (Qodo on #215) + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(6000); + byte[] last = streamPacket(3, 6000, 4000, Arrays.copyOfRange(j, 4000, 6000), 0); + byte[] mid = streamPacket(2, 6000, 2000, Arrays.copyOfRange(j, 2000, 4000), 0); + byte[] first = streamPacket(1, 6000, 0, Arrays.copyOfRange(j, 0, 2000), 0); + assertEquals(0, st.feed(last, 0, last.length)); + assertEquals(0, st.feed(mid, 0, mid.length)); + int n = st.feed(first, 0, first.length); + assertEquals(6000, n); + assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); + assertEquals(0, st.dropped); } @Test @@ -168,10 +200,11 @@ public void rtStreamDropsBrokenFrames() { // Start lost: a middle piece alone gives nothing byte[] mid = streamPacket(5, 9000, 4000, new byte[4000], 0); assertEquals(0, st.feed(mid, 0, mid.length)); - // Not a JPEG + assertEquals(0, st.dropped); + // Not a JPEG; it also ends the unfinished frame byte[] junk = streamPacket(6, 100, 0, new byte[100], 0); assertEquals(0, st.feed(junk, 0, junk.length)); - assertEquals(1, st.dropped); + assertEquals(2, st.dropped); assertTrue(st.lastDropReason, st.lastDropReason.contains("no FFD8")); assertEquals(100, st.lastDropped.length); // A frame cut short by the next one @@ -180,7 +213,7 @@ public void rtStreamDropsBrokenFrames() { assertEquals(0, st.feed(first, 0, first.length)); byte[] next = streamPacket(8, 4600, 0, jpeg(4600), 0); assertEquals(4600, st.feed(next, 0, next.length)); - assertEquals(2, st.dropped); + assertEquals(3, st.dropped); // Another type (audio?) and a bad length are ignored byte[] other = streamPacket(9, 4600, 0, jpeg(4600), 0); other[0] = 1;