diff --git a/README.md b/README.md index 56abfa69..0ec963e2 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Every item, the remote, sharing a URL from another app and **Reset**: [Usage](do | [Testing and releases](docs/testing.md) | branches, CI, unit tests, versions and releases, branch builds, `mapping.txt`, building locally | | [Security](docs/security.md) | credential store, debug server (LAN only), log redaction, no encryption on the network | | [Devices](docs/devices.md) | verified and not working devices, hardware decoder notes, device and camera reports | -| [Drones and Wi‑Fi toys](docs/toys.md) | how toys stream, FLOW-UFO (in testing), what to try with an unknown toy, what to send | +| [Drones and Wi‑Fi toys](docs/toys.md) | how toys stream, FLOW-UFO, KY UFO and Jieli drones, what to try with an unknown toy, what to send | The change history is the [merged pull requests](https://github.com/OpenIPC/decoder/pulls?q=is%3Apr+is%3Amerged+base%3Av2) and the [commits of v2](https://github.com/OpenIPC/decoder/commits/v2). diff --git a/README.ru.md b/README.ru.md index 6b47b53f..4040d840 100644 --- a/README.ru.md +++ b/README.ru.md @@ -112,7 +112,7 @@ | [Testing and releases](docs/testing.md) | ветки, CI, юнит-тесты, версии и релизы, сборки веток, `mapping.txt`, локальная сборка | | [Security](docs/security.md) | хранилище учётных данных, отладочный сервер (только локальная сеть), скрытие паролей в логах, отсутствие шифрования в сети | | [Devices](docs/devices.md) | проверенные и неработающие устройства, особенности аппаратных декодеров, отчёты об устройствах и камерах | -| [Drones and Wi‑Fi toys](docs/toys.md) | как передают видео игрушки, FLOW-UFO (в тестировании), что попробовать с неизвестной игрушкой, что прислать | +| [Drones and Wi‑Fi toys](docs/toys.md) | как передают видео игрушки, дроны FLOW-UFO, KY UFO и Jieli, что попробовать с неизвестной игрушкой, что прислать | История изменений — это [принятые pull request'ы](https://github.com/OpenIPC/decoder/pulls?q=is%3Apr+is%3Amerged+base%3Av2) и [коммиты ветки v2](https://github.com/OpenIPC/decoder/commits/v2). diff --git a/README.zh-CN.md b/README.zh-CN.md index 2b1c9b43..c3f08dec 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -112,7 +112,7 @@ | [Testing and releases](docs/testing.md) | 分支、CI、单元测试、版本与发布、分支构建、`mapping.txt`、本地构建 | | [Security](docs/security.md) | 凭据存储、调试服务器（仅限局域网）、日志脱敏、网络传输未加密 | | [Devices](docs/devices.md) | 已验证和无法使用的设备、硬件解码器说明、设备与摄像机报告 | -| [Drones and Wi‑Fi toys](docs/toys.md) | 玩具如何传输视频、FLOW-UFO（测试中）、遇到未知玩具时可以尝试什么、需要提交哪些信息 | +| [Drones and Wi‑Fi toys](docs/toys.md) | 玩具如何传输视频、FLOW-UFO、KY UFO 和 Jieli 无人机、遇到未知玩具时可以尝试什么、需要提交哪些信息 | 更新历史即[已合并的 pull request](https://github.com/OpenIPC/decoder/pulls?q=is%3Apr+is%3Amerged+base%3Av2) 和 [v2 分支的提交](https://github.com/OpenIPC/decoder/commits/v2)。 diff --git a/android/app/build.gradle b/android/app/build.gradle index 8fadf408..53d4226c 100755 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -23,8 +23,8 @@ android { applicationId "com.openipc.decoder" minSdk 21 targetSdk 36 - versionCode 38 - versionName "2.13" + versionCode 40 + versionName "2.15" buildConfigField "String", "GIT_HASH", "\"${getGitHash()}\"" } diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index bb26eb26..e224db1e 100755 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -2,6 +2,8 @@ + + { + if (statusText == null || "connected".equals(singleState)) return; + statusText.setText(text); + statusText.setVisibility(View.VISIBLE); + }); + } + private void updateStatus(String status) { singleState = status; runOnUiThread(() -> { @@ -1375,6 +1388,10 @@ private void removeQuadOverlay(int index) { private final DebugServer.Source debugSource = new DebugServer.Source() { @Override public Map snapshot() { return debugSnapshot(); } @Override public byte[] snapshotJpeg(boolean labels) { return debugSnapshotJpeg(labels); } + @Override public byte[] jieliRaw(String which) { + RtspClient c = rtspClient; + return c != null ? c.jieliRaw(which) : Jieli.Client.lastRaw(which); + } }; /** A cell counts as playing in a snapshot when it showed a frame this recently (#98). */ private static final long SNAPSHOT_LIVE_MS = 2000; @@ -1579,6 +1596,16 @@ private Map debugState() { // Frames shown in the last second and the playback mode of S (#153) single.put("out_fps", singleOutFps); single.put("decode", slides ? "key frames" : "full"); + // The toy drone's protocol counters (#187) + Map ufo = rtspClient != null ? rtspClient.flowUfoStats() : null; + if (ufo != null) single.put("flowufo", ufo); + Map jl = rtspClient != null ? rtspClient.jieliStats() : null; + if (jl != null) single.put("jieli", jl); + Map hb = rtspClient != null ? rtspClient.heartbeatStats() : null; + if (hb != null) single.put("heartbeat", hb); + // MJPEG turned counter-clockwise, "size" is the turned one (#200) + int rotate = rtspClient != null && rtspClient.isJpeg() ? rtspClient.rotate() : 0; + if (rotate != 0) single.put("rotate", rotate); m.put("single", single); } QuadCell[] cells = quadCells; @@ -1617,6 +1644,13 @@ private Map debugState() { ? cell.keyLoad() : cell.fullLoad()); c.put("paused", cell.isPaused()); c.put("audio", i == quadAudio); + Map ufo = cell.flowUfoStats(); + if (ufo != null) c.put("flowufo", ufo); + Map jl = cell.jieliStats(); + if (jl != null) c.put("jieli", jl); + Map hb = cell.heartbeatStats(); + if (hb != null) c.put("heartbeat", hb); + if (cell.rotate() != 0) c.put("rotate", cell.rotate()); } list.add(c); } @@ -1684,6 +1718,8 @@ private String slotTransport(int i, String url) { } if (RtpListener.Url.matches(url)) return "UDP listen"; if (HttpMjpeg.matches(url)) return "HTTP"; + if (FlowUfo.Url.matches(url)) return "FLOW-UFO UDP"; + if (Jieli.Url.matches(url)) return "Jieli RTP"; Transport.Mode m = StreamProbe.probeMode(mTransports[i]); return "check over " + (m == Transport.Mode.MULTICAST ? "multicast" : m.name()); } @@ -2230,7 +2266,10 @@ private int dp(float dp) { */ private boolean drawJpeg(Frame frame, int gen) { Surface s = mVideoSurface; - int[] size = jpegRenderer.render(frame, s, () -> gen == listenerGen && s == mVideoSurface); + RtspClient client = rtspClient; + int rotate = client != null ? client.rotate() : 0; + int[] size = jpegRenderer.render(frame, s, rotate, + () -> gen == listenerGen && s == mVideoSurface); if (size != null) jpegShown(size[0], size[1]); return size != null; } @@ -2293,6 +2332,14 @@ private void startListener() { try { Frame buffer = nalQueue.poll(5, TimeUnit.MILLISECONDS); if (buffer != null && rtspClient.isJpeg()) { + // Every JPEG frame is whole: when drawing falls behind (a turned frame + // is drawn in software, #187), show the newest and drop the older ones + // instead of letting the delay grow + Frame newer; + while ((newer = nalQueue.poll()) != null) { + framePool.recycle(buffer); + buffer = newer; + } // S: MJPEG has no key frames, one frame per second instead (#153); the // second counts from a frame really shown (Qodo on #155) long now = System.nanoTime() / 1_000_000; diff --git a/android/app/src/main/java/com/openipc/decoder/FlowUfo.java b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java new file mode 100644 index 00000000..c708b48c --- /dev/null +++ b/android/app/src/main/java/com/openipc/decoder/FlowUfo.java @@ -0,0 +1,1234 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * FlowUfo.java — video of FLOW-UFO toy drones: JPEG frames in UDP fragments, with acks (#187) + * + */ + +package com.openipc.decoder; + +import android.util.Log; + +import java.io.Closeable; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.InterfaceAddress; +import java.net.NetworkInterface; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Cheap Wi-Fi toy drones with a "FLOW-UFO-xxxxxx" / "FLOW_xxxxxx" access point (a BL60x + * camera) send MJPEG over UDP from 192.168.169.1 in a protocol of their own (#187). A short + * probe to port 8800 (BL608 engine) or 8801 (BL618 engine) wakes the drone; it then sends + * every JPEG frame, cut into fragments of 1024 bytes, from source port 1234 to the address the + * probe came from, and the client acknowledges the last four frames: what arrived, what is + * missing, and which frame to send next. A frame carries only the entropy-coded data of the + * scan: the client puts the JPEG headers in front. Video only: flight commands are never sent. + * + * The same protocol is implemented by the open-source project turbodrone (Apache-2.0, + * github.com/marshallrichards/turbodrone, "wifi_uav") and works there on real drones; the + * values taken over from its observations (quality parameters, both ports, which slots go into + * the acknowledgement, the raster-order tables of ?dqt=raster) say so where they are used. + * No code of it is used. Pure Java apart from the log. + */ +final class FlowUfo { + private static final String TAG = "OpenIPCDecoder"; + + static final String SCHEME = "flowufo://"; + static final String DEFAULT_HOST = "192.168.169.1"; + static final int DEFAULT_PORT = 8800; + /** + * The port of the BL618 engine. The vendor library starts both engines, on 8800 and 8801, + * and keeps the one that delivers frames (turbodrone docs/research/wifi_uav.md): a slot on + * 8800 (the default) probes both and locks onto the first that answers. + */ + static final int BL618_PORT = 8801; + + /** Wakes the drone up; sent every PROBE_MS (BL618: PROBE_MS_BL618) until it answers. */ + static final byte[] PROBE = {(byte) 0xEF, 0x00, 0x04, 0x00}; + static final long PROBE_MS = 100; + static final long PROBE_MS_BL618 = 200; + static final long ACK_MS = 25; + /** The BL618 engine acknowledges every 50 ms, and only when there is a record to send. */ + static final long ACK_MS_BL618 = 50; + /** No fragment for longer: the last frame is over, the acknowledgement asks for the next. */ + static final long QUIET_MS = 19; + /** + * No fragment for this long: everything is reset and the probes start again. Not final so + * unit tests can shorten it. + */ + static volatile long resetMs = 3000; + + // Drone packets: 0x93, type, length (u16 LE, the whole datagram) + static final int MAGIC = 0x93; + static final int TYPE_FRAGMENT = 1; + static final int TYPE_SERVICE = 4; + static final int MAX_PACKET = 0x438; + /** Header of a fragment; its data follows. */ + static final int FRAGMENT_HEADER = 0x38; + /** Fragment i holds the body bytes from i * FRAGMENT_DATA on. */ + static final int FRAGMENT_DATA = 1024; + /** More fragments than this (1 MiB) are not a frame of these cameras. */ + static final int MAX_FRAGMENTS = 1024; + + // The acknowledgement: a fixed header, then one record per frame + static final int ACK_HEADER = 0x58; + /** + * Quality parameters at 0x52 of the acknowledgement: 32 4B 14 2D, the bytes turbodrone + * sends to real drones (#187). The vendor app's own default was read as 19 32 23 4B; that + * one, or any other, can be given with ?qpara= (8 hex digits). + */ + static final byte[] QPARA = {0x32, 0x4B, 0x14, 0x2D}; + static final int SLOTS = 4; + /** + * A slot this many frames behind the newest is left out of the acknowledgement, as + * turbodrone does: the drone has moved on, asking for it again only holds it up. + */ + static final int LAG = 5; + /** Fragments in a row this far behind the newest frame: the drone counts from 1 again. */ + static final int RESTART_RUN = 8; + static final int STATUS_COLLECTING = 0; + static final int STATUS_COMPLETE = 1; + static final int STATUS_WHOLE = 3; + private static final int MAX_MASK_WORDS = (MAX_FRAGMENTS + 31) / 32; + static final int ACK_CAPACITY = ACK_HEADER + (SLOTS + 1) * (16 + 4 * MAX_MASK_WORDS); + + // The JPEG headers put in front of a body. Unverified choices, easy to change here: the + // luma sampling (0x11 = 4:4:4, like the chroma), no restart interval. With these the headers + // are 605 bytes and SOF0 has the height at 0x91 and the width at 0x93, the layout the vendor + // app is known to use. Cb and Cr share the second quantization and Huffman tables + static final int LUMA_SAMPLING = 0x11; + static final int RESTART_INTERVAL = 0; + /** A frame that names no usable quality gets this one. */ + static final int DEFAULT_QUALITY = 50; + static final int DEFAULT_WIDTH = 640; + static final int DEFAULT_HEIGHT = 360; + + /** + * Quantization tables of the header, ?dqt= (#187). ZIGZAG (default): the Annex K tables + * scaled to the quality byte of the fragment, in zig-zag order as the JPEG standard has + * DQT. RASTER: unscaled Annex K tables in raster (row-major) order, quality ignored: what + * turbodrone writes, and it shows real drones' frames. Q50: unscaled, zig-zag. Which one + * matches the drone's encoder is not known yet; the tester compares. + */ + static final int DQT_ZIGZAG = 0; + static final int DQT_RASTER = 1; + static final int DQT_Q50 = 2; + static final String[] DQT_NAMES = {"zigzag", "raster", "q50"}; + + /** Zig-zag position i to raster (row-major) position. */ + private static final int[] RASTER_OF_ZIGZAG = { + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63 + }; + + private FlowUfo() {} + + // ====================== URL ====================== + + /** + * flowufo://[host][:port][/...][?option=value&...]: the drone's address, 192.168.169.1:8800 + * when left out. Port 8800 (also when left out) probes 8801 as well; any other port only + * itself. Options, as in udp:// URLs (unknown ones are ignored, bad values refused): + * qpara=8 hex digits (the quality parameters of the acknowledgement, default 324B142D), + * dqt=zigzag|raster|q50 (the quantization tables of the JPEG header, default zigzag). + */ + static final class Url { + String host = DEFAULT_HOST; + int port = DEFAULT_PORT; + /** Probed as well as port until one answers, or 0. Not final so tests can move it. */ + int alternatePort; + byte[] qpara = QPARA.clone(); + int dqt = DQT_ZIGZAG; + + static boolean matches(String url) { + return url != null && url.trim().toLowerCase(Locale.ROOT).startsWith(SCHEME); + } + + static Url parse(String url) throws IOException { + if (!matches(url)) throw new IOException("Not a flowufo:// URL"); + String rest = url.trim().substring(SCHEME.length()); + int end = rest.length(); + for (int i = 0; i < rest.length(); i++) { + char ch = rest.charAt(i); + if (ch == '/' || ch == '?' || ch == '#') { + end = i; + break; + } + } + String authority = rest.substring(0, end).trim(); + String query = ""; + int q = rest.indexOf('?'); + int hash = rest.indexOf('#'); + if (q >= 0 && (hash < 0 || q < hash)) { + query = rest.substring(q + 1, hash >= 0 ? hash : rest.length()); + } + Url u = new Url(); + String host = authority; + int colon = authority.lastIndexOf(':'); + if (authority.startsWith("[")) { + int close = authority.indexOf(']'); + if (close < 0) throw new IOException("Bad host in URL"); + host = authority.substring(1, close); + colon = authority.indexOf(':', close); + } else if (colon >= 0) { + host = authority.substring(0, colon); + } + if (colon >= 0) { + try { + u.port = Integer.parseInt(authority.substring(colon + 1)); + } catch (NumberFormatException e) { + throw new IOException("Bad port in URL"); + } + if (u.port <= 0 || u.port > 65535) throw new IOException("Bad port in URL"); + } + if (!host.isEmpty()) u.host = host; + for (int i = 0; i < u.host.length(); i++) { + char ch = u.host.charAt(i); + if (ch <= 0x20 || ch == 0x7F || ch == '@') throw new IOException("Bad host in URL"); + } + u.alternatePort = u.port == DEFAULT_PORT ? BL618_PORT : 0; + for (String param : query.split("&")) { + int eq = param.indexOf('='); + if (eq < 0) continue; + String name = param.substring(0, eq).trim().toLowerCase(Locale.ROOT); + String value = param.substring(eq + 1).trim().toLowerCase(Locale.ROOT); + if (name.equals("qpara") || name.equals("q")) { + u.qpara = parseQpara(value); + } else if (name.equals("dqt")) { + u.dqt = Arrays.asList(DQT_NAMES).indexOf(value); + if (u.dqt < 0) { + throw new IOException("Unknown dqt: " + value + " (zigzag, raster or q50)"); + } + } + } + return u; + } + + /** Exactly 8 hex digits, e.g. 19322348, as 4 bytes. */ + static byte[] parseQpara(String v) throws IOException { + if (v.length() != 8) throw new IOException("qpara needs 8 hex digits: " + v); + byte[] b = new byte[4]; + for (int i = 0; i < 4; i++) { + int hi = Character.digit(v.charAt(2 * i), 16); + int lo = Character.digit(v.charAt(2 * i + 1), 16); + if (hi < 0 || lo < 0) throw new IOException("qpara needs 8 hex digits: " + v); + b[i] = (byte) (hi << 4 | lo); + } + return b; + } + + /** The ports probed: the given one, then the alternate one if any. */ + int[] ports() { + return alternatePort > 0 && alternatePort != port + ? new int[] {port, alternatePort} : new int[] {port}; + } + + String endpoint() { + return endpoint(port); + } + + String endpoint(int p) { + return (host.indexOf(':') >= 0 ? "[" + host + "]" : host) + ":" + p; + } + + /** "192.168.169.1:8800 and :8801", or the one endpoint. */ + String probed() { + int[] p = ports(); + return p.length > 1 ? endpoint(p[0]) + " and :" + p[1] : endpoint(p[0]); + } + } + + // ====================== Drone packets ====================== + + /** Type of a drone packet, or -1 when it is none: wrong first byte or length. */ + static int packetType(byte[] b, int off, int len) { + if (len < 4 || len > MAX_PACKET || (b[off] & 0xFF) != MAGIC) return -1; + if (u16(b, off + 2) != len) return -1; + return b[off + 1] & 0xFF; + } + + /** One frame fragment (type 1); fields as little-endian numbers. */ + static final class Fragment { + /** + * Frame number, counting from 1. Read as u32 at 0x08: whether 0x08 is a u64 or a u32 + * followed by another field at 0x0C is not known, and both read the same while the + * upper half is zero. The log shows 0x08..0x17 of the first fragments. + */ + long seq; + /** Frames up to this one are not acknowledged any more (low 32 bits of 0x18). */ + long base; + int index; + int total; + int bodyLength; + int width; + int height; + int quality; + /** + * Main and flow camera status at 0x34 and 0x35 (turbodrone docs: the vendor library + * passes them on with the frame). Shown in the stats, not used. + */ + int mainCamera; + int flowCamera; + /** The packet parsed last; its data starts at dataOffset. */ + byte[] buf; + int dataOffset; + int dataLength; + + /** Fills the fields from a type-1 packet; false when it is malformed. */ + boolean parse(byte[] b, int off, int len) { + if (len < FRAGMENT_HEADER || packetType(b, off, len) != TYPE_FRAGMENT) return false; + seq = u32(b, off + 0x08); + base = u32(b, off + 0x18); + long idx = u32(b, off + 0x20); + long tot = u32(b, off + 0x24); + long body = u32(b, off + 0x28); + width = u16(b, off + 0x2C); + height = u16(b, off + 0x2E); + quality = b[off + 0x30] & 0xFF; + mainCamera = b[off + 0x34] & 0xFF; + flowCamera = b[off + 0x35] & 0xFF; + buf = b; + dataOffset = off + FRAGMENT_HEADER; + dataLength = len - FRAGMENT_HEADER; + if (seq == 0 || tot < 1 || tot > MAX_FRAGMENTS || idx >= tot) return false; + if (body < 1 || body > tot * FRAGMENT_DATA || dataLength > FRAGMENT_DATA) return false; + index = (int) idx; + total = (int) tot; + bodyLength = (int) body; + return true; + } + } + + // ====================== Frames and acknowledgements ====================== + + /** One frame being collected, in slot (seq - 1) & 3. */ + static final class Slot { + long seq; + int status; + int total; + int received; + int bodyLength; + int width; + int height; + int quality; + int[] mask = new int[1]; + byte[] body = new byte[64 * 1024]; + + void clear() { + seq = 0; + status = STATUS_COLLECTING; + received = 0; + } + + void start(Fragment f) { + seq = f.seq; + status = STATUS_COLLECTING; + total = f.total; + received = 0; + bodyLength = f.bodyLength; + width = f.width; + height = f.height; + quality = f.quality; + int words = (total + 31) / 32; + if (mask.length < words) mask = new int[words]; + Arrays.fill(mask, 0); + if (body.length < total * FRAGMENT_DATA) body = new byte[total * FRAGMENT_DATA]; + } + + boolean has(int i) { + return (mask[i >> 5] & (1 << (i & 31))) != 0; + } + + /** Mask word w as acknowledged: fragments past the frame's last count as received. */ + int ackWord(int w) { + int v = mask[w]; + int first = w * 32; + if (first + 32 > total) v |= -1 << (total - first); + return v; + } + } + + /** + * The four frames in flight and the acknowledgement built from them. A frame goes into + * slot (seq - 1) & 3; a newer frame of the same slot replaces the one there, which counts + * as lost unless it was complete (the drone should only send seq + 4 there, frames skipped + * on the way are lost too). Single-threaded; the counters are read by the stats. + */ + static final class Window { + final Slot[] slots = new Slot[SLOTS]; + long maxSeq; + long base; + /** The frame whose fragments arrive now: not acknowledged while it is coming in. */ + long current; + long lastDelivered; + /** When a fragment last added data: the quiet time of the acknowledgement. */ + long lastFragmentMs; + /** Fragments in a row far behind the newest frame. */ + private int behind; + volatile int fragments; + volatile int duplicates; + volatile int stale; + volatile int mismatched; + /** Fragments with less data than their place in the body needs. */ + volatile int shortFragments; + volatile int completed; + volatile int delivered; + volatile int lost; + volatile int late; + /** Records left out because their slot lagged LAG or more frames behind. */ + volatile int lagging; + volatile int nextRequests; + /** Quality parameters of the acknowledgement. */ + byte[] qpara = QPARA; + /** The frame numbers started from 1 again (the drone restarted). */ + volatile int restarts; + + Window() { + for (int i = 0; i < SLOTS; i++) slots[i] = new Slot(); + } + + void reset() { + for (Slot s : slots) s.clear(); + maxSeq = 0; + base = 0; + current = 0; + lastDelivered = 0; + behind = 0; + } + + static int slotOf(long seq) { + return (int) ((seq - 1) & (SLOTS - 1)); + } + + /** + * Adds a fragment; returns its slot when the fragment completed a frame to show. Only a + * fragment that adds data counts for the quiet time: stale ones and duplicates do not + * (Qodo on #188). + */ + Slot add(Fragment f, long nowMs) { + if (maxSeq > 64 && f.seq + 64 < maxSeq) { + // Far behind the newest frame: a straggler, or the drone counts from 1 again; + // only a run of them is taken for a restart (Qodo on #188) + if (++behind < RESTART_RUN) { + stale++; + return null; + } + restarts++; + reset(); + } + behind = 0; + // Only a base behind the frame itself makes sense + if (f.base > base && f.base < f.seq) base = f.base; + Slot s = slots[slotOf(f.seq)]; + if (f.seq < s.seq) { + stale++; + return null; + } + if (f.seq > s.seq) { + if (s.seq > 0) { + if (s.status != STATUS_COMPLETE) lost++; + // Frames of this slot that never came at all + lost += (int) Math.min(1000, (f.seq - s.seq) / SLOTS - 1); + } + s.start(f); + } + fragments++; + if (f.seq > maxSeq) maxSeq = f.seq; + if (s.status == STATUS_COMPLETE || (f.total == s.total && s.has(f.index))) { + duplicates++; + return null; + } + if (f.total != s.total) { + mismatched++; + return null; + } + if (f.bodyLength != s.bodyLength) { + // Not seen as such; the longest is taken, the fragments fit either way + mismatched++; + s.bodyLength = Math.max(s.bodyLength, f.bodyLength); + } + int at = f.index * FRAGMENT_DATA; + if (f.dataLength < Math.min(FRAGMENT_DATA, s.bodyLength - at)) { + shortFragments++; + // A short one inside the body stays missing (Qodo on #188); the last one ends + // the body where its data ends + if (f.index < s.total - 1) return null; + s.bodyLength = at + f.dataLength; + } + int n = Math.min(f.dataLength, s.total * FRAGMENT_DATA - at); + System.arraycopy(f.buf, f.dataOffset, s.body, at, n); + s.mask[f.index >> 5] |= 1 << (f.index & 31); + s.received++; + lastFragmentMs = nowMs; + current = f.seq; + if (s.received < s.total) return null; + s.status = STATUS_COMPLETE; + completed++; + if (s.seq <= lastDelivered) { + // Completed after a newer frame was shown + late++; + return null; + } + lastDelivered = s.seq; + delivered++; + return s; + } + + /** + * The acknowledgement for now, into {@code out} (ACK_CAPACITY bytes); returns its + * length. Per slot: complete (status 1) or still collecting (0) with the mask of the + * fragments received; a slot LAG or more frames behind the newest is left out (as in + * turbodrone), and does not hold up the request for the next frame either. The frame + * coming in now is left out until the fragments pause; after QUIET_MS without any, when + * every frame is complete, a record asks for the frame after the newest (status 3). + * Unverified: the record order, and that the paused frame is included again. + */ + int buildAck(byte[] out, long nowMs) { + boolean quiet = nowMs - lastFragmentMs > QUIET_MS; + int pos = ACK_HEADER; + int records = 0; + boolean any = false; + boolean allComplete = true; + for (Slot s : slots) { + // At or below the base the drone is done with it: no record, and an incomplete + // one does not hold up the request for the next frame (Qodo on #188) + if (s.seq == 0 || s.seq <= base) continue; + if (maxSeq - s.seq >= LAG) { + lagging++; + continue; + } + any = true; + if (s.status != STATUS_COMPLETE) allComplete = false; + if (s.seq == current && !quiet) continue; + int words = (s.total + 31) / 32; + le64(out, pos, s.seq); + boolean complete = s.status == STATUS_COMPLETE; + le32(out, pos + 8, complete ? STATUS_COMPLETE : STATUS_COLLECTING); + le32(out, pos + 12, 16 + 4 * words); + pos += 16; + for (int w = 0; w < words; w++) { + le32(out, pos, complete ? -1 : s.ackWord(w)); + pos += 4; + } + records++; + } + if (quiet && any && allComplete) { + pos = wholeRecord(out, pos, maxSeq + 1); + records++; + nextRequests++; + } + ackHeader(out, pos, records, qpara); + return pos; + } + + private static int wholeRecord(byte[] out, int pos, long seq) { + le64(out, pos, seq); + le32(out, pos + 8, STATUS_WHOLE); + le32(out, pos + 12, 16); + return pos + 16; + } + } + + /** + * The fixed part of an acknowledgement: EF 02, length, 02 02 00 01, record count, then no + * command (counter 0, length 0, 0x40 zero bytes), the quality parameters and active camera + * 0. The vendor app and turbodrone put the current stick command in there, a neutral one + * when idle; this app never does: a viewer must not be able to move a toy, and a neutral + * command is still a command (it can hold a drone that the remote is flying). + */ + static void ackHeader(byte[] out, int length, int records, byte[] qpara) { + out[0] = (byte) 0xEF; + out[1] = 0x02; + le16(out, 2, length); + out[4] = 0x02; + out[5] = 0x02; + out[6] = 0x00; + out[7] = 0x01; + le32(out, 0x08, records); + le32(out, 0x0C, 0); + le16(out, 0x10, 0); + Arrays.fill(out, 0x12, 0x52, (byte) 0); + System.arraycopy(qpara, 0, out, 0x52, 4); + out[0x56] = 0; + out[0x57] = 0; + } + + // ====================== JPEG ====================== + + /** The headers in front of a body: SOI, DQT, SOF0, DHT, SOS (605 bytes as configured). */ + static byte[] jpegHeader(int quality, int width, int height) { + return jpegHeader(quality, width, height, DQT_ZIGZAG); + } + + static byte[] jpegHeader(int quality, int width, int height, int dqt) { + return JpegAssembler.headersSampled(LUMA_SAMPLING, width > 0 ? width : DEFAULT_WIDTH, + height > 0 ? height : DEFAULT_HEIGHT, dqtTables(dqt, quality), 0, + RESTART_INTERVAL); + } + + /** Luma then chroma table, 64 bytes each, in the order DQT carries them for {@code dqt}. */ + static byte[] dqtTables(int dqt, int quality) { + if (dqt == DQT_Q50) return JpegAssembler.ijgTables(50); + if (dqt == DQT_RASTER) { + // Q50 is the unscaled Annex K table; put each value at its raster position + byte[] zz = JpegAssembler.ijgTables(50); + byte[] out = new byte[128]; + for (int t = 0; t < 128; t += 64) { + for (int k = 0; k < 64; k++) out[t + RASTER_OF_ZIGZAG[k]] = zz[t + k]; + } + return out; + } + return JpegAssembler.ijgTables(quality >= 1 && quality <= 100 ? quality : DEFAULT_QUALITY); + } + + /** For the stats and the log: which tables, e.g. "zigzag Q75" or "raster (unscaled)". */ + static String dqtLabel(int dqt, int quality) { + if (dqt == DQT_ZIGZAG) { + return "zigzag Q" + (quality >= 1 && quality <= 100 ? quality : DEFAULT_QUALITY); + } + return DQT_NAMES[dqt] + " (unscaled)"; + } + + // ====================== Subnet hint ====================== + + /** + * A hint when the slot cannot be right for the phone's networks (#187), or null: the + * target is the phone itself, or the phone is on a network of another toy family than the + * address. {@code local}: the phone's IPv4 addresses. + */ + static String subnetHint(InetAddress target, List local) { + if (!(target instanceof Inet4Address) || local == null) return null; + boolean own = local.contains(target); + boolean on1 = false; + boolean on169 = false; + boolean on80 = false; + for (InetAddress a : local) { + on1 |= in24(a, 1); + on169 |= in24(a, 169); + on80 |= in24(a, 80); + } + StringBuilder parts = new StringBuilder(); + byte[] b = target.getAddress(); + if (own) { + sentence(parts, target.getHostAddress() + " is this phone; the toy is usually " + + (b[0] & 0xFF) + "." + (b[1] & 0xFF) + "." + (b[2] & 0xFF) + ".1"); + } + if (on1 && !on169 && (own || in24(target, 169))) { + sentence(parts, "This network looks like a KY UFO RTSP toy: try" + + " rtsp://192.168.1.1:7070/webcam"); + } + if (on169 && !on1 && in24(target, 1)) { + sentence(parts, "This phone is on 192.168.169.x: the toy is usually" + + " flowufo://192.168.169.1"); + } + if (on80 && !on169 && !on1) { + sentence(parts, "This phone is on 192.168.80.x (HF_ Wi-Fi), a Jieli toy: try" + + " jieli://192.168.80.1"); + } + return parts.length() > 0 ? parts.toString() : null; + } + + static void sentence(StringBuilder sb, String s) { + if (sb.length() > 0) sb.append(". "); + sb.append(s); + } + + /** {@code a} is in 192.168.{@code third}.0/24. */ + static boolean in24(InetAddress a, int third) { + if (!(a instanceof Inet4Address)) return false; + byte[] b = a.getAddress(); + return (b[0] & 0xFF) == 192 && (b[1] & 0xFF) == 168 && (b[2] & 0xFF) == third; + } + + /** The phone's IPv4 addresses on interfaces that are up. */ + static List localAddresses() { + List list = new ArrayList<>(); + try { + Enumeration e = NetworkInterface.getNetworkInterfaces(); + while (e != null && e.hasMoreElements()) { + NetworkInterface n = e.nextElement(); + if (!n.isUp() || n.isLoopback()) continue; + for (InterfaceAddress ia : n.getInterfaceAddresses()) { + if (ia.getAddress() instanceof Inet4Address) list.add(ia.getAddress()); + } + } + } catch (Exception ignored) { + // No hint then + } + return list; + } + + /** + * Header, body and EOI into {@code out} (grown when too small, returned in out[0]); + * returns the length. A body that is a whole JPEG already goes out as it is. + */ + static int jpeg(byte[] header, byte[] body, int bodyLength, byte[][] out) { + boolean whole = bodyLength >= 2 && (body[0] & 0xFF) == 0xFF && (body[1] & 0xFF) == 0xD8; + boolean eoi = bodyLength >= 2 && (body[bodyLength - 2] & 0xFF) == 0xFF + && (body[bodyLength - 1] & 0xFF) == 0xD9; + int head = whole ? 0 : header.length; + int total = head + bodyLength + (eoi ? 0 : 2); + if (out[0] == null || out[0].length < total) out[0] = new byte[total + 64 * 1024]; + byte[] d = out[0]; + if (!whole) System.arraycopy(header, 0, d, 0, head); + System.arraycopy(body, 0, d, head, bodyLength); + if (!eoi) { + d[total - 2] = (byte) 0xFF; + d[total - 1] = (byte) 0xD9; + } + return total; + } + + // ====================== Client ====================== + + interface Listener { + /** Probing (no fragments, or none for resetMs) or receiving; text for the status. */ + void onState(boolean receiving, String text); + + /** One whole JPEG in {@code buf}, valid until the call returns. */ + void onFrame(byte[] buf, int length); + } + + /** Called with the socket before the first send: binds it to the drone's Wi-Fi. */ + interface SocketSetup { + String prepare(DatagramSocket socket, InetAddress target); + } + + /** + * The receive loop on the calling thread: probes, fragments, acknowledgements and the + * timers between them, until {@code keepGoing} fails or close(). Each probed port has a + * socket of its own: the drone answers from source port 1234 whichever port was probed, + * so only the socket the packets arrive on tells which engine answered. The first 0x93 + * packet from the toy's address locks the client onto that socket and port (acks go + * there); the other socket is closed, and opened again when the drone goes quiet and the + * probes start over. + */ + static final class Client implements Closeable { + private final String tag; + private final Url url; + private final SocketSetup setup; + private final int[] ports; + /** One socket per probed port; null once another port is locked. Guarded by itself. */ + private final DatagramSocket[] sockets; + private volatile boolean closed; + /** Index of the port the drone answers on, or -1 while it has not. */ + private volatile int locked = -1; + /** The phone's addresses for the subnet hint; null: the real ones. For tests. */ + List localForHint; + + final Window window = new Window(); + private final Fragment fragment = new Fragment(); + private final byte[] ack = new byte[ACK_CAPACITY]; + private final DatagramPacket out = new DatagramPacket(ack, 0); + private final byte[][] jpegOut = new byte[1][]; + private byte[] header; + /** Quality and size the header was made for. */ + private int headerQuality; + private int headerWidth; + private int headerHeight; + + // For the stats; written by the loop thread only + private volatile boolean receiving; + private volatile boolean playing; + private volatile String network = "default"; + private volatile String hint; + private volatile int probes; + private volatile int acks; + private volatile int packets; + private volatile long bytes; + private volatile int badPackets; + private volatile int foreign; + private volatile int resets; + private volatile int sendErrors; + private volatile String lastError; + private volatile int services; + private volatile String lastService; + private volatile String firstHeader; + private volatile String size; + private volatile String dqtUsed; + private volatile int mainCamera = -1; + private volatile int flowCamera = -1; + private int cameraChanges; + private volatile float fragmentsPerS; + private volatile float framesPerS; + /** Any fragment from the drone, usable or not: it is still there. */ + private long lastFragmentAt; + + Client(String tag, Url url, SocketSetup setup) { + this.tag = tag; + this.url = url; + this.setup = setup; + ports = url.ports(); + sockets = new DatagramSocket[ports.length]; + window.qpara = url.qpara; + } + + @Override + public void close() { + closed = true; + synchronized (sockets) { + for (DatagramSocket s : sockets) { + if (s != null) s.close(); + } + } + } + + /** BL618 (port 8801) probes every 200 ms, BL608 and others every 100 ms. */ + static long probeInterval(int port) { + return port == BL618_PORT ? PROBE_MS_BL618 : PROBE_MS; + } + + /** BL618 acknowledges every 50 ms and only with a record; BL608 every 25 ms. */ + static long ackInterval(int port) { + return port == BL618_PORT ? ACK_MS_BL618 : ACK_MS; + } + + static boolean ackNeedsRecord(int port) { + return port == BL618_PORT; + } + + static String engine(int port) { + return port == DEFAULT_PORT ? "BL608" : port == BL618_PORT ? "BL618" : "port " + port; + } + + /** The port the drone answers on, or -1. */ + int lockedPort() { + int i = locked; + return i >= 0 ? ports[i] : -1; + } + + String probingText(boolean again) { + String t = (again ? "The toy went quiet: probing " : "Waiting for the toy: probing ") + + url.probed() + (again ? " again" : ""); + String h = hint; + return h != null ? t + ". " + h : t; + } + + private void open(int i, InetAddress target) throws IOException { + DatagramSocket s = new DatagramSocket(); + try { + s.setReceiveBufferSize(1024 * 1024); + } catch (IOException ignored) {} + try { + if (setup != null) network = setup.prepare(s, target); + } catch (RuntimeException e) { + // Not registered yet: close() would not see it (Qodo on #188) + s.close(); + throw e; + } + synchronized (sockets) { + if (sockets[i] != null) sockets[i].close(); + sockets[i] = s; + if (closed) s.close(); + } + } + + private void closeSocket(int i) { + synchronized (sockets) { + if (sockets[i] != null) sockets[i].close(); + sockets[i] = null; + } + } + + private DatagramSocket socket(int i) { + synchronized (sockets) { + return sockets[i]; + } + } + + private String localPorts() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < ports.length; i++) { + DatagramSocket s = socket(i); + if (s == null) continue; + if (sb.length() > 0) sb.append(", "); + sb.append(s.getLocalPort()).append(" for ").append(ports[i]); + } + return sb.toString(); + } + + void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException { + InetAddress address = InetAddress.getByName(url.host); + hint = subnetHint(address, localForHint != null ? localForHint : localAddresses()); + if (hint != null) Log.w(TAG, tag + ": FLOW-UFO: " + hint); + try { + for (int i = 0; i < ports.length; i++) open(i, address); + } catch (IOException | RuntimeException e) { + // The sockets opened before the one that failed (Qodo on #188) + close(); + throw e; + } + Log.i(TAG, tag + ": FLOW-UFO: probing " + url.probed() + " from local port " + + localPorts() + " (network: " + network + "), qpara " + + hex(url.qpara, 0, 4) + ", dqt " + DQT_NAMES[url.dqt]); + listener.onState(false, probingText(false)); + byte[] rx = new byte[2048]; + DatagramPacket in = new DatagramPacket(rx, rx.length); + long[] nextProbe = new long[ports.length]; + long nextAck = 0; + long rateMark = now(); + int rateFragments = 0; + int rateFrames = 0; + long nextSummary = rateMark + 30_000; + // Start of the current run of failed sends, or 0 + long failingSince = 0; + try { + while (!closed && keepGoing.holds()) { + long t = now(); + if (t - rateMark >= 1000) { + float sec = (t - rateMark) / 1000f; + fragmentsPerS = (window.fragments - rateFragments) / sec; + framesPerS = (window.delivered - rateFrames) / sec; + rateFragments = window.fragments; + rateFrames = window.delivered; + rateMark = t; + } + if (receiving && t - lastFragmentAt > resetMs) { + Log.w(TAG, tag + ": FLOW-UFO: no fragment for " + resetMs + + " ms, probing again (" + summary() + ")"); + probeAgain(listener, address); + Arrays.fill(nextProbe, t); + } + if (receiving && t >= nextSummary) { + nextSummary = t + 30_000; + Log.i(TAG, tag + ": FLOW-UFO: " + summary()); + } + boolean sendFailed = false; + int lk = locked; + if (!receiving) { + for (int i = 0; i < ports.length; i++) { + DatagramSocket s = socket(i); + if (s == null || t < nextProbe[i]) continue; + nextProbe[i] = t + probeInterval(ports[i]); + if (send(s, PROBE, PROBE.length, address, ports[i])) { + probes++; + } else { + sendFailed = true; + } + } + } else if (lk >= 0 && t >= nextAck) { + // Behind schedule (a slow thread) is caught up, not sent in a burst + int port = ports[lk]; + nextAck = Math.max(nextAck + ackInterval(port), t); + int len = window.buildAck(ack, t); + if (len > ACK_HEADER || !ackNeedsRecord(port)) { + if (send(socket(lk), ack, len, address, port)) { + acks++; + } else { + sendFailed = true; + } + } + } + if (sendFailed) { + // Off the toy's Wi-Fi, or its network went away: new sockets after + // 3 s of failures, bound again to whatever network reaches it now. The + // drone sends to the port of the probe: the new ports probe at once + // (Qodo on #188) + if (failingSince == 0) failingSince = t; + if (t - failingSince >= 3000) { + failingSince = 0; + if (receiving) { + probeAgain(listener, address); + } else { + locked = -1; + for (int i = 0; i < ports.length; i++) open(i, address); + } + Log.w(TAG, tag + ": FLOW-UFO: new sockets after send errors, local" + + " port " + localPorts() + " (network: " + network + ")"); + Arrays.fill(nextProbe, t); + } + } else { + failingSince = 0; + } + long wake = Long.MAX_VALUE; + if (receiving) { + wake = nextAck; + } else { + for (int i = 0; i < ports.length; i++) { + if (socket(i) != null) wake = Math.min(wake, nextProbe[i]); + } + } + int timeout = (int) Math.max(1, Math.min(100, wake - now())); + int got = -1; + lk = locked; + if (lk >= 0) { + if (receive(socket(lk), in, rx, timeout)) got = lk; + } else { + // Not locked yet: each socket in turn; what is not read waits in its + // buffer for the next turn + int each = Math.max(1, timeout / ports.length); + for (int i = 0; i < ports.length && got < 0; i++) { + DatagramSocket s = socket(i); + if (s != null && receive(s, in, rx, each)) got = i; + } + } + if (got < 0) continue; + // By source address only: the drone sends from port 1234, not the probed one + if (!address.equals(in.getAddress())) { + if (++foreign <= 3) { + Log.w(TAG, tag + ": FLOW-UFO: packet from " + in.getAddress() + + ", not the toy: ignored"); + } + continue; + } + // Only a frame fragment locks: a service message may come from the engine + // that sends no video (Qodo on #188) + if (locked < 0 && packetType(rx, 0, in.getLength()) == TYPE_FRAGMENT) { + lock(got, in); + } + handle(rx, in.getLength(), listener); + } + } catch (IOException e) { + if (closed) return; + throw e; + } finally { + close(); + } + } + + private boolean receive(DatagramSocket s, DatagramPacket in, byte[] rx, int timeout) + throws IOException { + if (s == null) return false; + try { + s.setSoTimeout(timeout); + in.setData(rx); + s.receive(in); + return true; + } catch (SocketTimeoutException e) { + return false; + } + } + + /** The drone answered on ports[i]: acks go there, the other probes stop. */ + private void lock(int i, DatagramPacket from) { + locked = i; + for (int j = 0; j < ports.length; j++) { + if (j != i) closeSocket(j); + } + Log.i(TAG, tag + ": FLOW-UFO: the toy answers the probe to port " + ports[i] + " (" + + engine(ports[i]) + "), from port " + from.getPort() + + (ports.length > 1 ? "; the other port is not probed any more" : "")); + } + + /** + * Back to probing on every port: the stream is gone. Every socket is new, bound again + * to whatever network reaches the toy now (Qodo on #188). + */ + private void probeAgain(Listener listener, InetAddress address) throws IOException { + resets++; + receiving = false; + playing = false; + window.reset(); + locked = -1; + for (int i = 0; i < ports.length; i++) open(i, address); + listener.onState(false, probingText(true)); + } + + private boolean send(DatagramSocket s, byte[] data, int len, InetAddress a, int port) { + if (s == null) return false; + try { + out.setData(data, 0, len); + out.setAddress(a); + out.setPort(port); + s.send(out); + return true; + } catch (IOException e) { + sendErrors++; + lastError = e.getMessage(); + if (sendErrors <= 3 || sendErrors % 100 == 0) { + Log.w(TAG, tag + ": FLOW-UFO: cannot send to " + url.endpoint(port) + " (" + + sendErrors + "): " + e.getMessage() + + "; is this device on the toy's Wi-Fi?"); + } + return false; + } + } + + private void handle(byte[] b, int len, Listener listener) { + packets++; + bytes += len; + int type = packetType(b, 0, len); + if (type == TYPE_SERVICE) { + // Logged only: the vendor app answers some of them with commands, we do not + services++; + lastService = hex(b, 0, Math.min(len, 64)); + if (services <= 5 || services % 100 == 0) { + Log.i(TAG, tag + ": FLOW-UFO: service message (" + services + "): " + + lastService); + } + return; + } + if (type != TYPE_FRAGMENT || !fragment.parse(b, 0, len)) { + badPackets++; + if (badPackets <= 5 || badPackets % 500 == 0) { + Log.w(TAG, tag + ": FLOW-UFO: unknown or malformed packet (" + badPackets + + ", " + len + " bytes): " + hex(b, 0, Math.min(len, 64))); + } + return; + } + long t = now(); + lastFragmentAt = t; + if (!receiving) { + receiving = true; + if (firstHeader == null) firstHeader = hex(b, 0, FRAGMENT_HEADER); + Log.i(TAG, tag + ": FLOW-UFO: first fragment after " + probes + " probes: " + + hex(b, 0, FRAGMENT_HEADER)); + listener.onState(true, "Receiving from the toy at " + + url.endpoint(lockedPort() > 0 ? lockedPort() : url.port) + "…"); + } + int n = window.fragments; + if (n < 5) { + Log.i(TAG, tag + ": FLOW-UFO: fragment seq " + fragment.seq + " base " + + fragment.base + " " + fragment.index + "/" + fragment.total + ", body " + + fragment.bodyLength + ", " + fragment.width + "x" + fragment.height + + " Q" + fragment.quality + ", bytes 0x08-0x17: " + hex(b, 0x08, 16)); + } + if (fragment.mainCamera != mainCamera || fragment.flowCamera != flowCamera) { + boolean first = mainCamera < 0; + mainCamera = fragment.mainCamera; + flowCamera = fragment.flowCamera; + if (++cameraChanges <= 5) { + Log.i(TAG, tag + ": FLOW-UFO: camera status (0x34/0x35): main " + mainCamera + + ", flow " + flowCamera + (first ? "" : " (changed)")); + } + } + Slot done = window.add(fragment, t); + if (done == null) return; + if (header == null || done.quality != headerQuality || done.width != headerWidth + || done.height != headerHeight) { + header = jpegHeader(done.quality, done.width, done.height, url.dqt); + headerQuality = done.quality; + headerWidth = done.width; + headerHeight = done.height; + size = done.width + "x" + done.height; + dqtUsed = dqtLabel(url.dqt, done.quality); + Log.i(TAG, tag + ": FLOW-UFO: frames " + size + " Q" + done.quality + + (done.quality >= 1 && done.quality <= 100 ? "" + : " (not a quality)") + ", tables " + dqtUsed); + } + int length = jpeg(header, done.body, done.bodyLength, jpegOut); + if (!playing) { + playing = true; + Log.i(TAG, tag + ": FLOW-UFO: first frame, seq " + done.seq + ", " + + done.total + " fragments, body " + done.bodyLength + " bytes: " + + hex(done.body, 0, Math.min(32, done.bodyLength))); + } + listener.onFrame(jpegOut[0], length); + } + + private String summary() { + Window w = window; + return String.format(Locale.ROOT, "%.1f fps, %.0f fragments/s, %d frames, %d lost," + + " %d late, %d lagging left out, %d next requests, %d acks, %d bad", + framesPerS, fragmentsPerS, w.delivered, w.lost, w.late, w.lagging, + w.nextRequests, acks, badPackets); + } + + /** For /stats.json. */ + Map stats() { + Window w = window; + Map m = new LinkedHashMap<>(); + m.put("state", playing ? "playing" : receiving ? "receiving" : "probing"); + m.put("target", url.probed()); + int lp = lockedPort(); + if (lp > 0) { + m.put("port", lp); + m.put("engine", engine(lp)); + } + m.put("network", network); + if (hint != null) m.put("hint", hint); + m.put("qpara", hex(url.qpara, 0, 4)); + m.put("dqt", dqtUsed != null ? dqtUsed : DQT_NAMES[url.dqt]); + m.put("frames_per_s", Math.round(framesPerS * 10) / 10f); + m.put("fragments_per_s", Math.round(fragmentsPerS)); + if (size != null) m.put("size", size); + if (size != null) m.put("quality", headerQuality); + if (mainCamera >= 0) { + Map cam = new LinkedHashMap<>(); + cam.put("main", mainCamera); + cam.put("flow", flowCamera); + m.put("camera_status", cam); + } + m.put("probes", probes); + m.put("acks", acks); + m.put("packets", packets); + m.put("bytes", bytes); + m.put("fragments", w.fragments); + m.put("duplicates", w.duplicates); + m.put("stale", w.stale); + m.put("mismatched", w.mismatched); + m.put("short_fragments", w.shortFragments); + m.put("bad_packets", badPackets); + m.put("frames", w.delivered); + m.put("frames_complete", w.completed); + m.put("frames_lost", w.lost); + m.put("frames_late", w.late); + m.put("lagging_left_out", w.lagging); + m.put("next_requests", w.nextRequests); + m.put("resets", resets); + m.put("seq_restarts", w.restarts); + m.put("seq", w.maxSeq); + m.put("base", w.base); + m.put("service_messages", services); + if (lastService != null) m.put("last_service", lastService); + if (firstHeader != null) m.put("first_fragment_header", firstHeader); + if (foreign > 0) m.put("foreign_packets", foreign); + if (sendErrors > 0) { + m.put("send_errors", sendErrors); + m.put("last_error", lastError); + } + return m; + } + } + + // ====================== Bytes ====================== + + private static long now() { + return System.nanoTime() / 1_000_000; + } + + static String hex(byte[] b, int off, int len) { + StringBuilder sb = new StringBuilder(len * 3); + for (int i = 0; i < len; i++) { + if (i > 0) sb.append(' '); + sb.append(String.format(Locale.ROOT, "%02X", b[off + i] & 0xFF)); + } + return sb.toString(); + } + + static int u16(byte[] b, int off) { + return (b[off] & 0xFF) | (b[off + 1] & 0xFF) << 8; + } + + static long u32(byte[] b, int off) { + return (u16(b, off) | (long) u16(b, off + 2) << 16) & 0xFFFFFFFFL; + } + + static void le16(byte[] b, int off, int v) { + b[off] = (byte) v; + b[off + 1] = (byte) (v >> 8); + } + + static void le32(byte[] b, int off, int v) { + le16(b, off, v); + le16(b, off + 2, v >> 16); + } + + static void le64(byte[] b, int off, long v) { + le32(b, off, (int) v); + le32(b, off + 4, (int) (v >>> 32)); + } +} diff --git a/android/app/src/main/java/com/openipc/decoder/Jieli.java b/android/app/src/main/java/com/openipc/decoder/Jieli.java new file mode 100644 index 00000000..6c3df414 --- /dev/null +++ b/android/app/src/main/java/com/openipc/decoder/Jieli.java @@ -0,0 +1,1425 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * Jieli.java — Jieli toy drones ("HF_xxxxxxxx" Wi-Fi): CTP over TCP, RTP to UDP 6666 (#187) + * + */ + +package com.openipc.decoder; + +import android.util.Log; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +/** + * Toy drones with an "HF_xxxxxxxx" access point (Jieli AC79 camera, vendor app "DVRunning 2") + * at 192.168.80.1 (#187, #196). The app talks CTP to TCP port 3333: "CTP:", the topic length + * (u16 LE), the topic, the JSON length (u32 LE) and the JSON, which may be empty. The session + * as a PCAPdroid capture of the vendor app shows it: APP_ACCESS; the toy announces + * KEEP_ALIVE_INTERVAL, VIDEO_PARAM and more; DATE_TIME; CTP_KEEP_ALIVE every 7.5 s; + * OPEN_RT_STREAM; CLOSE_RT_STREAM at the end. The video itself was not in the capture: after + * turbodrone's research on the same SDK (marshallrichards/turbodrone, Apache-2.0, used as a + * description only) it is RTP/JPEG (payload type 26) or H.264 (96) to the phone's UDP port + * 6666. The packets go into the app's normal RTP paths. A YL3 sent nothing there but its own + * stream to UDP 2224 instead (JPEG behind a 20-byte header, see {@link RtStream}): both ports + * are played. When neither has video, the log shows what arrives on the audio port 1234. + * Video only: FLIGHT_CONTROL is never sent, nothing goes to the toy's UDP port 3456. + * Pure Java apart from the log. + */ +final class Jieli { + private static final String TAG = "OpenIPCDecoder"; + + static final String SCHEME = "jieli://"; + static final String DEFAULT_HOST = "192.168.80.1"; + static final int CTP_PORT = 3333; + static final int RTP_PORT = 6666; + /** The toy's own stream (#187): a YL3 sends its JPEG frames here, not as RTP to 6666. */ + static final int STREAM_PORT = 2224; + /** The last frame of any session, and the last dropped one, for the debug page. */ + static volatile byte[] lastFrameAnySession; + static volatile byte[] lastDroppedAnySession; + /** The port listened on for that stream; STREAM_PORT but for the tests. */ + static volatile int streamPort = STREAM_PORT; + static final int DEFAULT_WIDTH = 640; + static final int DEFAULT_HEIGHT = 480; + static final int DEFAULT_FPS = 25; + static final int FORMAT_JPEG = 0; + static final int FORMAT_H264 = 1; + /** The YL3's frames come turned (portrait 240×320); 270 showed them upside down (#187). */ + static final int DEFAULT_ROTATE = 90; + static final int PT_JPEG = 26; + static final int PT_H264 = 96; + + static final byte[] MAGIC = {'C', 'T', 'P', ':'}; + /** Longer topics or JSON than this are not CTP: the reader looks for the next "CTP:". */ + static final int MAX_TOPIC = 256; + static final int MAX_JSON = 256 * 1024; + /** Distinct CTP topics counted one by one for the stats; the rest together. */ + static final int MAX_TOPICS = 64; + /** How long close() lets the session send CLOSE_RT_STREAM before it closes the TCP. */ + static volatile long closeGraceMs = 1000; + /** Diagnostic ports listen this long, or for this many packets, then are freed. */ + static volatile long diagnosticMs = 30_000; + static final int DIAGNOSTIC_PACKETS = 20; + + static final String APP_ACCESS = "APP_ACCESS"; + static final String DATE_TIME = "DATE_TIME"; + static final String KEEP_ALIVE = "CTP_KEEP_ALIVE"; + static final String KEEP_ALIVE_INTERVAL = "KEEP_ALIVE_INTERVAL"; + static final String VIDEO_PARAM = "VIDEO_PARAM"; + static final String OPEN_RT_STREAM = "OPEN_RT_STREAM"; + static final String CLOSE_RT_STREAM = "CLOSE_RT_STREAM"; + + private static final int CONNECT_TIMEOUT_MS = 5000; + /** Read timeout of the CTP socket: how often the timers are looked at. */ + private static final int TICK_MS = 100; + /** The keepalive of the vendor app; a shorter announced timeout makes it timeout / 4. */ + static final long KEEP_ALIVE_MAX_MS = 7500; + private static final long KEEP_ALIVE_MIN_MS = 250; + // Not final so unit tests can shorten them + /** How long APP_ACCESS waits for VIDEO_PARAM before the stream is asked for anyway. */ + static volatile long paramWaitMs = 1500; + /** No video this long after OPEN_RT_STREAM: said in the log and status, diagnostics on. */ + static volatile long noVideoMs = 5000; + /** No video this long (after a request or the last packet): OPEN_RT_STREAM again. */ + static volatile long reopenMs = 10_000; + /** Video that stops for this long is "quiet". */ + static volatile long quietMs = 3000; + /** The toy says nothing on CTP for this long (at least): the connection is given up. */ + static volatile long ctpSilenceMs = 20_000; + /** + * Ports opened only to log what arrives when the video ports stay silent: 1234 (audio in + * the vendor SDK). Nothing received there is played. + */ + static volatile int[] diagnosticPorts = {1234}; + + private Jieli() {} + + // ====================== URL ====================== + + /** + * jieli://[host][:port][/...][?option=value&...]: the toy's CTP address, 192.168.80.1:3333 + * when left out. Options, as in udp:// URLs (unknown ones are ignored, bad values + * refused): w, h, fps (the stream asked for; default what the toy announces in + * VIDEO_PARAM, else 640x480 at 25), rtp (the local UDP port for the video, 6666), + * format (0 JPEG, the default, or 1 H.264), rotate (degrees counter-clockwise for JPEG, + * 0, 90, 180 or 270; default 90: the YL3's camera is mounted sideways). + */ + static final class Url { + String host = DEFAULT_HOST; + int port = CTP_PORT; + int rtpPort = RTP_PORT; + /** 0: from VIDEO_PARAM, else the default. */ + int width; + int height; + int fps; + int format = FORMAT_JPEG; + int rotate = DEFAULT_ROTATE; + + static boolean matches(String url) { + return url != null && url.trim().toLowerCase(Locale.ROOT).startsWith(SCHEME); + } + + static Url parse(String url) throws IOException { + if (!matches(url)) throw new IOException("Not a jieli:// URL"); + String rest = url.trim().substring(SCHEME.length()); + int end = rest.length(); + for (int i = 0; i < rest.length(); i++) { + char ch = rest.charAt(i); + if (ch == '/' || ch == '?' || ch == '#') { + end = i; + break; + } + } + String authority = rest.substring(0, end).trim(); + String query = ""; + int q = rest.indexOf('?'); + int hash = rest.indexOf('#'); + if (q >= 0 && (hash < 0 || q < hash)) { + query = rest.substring(q + 1, hash >= 0 ? hash : rest.length()); + } + Url u = new Url(); + String host = authority; + int colon = authority.lastIndexOf(':'); + if (authority.startsWith("[")) { + int close = authority.indexOf(']'); + if (close < 0) throw new IOException("Bad host in URL"); + host = authority.substring(1, close); + colon = authority.indexOf(':', close); + } else if (colon >= 0) { + host = authority.substring(0, colon); + } + if (colon >= 0) u.port = number(authority.substring(colon + 1), 1, 65535, "port"); + if (!host.isEmpty()) u.host = host; + for (int i = 0; i < u.host.length(); i++) { + char ch = u.host.charAt(i); + if (ch <= 0x20 || ch == 0x7F || ch == '@') throw new IOException("Bad host in URL"); + } + for (String param : query.split("&")) { + int eq = param.indexOf('='); + if (eq < 0) continue; + String name = param.substring(0, eq).trim().toLowerCase(Locale.ROOT); + String value = param.substring(eq + 1).trim(); + switch (name) { + case "w": + u.width = number(value, 16, 4096, "w"); + break; + case "h": + u.height = number(value, 16, 4096, "h"); + break; + case "fps": + u.fps = number(value, 1, 120, "fps"); + break; + case "rtp": + u.rtpPort = number(value, 1, 65535, "rtp"); + break; + case "format": + u.format = number(value, 0, 1, "format (0 JPEG, 1 H.264)"); + break; + case "rotate": + u.rotate = RtspSession.Url.parseRotate(value); + break; + default: + break; + } + } + return u; + } + + private static int number(String s, int min, int max, String what) throws IOException { + int v; + try { + v = Integer.parseInt(s.trim()); + } catch (NumberFormatException e) { + throw new IOException("Bad " + what + " in URL: " + s); + } + if (v < min || v > max) throw new IOException("Bad " + what + " in URL: " + s); + return v; + } + + String endpoint() { + return (host.indexOf(':') >= 0 ? "[" + host + "]" : host) + ":" + port; + } + } + + // ====================== CTP messages ====================== + + /** One CTP message: a topic and its JSON ("" when there is none). */ + static final class Message { + final String topic; + final String json; + + Message(String topic, String json) { + this.topic = topic; + this.json = json; + } + + @Override + public String toString() { + return json.isEmpty() ? topic : topic + " " + json; + } + } + + /** "CTP:" + topic length (u16 LE) + topic + JSON length (u32 LE) + JSON. */ + static byte[] encode(String topic, String json) { + byte[] t = topic.getBytes(StandardCharsets.UTF_8); + byte[] j = json != null ? json.getBytes(StandardCharsets.UTF_8) : new byte[0]; + byte[] out = new byte[MAGIC.length + 2 + t.length + 4 + j.length]; + System.arraycopy(MAGIC, 0, out, 0, MAGIC.length); + int pos = MAGIC.length; + out[pos] = (byte) t.length; + out[pos + 1] = (byte) (t.length >> 8); + pos += 2; + System.arraycopy(t, 0, out, pos, t.length); + pos += t.length; + for (int i = 0; i < 4; i++) out[pos + i] = (byte) (j.length >> (8 * i)); + pos += 4; + System.arraycopy(j, 0, out, pos, j.length); + return out; + } + + /** + * Cuts the TCP byte stream into messages: a message may arrive in pieces or several in + * one read. Bytes that are not a message (no "CTP:", a length out of range) are skipped + * up to the next "CTP:" and counted. + */ + static final class Reader { + private byte[] buf = new byte[4096]; + private int start; + private int end; + long skipped; + + void feed(byte[] b, int off, int len) { + if (end + len > buf.length) { + int have = end - start; + if (have + len > buf.length) { + byte[] n = new byte[Math.max(buf.length * 2, have + len)]; + System.arraycopy(buf, start, n, 0, have); + buf = n; + } else { + System.arraycopy(buf, start, buf, 0, have); + } + start = 0; + end = have; + } + System.arraycopy(b, off, buf, end, len); + end += len; + } + + /** The next whole message, or null until more bytes arrive. */ + Message next() { + while (true) { + int have = end - start; + if (have < MAGIC.length) return null; + if (!startsWithMagic(start)) { + skipToMagic(); + continue; + } + if (have < MAGIC.length + 2) return null; + int topicLen = (buf[start + 4] & 0xFF) | (buf[start + 5] & 0xFF) << 8; + if (topicLen > MAX_TOPIC) { + skipOne(); + continue; + } + int jsonAt = start + MAGIC.length + 2 + topicLen; + if (end < jsonAt + 4) return null; + long jsonLen = (buf[jsonAt] & 0xFFL) | (buf[jsonAt + 1] & 0xFFL) << 8 + | (buf[jsonAt + 2] & 0xFFL) << 16 | (buf[jsonAt + 3] & 0xFFL) << 24; + if (jsonLen > MAX_JSON) { + skipOne(); + continue; + } + int total = MAGIC.length + 2 + topicLen + 4 + (int) jsonLen; + if (have < total) return null; + String topic = new String(buf, start + 6, topicLen, StandardCharsets.UTF_8); + String json = new String(buf, jsonAt + 4, (int) jsonLen, StandardCharsets.UTF_8); + start += total; + if (start == end) start = end = 0; + return new Message(topic, json); + } + } + + private boolean startsWithMagic(int at) { + for (int i = 0; i < MAGIC.length; i++) { + if (buf[at + i] != MAGIC[i]) return false; + } + return true; + } + + /** Drops the first byte (a "CTP:" with bad lengths) and looks for the next one. */ + private void skipOne() { + start++; + skipped++; + skipToMagic(); + } + + private void skipToMagic() { + int i = start; + while (i + MAGIC.length <= end && !startsWithMagic(i)) i++; + if (i + MAGIC.length > end) { + // Keep a tail that may be the start of "CTP:" + i = Math.max(start, end - (MAGIC.length - 1)); + } + skipped += i - start; + start = i; + } + } + + // ====================== JSON ====================== + + /** {"op":"PUT"}, or with "param" from name/value pairs, in their order. */ + static String put(String... nameValues) { + StringBuilder sb = new StringBuilder("{\"op\":\"PUT\""); + if (nameValues.length > 0) { + sb.append(",\"param\":{"); + for (int i = 0; i + 1 < nameValues.length; i += 2) { + if (i > 0) sb.append(','); + quote(sb, nameValues[i]); + sb.append(':'); + quote(sb, nameValues[i + 1]); + } + sb.append('}'); + } + return sb.append('}').toString(); + } + + private static void quote(StringBuilder sb, String s) { + sb.append('"'); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '"' || c == '\\') { + sb.append('\\').append(c); + } else if (c < 0x20) { + sb.append(String.format(Locale.ROOT, "\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + sb.append('"'); + } + + /** + * A small JSON reader (org.json is a stub in unit tests): objects become maps, arrays + * lists, strings strings, and numbers, true, false and null their text. Null when the + * text is not one JSON object. + */ + static Map parseJson(String s) { + if (s == null) return null; + try { + Json j = new Json(s); + j.space(); + if (j.peek() != '{') return null; + @SuppressWarnings("unchecked") + Map m = (Map) j.value(0); + j.space(); + return j.pos == s.length() ? m : null; + } catch (RuntimeException e) { + return null; + } + } + + private static final class Json { + final String s; + int pos; + + Json(String s) { + this.s = s; + } + + char peek() { + if (pos >= s.length()) throw new IllegalArgumentException("end"); + return s.charAt(pos); + } + + void space() { + while (pos < s.length() && Character.isWhitespace(s.charAt(pos))) pos++; + } + + void expect(char c) { + space(); + if (peek() != c) throw new IllegalArgumentException("expected " + c); + pos++; + } + + Object value(int depth) { + if (depth > 16) throw new IllegalArgumentException("too deep"); + space(); + char c = peek(); + if (c == '{') { + pos++; + Map m = new LinkedHashMap<>(); + space(); + if (peek() == '}') { + pos++; + return m; + } + while (true) { + space(); + String key = string(); + expect(':'); + m.put(key, value(depth + 1)); + space(); + if (peek() == ',') { + pos++; + continue; + } + expect('}'); + return m; + } + } + if (c == '[') { + pos++; + List l = new ArrayList<>(); + space(); + if (peek() == ']') { + pos++; + return l; + } + while (true) { + l.add(value(depth + 1)); + space(); + if (peek() == ',') { + pos++; + continue; + } + expect(']'); + return l; + } + } + if (c == '"') return string(); + int from = pos; + while (pos < s.length() && ",}] \t\r\n".indexOf(s.charAt(pos)) < 0) pos++; + if (pos == from) throw new IllegalArgumentException("value"); + return s.substring(from, pos); + } + + String string() { + if (peek() != '"') throw new IllegalArgumentException("string"); + pos++; + StringBuilder sb = new StringBuilder(); + while (true) { + char c = peek(); + pos++; + if (c == '"') return sb.toString(); + if (c != '\\') { + sb.append(c); + continue; + } + char e = peek(); + pos++; + switch (e) { + case 'n': sb.append('\n'); break; + case 't': sb.append('\t'); break; + case 'r': sb.append('\r'); break; + case 'b': sb.append('\b'); break; + case 'f': sb.append('\f'); break; + case 'u': + sb.append((char) Integer.parseInt(s.substring(pos, pos + 4), 16)); + pos += 4; + break; + default: sb.append(e); break; + } + } + } + } + + /** {@code name} in the message's "param" object, else at the top, as text; or null. */ + static String param(Map json, String name) { + if (json == null) return null; + Object p = json.get("param"); + if (p instanceof Map) { + Object v = ((Map) p).get(name); + if (v instanceof String) return (String) v; + } + Object v = json.get(name); + return v instanceof String ? (String) v : null; + } + + /** The integer {@code name} of a message, or -1. */ + static int intParam(Map json, String name) { + String v = param(json, name); + if (v == null) return -1; + try { + return Integer.parseInt(v.trim()); + } catch (NumberFormatException e) { + return -1; + } + } + + /** min(timeout / 4, 7.5 s), not below 250 ms; 7.5 s when the toy announced none. */ + static long keepAliveMs(long timeoutMs) { + if (timeoutMs <= 0) return KEEP_ALIVE_MAX_MS; + return Math.max(KEEP_ALIVE_MIN_MS, Math.min(timeoutMs / 4, KEEP_ALIVE_MAX_MS)); + } + + /** The request for the stream, in the field order of the vendor app. */ + static String openStream(int format, int width, int height, int fps) { + return put("format", String.valueOf(format), "h", String.valueOf(height), + "w", String.valueOf(width), "fps", String.valueOf(fps)); + } + + static String truncate(String s, int max) { + return s.length() <= max ? s : s.substring(0, max) + "…"; + } + + // ====================== Subnet hint ====================== + + /** + * A hint when a jieli:// slot cannot be right for the phone's networks, or null: the + * target is the phone itself, or the phone is on another toy family's network. + */ + static String subnetHint(InetAddress target, List local) { + if (!(target instanceof Inet4Address) || local == null) return null; + boolean own = local.contains(target); + boolean on1 = false; + boolean on169 = false; + boolean on80 = false; + for (InetAddress a : local) { + on1 |= FlowUfo.in24(a, 1); + on169 |= FlowUfo.in24(a, 169); + on80 |= FlowUfo.in24(a, 80); + } + StringBuilder parts = new StringBuilder(); + byte[] b = target.getAddress(); + if (own) { + FlowUfo.sentence(parts, target.getHostAddress() + " is this phone; the toy is usually " + + (b[0] & 0xFF) + "." + (b[1] & 0xFF) + "." + (b[2] & 0xFF) + ".1"); + } + if (on80 && !FlowUfo.in24(target, 80)) { + FlowUfo.sentence(parts, "This phone is on 192.168.80.x: the toy is usually" + + " jieli://192.168.80.1"); + } + if (!on80 && on169) { + FlowUfo.sentence(parts, "This network looks like a FLOW-UFO toy: try" + + " flowufo://192.168.169.1"); + } + if (!on80 && on1) { + FlowUfo.sentence(parts, "This network looks like a KY UFO RTSP toy: try" + + " rtsp://192.168.1.1:7070/webcam"); + } + return parts.length() > 0 ? parts.toString() : null; + } + + // ====================== RT stream (UDP 2224) ====================== + + /** + * The toy's own video stream on UDP 2224 (#187), as a YL3 sent it: a 20-byte header, + * then a piece of a JPEG. Header, little-endian: byte 0 the type (2 for JPEG, bit 7 set + * on the last piece of a frame; not relied on), bytes 2–3 the length of this piece, 4–7 + * a packet counter, 8–11 the length of the JPEG, 12–15 the offset of this piece in it. The + * datagram can be longer than the piece: what follows is left over from earlier frames + * (taking it gave a broken lower half), so only the piece is used. Pure Java, for the + * unit tests. + */ + static final class RtStream { + static final int HEADER = 20; + static final int TYPE_JPEG = 2; + static final int LAST_PIECE = 0x80; + static final int MAX_FRAME = 1 << 20; + + private byte[] frame = new byte[64 * 1024]; + private int size = -1; + /** Bytes of the frame received so far: duplicates and overlaps count once. */ + private final BitSet got = new BitSet(); + private int filled; + int frames; + int dropped; + int otherTypes; + /** Why the last frame was dropped, and its bytes as put together (for the debug page). */ + volatile String lastDropReason; + volatile byte[] lastDropped; + + private void drop(String reason, int n) { + dropped++; + lastDropReason = reason; + if (n > 0) { + lastDropped = Arrays.copyOf(frame, n); + lastDroppedAnySession = lastDropped; + } + } + + /** + * Takes one datagram; returns the length of a JPEG now complete in {@link #frame()} + * (from 0, FFD8 to its last FFD9), or 0. + */ + int feed(byte[] b, int off, int len) { + if (len <= HEADER) return 0; + if ((b[off] & 0x7F) != TYPE_JPEG) { + otherTypes++; + return 0; + } + int total = le32(b, off + 8); + int at = le32(b, off + 12); + int length = (b[off + 2] & 0xFF) | (b[off + 3] & 0xFF) << 8; + if (length == 0 || length > len - HEADER) length = len - HEADER; + if (total < 4 || total > MAX_FRAME || at < 0 || at >= total) { + drop("bad header: length " + total + ", offset " + at, 0); + return 0; + } + int piece = Math.min(length, total - at); + // Pieces may come twice or out of order (Qodo on #215): a piece belongs to the frame + // being put together if the length matches and it brings bytes not yet received; a + // repeat is ignored. Anything else starts the next frame, and only then is an + // unfinished one given up, even if its last piece came already. + if (size == total && got.get(at, at + piece).cardinality() == piece) { + if (same(b, off + HEADER, at, piece)) return 0; + startFrame(total); + } else if (size != total) { + startFrame(total); + } + System.arraycopy(b, off + HEADER, frame, at, piece); + filled += piece - got.get(at, at + piece).cardinality(); + got.set(at, at + piece); + if (filled < size) return 0; + int n = size; + size = -1; + got.clear(); + int end = jpegEnd(frame, n); + if (end == 0) { + drop(n + " bytes, starts " + FlowUfo.hex(frame, 0, Math.min(n, 4)) + ", ends " + + FlowUfo.hex(frame, Math.max(0, n - 8), Math.min(n, 8)) + + ": no FFD8 at the start or no FFD9", n); + return 0; + } + frames++; + return end; + } + + byte[] frame() { + return frame; + } + + private void startFrame(int total) { + if (size > 0) drop("incomplete: " + filled + " of " + size + " bytes", size); + size = total; + filled = 0; + got.clear(); + if (frame.length < total) frame = new byte[total]; + } + + private boolean same(byte[] b, int from, int at, int n) { + for (int i = 0; i < n; i++) { + if (frame[at + i] != b[from + i]) return false; + } + return true; + } + + /** Length up to the last FFD9 when {@code b} starts with FFD8, else 0. */ + static int jpegEnd(byte[] b, int n) { + if (n < 4 || (b[0] & 0xFF) != 0xFF || (b[1] & 0xFF) != 0xD8) return 0; + for (int end = n; end >= 4; end--) { + if ((b[end - 2] & 0xFF) == 0xFF && (b[end - 1] & 0xFF) == 0xD9) return end; + } + return 0; + } + + static int le32(byte[] b, int at) { + return (b[at] & 0xFF) | (b[at + 1] & 0xFF) << 8 | (b[at + 2] & 0xFF) << 16 + | (b[at + 3] & 0xFF) << 24; + } + } + + // ====================== Client ====================== + + interface Listener { + /** Waiting (no video) or receiving; text for the status. */ + void onState(boolean receiving, String text); + + /** + * One RTP packet with payload type 26 (JPEG) or 96 (H.264), valid during the call. + * Returns whether it was usable: only usable video keeps the toy "receiving" (Qodo on + * #198), so a toy sending broken packets is still asked for the stream again. + */ + boolean onRtp(byte[] buf, int offset, int length); + + /** One whole JPEG without RTP, valid during the call; returns whether it was shown. */ + default boolean onJpeg(byte[] buf, int offset, int length) { return false; } + } + + /** Binds the CTP socket to the toy's Wi-Fi before it connects; returns what was done. */ + interface TcpSetup { + String prepare(Socket socket, InetAddress target); + } + + /** + * One CTP session on the calling thread (connect, the commands, keepalives, the timers), + * and the video socket on a thread of its own, until {@code keepGoing} fails or close(). + * The stream is closed with CLOSE_RT_STREAM on the way out. A lost connection ends run() + * with an exception: the session starts again. + */ + static final class Client implements Closeable { + private final String tag; + private final Url url; + private final TcpSetup tcpSetup; + private final FlowUfo.SocketSetup udpSetup; + private volatile boolean closed; + /** The phone's addresses for the subnet hint; null: the real ones. For tests. */ + List localForHint; + + private final Object lock = new Object(); + private Socket tcp; + private boolean tcpConnected; + private final List udpSockets = new ArrayList<>(); + private volatile InetAddress address; + + // For the stats + private volatile String state = "connecting"; + private volatile String hint; + private volatile String networkTcp = "default"; + private volatile String networkUdp = "default"; + private volatile int ctpSent; + private volatile int ctpReceived; + private volatile int keepAlives; + private volatile int keepAliveReplies; + private volatile long keepAliveIntervalMs = KEEP_ALIVE_MAX_MS; + private volatile long announcedTimeoutMs = -1; + private volatile int openRequests; + private volatile String lastError; + private volatile String videoParam; + private volatile String requested; + private final ArrayDeque lastNotifies = new ArrayDeque<>(); + private final Map topics = new LinkedHashMap<>(); + /** Messages with a topic beyond the first MAX_TOPICS distinct ones. */ + private int otherTopics; + /** OPEN_RT_STREAM was sent: the session got as far as a PLAY would (Qodo on #198). */ + volatile boolean streamRequested; + /** Counted down when run() has cleaned up. */ + private final CountDownLatch ended = new CountDownLatch(1); + private final Reader reader = new Reader(); + private int ctpLogged; + + // Video socket; written by its thread + private volatile long lastVideoAt; + private volatile int packets; + private volatile long bytes; + private volatile int rtpPackets; + private volatile int rtpFrames; + private volatile int otherRtp; + private volatile int nonRtp; + private volatile int rawJpegs; + private volatile int foreign; + private volatile String firstPacket; + /** UDP port the video came from last: url.rtpPort or streamPort. */ + private volatile int videoPort; + private final RtStream rtStream = new RtStream(); + private volatile int streamPackets; + private volatile int streamForeign; + private volatile String streamFirst; + private int droppedLogged; + /** The last JPEG put together from UDP 2224, as received (for the debug page). */ + private volatile byte[] lastStreamFrame; + private volatile String lastSource; + private volatile boolean h264Logged; + private final Map payloadTypes = new LinkedHashMap<>(); + private final Map sources = new LinkedHashMap<>(); + private final Map diagnostic = new LinkedHashMap<>(); + private final Map diagnosticFirst = new LinkedHashMap<>(); + private int logged; + + Client(String tag, Url url, TcpSetup tcpSetup, FlowUfo.SocketSetup udpSetup) { + this.tag = tag; + this.url = url; + this.tcpSetup = tcpSetup; + this.udpSetup = udpSetup; + } + + /** + * Ends run() from any thread. The video sockets close at once; a connected CTP socket + * is left to the session thread, which sends CLOSE_RT_STREAM first (within TICK_MS). + */ + @Override + public void close() { + closed = true; + Socket connected = null; + synchronized (lock) { + for (DatagramSocket s : udpSockets) s.close(); + if (tcp != null && !tcpConnected) { + try { + tcp.close(); + } catch (IOException ignored) {} + } else if (tcp != null) { + connected = tcp; + } + } + if (connected == null) return; + // The session sends CLOSE_RT_STREAM and closes the TCP itself; a write blocked on a + // toy that stopped reading would keep it from that, so after a grace period the + // socket is closed from here (Qodo on #188) + final Socket s = connected; + Thread t = new Thread(() -> { + try { + if (!ended.await(closeGraceMs, java.util.concurrent.TimeUnit.MILLISECONDS)) { + s.close(); + } + } catch (InterruptedException | IOException ignored) {} + }, tag + "-jieli-close"); + t.setDaemon(true); + t.start(); + } + + private String withHint(String text) { + String h = hint; + return h != null ? text + ". " + h : text; + } + + /** Milliseconds since the last video packet from the toy, or Long.MAX_VALUE. */ + long videoAgeMs() { + long last = lastVideoAt; + return last > 0 ? now() - last : Long.MAX_VALUE; + } + + String connectingText() { + return withHint("Connecting to the toy (CTP " + url.endpoint() + ")"); + } + + String requestedText() { + return withHint("Stream requested, waiting for video on UDP " + videoPorts()); + } + + String noVideoText() { + return withHint("No video on UDP " + videoPorts() + " yet: the toy accepted the" + + " request; see /log"); + } + + String quietText() { + return withHint("The toy's video stopped: waiting on UDP " + videoPorts()); + } + + private String videoPorts() { + return url.rtpPort == streamPort ? String.valueOf(streamPort) + : url.rtpPort + " and " + streamPort; + } + + void run(UdpReceiver.Condition keepGoing, Listener listener) throws IOException { + InetAddress a = InetAddress.getByName(url.host); + address = a; + hint = subnetHint(a, localForHint != null ? localForHint : FlowUfo.localAddresses()); + if (hint != null) Log.w(TAG, tag + ": Jieli: " + hint); + OutputStream out = null; + boolean requestedStream = false; + Socket s = new Socket(); + try { + // The video port first: nothing the toy sends after OPEN_RT_STREAM is lost + DatagramSocket video = openUdp(url.rtpPort, a); + startThread(video, PORT_RTP, listener); + if (url.rtpPort != streamPort) { + try { + startThread(openUdp(streamPort, a), PORT_STREAM, listener); + } catch (IOException e) { + Log.w(TAG, tag + ": Jieli: cannot listen on UDP " + streamPort + ": " + + e.getMessage()); + } + } + state = "connecting"; + listener.onState(false, connectingText()); + if (tcpSetup != null) networkTcp = tcpSetup.prepare(s, a); + synchronized (lock) { + tcp = s; + if (closed) return; + } + Log.i(TAG, tag + ": Jieli: connecting to CTP " + url.endpoint() + " (network: " + + networkTcp + "), video on UDP " + videoPorts() + " (network: " + + networkUdp + ")"); + try { + s.connect(new InetSocketAddress(a, url.port), CONNECT_TIMEOUT_MS); + } catch (IOException e) { + if (closed) return; + throw new IOException("Cannot reach the toy's CTP port " + url.endpoint() + + ": " + e.getMessage()); + } + synchronized (lock) { + tcpConnected = true; + if (closed) return; + } + s.setTcpNoDelay(true); + s.setSoTimeout(TICK_MS); + out = s.getOutputStream(); + InputStream in = s.getInputStream(); + state = "ctp"; + Log.i(TAG, tag + ": Jieli: CTP connected to " + url.endpoint()); + send(out, APP_ACCESS, put("ver", "25", "type", "0")); + long t0 = now(); + long lastRx = t0; + // From the last one sent: an interval announced later applies at once (Qodo + // on #198) + long lastKeepAlive = 0; + long openAt = 0; + boolean diagnostics = false; + boolean receiving = false; + int reopens = 0; + byte[] buf = new byte[8192]; + while (!closed && keepGoing.holds()) { + long t = now(); + if (!requestedStream && (videoParam != null || t - t0 >= paramWaitMs)) { + send(out, DATE_TIME, put("date", new SimpleDateFormat("yyyyMMddHHmmss", + Locale.ROOT).format(new Date()))); + keepAlive(out); + lastKeepAlive = t; + requestStream(out); + requestedStream = true; + openAt = t; + state = "requested"; + listener.onState(false, requestedText()); + } + if (requestedStream && t - lastKeepAlive >= keepAliveIntervalMs) { + keepAlive(out); + lastKeepAlive = t; + } + if (requestedStream) { + long last = lastVideoAt; + boolean flowing = last > 0 && t - last < quietMs; + if (flowing && !receiving) { + receiving = true; + state = "playing"; + Log.i(TAG, tag + ": Jieli: video arrives on UDP " + videoPort + + " from " + lastSource); + listener.onState(true, "Receiving from the toy on UDP " + + videoPort + "…"); + } else if (!flowing && receiving) { + receiving = false; + state = "quiet"; + Log.w(TAG, tag + ": Jieli: no video for " + quietMs + " ms (" + + summary() + ")"); + listener.onState(false, quietText()); + } + if (!diagnostics && last == 0 && t - openAt >= noVideoMs) { + diagnostics = true; + Log.w(TAG, tag + ": Jieli: no video on UDP " + videoPorts() + " " + + noVideoMs + " ms after OPEN_RT_STREAM (" + summary() + + "); listening on " + ports(diagnosticPorts) + + " too, for the log only"); + for (int p : diagnosticPorts) { + if (p == url.rtpPort || p == streamPort) continue; + try { + startThread(openUdp(p, a), PORT_DIAGNOSTIC, listener); + } catch (IOException e) { + Log.w(TAG, tag + ": Jieli: cannot listen on UDP " + p + ": " + + e.getMessage()); + } + } + listener.onState(false, noVideoText()); + } + if (!receiving && t - Math.max(openAt, last) >= reopenMs) { + reopens++; + if (reopens <= 3 || reopens % 10 == 0) { + Log.w(TAG, tag + ": Jieli: still no video, OPEN_RT_STREAM again (" + + reopens + ")"); + } + requestStream(out); + openAt = t; + } + } + if (t - lastRx > Math.max(ctpSilenceMs, announcedTimeoutMs)) { + throw new IOException("The toy stopped answering on CTP " + + url.endpoint()); + } + int n; + try { + n = in.read(buf); + } catch (SocketTimeoutException e) { + continue; + } + if (n < 0) throw new IOException("The toy closed the CTP connection"); + lastRx = now(); + reader.feed(buf, 0, n); + Message m; + while ((m = reader.next()) != null) handle(m); + } + } catch (IOException e) { + lastError = e.getMessage(); + if (closed) return; + throw e; + } finally { + if (out != null && requestedStream && !s.isClosed()) { + try { + send(out, CLOSE_RT_STREAM, put("status", "1")); + } catch (IOException ignored) {} + } + try { + s.close(); + } catch (IOException ignored) {} + synchronized (lock) { + for (DatagramSocket d : udpSockets) d.close(); + udpSockets.clear(); + tcp = null; + tcpConnected = false; + } + Log.i(TAG, tag + ": Jieli: session ended (" + summary() + ")"); + ended.countDown(); + } + } + + private static String ports(int[] p) { + StringBuilder sb = new StringBuilder("UDP"); + for (int i = 0; i < p.length; i++) sb.append(i == 0 ? " " : ", ").append(p[i]); + return sb.toString(); + } + + private DatagramSocket openUdp(int port, InetAddress target) throws IOException { + DatagramSocket d = new DatagramSocket(null); + try { + d.bind(new InetSocketAddress(port)); + } catch (IOException e) { + d.close(); + throw new IOException("UDP port " + port + " is in use (another jieli:// slot" + + " or app?): " + e.getMessage()); + } + try { + d.setReceiveBufferSize(1024 * 1024); + } catch (IOException ignored) {} + String n = udpSetup != null ? udpSetup.prepare(d, target) : "default"; + if (port == url.rtpPort) networkUdp = n; + synchronized (lock) { + udpSockets.add(d); + if (closed) d.close(); + } + return d; + } + + private static final int PORT_DIAGNOSTIC = 0; + private static final int PORT_RTP = 1; + private static final int PORT_STREAM = 2; + + private void startThread(DatagramSocket d, int kind, Listener listener) { + int port = d.getLocalPort(); + Thread th = new Thread(() -> { + byte[] b = new byte[65536]; + DatagramPacket p = new DatagramPacket(b, b.length); + // A diagnostic port is only borrowed: freed after a while or a few packets, so + // another slot can use it (Qodo on #198) + long until = kind == PORT_DIAGNOSTIC ? now() + diagnosticMs : Long.MAX_VALUE; + int seen = 0; + try { + if (kind == PORT_DIAGNOSTIC) d.setSoTimeout(TICK_MS * 10); + while (!closed && !d.isClosed() && now() < until) { + p.setLength(b.length); + try { + d.receive(p); + } catch (SocketTimeoutException e) { + continue; + } + if (kind == PORT_RTP) { + video(p, listener); + } else if (kind == PORT_STREAM) { + stream(p, listener); + } else { + diagnostic(port, p); + if (++seen >= DIAGNOSTIC_PACKETS) break; + } + } + } catch (IOException e) { + // Closed + } + if (kind == PORT_DIAGNOSTIC) { + synchronized (lock) { + udpSockets.remove(d); + } + d.close(); + } + }, tag + "-jieli-" + port); + th.setDaemon(true); + th.start(); + } + + private void send(OutputStream out, String topic, String json) throws IOException { + out.write(encode(topic, json)); + out.flush(); + ctpSent++; + if (!KEEP_ALIVE.equals(topic) || keepAlives < 2) logCtp("→", topic, json); + } + + private void keepAlive(OutputStream out) throws IOException { + send(out, KEEP_ALIVE, put()); + keepAlives++; + } + + private void requestStream(OutputStream out) throws IOException { + streamRequested = true; + int[] p = videoParams(); + int w = url.width > 0 ? url.width : p[0] > 0 ? p[0] : DEFAULT_WIDTH; + int h = url.height > 0 ? url.height : p[1] > 0 ? p[1] : DEFAULT_HEIGHT; + int fps = url.fps > 0 ? url.fps : p[2] > 0 ? p[2] : DEFAULT_FPS; + requested = w + "x" + h + " at " + fps + " fps, format " + url.format + + (url.format == FORMAT_H264 ? " (H.264)" : " (JPEG)"); + send(out, OPEN_RT_STREAM, openStream(url.format, w, h, fps)); + openRequests++; + } + + /** Width, height and fps of VIDEO_PARAM, -1 each when not received. */ + private int[] videoParams() { + Map j = parseJson(videoParamJson); + return new int[] {intParam(j, "w"), intParam(j, "h"), intParam(j, "fps")}; + } + + private volatile String videoParamJson; + + private void logCtp(String dir, String topic, String json) { + int n = ++ctpLogged; + if (n <= 60 || n % 100 == 0) { + Log.i(TAG, tag + ": Jieli: CTP " + dir + " " + topic + + (json.isEmpty() ? "" : " " + truncate(json, 200))); + } + } + + private void handle(Message m) { + ctpReceived++; + synchronized (topics) { + Integer c = topics.get(m.topic); + // Bounded: a toy inventing topic names cannot grow it (Qodo on #198) + if (c != null || topics.size() < MAX_TOPICS) { + topics.put(m.topic, c == null ? 1 : c + 1); + } else { + otherTopics++; + } + if (!KEEP_ALIVE.equals(m.topic)) { + lastNotifies.addLast(truncate(m.toString(), 200)); + while (lastNotifies.size() > 8) lastNotifies.removeFirst(); + } + } + if (KEEP_ALIVE.equals(m.topic)) { + keepAliveReplies++; + if (keepAliveReplies <= 2) logCtp("←", m.topic, m.json); + return; + } + logCtp("←", m.topic, m.json); + Map j = parseJson(m.json); + if (KEEP_ALIVE_INTERVAL.equals(m.topic)) { + int timeout = intParam(j, "timeout"); + if (timeout > 0) { + announcedTimeoutMs = timeout; + keepAliveIntervalMs = keepAliveMs(timeout); + Log.i(TAG, tag + ": Jieli: keepalive every " + keepAliveIntervalMs + + " ms (timeout " + timeout + " ms)"); + } + } else if (VIDEO_PARAM.equals(m.topic)) { + int w = intParam(j, "w"); + int h = intParam(j, "h"); + if (w > 0 && h > 0) { + videoParamJson = m.json; + videoParam = w + "x" + h + " at " + intParam(j, "fps") + " fps, format " + + intParam(j, "format"); + } + } + } + + private void video(DatagramPacket p, Listener listener) { + byte[] b = p.getData(); + int len = p.getLength(); + packets++; + bytes += len; + String from = p.getAddress().getHostAddress() + ":" + p.getPort(); + lastSource = from; + synchronized (sources) { + Integer c = sources.get(from); + if (c != null || sources.size() < 8) { + sources.put(from, c == null ? 1 : c + 1); + } else { + Integer o = sources.get("other"); + sources.put("other", o == null ? 1 : o + 1); + } + } + boolean rtp = len >= 12 && (b[0] & 0xC0) == 0x80; + int pt = rtp ? b[1] & 0x7F : -1; + boolean toy = p.getAddress().equals(address); + if (firstPacket == null) { + firstPacket = from + ", " + len + " bytes: " + FlowUfo.hex(b, 0, Math.min(len, 32)); + } + int n = ++logged; + if (n <= 8 || n % 1000 == 0) { + Log.i(TAG, tag + ": Jieli: UDP " + url.rtpPort + " packet " + n + " from " + from + + (toy ? "" : " (not the toy: ignored)") + ", " + len + " bytes, " + + (rtp ? "RTP PT " + pt + " seq " + ((b[2] & 0xFF) << 8 | (b[3] & 0xFF)) + + ((b[1] & 0x80) != 0 ? " marker" : "") : "not RTP") + + ": " + FlowUfo.hex(b, 0, Math.min(len, 32))); + } + if (!toy) { + foreign++; + return; + } + if (rtp) { + synchronized (payloadTypes) { + Integer c = payloadTypes.get(pt); + payloadTypes.put(pt, c == null ? 1 : c + 1); + } + } + if (rtp && (pt == PT_JPEG || pt == PT_H264)) { + rtpPackets++; + if ((b[1] & 0x80) != 0) rtpFrames++; + if (pt == PT_H264 && !h264Logged) { + h264Logged = true; + Log.i(TAG, tag + ": Jieli: RTP payload type 96: played as H.264"); + } + if (listener.onRtp(b, 0, len)) { + videoPort = url.rtpPort; + lastVideoAt = now(); + } + } else if (!rtp && len >= 4 && (b[0] & 0xFF) == 0xFF && (b[1] & 0xFF) == 0xD8 + && (b[len - 2] & 0xFF) == 0xFF && (b[len - 1] & 0xFF) == 0xD9) { + // A whole JPEG without RTP: turbodrone takes those too + rawJpegs++; + if (listener.onJpeg(b, 0, len)) { + videoPort = url.rtpPort; + lastVideoAt = now(); + } + } else if (rtp) { + if (++otherRtp <= 3) { + Log.w(TAG, tag + ": Jieli: RTP payload type " + pt + " on UDP " + + url.rtpPort + ": not video we know, ignored"); + } + } else { + nonRtp++; + } + } + + /** One datagram of the toy's own stream on UDP 2224 (see {@link RtStream}). */ + private void stream(DatagramPacket p, Listener listener) { + byte[] b = p.getData(); + int len = p.getLength(); + int n = ++streamPackets; + String from = p.getAddress().getHostAddress() + ":" + p.getPort(); + boolean toy = p.getAddress().equals(address); + if (n <= 5 || n % 1000 == 0) { + // The whole header and the end: the format is only partly known (#187) + String head = FlowUfo.hex(b, 0, Math.min(len, RtStream.HEADER + 4)); + String tail = FlowUfo.hex(b, Math.max(0, len - 8), Math.min(len, 8)); + Log.i(TAG, tag + ": Jieli: UDP " + streamPort + " packet " + n + " from " + from + + (toy ? "" : " (not the toy: ignored)") + ", " + len + " bytes: " + head + + " … " + tail + (len > RtStream.HEADER + 12 ? " (type " + (b[0] & 0xFF) + + ", counter " + RtStream.le32(b, 4) + ", length " + RtStream.le32(b, 8) + + ", offset " + RtStream.le32(b, 12) + ")" : "")); + } + if (streamFirst == null) streamFirst = from + ", " + len + " bytes: " + + FlowUfo.hex(b, 0, Math.min(len, 32)); + if (!toy) { + streamForeign++; + return; + } + int jpeg = rtStream.feed(b, 0, len); + if (jpeg > 0) { + lastStreamFrame = Arrays.copyOf(rtStream.frame(), jpeg); + lastFrameAnySession = lastStreamFrame; + lastSource = from; + if (listener.onJpeg(rtStream.frame(), 0, jpeg)) { + videoPort = streamPort; + lastVideoAt = now(); + } + } else if (rtStream.dropped != droppedLogged && rtStream.dropped <= 3) { + droppedLogged = rtStream.dropped; + Log.w(TAG, tag + ": Jieli: UDP " + streamPort + ": frame " + droppedLogged + + " not put together: " + rtStream.lastDropReason); + } + } + + private void diagnostic(int port, DatagramPacket p) { + int count; + synchronized (diagnostic) { + Integer c = diagnostic.get(port); + count = c == null ? 1 : c + 1; + diagnostic.put(port, count); + if (count == 1) { + diagnosticFirst.put(port, p.getAddress().getHostAddress() + ":" + p.getPort() + + ", " + p.getLength() + " bytes: " + + FlowUfo.hex(p.getData(), 0, Math.min(p.getLength(), 32))); + } + } + if (count <= 5 || count % 1000 == 0) { + byte[] b = p.getData(); + int len = p.getLength(); + boolean rtp = len >= 12 && (b[0] & 0xC0) == 0x80; + Log.i(TAG, tag + ": Jieli: diagnostic UDP " + port + " packet " + count + " from " + + p.getAddress().getHostAddress() + ":" + p.getPort() + ", " + len + + " bytes" + (rtp ? ", RTP PT " + (b[1] & 0x7F) : "") + ": " + + FlowUfo.hex(b, 0, Math.min(len, 32)) + " (not played)"); + } + } + + private String summary() { + return String.format(Locale.ROOT, "CTP %d sent, %d received, %d keepalives;" + + " UDP %d: %d packets, %d RTP video, %d frames, %d not RTP," + + " %d from others; UDP %d: %d packets, %d frames, %d dropped", + ctpSent, ctpReceived, keepAlives, url.rtpPort, packets, rtpPackets, + rtpFrames, nonRtp, foreign, streamPort, streamPackets, rtStream.frames, + rtStream.dropped); + } + + /** + * For the debug page (#187): "frame" the last JPEG put together from UDP 2224, "dropped" + * the last one dropped, as the bytes came; null when there is none. + */ + byte[] raw(String which) { + return "frame".equals(which) ? lastStreamFrame + : "dropped".equals(which) ? rtStream.lastDropped : null; + } + + /** + * As {@link #raw} after the session ended too: opening the browser on the same phone + * pauses the app and ends the session (#187), the frames must outlive it. + */ + static byte[] lastRaw(String which) { + return "frame".equals(which) ? lastFrameAnySession + : "dropped".equals(which) ? lastDroppedAnySession : null; + } + + /** For /stats.json. */ + Map stats() { + Map m = new LinkedHashMap<>(); + m.put("state", state); + m.put("target", url.endpoint()); + m.put("network", networkTcp); + if (hint != null) m.put("hint", hint); + Map ctp = new LinkedHashMap<>(); + ctp.put("sent", ctpSent); + ctp.put("received", ctpReceived); + ctp.put("keepalives", keepAlives); + ctp.put("keepalive_replies", keepAliveReplies); + ctp.put("keepalive_ms", keepAliveIntervalMs); + if (announcedTimeoutMs > 0) ctp.put("announced_timeout_ms", announcedTimeoutMs); + ctp.put("open_requests", openRequests); + if (reader.skipped > 0) ctp.put("skipped_bytes", reader.skipped); + synchronized (topics) { + ctp.put("topics", new LinkedHashMap<>(topics)); + if (otherTopics > 0) ctp.put("other_topics", otherTopics); + ctp.put("last_notifies", new ArrayList<>(lastNotifies)); + } + m.put("ctp", ctp); + if (videoParam != null) m.put("video_param", videoParam); + if (requested != null) m.put("requested", requested); + Map rtp = new LinkedHashMap<>(); + rtp.put("port", url.rtpPort); + rtp.put("network", networkUdp); + rtp.put("packets", packets); + rtp.put("bytes", bytes); + rtp.put("video_packets", rtpPackets); + rtp.put("frames", rtpFrames); + if (rawJpegs > 0) rtp.put("raw_jpegs", rawJpegs); + rtp.put("not_rtp", nonRtp); + if (otherRtp > 0) rtp.put("other_rtp", otherRtp); + if (foreign > 0) rtp.put("from_others", foreign); + synchronized (payloadTypes) { + Map pts = new LinkedHashMap<>(); + for (Map.Entry e : payloadTypes.entrySet()) { + pts.put(String.valueOf(e.getKey()), e.getValue()); + } + rtp.put("payload_types", pts); + } + if (firstPacket != null) rtp.put("first_packet", firstPacket); + m.put("rtp", rtp); + Map stream = new LinkedHashMap<>(); + stream.put("port", streamPort); + stream.put("packets", streamPackets); + stream.put("frames", rtStream.frames); + stream.put("dropped", rtStream.dropped); + if (rtStream.otherTypes > 0) stream.put("other_types", rtStream.otherTypes); + if (rtStream.lastDropReason != null) { + stream.put("last_drop", rtStream.lastDropReason); + } + if (streamForeign > 0) stream.put("from_others", streamForeign); + if (streamFirst != null) stream.put("first_packet", streamFirst); + m.put("stream", stream); + if (videoPort > 0) m.put("video_port", videoPort); + synchronized (sources) { + m.put("sources", new LinkedHashMap<>(sources)); + } + synchronized (diagnostic) { + if (!diagnostic.isEmpty()) { + Map d = new LinkedHashMap<>(); + for (Map.Entry e : diagnostic.entrySet()) { + Map one = new LinkedHashMap<>(); + one.put("packets", e.getValue()); + one.put("first", diagnosticFirst.get(e.getKey())); + d.put(String.valueOf(e.getKey()), one); + } + m.put("diagnostic_ports", d); + } + } + if (lastError != null) m.put("last_error", lastError); + return m; + } + } + + private static long now() { + return System.nanoTime() / 1_000_000; + } +} diff --git a/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java b/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java index 06ebfbcf..a46d6fb7 100644 --- a/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java +++ b/android/app/src/main/java/com/openipc/decoder/JpegAssembler.java @@ -247,7 +247,15 @@ static int[] size(byte[] d, int length) { /** Quantization tables for Q 1..99 (RFC 2435 Appendix A), luminance then chrominance. */ static byte[] tables(int q) { - q = Math.max(1, Math.min(99, q)); + return ijgTables(Math.max(1, Math.min(99, q))); + } + + /** + * The standard tables scaled to an IJG quality 1..100 as libjpeg does it (100: all ones), + * zig-zag order, luminance then chrominance (#187). + */ + static byte[] ijgTables(int q) { + q = Math.max(1, Math.min(100, q)); int factor = q < 50 ? 5000 / q : 200 - q * 2; byte[] out = new byte[128]; for (int i = 0; i < 64; i++) { @@ -274,6 +282,16 @@ static byte[] headers(int type, int width, int height, byte[] tables, int restar static byte[] headers(int type, int width, int height, byte[] tables, int precision, int restartInterval) { + return headersSampled((type & 63) == 0 ? 0x21 : 0x22, width, height, tables, precision, + restartInterval); + } + + /** + * As headers(), with the luma sampling given ({@code 0x11} for 4:4:4, #187); chroma is + * always 1x1. + */ + static byte[] headersSampled(int lumaSampling, int width, int height, byte[] tables, + int precision, int restartInterval) { java.io.ByteArrayOutputStream o = new java.io.ByteArrayOutputStream(700); o.write(0xFF); o.write(0xD8); @@ -297,7 +315,7 @@ static byte[] headers(int type, int width, int height, byte[] tables, int precis o.write(3); int chromaTable = count > 1 ? 1 : 0; o.write(0); - o.write((type & 63) == 0 ? 0x21 : 0x22); + o.write(lumaSampling); o.write(0); o.write(1); o.write(0x11); diff --git a/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java b/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java index 999c02c0..0149da83 100644 --- a/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java +++ b/android/app/src/main/java/com/openipc/decoder/JpegRenderer.java @@ -13,6 +13,7 @@ import android.graphics.Color; import android.graphics.Paint; import android.graphics.Rect; +import android.graphics.RectF; import android.util.Log; import android.view.Surface; @@ -41,13 +42,16 @@ final class JpegRenderer { private Bitmap bitmap; /** - * Shows one frame on {@code s}. Returns its {width, height} when it is on screen, null when - * not (yet). {@code current} is false once the stream or the surface has been replaced. + * Shows one frame on {@code s}, turned {@code rotate} degrees counter-clockwise (0, 90, + * 180 or 270, #200). Returns its {width, height} as shown (turned) when it is on screen, + * null when not (yet). {@code current} is false once the stream or the surface has been + * replaced. */ - int[] render(Frame frame, Surface s, JpegCodec.Current current) { + int[] render(Frame frame, Surface s, int rotate, JpegCodec.Current current) { if (s == null || !s.isValid() || !current.holds()) return null; - // The hardware JPEG decoder first, where there is one (#132) - if (!hwFailed && !drawn && JpegCodec.candidate() != null) { + // The hardware JPEG decoder first, where there is one (#132); a turned frame is drawn + // by the app, which turns it while drawing (#200) + if (rotate == 0 && !hwFailed && !drawn && JpegCodec.candidate() != null) { int[] size = JpegAssembler.size(frame.data(), frame.length()); if (size != null) { try { @@ -83,7 +87,16 @@ int[] render(Frame frame, Surface s, JpegCodec.Current current) { Canvas c = s.lockCanvas(null); try { c.drawColor(Color.BLACK); - c.drawBitmap(b, null, new Rect(0, 0, c.getWidth(), c.getHeight()), paint); + if (rotate == 0) { + c.drawBitmap(b, null, new Rect(0, 0, c.getWidth(), c.getHeight()), paint); + } else { + float[] t = turn(rotate, c.getWidth(), c.getHeight()); + c.save(); + c.translate(t[0], t[1]); + c.rotate(t[2]); + c.drawBitmap(b, null, new RectF(t[3], t[4], t[5], t[6]), paint); + c.restore(); + } } finally { s.unlockCanvasAndPost(c); } @@ -92,7 +105,40 @@ int[] render(Frame frame, Surface s, JpegCodec.Current current) { return null; } drawn = true; - return new int[] {b.getWidth(), b.getHeight()}; + return shownSize(b.getWidth(), b.getHeight(), rotate); + } + + /** {width, height} of a {@code w}×{@code h} frame turned {@code rotate} degrees (#200). */ + static int[] shownSize(int w, int h, int rotate) { + return rotate == 90 || rotate == 270 ? new int[] {h, w} : new int[] {w, h}; + } + + /** + * How a frame turned {@code rotate} degrees counter-clockwise fills a {@code cw}×{@code ch} + * canvas (#200): translate by {t[0], t[1]} (the centre), rotate by t[2] degrees (Canvas + * turns clockwise, so negative), then draw the frame into the rectangle {t[3], t[4], t[5], + * t[6]} (left, top, right, bottom), which is the canvas with width and height swapped for + * a quarter turn. + */ + static float[] turn(int rotate, int cw, int ch) { + boolean quarter = rotate == 90 || rotate == 270; + float hw = (quarter ? ch : cw) / 2f; + float hh = (quarter ? cw : ch) / 2f; + return new float[] {cw / 2f, ch / 2f, -rotate, -hw, -hh, hw, hh}; + } + + /** + * Where the point ({@code u}, {@code v}) of the frame, as fractions of its width and height + * from the top left, lands on the canvas with {@link #turn}: the same transform as + * Canvas.translate() and rotate(), for tests without a bitmap. + */ + static float[] map(float[] t, float u, float v) { + float x = t[3] + u * (t[5] - t[3]); + float y = t[4] + v * (t[6] - t[4]); + double a = Math.toRadians(t[2]); + double cos = Math.cos(a); + double sin = Math.sin(a); + return new float[] {(float) (t[0] + x * cos - y * sin), (float) (t[1] + x * sin + y * cos)}; } /** diff --git a/android/app/src/main/java/com/openipc/decoder/NetInfo.java b/android/app/src/main/java/com/openipc/decoder/NetInfo.java index d953e876..28717f85 100644 --- a/android/app/src/main/java/com/openipc/decoder/NetInfo.java +++ b/android/app/src/main/java/com/openipc/decoder/NetInfo.java @@ -7,12 +7,20 @@ package com.openipc.decoder; +import android.content.Context; +import android.net.ConnectivityManager; +import android.net.LinkAddress; +import android.net.LinkProperties; +import android.net.Network; +import android.os.Build; import android.util.Log; +import java.net.DatagramSocket; import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; import java.net.NetworkInterface; +import java.net.Socket; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -126,4 +134,83 @@ static boolean isLocalClient(InetAddress a) { if (a instanceof Inet4Address) return a.isSiteLocalAddress(); return a instanceof Inet6Address && (a.getAddress()[0] & 0xfe) == 0xfc; } + + /** + * Binds {@code socket} to the network whose own subnet holds {@code target} (#187). A toy's + * Wi-Fi has no internet: with mobile data on, Android may send even on-link traffic over the + * default network instead. Returns what was done, for the log: "default" when no such + * network is found (or before Android 6: Network.bindSocket(DatagramSocket) is API 23, + * Qodo on #188), else the interface bound to. + */ + static String bindToNetworkOf(DatagramSocket socket, InetAddress target) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) return "default"; + try { + Found f = networkOf(target); + if (f == null) return "default"; + f.network.bindSocket(socket); + return "bound to " + f.name; + } catch (Exception e) { + Log.w(TAG, "No network bound for " + target.getHostAddress() + ": " + e.getMessage()); + return "default"; + } + } + + /** + * The same for a TCP socket that is not connected yet (the CTP connection of a jieli:// + * slot, #187); binding one needs Android 6. + */ + static String bindToNetworkOf(Socket socket, InetAddress target) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) return "default"; + try { + Found f = networkOf(target); + if (f == null) return "default"; + f.network.bindSocket(socket); + return "bound to " + f.name; + } catch (Exception e) { + Log.w(TAG, "No network bound for " + target.getHostAddress() + ": " + e.getMessage()); + return "default"; + } + } + + private static final class Found { + final Network network; + final String name; + + Found(Network network, String name) { + this.network = network; + this.name = name; + } + } + + /** The network whose own subnet holds {@code target}, or null. */ + private static Found networkOf(InetAddress target) { + Context ctx = App.context; + if (ctx == null) return null; + ConnectivityManager cm = + (ConnectivityManager) ctx.getSystemService(Context.CONNECTIVITY_SERVICE); + if (cm == null) return null; + for (Network n : cm.getAllNetworks()) { + LinkProperties lp = cm.getLinkProperties(n); + if (lp == null) continue; + for (LinkAddress la : lp.getLinkAddresses()) { + if (sameSubnet(la.getAddress(), la.getPrefixLength(), target)) { + return new Found(n, lp.getInterfaceName()); + } + } + } + return null; + } + + /** {@code a} and {@code b} share the first {@code prefix} bits (and the address family). */ + static boolean sameSubnet(InetAddress a, int prefix, InetAddress b) { + if (a == null || b == null) return false; + byte[] x = a.getAddress(); + byte[] y = b.getAddress(); + if (x.length != y.length || prefix < 0 || prefix > x.length * 8) return false; + for (int i = 0; i < prefix; i++) { + int mask = 0x80 >> (i & 7); + if ((x[i >> 3] & mask) != (y[i >> 3] & mask)) return false; + } + return true; + } } diff --git a/android/app/src/main/java/com/openipc/decoder/QuadCell.java b/android/app/src/main/java/com/openipc/decoder/QuadCell.java index e4e776d9..aa8b4ef7 100644 --- a/android/app/src/main/java/com/openipc/decoder/QuadCell.java +++ b/android/app/src/main/java/com/openipc/decoder/QuadCell.java @@ -56,6 +56,8 @@ class QuadCell implements RtspSession.Callback { private volatile String state = "connecting"; /** What keeps the cell from playing (connection error, decoder limit), or null. */ private volatile String problem; + /** What a flowufo:// cell waits for (#187), shown instead of "Buffering…", or null. */ + private volatile String detail; /** Why the cell was paused to relieve the decoder (#83), or null. */ private volatile String pausedReason; private volatile long decoderStartedMs; @@ -286,12 +288,19 @@ public void onStatus(String status) { // stalls again (Qodo on #146) if ("disconnected".equals(status)) nalAssembler.setSalvage(false); state = status; + // A flowufo:// session sends its detail after the status it belongs to (Qodo on #188) + detail = null; // A connection that works again clears the last connection error if ("connected".equals(status) && problem != null && !problem.startsWith("Decoder")) { problem = null; } } + @Override + public void onDetail(String text) { + detail = text; + } + @Override public void onFailure(String message) { // A decoder problem stays visible; the connection retries on its own @@ -351,6 +360,8 @@ String sample(long nowMs) { if (p != null) return p; String st = state; if (!"connected".equals(st)) { + String d = detail; + if (d != null) return d; if ("disconnected".equals(st)) return "Disconnected"; return "buffering".equals(st) ? "Buffering…" : "Connecting…"; } @@ -417,6 +428,18 @@ static boolean isKeyFrameNal(int type, boolean h265) { return h265 ? type >= 16 && type <= 21 : type == 5; } + /** FLOW-UFO statistics of a flowufo:// cell (#187), else null. */ + java.util.Map flowUfoStats() { return session.flowUfoStats(); } + + /** Jieli statistics of a jieli:// cell (#187), else null. */ + java.util.Map jieliStats() { return session.jieliStats(); } + + /** The cooingdv heartbeat of an RTSP cell once it used one (#187), else null. */ + java.util.Map heartbeatStats() { return session.heartbeatStats(); } + + /** Degrees counter-clockwise the MJPEG frames are turned (#200), 0 for none. */ + int rotate() { return jpeg ? session.rotate() : 0; } + /** Transport in use, with the mode after a fallback (#112). */ String transportLabel() { return Transport.label(session.transportInUse(), transport); } @@ -710,7 +733,7 @@ private void decodeJpeg(Frame frame) { return; } Surface s = surface; - int[] size = jpegRenderer.render(frame, s, + int[] size = jpegRenderer.render(frame, s, session.rotate(), () -> running && jpeg && s == surface); if (size == null) return; lastJpegDecodeMs = now; diff --git a/android/app/src/main/java/com/openipc/decoder/RtspClient.java b/android/app/src/main/java/com/openipc/decoder/RtspClient.java index 287e4a67..d0f550cb 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspClient.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspClient.java @@ -43,6 +43,8 @@ interface Listener { void onStatusChanged(String status); void onCodecChanged(boolean codecH265, AudioConfig audio); void onJitterSample(int avgJitter); + /** What a flowufo:// slot waits for (#187), shown instead of the plain status. */ + default void onDetail(String text) {} } private volatile Listener listenerCb; @@ -85,6 +87,16 @@ void configure(String host, String userAgent, Transport.Mode transport) { /** The stream is MJPEG: the queue carries complete JPEG frames (#119). */ boolean isJpeg() { return videoJpeg; } long getBitrateMarkNs() { return session.getBitrateMarkNs(); } + /** FLOW-UFO statistics while a flowufo:// slot plays (#187), else null. */ + java.util.Map flowUfoStats() { return session.flowUfoStats(); } + /** Jieli statistics while a jieli:// slot plays (#187), else null. */ + java.util.Map jieliStats() { return session.jieliStats(); } + byte[] jieliRaw(String which) { return session.jieliRaw(which); } + /** The cooingdv heartbeat once this slot used one (#187), else null. */ + java.util.Map heartbeatStats() { return session.heartbeatStats(); } + + /** Degrees counter-clockwise the MJPEG frames are turned (#200), 0 for none. */ + int rotate() { return session.rotate(); } void start() { nalAssembler.reset(); @@ -109,6 +121,12 @@ public void onStatus(String status) { if (l != null) l.onStatusChanged(status); } + @Override + public void onDetail(String text) { + Listener l = listenerCb; + if (l != null) l.onDetail(text); + } + @Override public void onSdp(Sdp sdp) { nalAssembler.reset(); diff --git a/android/app/src/main/java/com/openipc/decoder/RtspSession.java b/android/app/src/main/java/com/openipc/decoder/RtspSession.java index 40bd0581..dd496e98 100644 --- a/android/app/src/main/java/com/openipc/decoder/RtspSession.java +++ b/android/app/src/main/java/com/openipc/decoder/RtspSession.java @@ -24,6 +24,7 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; /** @@ -71,6 +72,12 @@ default void onFailure(String message) {} * Returns true if it was usable; only those feed the watchdog. */ default boolean onJpegFrame(byte[] buf, int offset, int length) { return false; } + + /** + * What a flowufo:// slot waits for while it has no picture (#187), e.g. "Waiting for + * the toy: probing 192.168.169.1:8800"; shown instead of the plain status. + */ + default void onDetail(String text) {} } private final String tag; @@ -99,6 +106,20 @@ default void onFailure(String message) {} private RtpListener listener; /** A udp:// slot is listening: silence is reported, never "fixed" by a rebind. */ private volatile boolean listening; + /** Client of a flowufo:// slot (#187), else null: it probes on its own when silent. */ + private volatile FlowUfo.Client ufo; + /** Client of a jieli:// slot (#187), else null: it asks for the stream again on its own. */ + private volatile Jieli.Client jieli; + /** Heartbeat of the RTSP connection now (#187), else null; counts of the earlier ones. */ + private volatile UdpHeartbeat heartbeat; + private volatile int heartbeatsBefore; + private volatile int heartbeatErrorsBefore; + private volatile boolean heartbeatUsed; + private volatile boolean heartbeatLogged; + /** Degrees counter-clockwise the MJPEG frames of this connection are turned (#200). */ + private volatile int rotate; + private volatile boolean rotateLogged; + private volatile boolean rotateIgnoredLogged; private Thread networkThread; private Thread watchdogThread; @@ -120,6 +141,20 @@ void configure(String url, String userAgent, boolean withAudio) { } void configure(String url, String userAgent, boolean withAudio, Transport.Mode mode) { + if (!java.util.Objects.equals(url, this.url)) { + // Another camera: its rotation is logged again (#200), and the heartbeat counts + // of the previous one leave /stats.json (Qodo on #188). A heartbeat still running + // is closed by its connection, without adding to the new totals. + rotateLogged = false; + rotateIgnoredLogged = false; + synchronized (lock) { + heartbeat = null; + heartbeatUsed = false; + heartbeatsBefore = 0; + heartbeatErrorsBefore = 0; + heartbeatLogged = false; + } + } this.url = url; this.userAgent = userAgent != null ? userAgent : ""; this.withAudio = withAudio; @@ -201,6 +236,9 @@ void stop() { */ private void closeSocketLocked() { if (listener != null) listener.close(); + if (ufo != null) ufo.close(); + if (jieli != null) jieli.close(); + if (heartbeat != null) heartbeat.close(); if (udp != null) { udp.close(); if (udpPlaying) return; @@ -273,7 +311,7 @@ private void watchdogLoop(int gen) { } } } - if (active && !listening + if (active && !listening && ufo == null && jieli == null && System.nanoTime() / 1_000_000 - lastPacketMs > watchdogMs) { Log.w(TAG, tag + ": stream is inactive, reconnecting"); active = false; @@ -291,8 +329,12 @@ private void watchdogLoop(int gen) { /** One session from connect to disconnect. Returns true if PLAY succeeded. */ private boolean connect(int gen) throws IOException { + // Only an RTSP MJPEG stream is turned (#200); the other transports draw it as it comes + rotate = 0; if (RtpListener.Url.matches(url)) return listen(gen, RtpListener.Url.parse(url)); if (HttpMjpeg.matches(url)) return httpMjpeg(gen, HttpMjpeg.Url.parse(url)); + if (FlowUfo.Url.matches(url)) return flowUfo(gen, FlowUfo.Url.parse(url)); + if (Jieli.Url.matches(url)) return jieli(gen, Jieli.Url.parse(url)); Url u = Url.parse(url); Socket s = new Socket(); synchronized (lock) { @@ -301,9 +343,11 @@ private boolean connect(int gen) throws IOException { } Conn c = null; UdpReceiver rx = null; + UdpHeartbeat hb = null; String session = null; try { s.connect(new InetSocketAddress(u.host, u.port), CONNECT_TIMEOUT_MS); + if (u.heartbeatPort > 0) hb = startHeartbeat(gen, s, u.heartbeatPort); s.setTcpNoDelay(true); s.setSoTimeout(HANDSHAKE_TIMEOUT_MS); BufferedInputStream in = new BufferedInputStream(s.getInputStream(), 65536); @@ -444,6 +488,7 @@ private boolean connect(int gen) throws IOException { } if (gen != generation) return false; + rotate = rotation(u.rotate, sdp.videoJpeg); callback.onSdp(sdp); // PLAY on the request URL, as before: accepted by the servers seen so far @@ -483,6 +528,7 @@ private boolean connect(int gen) throws IOException { Keepalive k = keepalive; if (k != null && k.conn == c) keepalive = null; } + stopHeartbeat(hb); if (rx != null) { rx.close(); if (session != null && !s.isClosed()) { @@ -589,6 +635,270 @@ private boolean httpMjpeg(int gen, HttpMjpeg.Url u) throws IOException { throw new IOException("401 Unauthorized"); } + /** + * The cooingdv heartbeat (#187): 01 01 to the camera's UDP port every second, from the + * address of the RTSP connection, while the connection lasts. Logged at the first start. + */ + private UdpHeartbeat startHeartbeat(int gen, Socket s, int port) { + UdpHeartbeat h = new UdpHeartbeat(tag, s.getLocalAddress(), + new InetSocketAddress(s.getInetAddress(), port)); + synchronized (lock) { + if (gen != generation) return null; + stopHeartbeat(heartbeat); + heartbeat = h; + heartbeatUsed = true; + } + try { + h.start(!heartbeatLogged); + heartbeatLogged = true; + } catch (IOException e) { + Log.w(TAG, tag + ": no heartbeat socket: " + e.getMessage()); + } + return h; + } + + /** + * Ends the heartbeat {@code h} of a connection; when it is the current one its counts + * go to the totals of the stats. A newer connection's heartbeat is left alone. + */ + private void stopHeartbeat(UdpHeartbeat h) { + if (h == null) return; + h.close(); + synchronized (lock) { + if (heartbeat != h) return; + heartbeatsBefore += h.sent(); + heartbeatErrorsBefore += h.errors(); + heartbeat = null; + } + } + + /** + * The turn for frames of this connection: {@code wanted} for MJPEG, 0 for H.264/H.265, + * which have no rotation yet (#201). Logged once each. + */ + private int rotation(int wanted, boolean jpeg) { + if (wanted == 0) return 0; + if (!jpeg) { + if (!rotateIgnoredLogged) { + rotateIgnoredLogged = true; + Log.i(TAG, tag + ": rotate= only applies to MJPEG streams for now (#201)"); + } + return 0; + } + if (!rotateLogged) { + rotateLogged = true; + Log.i(TAG, tag + ": MJPEG frames turned " + wanted + "\u00b0 counter-clockwise" + + " (#200; ?rotate=0 turns it off)"); + } + return wanted; + } + + /** Degrees counter-clockwise the MJPEG frames are turned now (#200), 0 for none. */ + int rotate() { return rotate; } + + /** Heartbeat statistics for /stats.json once an RTSP slot used one (#187), else null. */ + Map heartbeatStats() { + if (!heartbeatUsed) return null; + synchronized (lock) { + return UdpHeartbeat.stats(heartbeat, heartbeatsBefore, heartbeatErrorsBefore); + } + } + + /** FLOW-UFO statistics for /stats.json while a flowufo:// slot runs (#187), else null. */ + Map flowUfoStats() { + FlowUfo.Client c = ufo; + return c != null ? c.stats() : null; + } + + /** + * A flowufo:// slot (#187): the toy drone's own protocol over UDP, JPEG frames to + * onJpegFrame. The client probes until the drone answers and again after it went quiet, + * so the watchdog leaves it alone. Returns true once it ran, like a PLAY that succeeded. + */ + private boolean flowUfo(int gen, FlowUfo.Url u) throws IOException { + FlowUfo.Client c = new FlowUfo.Client(tag, u, NetInfo::bindToNetworkOf); + synchronized (lock) { + if (gen != generation) return false; + ufo = c; + keepalive = null; + lastPacketMs = System.nanoTime() / 1_000_000; + streamBytes = 0; + bitrateMarkNs = System.nanoTime(); + active = true; + } + try { + Sdp sdp = new Sdp(); + sdp.videoJpeg = true; + sdp.videoPt = Sdp.PT_JPEG; + callback.onSdp(sdp); + final boolean[] first = {true}; + final boolean[] started = {false}; + c.run(() -> active && gen == generation, new FlowUfo.Listener() { + @Override + public void onState(boolean receiving, String text) { + if (gen != generation) return; + if (receiving) { + inUse = "FLOW-UFO UDP"; + } else { + inUse = null; + // The toy went quiet: say so, the next frame reports "connected" again + status(gen, started[0] ? "disconnected" : "buffering"); + started[0] = true; + first[0] = true; + } + callback.onDetail(text); + } + + @Override + public void onFrame(byte[] buf, int length) { + if (gen != generation) return; + streamBytes += length; + if (callback.onJpegFrame(buf, 0, length)) { + lastPacketMs = System.nanoTime() / 1_000_000; + if (first[0]) { + first[0] = false; + status(gen, "connected"); + } + } + } + }); + return true; + } finally { + synchronized (lock) { + if (ufo == c) { + ufo = null; + inUse = null; + } + } + c.close(); + } + } + + /** Jieli statistics for /stats.json while a jieli:// slot runs (#187), else null. */ + Map jieliStats() { + Jieli.Client c = jieli; + return c != null ? c.stats() : null; + } + + /** Raw bytes of a jieli:// slot for the debug page (#187), see Jieli.Client.raw(). */ + byte[] jieliRaw(String which) { + Jieli.Client c = jieli; + byte[] b = c != null ? c.raw(which) : null; + return b != null ? b : Jieli.Client.lastRaw(which); + } + + /** + * A jieli:// slot (#187): CTP over TCP asks the toy for the stream, which comes as RTP to + * a fixed local UDP port: payload type 26 goes to the RTP/JPEG path, 96 to the H.264 one, + * as on a udp:// slot. The client asks again when no video comes, so the watchdog leaves + * it alone. Returns true once it ran, like a PLAY that succeeded. + */ + private boolean jieli(int gen, Jieli.Url u) throws IOException { + Jieli.Client c = new Jieli.Client(tag, u, NetInfo::bindToNetworkOf, + NetInfo::bindToNetworkOf); + synchronized (lock) { + if (gen != generation) return false; + jieli = c; + keepalive = null; + lastPacketMs = System.nanoTime() / 1_000_000; + streamBytes = 0; + bitrateMarkNs = System.nanoTime(); + active = true; + } + try { + final int[] pt = {u.format == Jieli.FORMAT_H264 ? Jieli.PT_H264 : Jieli.PT_JPEG}; + rotate = rotation(u.rotate, true); + callback.onSdp(jieliSdp(pt[0])); + // onState comes from the CTP thread, the video from the UDP one + final AtomicBoolean first = new AtomicBoolean(true); + final AtomicBoolean received = new AtomicBoolean(false); + final Object order = new Object(); + try { + c.run(() -> active && gen == generation, new Jieli.Listener() { + @Override + public void onState(boolean receiving, String text) { + if (gen != generation) return; + // In order with usable() on the UDP thread: a packet that came in since + // the CTP thread saw the video stop wins (Qodo on #198) + synchronized (order) { + if (receiving) { + inUse = "Jieli RTP"; + received.set(true); + } else { + if (c.videoAgeMs() < Jieli.quietMs) return; + inUse = null; + // Video that stopped says so; the next frame reports "connected" + status(gen, received.get() ? "disconnected" : "buffering"); + first.set(true); + } + callback.onDetail(text); + } + } + + @Override + public boolean onRtp(byte[] buf, int offset, int length) { + if (gen != generation) return false; + int type = buf[offset + 1] & 0x7F; + if (type != pt[0]) { + // The toy sends the other codec than asked for: follow it + pt[0] = type; + Log.i(TAG, tag + ": Jieli: video is " + + (type == Jieli.PT_H264 ? "H.264" : "JPEG") + + " (RTP PT " + type + ")"); + callback.onSdp(jieliSdp(type)); + } + streamBytes += length; + if (!callback.onPacket(CHANNEL_VIDEO, buf, offset, length)) return false; + usable(); + return true; + } + + @Override + public boolean onJpeg(byte[] buf, int offset, int length) { + if (gen != generation) return false; + if (pt[0] != Jieli.PT_JPEG) { + pt[0] = Jieli.PT_JPEG; + callback.onSdp(jieliSdp(Jieli.PT_JPEG)); + } + streamBytes += length; + if (!callback.onJpegFrame(buf, offset, length)) return false; + usable(); + return true; + } + + private void usable() { + lastPacketMs = System.nanoTime() / 1_000_000; + synchronized (order) { + if (first.compareAndSet(true, false)) status(gen, "connected"); + } + } + }); + } catch (IOException e) { + // A toy that dropped after the stream was asked for ends like a PLAY that + // worked: the next attempt comes soon, not after a growing back-off (Qodo on #198) + if (!c.streamRequested || gen != generation) throw e; + Log.w(TAG, tag + ": Jieli: " + e.getMessage() + "; connecting again"); + callback.onFailure(e.getMessage()); + } + return true; + } finally { + synchronized (lock) { + if (jieli == c) { + jieli = null; + inUse = null; + } + } + c.close(); + } + } + + private static Sdp jieliSdp(int pt) { + Sdp sdp = new Sdp(); + sdp.videoPt = pt; + sdp.videoJpeg = pt == Jieli.PT_JPEG; + return sdp; + } + private boolean listen(int gen, RtpListener.Url u) throws IOException { RtpListener l = new RtpListener(tag, u); synchronized (lock) { @@ -1111,6 +1421,18 @@ static final class Url { String userInfo; /** URL without credentials, used in requests. */ String requestUrl; + /** + * UDP port for the cooingdv heartbeat (#187), or 0 for none: 7099 for + * rtsp://{host}:7070/webcam, else ?heartbeat={port}; ?heartbeat=off turns it off. The + * option is ours: it is taken out of the URL sent to the camera. + */ + int heartbeatPort; + /** + * Degrees counter-clockwise to turn MJPEG frames when drawing them (#200): 270 for + * rtsp://{host}:7070/webcam, whose toy camera is mounted sideways, else 0; + * ?rotate=0|90|180|270 overrides it. Ours as well: not sent to the camera. + */ + int rotate; static Url parse(String url) throws IOException { if (url == null) throw new IOException("Camera slot not configured"); @@ -1157,10 +1479,71 @@ static Url parse(String url) throws IOException { u.host = hostPort; } if (u.host.isEmpty()) throw new IOException("Invalid RTSP URL: host is missing"); + rest = takeOptions(u, rest); u.requestUrl = scheme + "://" + authority + rest; return u; } + /** + * Sets heartbeatPort and rotate from path and query; returns them without our + * options. + */ + private static String takeOptions(Url u, String rest) throws IOException { + int q = rest.indexOf('?'); + String path = q >= 0 ? rest.substring(0, q) : rest; + if (u.port == 7070 && (path.equalsIgnoreCase("/webcam") + || path.equalsIgnoreCase("/webcam/"))) { + // The cooingdv toys (#187): heartbeat, and the K200's sideways camera (#200) + u.heartbeatPort = UdpHeartbeat.DEFAULT_PORT; + u.rotate = 270; + } + if (q < 0) return rest; + StringBuilder kept = new StringBuilder(); + boolean found = false; + for (String param : rest.substring(q + 1).split("&", -1)) { + int eq = param.indexOf('='); + String name = (eq < 0 ? param : param.substring(0, eq)).toLowerCase(Locale.ROOT); + if (!name.equals("heartbeat") && !name.equals("rotate")) { + if (kept.length() > 0) kept.append('&'); + kept.append(param); + continue; + } + found = true; + String v = eq < 0 ? "" : param.substring(eq + 1).trim().toLowerCase(Locale.ROOT); + if (name.equals("rotate")) { + u.rotate = parseRotate(v); + } else if (v.equals("off") || v.equals("0")) { + u.heartbeatPort = 0; + } else { + try { + u.heartbeatPort = Integer.parseInt(v); + } catch (NumberFormatException e) { + u.heartbeatPort = -1; + } + if (u.heartbeatPort <= 0 || u.heartbeatPort > 65535) { + throw new IOException("Invalid heartbeat port: " + v + + " (a UDP port, e.g. heartbeat=7099, or heartbeat=off)"); + } + } + } + // Untouched unless our option was there + if (!found) return rest; + return kept.length() > 0 ? path + "?" + kept : path; + } + + /** ?rotate= in degrees counter-clockwise: 0, 90, 180 or 270 (#200). */ + static int parseRotate(String v) throws IOException { + switch (v) { + case "0": return 0; + case "90": return 90; + case "180": return 180; + case "270": return 270; + default: + throw new IOException("Invalid rotate: " + v + + " (degrees counter-clockwise: rotate=0, 90, 180 or 270)"); + } + } + private static int parsePort(String s, String url) throws IOException { if (s.isEmpty()) return 554; try { diff --git a/android/app/src/main/java/com/openipc/decoder/StreamProbes.java b/android/app/src/main/java/com/openipc/decoder/StreamProbes.java index 9e3d0946..7a3c08a6 100644 --- a/android/app/src/main/java/com/openipc/decoder/StreamProbes.java +++ b/android/app/src/main/java/com/openipc/decoder/StreamProbes.java @@ -49,6 +49,10 @@ static String skipReason(String url, String defaultUrl, String screenUrl) { if (url == null || url.trim().isEmpty() || url.equals(defaultUrl)) return NOT_CONFIGURED; // A listening slot would take the port; the camera pushes to it, nothing to ask if (RtpListener.Url.matches(url)) return "UDP listen slot: not checked in single view"; + // A toy drone sends to one client; a check would take the stream from the screen + if (FlowUfo.Url.matches(url) || Jieli.Url.matches(url)) { + return "toy drone slot: not checked in single view"; + } String e = endpoint(url); if (e == null) return "invalid URL"; if (e.equals(endpoint(screenUrl))) return "same camera as on screen"; diff --git a/android/app/src/main/java/com/openipc/decoder/UdpHeartbeat.java b/android/app/src/main/java/com/openipc/decoder/UdpHeartbeat.java new file mode 100644 index 00000000..72fe420f --- /dev/null +++ b/android/app/src/main/java/com/openipc/decoder/UdpHeartbeat.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * UdpHeartbeat.java — the "01 01" heartbeat that cooingdv RTSP toys expect on UDP 7099 (#187) + * + */ + +package com.openipc.decoder; + +import android.util.Log; + +import java.io.Closeable; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Toys of the cooingdv apps (KY UFO, RC UFO, ...; Wi-Fi "FLY-UFO-…" at 192.168.1.1) stream + * RTSP from rtsp://192.168.1.1:7070/webcam, and their apps send {@code 01 01} to UDP port + * 7099 once a second while they show it (described by turbodrone, docs/research/cooingdv.md). + * This sends exactly that heartbeat, from an ephemeral socket on the address the RTSP + * connection uses, and nothing else: no flight command and no "08 01" either. One thread per + * heartbeat, until close(). + */ +final class UdpHeartbeat implements Closeable { + private static final String TAG = "OpenIPCDecoder"; + + static final byte[] PAYLOAD = {0x01, 0x01}; + static final int DEFAULT_PORT = 7099; + /** Not final so unit tests can shorten it. */ + static volatile long intervalMs = 1000; + + private final String tag; + private final InetAddress local; + private final InetSocketAddress target; + private volatile boolean closed; + private volatile DatagramSocket socket; + private Thread thread; + private volatile int sent; + private volatile int errors; + private volatile String lastError; + + /** @param local address to send from (the RTSP connection's), or null for any. */ + UdpHeartbeat(String tag, InetAddress local, InetSocketAddress target) { + this.tag = tag; + this.local = local; + this.target = target; + } + + /** Opens the socket and starts sending, the first heartbeat at once. */ + synchronized void start(boolean log) throws IOException { + if (closed || thread != null) return; + DatagramSocket s = local != null + ? new DatagramSocket(new InetSocketAddress(local, 0)) : new DatagramSocket(); + String network = NetInfo.bindToNetworkOf(s, target.getAddress()); + socket = s; + if (log) { + Log.i(TAG, tag + ": heartbeat 01 01 every " + intervalMs + " ms to " + + target.getAddress().getHostAddress() + ":" + target.getPort() + + " from local port " + s.getLocalPort() + " (network: " + network + ")"); + } + thread = new Thread(() -> loop(s), tag + "-hb"); + thread.setDaemon(true); + thread.start(); + } + + private void loop(DatagramSocket s) { + DatagramPacket p = new DatagramPacket(PAYLOAD, PAYLOAD.length, target); + long next = System.nanoTime() / 1_000_000; + while (!closed) { + try { + s.send(p); + sent++; + } catch (IOException e) { + if (closed) break; + errors++; + lastError = e.getMessage(); + if (errors <= 3 || errors % 100 == 0) { + Log.w(TAG, tag + ": heartbeat failed (" + errors + "): " + e.getMessage()); + } + } + // A fixed schedule: a slow send does not shift the following ones + next += intervalMs; + long wait = next - System.nanoTime() / 1_000_000; + if (wait < 0) { + next = System.nanoTime() / 1_000_000; + wait = 0; + } + try { + Thread.sleep(wait); + } catch (InterruptedException e) { + break; + } + } + s.close(); + } + + int sent() { return sent; } + + int errors() { return errors; } + + String lastError() { return lastError; } + + InetSocketAddress target() { return target; } + + @Override + public synchronized void close() { + closed = true; + DatagramSocket s = socket; + if (s != null) s.close(); + if (thread != null) thread.interrupt(); + } + + /** Running or not: true until close(). */ + boolean running() { return !closed && thread != null; } + + static Map stats(UdpHeartbeat h, int sentBefore, int errorsBefore) { + Map m = new LinkedHashMap<>(); + m.put("payload", "01 01"); + if (h != null) { + m.put("target", h.target.getAddress().getHostAddress() + ":" + h.target.getPort()); + m.put("running", h.running()); + if (h.lastError != null) m.put("last_error", h.lastError); + } else { + m.put("running", false); + } + m.put("sent", sentBefore + (h != null ? h.sent : 0)); + int e = errorsBefore + (h != null ? h.errors : 0); + if (e > 0) m.put("errors", e); + return m; + } +} diff --git a/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java new file mode 100644 index 00000000..a281bff4 --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/FlowUfoTest.java @@ -0,0 +1,1174 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * FlowUfoTest.java — flowufo:// slots: packets, acknowledgements, JPEG headers, a fake drone + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.After; +import org.junit.Test; +import org.w3c.dom.NodeList; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.SocketAddress; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class FlowUfoTest { + /** Zig-zag position i to natural (row-major) position. */ + private static final int[] NATURAL = { + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63 + }; + + @After + public void restoreTimers() { + FlowUfo.resetMs = 3000; + } + + // ====================== Helpers ====================== + + private static byte[] bytes(String hex) { + String h = hex.replaceAll("\\s", ""); + byte[] b = new byte[h.length() / 2]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) Integer.parseInt(h.substring(2 * i, 2 * i + 2), 16); + } + return b; + } + + /** A type-1 packet as the drone sends it. */ + static byte[] fragment(long seq, long base, int index, int total, int bodyLength, int width, + int height, int quality, byte[] data, int dataOff, int dataLen) { + byte[] p = new byte[FlowUfo.FRAGMENT_HEADER + dataLen]; + p[0] = (byte) 0x93; + p[1] = 0x01; + FlowUfo.le16(p, 2, p.length); + FlowUfo.le64(p, 0x08, seq); + FlowUfo.le64(p, 0x18, base); + FlowUfo.le32(p, 0x20, index); + FlowUfo.le32(p, 0x24, total); + FlowUfo.le32(p, 0x28, bodyLength); + FlowUfo.le16(p, 0x2C, width); + FlowUfo.le16(p, 0x2E, height); + p[0x30] = (byte) quality; + p[0x34] = 1; + p[0x35] = 1; + System.arraycopy(data, dataOff, p, FlowUfo.FRAGMENT_HEADER, dataLen); + return p; + } + + /** Fragment {@code index} of {@code body} for frame {@code seq}. */ + static byte[] piece(long seq, byte[] body, int index) { + return piece(seq, body, index, 640, 360, 50); + } + + static byte[] piece(long seq, byte[] body, int index, int width, int height, int quality) { + int total = (body.length + FlowUfo.FRAGMENT_DATA - 1) / FlowUfo.FRAGMENT_DATA; + int at = index * FlowUfo.FRAGMENT_DATA; + int len = Math.min(FlowUfo.FRAGMENT_DATA, body.length - at); + return fragment(seq, 0, index, total, body.length, width, height, quality, body, at, + len); + } + + private static FlowUfo.Fragment parsed(byte[] p) { + FlowUfo.Fragment f = new FlowUfo.Fragment(); + assertTrue(f.parse(p, 0, p.length)); + return f; + } + + private static byte[] ack(FlowUfo.Window w, long nowMs) { + byte[] out = new byte[FlowUfo.ACK_CAPACITY]; + int len = w.buildAck(out, nowMs); + return Arrays.copyOf(out, len); + } + + /** {seq, status, record length} of each record of an acknowledgement. */ + private static List records(byte[] a) { + List list = new ArrayList<>(); + int n = (int) FlowUfo.u32(a, 0x08); + int pos = FlowUfo.ACK_HEADER; + for (int i = 0; i < n; i++) { + long seq = FlowUfo.u32(a, pos) | FlowUfo.u32(a, pos + 4) << 32; + long status = FlowUfo.u32(a, pos + 8); + long len = FlowUfo.u32(a, pos + 12); + list.add(new long[] {seq, status, len, pos}); + pos += (int) len; + } + assertEquals("records fill the packet", a.length, pos); + return list; + } + + // ====================== URL ====================== + + @Test + public void urlDefaultsAndPort() throws IOException { + FlowUfo.Url u = FlowUfo.Url.parse("flowufo://192.168.169.1:8800"); + assertEquals("192.168.169.1", u.host); + assertEquals(8800, u.port); + u = FlowUfo.Url.parse("flowufo://"); + assertEquals(FlowUfo.DEFAULT_HOST, u.host); + assertEquals(FlowUfo.DEFAULT_PORT, u.port); + u = FlowUfo.Url.parse(" FLOWUFO://10.0.0.5/ "); + assertEquals("10.0.0.5", u.host); + assertEquals(8800, u.port); + u = FlowUfo.Url.parse("flowufo://:9000"); + assertEquals(FlowUfo.DEFAULT_HOST, u.host); + assertEquals(9000, u.port); + assertEquals("192.168.169.1:9000", u.endpoint()); + assertTrue(FlowUfo.Url.matches("flowufo://")); + assertFalse(FlowUfo.Url.matches("udp://@:5600")); + } + + @Test + public void badUrlsAreRefused() { + for (String bad : new String[] {"flowufo://host:0", "flowufo://host:x", + "flowufo://host:70000", "flowufo://a b", "flowufo://u@host", "http://x"}) { + try { + FlowUfo.Url.parse(bad); + fail(bad); + } catch (IOException expected) { + // ok + } + } + } + + @Test + public void notCheckedBySingleViewProbes() { + assertEquals("toy drone slot: not checked in single view", + StreamProbes.skipReason("flowufo://", "rtsp://default", null)); + } + + // ====================== Packets ====================== + + @Test + public void packetTypeNeedsMagicAndLength() { + byte[] service = bytes("93 04 0E 00 01 00 00 00 02 00 00 00 55 5A"); + assertEquals(FlowUfo.TYPE_SERVICE, FlowUfo.packetType(service, 0, service.length)); + // The length field must be the datagram size + assertEquals(-1, FlowUfo.packetType(service, 0, service.length - 1)); + byte[] other = service.clone(); + other[0] = (byte) 0x94; + assertEquals(-1, FlowUfo.packetType(other, 0, other.length)); + byte[] big = new byte[FlowUfo.MAX_PACKET + 1]; + big[0] = (byte) 0x93; + FlowUfo.le16(big, 2, big.length); + assertEquals(-1, FlowUfo.packetType(big, 0, big.length)); + } + + @Test + public void fragmentFields() { + byte[] data = new byte[1024]; + for (int i = 0; i < data.length; i++) data[i] = (byte) i; + byte[] p = fragment(7, 3, 2, 5, 4500, 640, 360, 75, data, 0, 1024); + // An unknown field at 0x0C (if 0x08 is a u32): the frame number stays 7 + p[0x0C] = 0x55; + p[0x0F] = 0x66; + FlowUfo.Fragment f = parsed(p); + assertEquals(7, f.seq); + assertEquals(3, f.base); + assertEquals(2, f.index); + assertEquals(5, f.total); + assertEquals(4500, f.bodyLength); + assertEquals(640, f.width); + assertEquals(360, f.height); + assertEquals(75, f.quality); + // Camera status bytes: main and flow camera + assertEquals(1, f.mainCamera); + assertEquals(1, f.flowCamera); + p[0x34] = 0; + p[0x35] = 2; + f = parsed(p); + assertEquals(0, f.mainCamera); + assertEquals(2, f.flowCamera); + assertEquals(FlowUfo.FRAGMENT_HEADER, f.dataOffset); + assertEquals(1024, f.dataLength); + assertSame(p, f.buf); + } + + @Test + public void malformedFragmentsAreRefused() { + byte[] d = new byte[1024]; + FlowUfo.Fragment f = new FlowUfo.Fragment(); + // Index beyond the count, no fragments, body longer than the fragments, no frame number + byte[][] bad = { + fragment(1, 0, 3, 3, 3000, 640, 360, 50, d, 0, 10), + fragment(1, 0, 0, 0, 3000, 640, 360, 50, d, 0, 10), + fragment(1, 0, 0, 2, 2049, 640, 360, 50, d, 0, 10), + fragment(0, 0, 0, 1, 10, 640, 360, 50, d, 0, 10), + fragment(1, 0, 0, FlowUfo.MAX_FRAGMENTS + 1, 10, 640, 360, 50, d, 0, 10), + }; + for (byte[] p : bad) assertFalse(f.parse(p, 0, p.length)); + byte[] shortOne = Arrays.copyOf(fragment(1, 0, 0, 1, 10, 0, 0, 50, d, 0, 10), 0x30); + FlowUfo.le16(shortOne, 2, shortOne.length); + assertFalse(f.parse(shortOne, 0, shortOne.length)); + byte[] service = fragment(1, 0, 0, 1, 10, 0, 0, 50, d, 0, 10); + service[1] = 4; + assertFalse(f.parse(service, 0, service.length)); + } + + // ====================== Window and acknowledgements ====================== + + @Test + public void ackBytesExact() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] d = new byte[3 * 1024]; + // Frame 1 of two fragments, complete; frame 2 of three with the middle one missing + assertNull(w.add(parsed(piece(1, Arrays.copyOf(d, 2000), 0)), 1000)); + assertNotNull(w.add(parsed(piece(1, Arrays.copyOf(d, 2000), 1)), 1000)); + assertNull(w.add(parsed(piece(2, Arrays.copyOf(d, 2100), 0)), 1010)); + assertNull(w.add(parsed(piece(2, Arrays.copyOf(d, 2100), 2)), 1010)); + + // Fragments still coming: frame 2 is the one arriving, left out + String header = "EF 02 %s 02 02 00 01 %s 00 00 00 00 00 00" + + " 00000000000000000000000000000000 00000000000000000000000000000000" + + " 00000000000000000000000000000000 00000000000000000000000000000000" + + " 32 4B 14 2D 00 00"; + byte[] expected = bytes(String.format(header, "6C 00", "01 00 00 00") + + " 01 00 00 00 00 00 00 00 01 00 00 00 14 00 00 00 FF FF FF FF"); + assertArrayEquals(expected, ack(w, 1015)); + + // A pause: frame 2 is in again, collecting, fragment 1 missing (bits past 3 set) + expected = bytes(String.format(header, "80 00", "02 00 00 00") + + " 01 00 00 00 00 00 00 00 01 00 00 00 14 00 00 00 FF FF FF FF" + + " 02 00 00 00 00 00 00 00 00 00 00 00 14 00 00 00 FD FF FF FF"); + assertArrayEquals(expected, ack(w, 1030)); + + // All complete and quiet: frame 3 is asked for whole (status 3, no mask) + assertNotNull(w.add(parsed(piece(2, Arrays.copyOf(d, 2100), 1)), 1040)); + expected = bytes(String.format(header, "90 00", "03 00 00 00") + + " 01 00 00 00 00 00 00 00 01 00 00 00 14 00 00 00 FF FF FF FF" + + " 02 00 00 00 00 00 00 00 01 00 00 00 14 00 00 00 FF FF FF FF" + + " 03 00 00 00 00 00 00 00 03 00 00 00 10 00 00 00"); + assertArrayEquals(expected, ack(w, 1060)); + assertEquals(1, w.nextRequests); + } + + @Test + public void maskOfManyFragments() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[40 * 1024 - 100]; + // 40 fragments: two mask words; only 0 and 33 arrive + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(piece(1, body, 33)), 0); + byte[] a = ack(w, 100); + List r = records(a); + assertEquals(1, r.size()); + assertEquals(16 + 8, r.get(0)[2]); + int pos = (int) r.get(0)[3] + 16; + assertEquals(0x00000001L, FlowUfo.u32(a, pos)); + // Bit 1 is fragment 33; bits 8.. are past fragment 39 + assertEquals(0xFFFFFF02L, FlowUfo.u32(a, pos + 4)); + } + + @Test + public void lateSlotIsLeftOut() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[100]; + // Frames 1..4 and 6, each one fragment; frame 5 (slot of frame 1) never comes + for (long seq : new long[] {1, 2, 3, 4, 6}) { + assertNotNull(w.add(parsed(piece(seq, body, 0)), 0)); + } + List r = records(ack(w, 5)); + // Slot 0 (frame 1) is 5 behind: no record (as in turbodrone); 3 and 4 complete; 6 (in + // the slot of 2) is arriving + assertEquals(2, r.size()); + assertArrayEquals(new long[] {3, FlowUfo.STATUS_COMPLETE, 20}, Arrays.copyOf(r.get(0), 3)); + assertArrayEquals(new long[] {4, FlowUfo.STATUS_COMPLETE, 20}, Arrays.copyOf(r.get(1), 3)); + assertEquals(1, w.lagging); + for (long[] x : r) assertFalse("no seq+4 request", x[1] == FlowUfo.STATUS_WHOLE); + // Frame 9 takes slot 0: frame 1 was complete, frame 5 never came + w.add(parsed(piece(9, body, 0)), 10); + assertEquals(1, w.lost); + } + + @Test + public void incompleteLateSlotDoesNotHoldUpTheNextRequest() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + byte[] one = Arrays.copyOf(body, 10); + // Frame 1 misses its second fragment; frames 2, 3, 4 and 6 complete + w.add(parsed(piece(1, body, 0)), 0); + for (long seq : new long[] {2, 3, 4, 6}) assertNotNull(w.add(parsed(piece(seq, one, 0)), 0)); + List r = records(ack(w, 100)); + // Quiet: 6, 3, 4 complete, then frame 7 asked for; frame 1 (5 behind) left out + long[] last = r.get(r.size() - 1); + assertArrayEquals(new long[] {7, FlowUfo.STATUS_WHOLE, 16}, Arrays.copyOf(last, 3)); + for (long[] x : r) assertFalse("frame 1 left out", x[0] == 1); + assertEquals(1, w.nextRequests); + } + + @Test + public void baseHidesOldFrames() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[100]; + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(piece(2, body, 0)), 0); + byte[] p = fragment(3, 1, 0, 1, body.length, 640, 360, 50, body, 0, body.length); + w.add(parsed(p), 0); + List r = records(ack(w, 1)); + // 1 is at or below the base, 3 is arriving: only 2 + assertEquals(1, r.size()); + assertEquals(2, r.get(0)[0]); + } + + @Test + public void incompleteFrameReplacedCountsAsLost() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + w.add(parsed(piece(1, body, 0)), 0); + // Frame 5 takes slot 0 while frame 1 misses a fragment + assertNull(w.add(parsed(piece(5, body, 0)), 1)); + assertEquals(1, w.lost); + // A fragment of frame 1 now is stale + assertNull(w.add(parsed(piece(1, body, 1)), 2)); + assertEquals(1, w.stale); + assertNotNull(w.add(parsed(piece(5, body, 1)), 3)); + } + + @Test + public void duplicatesAndLateFrames() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(piece(1, body, 0)), 0); + assertEquals(1, w.duplicates); + // Frame 2 completes first, frame 1 after it is not shown + assertNotNull(w.add(parsed(piece(2, Arrays.copyOf(body, 10), 0)), 1)); + assertNull(w.add(parsed(piece(1, body, 1)), 2)); + assertEquals(1, w.late); + assertEquals(2, w.completed); + assertEquals(1, w.delivered); + } + + @Test + public void bodyIsAssembledInOrderOfIndex() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2500]; + for (int i = 0; i < body.length; i++) body[i] = (byte) (i * 7); + w.add(parsed(piece(1, body, 2)), 0); + w.add(parsed(piece(1, body, 0)), 0); + FlowUfo.Slot s = w.add(parsed(piece(1, body, 1)), 0); + assertNotNull(s); + assertArrayEquals(body, Arrays.copyOf(s.body, s.bodyLength)); + } + + @Test + public void restartedDroneCountsFromOne() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] small = new byte[10]; + assertNotNull(w.add(parsed(piece(500, small, 0)), 0)); + // A run of fragments far behind: the drone started again from 1 + byte[] body = new byte[FlowUfo.RESTART_RUN * 1024]; + for (int i = 0; i < FlowUfo.RESTART_RUN; i++) w.add(parsed(piece(1, body, i)), 1); + assertEquals(1, w.restarts); + assertEquals(1, w.maxSeq); + assertNotNull(w.add(parsed(piece(2, small, 0)), 2)); + } + + @Test + public void oneStragglerDoesNotReset() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + assertNotNull(w.add(parsed(piece(100, Arrays.copyOf(body, 10), 0)), 0)); + assertNull(w.add(parsed(piece(101, body, 0)), 10)); + // A delayed fragment of frame 35 is only stale + assertNull(w.add(parsed(piece(35, body, 0)), 15)); + assertEquals(0, w.restarts); + assertEquals(1, w.stale); + assertEquals(101, w.maxSeq); + assertNotNull(w.add(parsed(piece(101, body, 1)), 20)); + } + + @Test + public void staleFragmentsDoNotHoldBackTheMask() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(piece(5, body, 0)), 0); + // Fragments of the replaced frame 1 keep coming: frame 5 is still due its mask + for (int t = 5; t <= 40; t += 5) w.add(parsed(piece(1, body, 1)), t); + List r = records(ack(w, 40)); + assertEquals(1, r.size()); + assertEquals(5, r.get(0)[0]); + assertEquals(FlowUfo.STATUS_COLLECTING, r.get(0)[1]); + } + + @Test + public void baseHiddenFrameDoesNotHoldUpTheNextRequest() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2000]; + byte[] one = Arrays.copyOf(body, 10); + // Frame 1 incomplete; frame 2 complete and says base 1 + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(fragment(2, 1, 0, 1, one.length, 640, 360, 50, one, 0, one.length)), 0); + List r = records(ack(w, 100)); + assertEquals(2, r.size()); + assertArrayEquals(new long[] {2, FlowUfo.STATUS_COMPLETE, 20}, Arrays.copyOf(r.get(0), 3)); + assertArrayEquals(new long[] {3, FlowUfo.STATUS_WHOLE, 16}, Arrays.copyOf(r.get(1), 3)); + } + + @Test + public void shortFragments() { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] body = new byte[2500]; + // Fragment 0 with 100 bytes instead of 1024: stays missing + byte[] p = fragment(1, 0, 0, 3, body.length, 640, 360, 50, body, 0, 100); + assertNull(w.add(parsed(p), 0)); + assertEquals(1, w.shortFragments); + assertEquals(0, w.slots[0].received); + w.add(parsed(piece(1, body, 0)), 0); + w.add(parsed(piece(1, body, 1)), 0); + // The last one with 2 of its 452 bytes: the body ends there + p = fragment(1, 0, 2, 3, body.length, 640, 360, 50, body, 2048, 2); + FlowUfo.Slot s = w.add(parsed(p), 0); + assertNotNull(s); + assertEquals(2050, s.bodyLength); + assertEquals(2, w.shortFragments); + } + + // ====================== JPEG headers ====================== + + @Test + public void headerLayout() { + byte[] h = FlowUfo.jpegHeader(50, 1280, 720); + assertEquals(605, h.length); + // Height and width in SOF0 at 0x91 and 0x93, big-endian + assertEquals(720, (h[0x91] & 0xFF) << 8 | h[0x92] & 0xFF); + assertEquals(1280, (h[0x93] & 0xFF) << 8 | h[0x94] & 0xFF); + assertEquals(0xFF, h[0] & 0xFF); + assertEquals(0xD8, h[1] & 0xFF); + // Segments by their lengths + List markers = new ArrayList<>(); + int i = 2; + while (i < h.length) { + assertEquals(0xFF, h[i] & 0xFF); + int marker = h[i + 1] & 0xFF; + int len = (h[i + 2] & 0xFF) << 8 | h[i + 3] & 0xFF; + markers.add(marker); + if (marker == 0xC0) { + assertEquals(17, len); + assertEquals(8, h[i + 4]); + assertEquals(3, h[i + 9]); + // Every component 1x1 (4:4:4); Y on table 0, Cb and Cr on table 1 + assertArrayEquals(bytes("00 11 00 01 11 01 02 11 01"), + Arrays.copyOfRange(h, i + 10, i + 19)); + } + if (marker == 0xDB) assertEquals(67, len); + if (marker == 0xDA) { + assertArrayEquals(bytes("03 00 00 01 11 02 11 00 3F 00"), + Arrays.copyOfRange(h, i + 4, i + 14)); + } + i += 2 + len; + } + assertEquals(h.length, i); + assertEquals(Arrays.asList(0xDB, 0xDB, 0xC0, 0xC4, 0xC4, 0xC4, 0xC4, 0xDA), markers); + // DHT: Y DC, Y AC, chroma DC, chroma AC + assertEquals(0x00, h[159 + 4]); + assertEquals(0x10, h[159 + 33 + 4]); + } + + @Test + public void ijgQualityScaling() { + // Zig-zag order: luma 16 11 12 14 ..., chroma 17 18 18 24 ... + byte[] q50 = JpegAssembler.ijgTables(50); + assertEquals(16, q50[0]); + assertEquals(11, q50[1]); + assertEquals(99, q50[63] & 0xFF); + assertEquals(17, q50[64]); + assertEquals(99, q50[127] & 0xFF); + // libjpeg: scale 5000/q below 50, else 200 - 2q; (v * scale + 50) / 100 in 1..255 + assertEquals(8, JpegAssembler.ijgTables(75)[0]); + assertEquals(6, JpegAssembler.ijgTables(75)[1]); + assertEquals(32, JpegAssembler.ijgTables(25)[0] & 0xFF); + assertEquals(80, JpegAssembler.ijgTables(10)[0] & 0xFF); + assertEquals(160, JpegAssembler.ijgTables(5)[0] & 0xFF); + assertEquals(170, JpegAssembler.ijgTables(5)[64] & 0xFF); + assertEquals(255, JpegAssembler.ijgTables(5)[127] & 0xFF); + for (byte v : JpegAssembler.ijgTables(100)) assertEquals(1, v); + // The headers carry them: luma after "FF DB 00 43 00", chroma after "FF DB 00 43 01" + byte[] h = FlowUfo.jpegHeader(75, 0, 0); + assertArrayEquals(Arrays.copyOf(JpegAssembler.ijgTables(75), 64), + Arrays.copyOfRange(h, 7, 71)); + assertArrayEquals(Arrays.copyOfRange(JpegAssembler.ijgTables(75), 64, 128), + Arrays.copyOfRange(h, 76, 140)); + // No size: 640x360; no usable quality: Q50 + assertEquals(360, (h[0x91] & 0xFF) << 8 | h[0x92] & 0xFF); + assertEquals(640, (h[0x93] & 0xFF) << 8 | h[0x94] & 0xFF); + assertArrayEquals(FlowUfo.jpegHeader(50, 640, 360), FlowUfo.jpegHeader(0, 640, 360)); + // RFC 2435 keeps its own clamp to 99 + assertArrayEquals(JpegAssembler.ijgTables(99), JpegAssembler.tables(100)); + } + + @Test + public void jpegAddsEoiOnceAndPassesWholeJpegs() { + byte[] header = FlowUfo.jpegHeader(50, 8, 8); + byte[][] out = new byte[1][]; + int n = FlowUfo.jpeg(header, new byte[] {1, 2, 3}, 3, out); + assertEquals(header.length + 5, n); + assertEquals(0xD9, out[0][n - 1] & 0xFF); + n = FlowUfo.jpeg(header, new byte[] {1, (byte) 0xFF, (byte) 0xD9}, 3, out); + assertEquals(header.length + 3, n); + byte[] whole = {(byte) 0xFF, (byte) 0xD8, 5, (byte) 0xFF, (byte) 0xD9}; + n = FlowUfo.jpeg(header, whole, whole.length, out); + assertArrayEquals(whole, Arrays.copyOf(out[0], n)); + } + + // ====================== Round trip through a real JPEG codec ====================== + + // Unit tests compile against android.jar, which has no java.awt or javax.imageio: the + // JDK's ImageIO (there at run time) is reached by reflection + + /** + * Calls the public method {@code name} whose parameters take {@code args}, through a + * public type: the classes behind ImageIO's interfaces are in packages of the JDK that are + * not exported, so their own Method objects cannot be invoked. + */ + private static Object call(Object target, String name, Object... args) throws Exception { + boolean statik = target instanceof Class; + Class c = statik ? (Class) target : target.getClass(); + List> types = new ArrayList<>(); + for (Class k = c; k != null; k = k.getSuperclass()) { + types.add(k); + Collections.addAll(types, k.getInterfaces()); + } + Exception last = new NoSuchMethodException(c.getName() + "." + name); + for (Class k : types) { + if (!java.lang.reflect.Modifier.isPublic(k.getModifiers())) continue; + for (java.lang.reflect.Method m : k.getMethods()) { + if (!m.getName().equals(name) || !fits(m.getParameterTypes(), args)) continue; + try { + return m.invoke(statik ? null : target, args); + } catch (IllegalAccessException e) { + last = e; + } + } + } + throw last; + } + + private static Object create(String className, Object... args) throws Exception { + for (java.lang.reflect.Constructor k : Class.forName(className).getConstructors()) { + if (fits(k.getParameterTypes(), args)) return k.newInstance(args); + } + throw new NoSuchMethodException(className); + } + + private static boolean fits(Class[] types, Object[] args) { + if (types.length != args.length) return false; + for (int i = 0; i < types.length; i++) { + if (args[i] == null) { + if (types[i].isPrimitive()) return false; + continue; + } + Class t = types[i] == int.class ? Integer.class + : types[i] == boolean.class ? Boolean.class : types[i]; + if (!t.isInstance(args[i])) return false; + } + return true; + } + + private static Class imageIo() throws Exception { + return Class.forName("javax.imageio.ImageIO"); + } + + /** A java.awt.image.BufferedImage. */ + static final class Img { + final Object image; + + Img(Object image) { + this.image = image; + } + + int width() throws Exception { + return (Integer) call(image, "getWidth"); + } + + int height() throws Exception { + return (Integer) call(image, "getHeight"); + } + + int[] pixels() throws Exception { + int w = width(); + return (int[]) call(image, "getRGB", 0, 0, w, height(), null, 0, w); + } + + /** Decoded by ImageIO, or null when it cannot. */ + static Img read(byte[] jpeg, int length) throws Exception { + Object img = call(imageIo(), "read", new ByteArrayInputStream(jpeg, 0, length)); + return img != null ? new Img(img) : null; + } + } + + private static Img picture(int w, int h) throws Exception { + // BufferedImage.TYPE_INT_RGB + Object img = create("java.awt.image.BufferedImage", w, h, 1); + int[] rgb = new int[w * h]; + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + int r = x * 255 / Math.max(1, w - 1); + int g = y * 255 / Math.max(1, h - 1); + int b = ((x / 8 + y / 8) & 1) != 0 ? 200 : 40; + rgb[y * w + x] = r << 16 | g << 8 | b; + } + } + call(img, "setRGB", 0, 0, w, h, rgb, 0, w); + return new Img(img); + } + + /** Baseline 4:4:4 JPEG with the IJG tables at {@code quality}, by ImageIO. */ + static byte[] encode444(Img img, int quality) throws Exception { + Object writer = ((java.util.Iterator) call(imageIo(), "getImageWritersByFormatName", + "jpeg")).next(); + Object param = call(writer, "getDefaultWriteParam"); + Object type = call(Class.forName("javax.imageio.ImageTypeSpecifier"), + "createFromRenderedImage", img.image); + Object meta = call(writer, "getDefaultImageMetadata", type, param); + String format = "javax_imageio_jpeg_image_1.0"; + org.w3c.dom.Element root = (org.w3c.dom.Element) call(meta, "getAsTree", format); + byte[] tables = JpegAssembler.ijgTables(quality); + NodeList dqt = root.getElementsByTagName("dqtable"); + for (int i = 0; i < dqt.getLength(); i++) { + org.w3c.dom.Element n = (org.w3c.dom.Element) dqt.item(i); + int id = Integer.parseInt(n.getAttribute("qtableId")); + int[] natural = new int[64]; + for (int k = 0; k < 64; k++) natural[NATURAL[k]] = tables[id * 64 + k] & 0xFF; + call(n, "setUserObject", create("javax.imageio.plugins.jpeg.JPEGQTable", + (Object) natural)); + } + NodeList comps = root.getElementsByTagName("componentSpec"); + for (int i = 0; i < comps.getLength(); i++) { + org.w3c.dom.Element n = (org.w3c.dom.Element) comps.item(i); + n.setAttribute("HsamplingFactor", "1"); + n.setAttribute("VsamplingFactor", "1"); + } + call(meta, "setFromTree", format, root); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + Object out = call(imageIo(), "createImageOutputStream", bytes); + try { + call(writer, "setOutput", out); + Object image = create("javax.imageio.IIOImage", img.image, null, meta); + call(writer, "write", null, image, param); + } finally { + call(out, "close"); + call(writer, "dispose"); + } + return bytes.toByteArray(); + } + + /** The entropy-coded data between SOS and EOI: what the drone sends. */ + static byte[] scanOf(byte[] jpeg) { + int i = 2; + while (i + 4 <= jpeg.length) { + int marker = jpeg[i + 1] & 0xFF; + int len = (jpeg[i + 2] & 0xFF) << 8 | jpeg[i + 3] & 0xFF; + if (marker == 0xC0) { + // The encoder really wrote 4:4:4 + for (int c = 0; c < 3; c++) assertEquals(0x11, jpeg[i + 11 + 3 * c]); + } + if (marker == 0xDA) { + int start = i + 2 + len; + assertEquals(0xD9, jpeg[jpeg.length - 1] & 0xFF); + return Arrays.copyOfRange(jpeg, start, jpeg.length - 2); + } + i += 2 + len; + } + throw new AssertionError("no SOS"); + } + + /** + * ImageIO's own guess for a JPEG without JFIF or Adobe marker and no chroma subsampling is + * RGB; libjpeg(-turbo), as on Android, takes YCbCr. For ImageIO the headers get a JFIF + * APP0 after SOI, which says YCbCr; the rest is left as it is. + */ + static byte[] withJfif(byte[] jpeg, int length) { + byte[] app0 = bytes("FF E0 00 10 4A 46 49 46 00 01 01 00 00 01 00 01 00 00"); + byte[] out = new byte[length + app0.length]; + System.arraycopy(jpeg, 0, out, 0, 2); + System.arraycopy(app0, 0, out, 2, app0.length); + System.arraycopy(jpeg, 2, out, 2 + app0.length, length - 2); + return out; + } + + private static Img decode(byte[] jpeg, int length) throws Exception { + byte[] j = withJfif(jpeg, length); + return Img.read(j, j.length); + } + + @Test + public void scanWithOurHeadersDecodesLikeTheOriginal() throws Exception { + for (int q : new int[] {5, 10, 25, 50, 75, 100}) { + for (int[] size : new int[][] {{64, 48}, {50, 30}}) { + Img src = picture(size[0], size[1]); + byte[] original = encode444(src, q); + byte[] scan = scanOf(original); + byte[][] out = new byte[1][]; + int n = FlowUfo.jpeg(FlowUfo.jpegHeader(q, size[0], size[1]), scan, scan.length, + out); + Img a = Img.read(original, original.length); + Img b = decode(out[0], n); + assertNotNull("Q" + q, b); + assertEquals(size[0], b.width()); + assertEquals(size[1], b.height()); + assertArrayEquals("Q" + q + " " + size[0] + "x" + size[1], a.pixels(), b.pixels()); + if (q >= 75) { + // And close to the picture itself: colours are not mixed up + int[] p = src.pixels(); + int[] r = b.pixels(); + long err = 0; + for (int k = 0; k < p.length; k++) { + for (int sh = 0; sh <= 16; sh += 8) { + err += Math.abs((p[k] >> sh & 0xFF) - (r[k] >> sh & 0xFF)); + } + } + assertTrue("Q" + q + " error " + err, err / (p.length * 3.0) < 8); + } + } + } + } + + // ====================== A fake drone on loopback ====================== + + /** Next packet from the client that starts with EF 02, skipping probes. */ + private static byte[] nextAck(DatagramSocket drone, long deadlineMs) throws IOException { + byte[] buf = new byte[4096]; + while (System.currentTimeMillis() < deadlineMs) { + DatagramPacket p = new DatagramPacket(buf, buf.length); + try { + drone.receive(p); + } catch (SocketTimeoutException e) { + continue; + } + if (p.getLength() >= FlowUfo.ACK_HEADER && (buf[0] & 0xFF) == 0xEF && buf[1] == 2) { + return Arrays.copyOf(buf, p.getLength()); + } + } + throw new AssertionError("no acknowledgement"); + } + + @Test + public void fakeDroneOnLoopback() throws Exception { + FlowUfo.resetMs = 400; + Img src = picture(160, 120); + byte[] original = encode444(src, 75); + byte[] body = scanOf(original); + int total = (body.length + 1023) / 1024; + assertTrue("several fragments: " + total, total >= 3); + + try (DatagramSocket drone = new DatagramSocket(0, InetAddress.getLoopbackAddress())) { + drone.setSoTimeout(200); + FlowUfo.Url u = FlowUfo.Url.parse("flowufo://127.0.0.1:" + drone.getLocalPort()); + java.util.concurrent.atomic.AtomicInteger setups = + new java.util.concurrent.atomic.AtomicInteger(); + FlowUfo.Client client = new FlowUfo.Client("test", u, (s, target) -> { + setups.incrementAndGet(); + return "test"; + }); + List frames = Collections.synchronizedList(new ArrayList<>()); + List states = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch frame = new CountDownLatch(1); + CountDownLatch again = new CountDownLatch(1); + Throwable[] error = new Throwable[1]; + Thread t = new Thread(() -> { + try { + client.run(() -> true, new FlowUfo.Listener() { + @Override + public void onState(boolean receiving, String text) { + states.add(text); + if (text.contains("went quiet")) again.countDown(); + } + + @Override + public void onFrame(byte[] buf, int length) { + frames.add(Arrays.copyOf(buf, length)); + frame.countDown(); + } + }); + } catch (Throwable e) { + error[0] = e; + } + }); + t.start(); + try { + // Probes first, every 100 ms + byte[] buf = new byte[64]; + DatagramPacket p = new DatagramPacket(buf, buf.length); + drone.setSoTimeout(2000); + drone.receive(p); + assertArrayEquals(FlowUfo.PROBE, Arrays.copyOf(buf, p.getLength())); + SocketAddress phone = p.getSocketAddress(); + drone.setSoTimeout(200); + assertTrue(states.get(0).startsWith("Waiting for the toy: probing 127.0.0.1:")); + + // Frame 1 without fragment 1 + for (int i = 0; i < total; i++) { + if (i == 1) continue; + byte[] f = piece(1, body, i, 160, 120, 75); + drone.send(new DatagramPacket(f, f.length, phone)); + } + // A service message is only logged + byte[] service = bytes("93 04 0E 00 01 00 00 00 02 00 00 00 55 5A"); + drone.send(new DatagramPacket(service, service.length, phone)); + + // After the pause the acknowledgement asks for the missing fragment + long deadline = System.currentTimeMillis() + 3000; + long[] rec; + byte[] a; + do { + a = nextAck(drone, deadline); + List r = records(a); + rec = r.isEmpty() ? null : r.get(0); + } while (rec == null); + assertEquals(1, rec[0]); + assertEquals(FlowUfo.STATUS_COLLECTING, rec[1]); + long mask = FlowUfo.u32(a, (int) rec[3] + 16); + assertEquals("fragment 1 missing", 0, mask & 2); + assertEquals(1, mask & 1); + assertEquals(1, frame.getCount()); + + byte[] f = piece(1, body, 1, 160, 120, 75); + drone.send(new DatagramPacket(f, f.length, phone)); + assertTrue(frame.await(3, TimeUnit.SECONDS)); + byte[] jpeg = frames.get(0); + byte[][] expected = new byte[1][]; + int n = FlowUfo.jpeg(FlowUfo.jpegHeader(75, 160, 120), body, body.length, expected); + assertArrayEquals(Arrays.copyOf(expected[0], n), jpeg); + Img got = decode(jpeg, jpeg.length); + assertEquals(160, got.width()); + assertArrayEquals(Img.read(original, original.length).pixels(), got.pixels()); + + // Then: frame 1 complete, frame 2 asked for whole + boolean asked = false; + deadline = System.currentTimeMillis() + 3000; + while (!asked) { + for (long[] r : records(nextAck(drone, deadline))) { + if (r[0] == 2 && r[1] == FlowUfo.STATUS_WHOLE) asked = true; + } + } + assertEquals(1, client.window.delivered); + assertEquals(1, client.stats().get("service_messages")); + assertEquals("playing", client.stats().get("state")); + + // Silence: probing again + assertTrue(again.await(3, TimeUnit.SECONDS)); + deadline = System.currentTimeMillis() + 2000; + boolean probed = false; + while (!probed && System.currentTimeMillis() < deadline) { + DatagramPacket q = new DatagramPacket(buf, buf.length); + try { + drone.receive(q); + } catch (SocketTimeoutException e) { + continue; + } + probed = Arrays.equals(FlowUfo.PROBE, Arrays.copyOf(buf, q.getLength())); + } + assertTrue(probed); + assertEquals("probing", client.stats().get("state")); + // A new socket, set up for the network again (Qodo on #188) + assertEquals(2, setups.get()); + } finally { + client.close(); + t.join(2000); + } + assertFalse(t.isAlive()); + assertNull(error[0]); + } + } + + @Test + public void sameSubnet() throws Exception { + InetAddress phone = InetAddress.getByName("192.168.169.2"); + InetAddress toy = InetAddress.getByName("192.168.169.1"); + assertTrue(NetInfo.sameSubnet(phone, 24, toy)); + assertFalse(NetInfo.sameSubnet(InetAddress.getByName("192.168.1.2"), 24, toy)); + assertTrue(NetInfo.sameSubnet(InetAddress.getByName("192.168.1.2"), 16, toy)); + assertFalse(NetInfo.sameSubnet(InetAddress.getByName("fe80::1"), 64, toy)); + assertFalse(NetInfo.sameSubnet(phone, 33, toy)); + } + + // ====================== Options, ports, hints (#187, after turbodrone) ====================== + + @Test + public void urlOptions() throws IOException { + FlowUfo.Url u = FlowUfo.Url.parse("flowufo://"); + assertArrayEquals(bytes("32 4B 14 2D"), u.qpara); + assertEquals(FlowUfo.DQT_ZIGZAG, u.dqt); + u = FlowUfo.Url.parse("flowufo://192.168.169.1:8800?qpara=19322348&dqt=raster"); + assertEquals("192.168.169.1", u.host); + assertEquals(8800, u.port); + assertArrayEquals(bytes("19 32 23 48"), u.qpara); + assertEquals(FlowUfo.DQT_RASTER, u.dqt); + u = FlowUfo.Url.parse("flowufo://192.168.169.1:8800?q=19322348"); + assertArrayEquals(bytes("19 32 23 48"), u.qpara); + u = FlowUfo.Url.parse("flowufo://?DQT=Q50&other=1#x"); + assertEquals(FlowUfo.DEFAULT_HOST, u.host); + assertEquals(FlowUfo.DQT_Q50, u.dqt); + u = FlowUfo.Url.parse("flowufo://10.0.0.5/?dqt=zigzag&qpara=aabbccdd"); + assertEquals("10.0.0.5", u.host); + assertEquals(FlowUfo.DQT_ZIGZAG, u.dqt); + assertArrayEquals(bytes("AA BB CC DD"), u.qpara); + for (String bad : new String[] {"flowufo://?qpara=1932234", "flowufo://?qpara=1932234G", + "flowufo://?qpara=193223480", "flowufo://?dqt=natural", "flowufo://?dqt="}) { + try { + FlowUfo.Url.parse(bad); + fail(bad); + } catch (IOException expected) { + // ok + } + } + } + + @Test + public void portsProbed() throws IOException { + // 8800, given or not, probes the BL618 port 8801 too; any other port only itself + assertArrayEquals(new int[] {8800, 8801}, FlowUfo.Url.parse("flowufo://").ports()); + assertArrayEquals(new int[] {8800, 8801}, + FlowUfo.Url.parse("flowufo://192.168.169.1:8800").ports()); + assertArrayEquals(new int[] {8801}, + FlowUfo.Url.parse("flowufo://192.168.169.1:8801").ports()); + assertArrayEquals(new int[] {9000}, FlowUfo.Url.parse("flowufo://:9000").ports()); + assertEquals("192.168.169.1:8800 and :8801", FlowUfo.Url.parse("flowufo://").probed()); + assertEquals("192.168.169.1:8801", FlowUfo.Url.parse("flowufo://:8801").probed()); + } + + @Test + public void bl618Timing() { + assertEquals(100, FlowUfo.Client.probeInterval(8800)); + assertEquals(200, FlowUfo.Client.probeInterval(8801)); + assertEquals(25, FlowUfo.Client.ackInterval(8800)); + assertEquals(50, FlowUfo.Client.ackInterval(8801)); + assertEquals(25, FlowUfo.Client.ackInterval(9000)); + assertFalse(FlowUfo.Client.ackNeedsRecord(8800)); + assertTrue(FlowUfo.Client.ackNeedsRecord(8801)); + assertEquals("BL608", FlowUfo.Client.engine(8800)); + assertEquals("BL618", FlowUfo.Client.engine(8801)); + assertEquals("port 9000", FlowUfo.Client.engine(9000)); + } + + @Test + public void ackCarriesTheQualityParametersAndNoCommand() throws IOException { + FlowUfo.Window w = new FlowUfo.Window(); + byte[] a = ack(w, 0); + assertEquals(FlowUfo.ACK_HEADER, a.length); + assertArrayEquals(bytes("32 4B 14 2D 00 00"), Arrays.copyOfRange(a, 0x52, 0x58)); + // Command counter, command length and the command itself: all zero + for (int i = 0x0C; i < 0x52; i++) assertEquals("byte " + i, 0, a[i]); + w.qpara = FlowUfo.Url.parse("flowufo://?qpara=19322348").qpara; + assertArrayEquals(bytes("19 32 23 48 00 00"), Arrays.copyOfRange(ack(w, 0), 0x52, 0x58)); + } + + @Test + public void dqtOrders() { + byte[] raster = FlowUfo.dqtTables(FlowUfo.DQT_RASTER, 75); + // Annex K row by row: luma 16 11 10 16 24 40 51 61 / 12 12 14 19 26 58 60 55 + assertArrayEquals(bytes("10 0B 0A 10 18 28 33 3D 0C 0C 0E 13 1A 3A 3C 37"), + Arrays.copyOf(raster, 16)); + // Chroma 17 18 24 47 99 99 99 99 / 18 21 26 66 99 99 99 99 + assertArrayEquals(bytes("11 12 18 2F 63 63 63 63 12 15 1A 42 63 63 63 63"), + Arrays.copyOfRange(raster, 64, 80)); + // Unscaled whatever the quality byte says + assertArrayEquals(raster, FlowUfo.dqtTables(FlowUfo.DQT_RASTER, 20)); + byte[] zz = JpegAssembler.ijgTables(50); + for (int k = 0; k < 64; k++) { + assertEquals(zz[k], raster[NATURAL[k]]); + assertEquals(zz[64 + k], raster[64 + NATURAL[k]]); + } + assertArrayEquals(zz, FlowUfo.dqtTables(FlowUfo.DQT_Q50, 90)); + assertArrayEquals(JpegAssembler.ijgTables(90), FlowUfo.dqtTables(FlowUfo.DQT_ZIGZAG, 90)); + assertArrayEquals(zz, FlowUfo.dqtTables(FlowUfo.DQT_ZIGZAG, 0)); + // In the header: luma after "FF DB 00 43 00", chroma after "FF DB 00 43 01" + byte[] h = FlowUfo.jpegHeader(75, 640, 360, FlowUfo.DQT_RASTER); + assertEquals(605, h.length); + assertArrayEquals(Arrays.copyOf(raster, 64), Arrays.copyOfRange(h, 7, 71)); + assertArrayEquals(Arrays.copyOfRange(raster, 64, 128), Arrays.copyOfRange(h, 76, 140)); + assertArrayEquals(FlowUfo.jpegHeader(75, 640, 360), + FlowUfo.jpegHeader(75, 640, 360, FlowUfo.DQT_ZIGZAG)); + assertEquals("zigzag Q75", FlowUfo.dqtLabel(FlowUfo.DQT_ZIGZAG, 75)); + assertEquals("zigzag Q50", FlowUfo.dqtLabel(FlowUfo.DQT_ZIGZAG, 0)); + assertEquals("raster (unscaled)", FlowUfo.dqtLabel(FlowUfo.DQT_RASTER, 75)); + assertEquals("q50 (unscaled)", FlowUfo.dqtLabel(FlowUfo.DQT_Q50, 75)); + } + + private static InetAddress ip(String s) throws IOException { + return InetAddress.getByName(s); + } + + @Test + public void subnetHints() throws Exception { + String ky = "This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam"; + // #187: the K200 slot pointed at the phone itself, on a 192.168.1.x network + List on1 = Collections.singletonList(ip("192.168.1.100")); + assertEquals("192.168.1.100 is this phone; the toy is usually 192.168.1.1. " + ky, + FlowUfo.subnetHint(ip("192.168.1.100"), on1)); + assertEquals(ky, FlowUfo.subnetHint(ip("192.168.169.1"), on1)); + assertNull(FlowUfo.subnetHint(ip("192.168.1.1"), on1)); + assertNull(FlowUfo.subnetHint(ip("10.0.0.5"), on1)); + List on169 = Collections.singletonList(ip("192.168.169.2")); + assertNull(FlowUfo.subnetHint(ip("192.168.169.1"), on169)); + assertEquals("192.168.169.2 is this phone; the toy is usually 192.168.169.1", + FlowUfo.subnetHint(ip("192.168.169.2"), on169)); + assertEquals("This phone is on 192.168.169.x: the toy is usually flowufo://192.168.169.1", + FlowUfo.subnetHint(ip("192.168.1.1"), on169)); + assertEquals("This phone is on 192.168.80.x (HF_ Wi-Fi), a Jieli toy: try" + + " jieli://192.168.80.1", + FlowUfo.subnetHint(ip("192.168.169.1"), + Collections.singletonList(ip("192.168.80.3")))); + // On both networks: no guess + assertNull(FlowUfo.subnetHint(ip("192.168.169.1"), + Arrays.asList(ip("192.168.1.5"), ip("192.168.169.2")))); + assertNull(FlowUfo.subnetHint(ip("192.168.169.1"), Collections.emptyList())); + assertNull(FlowUfo.subnetHint(ip("::1"), on1)); + } + + @Test + public void hintInTheStatusText() throws Exception { + try (DatagramSocket drone = new DatagramSocket(0, InetAddress.getLoopbackAddress())) { + FlowUfo.Url u = FlowUfo.Url.parse("flowufo://127.0.0.1:" + drone.getLocalPort()); + FlowUfo.Client client = new FlowUfo.Client("test", u, null); + client.localForHint = Collections.singletonList(InetAddress.getByName("127.0.0.1")); + List states = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch first = new CountDownLatch(1); + Thread t = new Thread(() -> { + try { + client.run(() -> true, new FlowUfo.Listener() { + @Override + public void onState(boolean receiving, String text) { + states.add(text); + first.countDown(); + } + + @Override + public void onFrame(byte[] buf, int length) {} + }); + } catch (IOException ignored) { + // closed + } + }); + t.start(); + try { + assertTrue(first.await(3, TimeUnit.SECONDS)); + assertEquals("Waiting for the toy: probing 127.0.0.1:" + drone.getLocalPort() + + ". 127.0.0.1 is this phone; the toy is usually 127.0.0.1", states.get(0)); + assertNotNull(client.stats().get("hint")); + } finally { + client.close(); + t.join(2000); + } + } + } + + /** The next probe arriving at {@code drone}; returns where it came from. */ + private static SocketAddress probeAt(DatagramSocket drone) throws IOException { + byte[] buf = new byte[64]; + long deadline = System.currentTimeMillis() + 3000; + while (System.currentTimeMillis() < deadline) { + DatagramPacket p = new DatagramPacket(buf, buf.length); + try { + drone.receive(p); + } catch (SocketTimeoutException e) { + continue; + } + if (Arrays.equals(FlowUfo.PROBE, Arrays.copyOf(buf, p.getLength()))) { + return p.getSocketAddress(); + } + } + throw new AssertionError("no probe"); + } + + @Test + public void probesBothPortsAndLocksOntoTheOneThatAnswers() throws Exception { + InetAddress lo = InetAddress.getLoopbackAddress(); + try (DatagramSocket a = new DatagramSocket(0, lo); + DatagramSocket b = new DatagramSocket(0, lo); + DatagramSocket from1234 = new DatagramSocket(0, lo)) { + a.setSoTimeout(200); + b.setSoTimeout(200); + // a stands for 8800, b for 8801 + FlowUfo.Url u = FlowUfo.Url.parse("flowufo://127.0.0.1:" + a.getLocalPort()); + u.alternatePort = b.getLocalPort(); + assertArrayEquals(new int[] {a.getLocalPort(), b.getLocalPort()}, u.ports()); + FlowUfo.Client client = new FlowUfo.Client("test", u, null); + client.localForHint = Collections.emptyList(); + Thread t = new Thread(() -> { + try { + client.run(() -> true, new FlowUfo.Listener() { + @Override + public void onState(boolean receiving, String text) {} + + @Override + public void onFrame(byte[] buf, int length) {} + }); + } catch (IOException ignored) { + // closed + } + }); + t.start(); + try { + // Both ports are probed, each from a socket of its own + SocketAddress phoneA = probeAt(a); + SocketAddress phoneB = probeAt(b); + assertFalse(phoneA.equals(phoneB)); + assertEquals(-1, client.lockedPort()); + // A service message on a's socket does not lock it (Qodo on #188) + byte[] service = bytes("93 04 0E 00 01 00 00 00 02 00 00 00 55 5A"); + from1234.send(new DatagramPacket(service, service.length, phoneA)); + long until = System.currentTimeMillis() + 3000; + while (client.stats().get("service_messages").equals(0) + && System.currentTimeMillis() < until) { + Thread.sleep(10); + } + assertEquals(1, client.stats().get("service_messages")); + assertEquals(-1, client.lockedPort()); + probeAt(b); + // The engine behind b answers, from another source port (the drone's 1234) + byte[] f = piece(1, new byte[2000], 0); + from1234.send(new DatagramPacket(f, f.length, phoneB)); + // Acknowledgements go to b, to the port that was probed + nextAck(b, System.currentTimeMillis() + 3000); + assertEquals(b.getLocalPort(), client.lockedPort()); + assertEquals(b.getLocalPort(), client.stats().get("port")); + // a gets nothing any more once what was sent before the lock is read + a.setSoTimeout(50); + try { + while (true) a.receive(new DatagramPacket(new byte[64], 64)); + } catch (SocketTimeoutException expected) { + // drained + } + a.setSoTimeout(400); + try { + a.receive(new DatagramPacket(new byte[64], 64)); + fail("probe to the other port after the lock"); + } catch (SocketTimeoutException expected) { + // ok + } + } finally { + client.close(); + t.join(2000); + } + assertFalse(t.isAlive()); + } + } +} diff --git a/android/app/src/test/java/com/openipc/decoder/JieliTest.java b/android/app/src/test/java/com/openipc/decoder/JieliTest.java new file mode 100644 index 00000000..9d99f5e0 --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/JieliTest.java @@ -0,0 +1,866 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * JieliTest.java — jieli:// slots: CTP framing, the session against a fake toy, RTP/JPEG on UDP + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class JieliTest { + private static final InetAddress LOOPBACK = InetAddress.getLoopbackAddress(); + + @Before + public void shortTimers() { + Jieli.paramWaitMs = 300; + Jieli.diagnosticPorts = new int[0]; + Jieli.streamPort = freeUdpPortOrZero(); + } + + @After + public void restoreTimers() { + Jieli.paramWaitMs = 1500; + Jieli.noVideoMs = 5000; + Jieli.reopenMs = 10_000; + Jieli.quietMs = 3000; + Jieli.ctpSilenceMs = 20_000; + Jieli.diagnosticPorts = new int[] {1234}; + Jieli.diagnosticMs = 30_000; + Jieli.streamPort = Jieli.STREAM_PORT; + } + + private static int freeUdpPortOrZero() { + try (DatagramSocket d = new DatagramSocket(0, LOOPBACK)) { + return d.getLocalPort(); + } catch (IOException e) { + return 0; + } + } + + /** A datagram of the toy's stream on UDP 2224 as a YL3 sent it (#187). */ + private static byte[] streamPacket(int counter, int total, int offset, byte[] piece, int tail) { + byte[] b = new byte[Jieli.RtStream.HEADER + piece.length + tail]; + // Type 2, bit 7 on the last piece; bytes 2-3 the length of this piece + b[0] = (byte) (offset + piece.length >= total ? 0x82 : 0x02); + b[2] = (byte) piece.length; + b[3] = (byte) (piece.length >>> 8); + le32(b, 4, counter); + le32(b, 8, total); + le32(b, 12, offset); + System.arraycopy(piece, 0, b, Jieli.RtStream.HEADER, piece.length); + return b; + } + + private static void le32(byte[] b, int at, int v) { + for (int i = 0; i < 4; i++) b[at + i] = (byte) (v >>> (8 * i)); + } + + private static byte[] jpeg(int length) { + byte[] j = new byte[length]; + j[0] = (byte) 0xFF; + j[1] = (byte) 0xD8; + for (int i = 2; i < length - 2; i++) j[i] = (byte) (i * 7); + j[length - 2] = (byte) 0xFF; + j[length - 1] = (byte) 0xD9; + return j; + } + + // ====================== RT stream ====================== + + @Test + public void rtStreamOneDatagramFrame() { + // As seen from a YL3: 4600 bytes of JPEG, 20 more bytes after it in the datagram + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(4600); + byte[] p = streamPacket(42, 4600, 0, j, 20); + assertEquals(4640, p.length); + int n = st.feed(p, 0, p.length); + assertEquals(4600, n); + assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); + assertEquals(1, st.frames); + assertEquals(0, st.dropped); + } + + @Test + public void rtStreamFrameInPieces() { + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(10000); + assertEquals(0, st.feed(streamPacket(1, 10000, 0, Arrays.copyOfRange(j, 0, 4000), 0), + 0, Jieli.RtStream.HEADER + 4000)); + assertEquals(0, st.feed(streamPacket(2, 10000, 4000, Arrays.copyOfRange(j, 4000, 8000), 0), + 0, Jieli.RtStream.HEADER + 4000)); + byte[] last = streamPacket(3, 10000, 8000, Arrays.copyOfRange(j, 8000, 10000), 0); + int n = st.feed(last, 0, last.length); + assertEquals(10000, n); + assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); + } + + @Test + public void rtStreamUsesOnlyThePiece() { + // As test build 3 showed it: 2964 of 4600 bytes in a 4640-byte datagram whose rest is + // left over from an earlier frame (even with an FFD9); the rest comes at offset 2964 + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(4600); + byte[] first = streamPacket(1, 4600, 0, Arrays.copyOfRange(j, 0, 2964), 0); + byte[] dgram = Arrays.copyOf(first, 4640); + byte[] stale = jpeg(4600 - 2964 + 20); + System.arraycopy(stale, 0, dgram, first.length, stale.length); + assertEquals(0, st.feed(dgram, 0, dgram.length)); + byte[] second = streamPacket(2, 4600, 2964, Arrays.copyOfRange(j, 2964, 4600), 40); + int n = st.feed(second, 0, second.length); + assertEquals(4600, n); + assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); + assertEquals(0, st.dropped); + } + + @Test + public void rtStreamDropsAFrameWithAPieceMissing() { + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(6000); + byte[] first = streamPacket(1, 6000, 0, Arrays.copyOfRange(j, 0, 2000), 0); + assertEquals(0, st.feed(first, 0, first.length)); + // The middle piece is lost; the last one alone does not finish the frame + byte[] last = streamPacket(3, 6000, 4000, Arrays.copyOfRange(j, 4000, 6000), 0); + assertEquals(0, st.feed(last, 0, last.length)); + assertEquals(0, st.dropped); + // The next frame plays, the unfinished one is given up + byte[] next = streamPacket(4, 3000, 0, jpeg(3000), 0); + assertEquals(3000, st.feed(next, 0, next.length)); + assertEquals(1, st.dropped); + assertTrue(st.lastDropReason, st.lastDropReason.startsWith("incomplete")); + } + + @Test + public void rtStreamIgnoresRepeatedPieces() { + // A repeated piece must not count twice (Qodo on #215) + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(6000); + byte[] a = streamPacket(1, 6000, 0, Arrays.copyOfRange(j, 0, 3000), 0); + assertEquals(0, st.feed(a, 0, a.length)); + assertEquals(0, st.feed(a, 0, a.length)); + byte[] b = streamPacket(2, 6000, 3000, Arrays.copyOfRange(j, 3000, 6000), 0); + int n = st.feed(b, 0, b.length); + assertEquals(6000, n); + assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); + assertEquals(0, st.dropped); + } + + @Test + public void rtStreamTakesPiecesOutOfOrder() { + // The last piece before the middle one, the first one last (Qodo on #215) + Jieli.RtStream st = new Jieli.RtStream(); + byte[] j = jpeg(6000); + byte[] last = streamPacket(3, 6000, 4000, Arrays.copyOfRange(j, 4000, 6000), 0); + byte[] mid = streamPacket(2, 6000, 2000, Arrays.copyOfRange(j, 2000, 4000), 0); + byte[] first = streamPacket(1, 6000, 0, Arrays.copyOfRange(j, 0, 2000), 0); + assertEquals(0, st.feed(last, 0, last.length)); + assertEquals(0, st.feed(mid, 0, mid.length)); + int n = st.feed(first, 0, first.length); + assertEquals(6000, n); + assertArrayEquals(j, Arrays.copyOf(st.frame(), n)); + assertEquals(0, st.dropped); + } + + @Test + public void rtStreamDropsBrokenFrames() { + Jieli.RtStream st = new Jieli.RtStream(); + // Start lost: a middle piece alone gives nothing + byte[] mid = streamPacket(5, 9000, 4000, new byte[4000], 0); + assertEquals(0, st.feed(mid, 0, mid.length)); + assertEquals(0, st.dropped); + // Not a JPEG; it also ends the unfinished frame + byte[] junk = streamPacket(6, 100, 0, new byte[100], 0); + assertEquals(0, st.feed(junk, 0, junk.length)); + assertEquals(2, st.dropped); + assertTrue(st.lastDropReason, st.lastDropReason.contains("no FFD8")); + assertEquals(100, st.lastDropped.length); + // A frame cut short by the next one + byte[] j = jpeg(8000); + byte[] first = streamPacket(7, 8000, 0, Arrays.copyOfRange(j, 0, 4000), 0); + assertEquals(0, st.feed(first, 0, first.length)); + byte[] next = streamPacket(8, 4600, 0, jpeg(4600), 0); + assertEquals(4600, st.feed(next, 0, next.length)); + assertEquals(3, st.dropped); + // Another type (audio?) and a bad length are ignored + byte[] other = streamPacket(9, 4600, 0, jpeg(4600), 0); + other[0] = 1; + assertEquals(0, st.feed(other, 0, other.length)); + assertEquals(1, st.otherTypes); + byte[] bad = streamPacket(10, -5, 0, jpeg(100), 0); + assertEquals(0, st.feed(bad, 0, bad.length)); + assertEquals(1, st.frames); + } + + // ====================== Helpers ====================== + + private static byte[] bytes(String hex) { + String h = hex.replaceAll("\\s", ""); + byte[] b = new byte[h.length() / 2]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) Integer.parseInt(h.substring(2 * i, 2 * i + 2), 16); + } + return b; + } + + private static byte[] concat(byte[]... parts) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (byte[] p : parts) out.write(p, 0, p.length); + return out.toByteArray(); + } + + /** RTP/JPEG packets of a 64x32 test picture (see JpegAssemblerTest). */ + private static List jpegPackets() throws IOException { + ByteArrayOutputStream all = new ByteArrayOutputStream(); + try (InputStream in = JieliTest.class.getResourceAsStream("/rtp/jpeg_q50_64x32.rtp")) { + assertNotNull(in); + byte[] b = new byte[4096]; + int n; + while ((n = in.read(b)) > 0) all.write(b, 0, n); + } + byte[] d = all.toByteArray(); + List out = new ArrayList<>(); + for (int i = 0; i < d.length; ) { + int len = ((d[i] & 0xFF) << 8) | (d[i + 1] & 0xFF); + out.add(Arrays.copyOfRange(d, i + 2, i + 2 + len)); + i += 2 + len; + } + return out; + } + + private static byte[] resource(String name) throws IOException { + ByteArrayOutputStream all = new ByteArrayOutputStream(); + try (InputStream in = JieliTest.class.getResourceAsStream("/rtp/" + name)) { + assertNotNull(in); + byte[] b = new byte[4096]; + int n; + while ((n = in.read(b)) > 0) all.write(b, 0, n); + } + return all.toByteArray(); + } + + private static int freeUdpPort() throws IOException { + try (DatagramSocket s = new DatagramSocket(0)) { + return s.getLocalPort(); + } + } + + private static long now() { + return System.nanoTime() / 1_000_000; + } + + // ====================== Framing ====================== + + @Test + public void encodesAsTheVendorApp() { + // CTP_KEEP_ALIVE {"op":"PUT"} as in the capture of #196 + assertArrayEquals(concat("CTP:".getBytes(StandardCharsets.US_ASCII), bytes("0E 00"), + "CTP_KEEP_ALIVE".getBytes(StandardCharsets.US_ASCII), bytes("0C 00 00 00"), + "{\"op\":\"PUT\"}".getBytes(StandardCharsets.US_ASCII)), + Jieli.encode(Jieli.KEEP_ALIVE, Jieli.put())); + // An empty JSON has length 0 + assertArrayEquals(concat("CTP:".getBytes(StandardCharsets.US_ASCII), bytes("02 00"), + "AB".getBytes(StandardCharsets.US_ASCII), bytes("00 00 00 00")), + Jieli.encode("AB", "")); + } + + @Test + public void commandsAsTheVendorApp() { + assertEquals("{\"op\":\"PUT\",\"param\":{\"ver\":\"25\",\"type\":\"0\"}}", + Jieli.put("ver", "25", "type", "0")); + assertEquals("{\"op\":\"PUT\",\"param\":{\"format\":\"0\",\"h\":\"480\",\"w\":\"640\"," + + "\"fps\":\"25\"}}", Jieli.openStream(0, 640, 480, 25)); + assertEquals("{\"op\":\"PUT\",\"param\":{\"status\":\"1\"}}", Jieli.put("status", "1")); + assertEquals("{\"op\":\"PUT\",\"param\":{\"a\\\"b\":\"c\\\\\"}}", Jieli.put("a\"b", "c\\")); + } + + @Test + public void readsMessagesInPiecesAndTogether() { + byte[] a = Jieli.encode("VIDEO_PARAM", + "{\"op\":\"NOTIFY\",\"param\":{\"w\":\"640\",\"h\":\"480\"}}"); + byte[] b = Jieli.encode(Jieli.KEEP_ALIVE, ""); + byte[] c = Jieli.encode("KEEP_ALIVE_INTERVAL", "{\"param\":{\"timeout\":\"60000\"}}"); + byte[] all = concat(a, b, c); + // Byte by byte + Jieli.Reader r = new Jieli.Reader(); + List got = new ArrayList<>(); + for (int i = 0; i < all.length; i++) { + r.feed(all, i, 1); + Jieli.Message m; + while ((m = r.next()) != null) got.add(m); + } + assertEquals(3, got.size()); + assertEquals("VIDEO_PARAM", got.get(0).topic); + assertEquals("{\"op\":\"NOTIFY\",\"param\":{\"w\":\"640\",\"h\":\"480\"}}", + got.get(0).json); + assertEquals(Jieli.KEEP_ALIVE, got.get(1).topic); + assertEquals("", got.get(1).json); + assertEquals("KEEP_ALIVE_INTERVAL", got.get(2).topic); + assertEquals(0, r.skipped); + // All in one read, split across a length field + r = new Jieli.Reader(); + r.feed(all, 0, a.length + 5); + assertEquals("VIDEO_PARAM", r.next().topic); + assertNull(r.next()); + r.feed(all, a.length + 5, all.length - a.length - 5); + assertEquals(Jieli.KEEP_ALIVE, r.next().topic); + assertEquals("KEEP_ALIVE_INTERVAL", r.next().topic); + assertNull(r.next()); + } + + @Test + public void skipsWhatIsNotCtp() { + byte[] good = Jieli.encode("SD_STATUS", "{\"online\":\"1\"}"); + // Garbage, then a "CTP:" with a topic length out of range, then a real message + byte[] bad = concat("xyCT".getBytes(StandardCharsets.US_ASCII), + "CTP:".getBytes(StandardCharsets.US_ASCII), bytes("FF FF"), good); + Jieli.Reader r = new Jieli.Reader(); + r.feed(bad, 0, bad.length); + Jieli.Message m = r.next(); + assertNotNull(m); + assertEquals("SD_STATUS", m.topic); + assertEquals(10, r.skipped); + assertNull(r.next()); + // A large message grows the buffer + StringBuilder big = new StringBuilder("{\"x\":\""); + for (int i = 0; i < 20000; i++) big.append('a'); + big.append("\"}"); + byte[] large = Jieli.encode("BIG", big.toString()); + r.feed(large, 0, large.length); + assertEquals(big.toString(), r.next().json); + } + + @Test + public void readsJson() { + Map j = Jieli.parseJson("{\"errno\":0,\"op\":\"NOTIFY\",\"param\":" + + "{\"w\":\"640\",\"h\":\"480\",\"format\":\"0\",\"fps\":\"25\",\"l\":[1,\"x\"]}}"); + assertNotNull(j); + assertEquals("NOTIFY", j.get("op")); + assertEquals("0", j.get("errno")); + assertEquals("640", Jieli.param(j, "w")); + assertEquals(25, Jieli.intParam(j, "fps")); + assertEquals("NOTIFY", Jieli.param(j, "op")); + assertEquals(-1, Jieli.intParam(j, "missing")); + assertEquals("a\"b\u00e9", Jieli.parseJson("{\"k\":\"a\\\"b\\u00e9\"}").get("k")); + assertNull(Jieli.parseJson("")); + assertNull(Jieli.parseJson("{\"a\":")); + assertNull(Jieli.parseJson("[1]")); + assertNull(Jieli.parseJson("{} x")); + } + + @Test + public void keepAliveInterval() { + assertEquals(7500, Jieli.keepAliveMs(60000)); + assertEquals(7500, Jieli.keepAliveMs(-1)); + assertEquals(2500, Jieli.keepAliveMs(10000)); + assertEquals(250, Jieli.keepAliveMs(100)); + } + + // ====================== URL ====================== + + @Test + public void urls() throws Exception { + Jieli.Url u = Jieli.Url.parse("jieli://"); + assertEquals("192.168.80.1", u.host); + assertEquals(3333, u.port); + assertEquals(6666, u.rtpPort); + assertEquals(90, u.rotate); + assertEquals(0, Jieli.Url.parse("jieli://?rotate=0").rotate); + assertEquals(90, Jieli.Url.parse("jieli://?rotate=90").rotate); + assertEquals(0, u.width); + assertEquals(Jieli.FORMAT_JPEG, u.format); + u = Jieli.Url.parse(" JIELI://192.168.80.1:3334/?w=1280&h=720&fps=30&rtp=7000&format=1 "); + assertEquals(3334, u.port); + assertEquals(1280, u.width); + assertEquals(720, u.height); + assertEquals(30, u.fps); + assertEquals(7000, u.rtpPort); + assertEquals(Jieli.FORMAT_H264, u.format); + assertEquals("192.168.80.1:3334", u.endpoint()); + assertTrue(Jieli.Url.matches("jieli://x")); + assertTrue(!Jieli.Url.matches("flowufo://")); + for (String bad : new String[] {"jieli://h:0", "jieli://h:x", "jieli://?w=8", + "jieli://?fps=0", "jieli://?rtp=70000", "jieli://?format=2", "jieli://?rotate=45", "jieli://a b", + "jieli://[::1"}) { + try { + Jieli.Url.parse(bad); + fail(bad); + } catch (IOException expected) { + // refused + } + } + assertEquals("toy drone slot: not checked in single view", + StreamProbes.skipReason("jieli://", "rtsp://default", null)); + } + + @Test + public void subnetHints() throws Exception { + InetAddress toy = InetAddress.getByName("192.168.80.1"); + List on80 = Collections.singletonList(InetAddress.getByName("192.168.80.2")); + assertNull(Jieli.subnetHint(toy, on80)); + assertEquals("192.168.80.2 is this phone; the toy is usually 192.168.80.1", + Jieli.subnetHint(InetAddress.getByName("192.168.80.2"), on80)); + assertEquals("This phone is on 192.168.80.x: the toy is usually jieli://192.168.80.1", + Jieli.subnetHint(InetAddress.getByName("192.168.169.1"), on80)); + assertEquals("This network looks like a FLOW-UFO toy: try flowufo://192.168.169.1", + Jieli.subnetHint(toy, + Collections.singletonList(InetAddress.getByName("192.168.169.3")))); + assertEquals("This network looks like a KY UFO RTSP toy: try" + + " rtsp://192.168.1.1:7070/webcam", + Jieli.subnetHint(toy, + Collections.singletonList(InetAddress.getByName("192.168.1.100")))); + assertNull(Jieli.subnetHint(toy, Collections.emptyList())); + } + + // ====================== A fake toy on loopback ====================== + + /** One message the fake toy received, with the time. */ + private static final class Got { + final String topic; + final String json; + final long at; + + Got(Jieli.Message m) { + topic = m.topic; + json = m.json; + at = now(); + } + } + + /** + * Answers like the toy of #196: KEEP_ALIVE_INTERVAL and VIDEO_PARAM after APP_ACCESS, + * empty keepalive replies, an OPEN_RT_STREAM notify; then the video to {@code rtpPort}. + */ + private static final class FakeToy implements AutoCloseable { + final ServerSocket server; + final List got = new CopyOnWriteArrayList<>(); + final CountDownLatch closeStream = new CountDownLatch(1); + final CountDownLatch opened = new CountDownLatch(1); + final int timeoutMs; + final boolean sendVideoParam; + volatile int rtpPort; + volatile boolean sendVideo = true; + /** KEEP_ALIVE_INTERVAL only after the first keepalive, not after APP_ACCESS. */ + volatile boolean lateInterval; + private boolean intervalSent; + private final Thread thread; + + FakeToy(int timeoutMs, boolean sendVideoParam) throws IOException { + this.timeoutMs = timeoutMs; + this.sendVideoParam = sendVideoParam; + server = new ServerSocket(0, 1, LOOPBACK); + thread = new Thread(this::serve, "fake-jieli"); + thread.setDaemon(true); + thread.start(); + } + + int port() { + return server.getLocalPort(); + } + + private void serve() { + try (Socket s = server.accept()) { + InputStream in = s.getInputStream(); + OutputStream out = s.getOutputStream(); + Jieli.Reader r = new Jieli.Reader(); + byte[] buf = new byte[4096]; + int n; + while ((n = in.read(buf)) > 0) { + r.feed(buf, 0, n); + Jieli.Message m; + while ((m = r.next()) != null) { + got.add(new Got(m)); + switch (m.topic) { + case Jieli.APP_ACCESS: + if (!lateInterval) interval(out); + out.write(Jieli.encode("BAT_STATUS", + "{\"op\":\"NOTIFY\",\"param\":{\"level\":\"4\"}}")); + if (sendVideoParam) { + out.write(Jieli.encode(Jieli.VIDEO_PARAM, + "{\"op\":\"NOTIFY\",\"param\":{\"w\":\"320\"," + + "\"h\":\"240\",\"format\":\"0\"," + + "\"fps\":\"15\"}}")); + } + break; + case Jieli.KEEP_ALIVE: + out.write(Jieli.encode(Jieli.KEEP_ALIVE, "")); + if (lateInterval && !intervalSent) interval(out); + break; + case Jieli.OPEN_RT_STREAM: + out.write(Jieli.encode(Jieli.OPEN_RT_STREAM, + "{\"op\":\"NOTIFY\",\"param\":{\"status\":\"1\"}}")); + if (opened.getCount() > 0 && sendVideo) sendVideo(); + opened.countDown(); + break; + case Jieli.CLOSE_RT_STREAM: + closeStream.countDown(); + break; + default: + break; + } + out.flush(); + } + } + } catch (IOException e) { + // Closed + } + } + + private void interval(OutputStream out) throws IOException { + intervalSent = true; + out.write(Jieli.encode(Jieli.KEEP_ALIVE_INTERVAL, + "{\"op\":\"NOTIFY\",\"param\":{\"timeout\":\"" + timeoutMs + "\"}}")); + } + + private void sendVideo() throws IOException { + try (DatagramSocket u = new DatagramSocket(0, LOOPBACK)) { + InetSocketAddress to = new InetSocketAddress(LOOPBACK, rtpPort); + // Something that is not RTP first: counted and logged, not played + byte[] junk = {1, 2, 3}; + u.send(new DatagramPacket(junk, junk.length, to)); + for (byte[] p : jpegPackets()) u.send(new DatagramPacket(p, p.length, to)); + } + } + + List topics() { + List t = new ArrayList<>(); + for (Got g : got) t.add(g.topic); + return t; + } + + @Override + public void close() throws IOException { + server.close(); + } + } + + /** Runs the client on a thread; frames assembled from the RTP it delivers. */ + private static final class Running { + final Jieli.Client client; + final AtomicBoolean keepGoing = new AtomicBoolean(true); + final List frames = new CopyOnWriteArrayList<>(); + final List jpegs = new CopyOnWriteArrayList<>(); + final List states = new CopyOnWriteArrayList<>(); + final CountDownLatch frame = new CountDownLatch(1); + final CountDownLatch ended = new CountDownLatch(1); + volatile Exception failure; + final Thread thread; + + /** False: the "decoder" takes nothing, as with broken packets. */ + final boolean usable; + + Running(String url) throws IOException { + this(url, true); + } + + Running(String url, boolean usable) throws IOException { + this.usable = usable; + client = new Jieli.Client("test", Jieli.Url.parse(url), null, null); + client.localForHint = Collections.emptyList(); + JpegAssembler assembler = new JpegAssembler(new FramePool(4)); + thread = new Thread(() -> { + try { + client.run(keepGoing::get, new Jieli.Listener() { + @Override + public void onState(boolean receiving, String text) { + states.add(text); + } + + @Override + public boolean onRtp(byte[] buf, int offset, int length) { + if (!usable) return false; + assembler.assemble(buf, offset, length, f -> { + frames.add(Arrays.copyOf(f.data(), f.length())); + frame.countDown(); + }); + return true; + } + + @Override + public boolean onJpeg(byte[] buf, int offset, int length) { + if (!usable) return false; + jpegs.add(Arrays.copyOfRange(buf, offset, offset + length)); + frame.countDown(); + return true; + } + }); + } catch (Exception e) { + failure = e; + } finally { + ended.countDown(); + } + }, "jieli-client"); + thread.start(); + } + } + + @Test + public void sessionAsTheVendorApp() throws Exception { + int rtp = freeUdpPort(); + try (FakeToy toy = new FakeToy(2000, true)) { + toy.rtpPort = rtp; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp); + assertTrue("no frame", r.frame.await(5, TimeUnit.SECONDS)); + // A few keepalives at 2000 / 4 = 500 ms + Thread.sleep(1700); + r.keepGoing.set(false); + assertTrue("no CLOSE_RT_STREAM", toy.closeStream.await(3, TimeUnit.SECONDS)); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + assertNull(r.failure); + assertTrue(r.client.streamRequested); + + List t = toy.topics(); + assertEquals(Arrays.asList(Jieli.APP_ACCESS, Jieli.DATE_TIME, Jieli.KEEP_ALIVE, + Jieli.OPEN_RT_STREAM), t.subList(0, 4)); + assertEquals(Jieli.CLOSE_RT_STREAM, t.get(t.size() - 1)); + assertEquals("{\"op\":\"PUT\",\"param\":{\"ver\":\"25\",\"type\":\"0\"}}", + toy.got.get(0).json); + assertTrue(toy.got.get(1).json, + toy.got.get(1).json.matches("\\{\"op\":\"PUT\",\"param\":\\{\"date\":" + + "\"\\d{14}\"}}")); + // The size the toy announced in VIDEO_PARAM + assertEquals(Jieli.openStream(0, 320, 240, 15), toy.got.get(3).json); + assertEquals("{\"op\":\"PUT\",\"param\":{\"status\":\"1\"}}", + toy.got.get(t.size() - 1).json); + assertEquals(1, Collections.frequency(t, Jieli.OPEN_RT_STREAM)); + + List keepAlives = new ArrayList<>(); + for (Got g : toy.got) { + if (g.topic.equals(Jieli.KEEP_ALIVE)) keepAlives.add(g.at); + } + assertTrue("keepalives: " + keepAlives.size(), keepAlives.size() >= 3); + for (int i = 1; i < keepAlives.size(); i++) { + long gap = keepAlives.get(i) - keepAlives.get(i - 1); + assertTrue("gap " + gap, gap >= 400 && gap <= 900); + } + + // The JPEG the RTP/JPEG path makes of the packets + assertArrayEquals(resource("jpeg_q50_64x32.jpg"), r.frames.get(0)); + Map st = r.client.stats(); + @SuppressWarnings("unchecked") + Map ctp = (Map) st.get("ctp"); + assertEquals(500L, ctp.get("keepalive_ms")); + assertEquals("320x240 at 15 fps, format 0", st.get("video_param")); + @SuppressWarnings("unchecked") + Map rtpStats = (Map) st.get("rtp"); + assertEquals(1, rtpStats.get("not_rtp")); + assertTrue((Integer) rtpStats.get("frames") >= 1); + @SuppressWarnings("unchecked") + Map sources = (Map) st.get("sources"); + assertEquals(1, sources.size()); + assertTrue(r.states.get(0), r.states.get(0).startsWith("Connecting to the toy (CTP")); + assertTrue(r.states.toString(), r.states.contains( + "Stream requested, waiting for video on UDP " + rtp + " and " + Jieli.streamPort)); + assertEquals(rtp, st.get("video_port")); + } + } + + @Test + public void defaultSizeAndCloseOnStop() throws Exception { + int rtp = freeUdpPort(); + try (FakeToy toy = new FakeToy(60000, false)) { + toy.rtpPort = rtp; + toy.sendVideo = false; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp); + assertTrue(toy.opened.await(5, TimeUnit.SECONDS)); + // No VIDEO_PARAM: 640x480 at 25 after the wait + Got open = null; + for (Got g : toy.got) if (g.topic.equals(Jieli.OPEN_RT_STREAM)) open = g; + assertNotNull(open); + assertEquals(Jieli.openStream(0, 640, 480, 25), open.json); + assertTrue(open.at - toy.got.get(0).at >= 250); + // close() as RtspSession.stop() does it: CLOSE_RT_STREAM still goes out + r.client.close(); + assertTrue("no CLOSE_RT_STREAM", toy.closeStream.await(3, TimeUnit.SECONDS)); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + assertNull(r.failure); + @SuppressWarnings("unchecked") + Map ctp = (Map) r.client.stats().get("ctp"); + assertEquals(7500L, ctp.get("keepalive_ms")); + } + } + + @Test + public void lateIntervalAppliesAtOnce() throws Exception { + int rtp = freeUdpPort(); + try (FakeToy toy = new FakeToy(2000, true)) { + toy.rtpPort = rtp; + toy.sendVideo = false; + toy.lateInterval = true; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp); + assertTrue(toy.opened.await(5, TimeUnit.SECONDS)); + long until = now() + 3000; + while (now() < until && Collections.frequency(toy.topics(), Jieli.KEEP_ALIVE) < 2) { + Thread.sleep(20); + } + r.keepGoing.set(false); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + List keepAlives = new ArrayList<>(); + for (Got g : toy.got) { + if (g.topic.equals(Jieli.KEEP_ALIVE)) keepAlives.add(g.at); + } + // The second one 500 ms after the first, not 7.5 s + assertTrue("keepalives: " + keepAlives.size(), keepAlives.size() >= 2); + long gap = keepAlives.get(1) - keepAlives.get(0); + assertTrue("gap " + gap, gap >= 400 && gap <= 900); + } + } + + @Test + public void noVideoListensElsewhereAndAsksAgain() throws Exception { + int rtp = freeUdpPort(); + int diag = freeUdpPort(); + Jieli.noVideoMs = 300; + Jieli.reopenMs = 700; + Jieli.diagnosticPorts = new int[] {diag}; + Jieli.diagnosticMs = 500; + try (FakeToy toy = new FakeToy(60000, true)) { + toy.rtpPort = rtp; + toy.sendVideo = false; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp); + assertTrue(toy.opened.await(5, TimeUnit.SECONDS)); + long until = now() + 3000; + while (now() < until && !r.states.contains(r.client.noVideoText())) Thread.sleep(20); + assertTrue(r.states.toString(), r.states.contains(r.client.noVideoText())); + Thread.sleep(100); + try (DatagramSocket u = new DatagramSocket(0, LOOPBACK)) { + byte[] p = {(byte) 0x80, 97, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 9}; + u.send(new DatagramPacket(p, p.length, new InetSocketAddress(LOOPBACK, diag))); + } + until = now() + 3000; + while (now() < until && (Collections.frequency(toy.topics(), Jieli.OPEN_RT_STREAM) < 2 + || r.client.stats().get("diagnostic_ports") == null)) { + Thread.sleep(20); + } + assertTrue(Collections.frequency(toy.topics(), Jieli.OPEN_RT_STREAM) >= 2); + Map st = r.client.stats(); + @SuppressWarnings("unchecked") + Map d = (Map) st.get("diagnostic_ports"); + assertNotNull(d); + @SuppressWarnings("unchecked") + Map one = (Map) d.get(String.valueOf(diag)); + assertEquals(1, one.get("packets")); + // The diagnostic port is freed after diagnosticMs while the session goes on + until = now() + 3000; + boolean freed = false; + while (now() < until && !freed) { + try (DatagramSocket again = new DatagramSocket(diag, LOOPBACK)) { + freed = true; + } catch (IOException e) { + Thread.sleep(50); + } + } + assertTrue("diagnostic port still taken", freed); + r.keepGoing.set(false); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + assertNull(r.failure); + // Nothing from a diagnostic port reaches the video path + assertTrue(r.frames.isEmpty()); + } + } + + @Test + public void unusableVideoIsNotReceiving() throws Exception { + int rtp = freeUdpPort(); + Jieli.reopenMs = 700; + try (FakeToy toy = new FakeToy(60000, true)) { + toy.rtpPort = rtp; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp, false); + assertTrue(toy.opened.await(5, TimeUnit.SECONDS)); + // Packets arrive but the decoder takes none: the toy is asked again (Qodo on #198) + long until = now() + 3000; + while (now() < until + && Collections.frequency(toy.topics(), Jieli.OPEN_RT_STREAM) < 2) { + Thread.sleep(20); + } + assertTrue(Collections.frequency(toy.topics(), Jieli.OPEN_RT_STREAM) >= 2); + for (String st : r.states) assertFalse(st, st.startsWith("Receiving")); + r.keepGoing.set(false); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + } + } + + @Test + public void streamPortVideoIsPlayed() throws Exception { + int rtp = freeUdpPort(); + try (FakeToy toy = new FakeToy(60000, true)) { + toy.rtpPort = rtp; + toy.sendVideo = false; + Running r = new Running("jieli://127.0.0.1:" + toy.port() + "?rtp=" + rtp); + assertTrue(toy.opened.await(5, TimeUnit.SECONDS)); + byte[] j = jpeg(4600); + try (DatagramSocket u = new DatagramSocket(0, LOOPBACK)) { + byte[] p = streamPacket(1, 4600, 0, j, 20); + u.send(new DatagramPacket(p, p.length, + new InetSocketAddress(LOOPBACK, Jieli.streamPort))); + } + assertTrue(r.frame.await(3, TimeUnit.SECONDS)); + assertArrayEquals(j, r.jpegs.get(0)); + assertTrue(r.frames.isEmpty()); + long until = now() + 3000; + while (now() < until && !r.client.stats().containsKey("video_port")) Thread.sleep(20); + assertEquals(Jieli.streamPort, r.client.stats().get("video_port")); + r.keepGoing.set(false); + assertTrue(r.ended.await(3, TimeUnit.SECONDS)); + assertNull(r.failure); + } + } + + @Test + public void unreachableToyFails() throws Exception { + int port; + try (ServerSocket s = new ServerSocket(0, 1, LOOPBACK)) { + port = s.getLocalPort(); + } + Jieli.Client c = new Jieli.Client("test", + Jieli.Url.parse("jieli://127.0.0.1:" + port + "?rtp=" + freeUdpPort()), null, null); + c.localForHint = Collections.emptyList(); + try { + c.run(() -> true, new Jieli.Listener() { + @Override + public void onState(boolean receiving, String text) {} + + @Override + public boolean onRtp(byte[] buf, int offset, int length) { + return false; + } + }); + fail("connected"); + } catch (IOException e) { + assertTrue(e.getMessage(), + e.getMessage().startsWith("Cannot reach the toy's CTP port")); + } + } +} diff --git a/android/app/src/test/java/com/openipc/decoder/RotateTest.java b/android/app/src/test/java/com/openipc/decoder/RotateTest.java new file mode 100644 index 00000000..b84d9998 --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/RotateTest.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * RotateTest.java — MJPEG frames turned for the sideways KY UFO camera (#200) + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import org.junit.Test; + +import java.io.IOException; + +public class RotateTest { + private static final float EPS = 0.01f; + + @Test + public void ninetyForTheCooingdvWebcamPathOnly() throws IOException { + assertEquals(270, RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam").rotate); + assertEquals(270, RtspSession.Url.parse("rtsp://10.0.0.9:7070/WEBCAM/").rotate); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1:554/webcam").rotate); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1:7070/live").rotate); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.10:554/stream=0").rotate); + } + + @Test + public void optionOverridesAndIsNotSentToTheCamera() throws IOException { + RtspSession.Url u = RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam?rotate=0"); + assertEquals(0, u.rotate); + assertEquals(7099, u.heartbeatPort); + assertEquals("rtsp://192.168.1.1:7070/webcam", u.requestUrl); + u = RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam?Rotate=270&heartbeat=off"); + assertEquals(270, u.rotate); + assertEquals(0, u.heartbeatPort); + assertEquals("rtsp://192.168.1.1:7070/webcam", u.requestUrl); + // Any camera, other parameters kept in their order + u = RtspSession.Url.parse("rtsp://u:p@cam/x?a=1&rotate=180&b=2"); + assertEquals(180, u.rotate); + assertEquals("rtsp://cam/x?a=1&b=2", u.requestUrl); + u = RtspSession.Url.parse("rtsp://cam/x?rotate=90"); + assertEquals(90, u.rotate); + assertEquals("rtsp://cam/x", u.requestUrl); + for (String bad : new String[] {"rtsp://cam/x?rotate=45", "rtsp://cam/x?rotate=", + "rtsp://cam/x?rotate", "rtsp://cam/x?rotate=-90", "rtsp://cam/x?rotate=360", + "rtsp://cam/x?rotate=left"}) { + try { + RtspSession.Url.parse(bad); + fail(bad); + } catch (IOException expected) { + // ok + } + } + } + + @Test + public void quarterTurnSwapsTheShownSize() { + assertArrayEquals(new int[] {640, 480}, JpegRenderer.shownSize(480, 640, 90)); + assertArrayEquals(new int[] {640, 480}, JpegRenderer.shownSize(480, 640, 270)); + assertArrayEquals(new int[] {480, 640}, JpegRenderer.shownSize(480, 640, 180)); + assertArrayEquals(new int[] {480, 640}, JpegRenderer.shownSize(480, 640, 0)); + } + + @Test + public void turnedPortraitSourceFillsALandscapeScreen() { + // The K200's 480x640 on a 1600x720 phone: 960x720, not a 540x720 strip + int[] shown = JpegRenderer.shownSize(480, 640, 90); + assertArrayEquals(new int[] {960, 720}, Decoder.fitInside(shown[0], shown[1], 1600, 720)); + assertArrayEquals(new int[] {540, 720}, Decoder.fitInside(480, 640, 1600, 720)); + } + + @Test + public void ninetyCounterClockwiseMovesTheCorners() { + // A 960x720 surface; the frame's top left goes to the bottom left, its top right to + // the top left: the top edge becomes the left one + float[] t = JpegRenderer.turn(90, 960, 720); + assertPoint(0, 720, JpegRenderer.map(t, 0, 0)); + assertPoint(0, 0, JpegRenderer.map(t, 1, 0)); + assertPoint(960, 720, JpegRenderer.map(t, 0, 1)); + assertPoint(960, 0, JpegRenderer.map(t, 1, 1)); + assertPoint(480, 360, JpegRenderer.map(t, 0.5f, 0.5f)); + } + + @Test + public void otherTurnsFillTheSurfaceToo() { + float[] t = JpegRenderer.turn(270, 960, 720); + assertPoint(960, 0, JpegRenderer.map(t, 0, 0)); + assertPoint(0, 720, JpegRenderer.map(t, 1, 1)); + t = JpegRenderer.turn(180, 960, 720); + assertPoint(960, 720, JpegRenderer.map(t, 0, 0)); + assertPoint(0, 0, JpegRenderer.map(t, 1, 1)); + t = JpegRenderer.turn(0, 960, 720); + assertPoint(0, 0, JpegRenderer.map(t, 0, 0)); + assertPoint(960, 720, JpegRenderer.map(t, 1, 1)); + } + + private static void assertPoint(float x, float y, float[] p) { + assertEquals(x, p[0], EPS); + assertEquals(y, p[1], EPS); + } +} diff --git a/android/app/src/test/java/com/openipc/decoder/RtspSessionTest.java b/android/app/src/test/java/com/openipc/decoder/RtspSessionTest.java index 1c1e6799..3e282f4b 100644 --- a/android/app/src/test/java/com/openipc/decoder/RtspSessionTest.java +++ b/android/app/src/test/java/com/openipc/decoder/RtspSessionTest.java @@ -428,6 +428,51 @@ public void handshakeWithAudio() throws Exception { assertEquals(1, rec.audio.get()); } + @Test + public void heartbeatWhilePlayingStopsWithTheSlot() throws Exception { + // #187: cooingdv toys get 01 01 on UDP while their RTSP stream plays + UdpHeartbeat.intervalMs = 50; + try (DatagramSocket toy = new DatagramSocket(0, InetAddress.getLoopbackAddress())) { + toy.setSoTimeout(3000); + runUntilVideo(server.url(null) + "?heartbeat=" + toy.getLocalPort(), false); + byte[] buf = new byte[16]; + for (int i = 0; i < 2; i++) { + DatagramPacket p = new DatagramPacket(buf, buf.length); + toy.receive(p); + assertEquals(2, p.getLength()); + assertEquals(1, buf[0]); + assertEquals(1, buf[1]); + } + // The option is ours: the camera never sees it + assertEquals("DESCRIBE " + base() + " RTSP/1.0", server.methods().get(0)); + java.util.Map m = session.heartbeatStats(); + assertTrue((Integer) m.get("sent") >= 2); + assertEquals(true, m.get("running")); + session.stop(); + toy.setSoTimeout(100); + try { + while (true) toy.receive(new DatagramPacket(buf, buf.length)); + } catch (java.net.SocketTimeoutException expected) { + // drained + } + toy.setSoTimeout(300); + try { + toy.receive(new DatagramPacket(buf, buf.length)); + throw new AssertionError("heartbeat after stop"); + } catch (java.net.SocketTimeoutException expected) { + // ok + } + } finally { + UdpHeartbeat.intervalMs = 1000; + } + } + + @Test + public void noHeartbeatForOrdinaryCameras() throws Exception { + runUntilVideo(server.url(null), false); + assertEquals(null, session.heartbeatStats()); + } + @Test public void noAudioTrackMeansNoAudioSetup() throws Exception { // #18 diff --git a/android/app/src/test/java/com/openipc/decoder/UdpHeartbeatTest.java b/android/app/src/test/java/com/openipc/decoder/UdpHeartbeatTest.java new file mode 100644 index 00000000..5004a161 --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/UdpHeartbeatTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * UdpHeartbeatTest.java — the cooingdv heartbeat: when RTSP slots send it, and its schedule + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketTimeoutException; +import java.util.Arrays; +import java.util.Map; + +public class UdpHeartbeatTest { + @After + public void restoreInterval() { + UdpHeartbeat.intervalMs = 1000; + } + + @Test + public void onForTheCooingdvWebcamPath() throws IOException { + // Any host: the toy is usually 192.168.1.1, but not always + RtspSession.Url u = RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam"); + assertEquals(7099, u.heartbeatPort); + assertEquals("rtsp://192.168.1.1:7070/webcam", u.requestUrl); + assertEquals(7099, RtspSession.Url.parse("rtsp://10.0.0.9:7070/webcam").heartbeatPort); + assertEquals(7099, RtspSession.Url.parse("rtsp://10.0.0.9:7070/webcam/").heartbeatPort); + // Not the path or not the port: off + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1:554/webcam").heartbeatPort); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1:7070/live").heartbeatPort); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.1/webcam").heartbeatPort); + assertEquals(0, RtspSession.Url.parse("rtsp://192.168.1.10:554/stream=0").heartbeatPort); + } + + @Test + public void optionTurnsItOnOrOffAndIsNotSentToTheCamera() throws IOException { + RtspSession.Url u = RtspSession.Url.parse("rtsp://192.168.1.1:554/live?heartbeat=7099"); + assertEquals(7099, u.heartbeatPort); + assertEquals("rtsp://192.168.1.1:554/live", u.requestUrl); + u = RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam?heartbeat=off"); + assertEquals(0, u.heartbeatPort); + assertEquals("rtsp://192.168.1.1:7070/webcam", u.requestUrl); + u = RtspSession.Url.parse("rtsp://192.168.1.1:7070/webcam?HeartBeat=0"); + assertEquals(0, u.heartbeatPort); + // Other parameters stay, in their order + u = RtspSession.Url.parse("rtsp://u:p@cam/x?a=1&heartbeat=7000&b=2#frag"); + assertEquals(7000, u.heartbeatPort); + assertEquals("rtsp://cam/x?a=1&b=2", u.requestUrl); + u = RtspSession.Url.parse("rtsp://cam/x?a=1"); + assertEquals(0, u.heartbeatPort); + assertEquals("rtsp://cam/x?a=1", u.requestUrl); + for (String bad : new String[] {"rtsp://cam/x?heartbeat=abc", "rtsp://cam/x?heartbeat=", + "rtsp://cam/x?heartbeat=70000", "rtsp://cam/x?heartbeat=-5"}) { + try { + RtspSession.Url.parse(bad); + fail(bad); + } catch (IOException expected) { + // ok + } + } + } + + @Test + public void sendsOneOneEverySecondUntilClosed() throws Exception { + UdpHeartbeat.intervalMs = 100; + InetAddress lo = InetAddress.getLoopbackAddress(); + try (DatagramSocket toy = new DatagramSocket(0, lo)) { + toy.setSoTimeout(2000); + UdpHeartbeat h = new UdpHeartbeat("test", lo, + new InetSocketAddress(lo, toy.getLocalPort())); + h.start(true); + assertTrue(h.running()); + long[] at = new long[4]; + byte[] buf = new byte[16]; + int port = -1; + for (int i = 0; i < at.length; i++) { + DatagramPacket p = new DatagramPacket(buf, buf.length); + toy.receive(p); + at[i] = System.nanoTime() / 1_000_000; + assertArrayEquals(UdpHeartbeat.PAYLOAD, Arrays.copyOf(buf, p.getLength())); + // Always from the same ephemeral port + if (port < 0) port = p.getPort(); + assertEquals(port, p.getPort()); + } + // About one interval apart: the scheduler neither bursts nor stalls + long span = at[at.length - 1] - at[0]; + assertTrue("span " + span, span >= 3 * 100 - 40 && span < 3 * 100 + 400); + assertTrue(h.sent() >= 4); + h.close(); + assertFalse(h.running()); + // What was on its way is read; then nothing more + toy.setSoTimeout(50); + try { + while (true) toy.receive(new DatagramPacket(buf, buf.length)); + } catch (SocketTimeoutException expected) { + // drained + } + toy.setSoTimeout(400); + try { + toy.receive(new DatagramPacket(buf, buf.length)); + fail("heartbeat after close"); + } catch (SocketTimeoutException expected) { + // ok + } + Map m = UdpHeartbeat.stats(h, 3, 0); + assertEquals(3 + h.sent(), m.get("sent")); + assertEquals(false, m.get("running")); + assertEquals("01 01", m.get("payload")); + assertEquals(5, UdpHeartbeat.stats(null, 5, 0).get("sent")); + } + } +} diff --git a/docs/debug.md b/docs/debug.md index 12470c0e..ead0156c 100644 --- a/docs/debug.md +++ b/docs/debug.md @@ -14,13 +14,14 @@ While the app runs it serves its debug information over HTTP on port **8085**, for a browser or `curl` on the same network. It is always on and read-only. -| Path | Content | -|---|---| -| `/` (or `/index.html`) | the [page](#page): snapshot with the text of each cell | -| `/stats.json` | the state as JSON, pretty-printed, see [fields](#statsjson-fields) | -| `/snapshot.jpg` | a picture of the screen, see [Snapshot](#snapshot); `?labels=0`: frames only | -| `/log` | the last 500 lines of the app's log, see [Log](#log) | -| `/report.txt` | a header, `/stats.json` and `/log` in one file, downloaded as `report-YYYYMMDD-HHMMSS.txt`, see [Save report](#save-report) | +| Path | Content | +|------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `/` (or `/index.html`) | the [page](#page): snapshot with the text of each cell | +| `/stats.json` | the state as JSON, pretty-printed, see [fields](#statsjson-fields) | +| `/snapshot.jpg` | a picture of the screen, see [Snapshot](#snapshot); `?labels=0`: frames only | +| `/log` | the last 500 lines of the app's log, see [Log](#log) | +| `/report.txt` | a header, `/stats.json` and `/log` in one file, downloaded as `report-YYYYMMDD-HHMMSS.txt`, see [Save report](#save-report) | +| `/jieli-frame.jpg`, `/jieli-dropped.bin` | the last JPEG of a `jieli://` slot in the single view as the toy sent it, and the last frame that could not be put together ([toys.md](toys.md#hf--jieli-toys)) | - Only clients on the local network are answered; others get `403`. The rule and what is hidden: [security.md](security.md#debug-server). - `GET` only (`405` otherwise), one request per connection, `Cache-Control: no-store`. @@ -46,13 +47,13 @@ The device's addresses are in `addresses` of `/stats.json`, and the [U cell](usa - **Snapshot 3 s / 5 s / 10 s**: how often the picture is refreshed (default 5 s). `/stats.json` is fetched every 1.5 s. Both only while the tab is visible. - On a screen narrower than 720 px the snapshot takes the full width and the cells are listed below it. -| Status dot / header note | Meaning | -|---|---| -| green | the app answers | -| orange, `UI not responding within 1000 ms` | the UI thread is stuck (an ANR); only the device facts are reported | -| orange, `no activity` / `App in background` | no activity behind the server, or the app is paused | +| Status dot / header note | Meaning | +|----------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------| +| green | the app answers | +| orange, `UI not responding within 1000 ms` | the UI thread is stuck (an ANR); only the device facts are reported | +| orange, `no activity` / `App in background` | no activity behind the server, or the app is paused | | red, `video decoder not responding — restart the device` | `decoder_stuck`: frames arrive in several cells, none is decoded for 20 s ([#145](https://github.com/OpenIPC/decoder/issues/145)) | -| red, `offline` | the page cannot reach the app | +| red, `offline` | the page cannot reach the app | ## Snapshot @@ -76,101 +77,101 @@ In the single view (and the U view) the page and the snapshot still show the who ### Top level -| Field | Type | Meaning | -|---|---|---| -| `app` | string | `OpenIPC Decoder` | -| `version` | string | versionName, e.g. `2.10` | -| `git` | string | short commit hash of the build | -| `time` | string | device time, `yyyy-MM-dd HH:mm:ss` | -| `uptime_s` | number | seconds since the device booted | -| `device` | string | manufacturer, model, SoC (Android 12+; else `Build.HARDWARE`) and board | -| `android` | string | release and SDK level, e.g. `10 (SDK 29)` | -| `screen` | string | display size in pixels, `WxH` | -| `addresses` | list | `http://:8085/` for every address of the device | -| `server` | string | `listening`, or why port 8085 could not be opened | -| `decoders` | list or `"loading"` | the device's video decoders, see [decoders](#decoders) | -| `ui_thread` | string | `ok`, or `not responding within 1000 ms` (then only the fields above are present) | -| `mode` | string | `single`, `multi-view`, `slides` (multi-view with Slides on), `udp` (U cell full-screen), `paused` (app in background); `no activity` when no activity is attached | -| `active_slot` | number | active slot, 1–8 | -| `decoder` | object | multi-view only: `capacity` (macroblocks/s), `learnt` (bool), `failed_load` (macroblocks/s, 0: none), see [architecture.md](architecture.md#decode-planner) | -| `decoder_stuck` | bool | multi-view only: frames arrive in at least two cells and none is decoded for 20 s | -| `udp` | object | U view only: `port` (5600) and `addresses` (the device's IPs to send to) | -| `single` | object | single view only, see [single](#single) | -| `cells` | list | multi-view and U view, see [cells](#cells) | -| `slots` | list | single view and U view: every grid cell with the stream checks, see [slots](#slots) | +| Field | Type | Meaning | +|-----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `app` | string | `OpenIPC Decoder` | +| `version` | string | versionName, e.g. `2.10` | +| `git` | string | short commit hash of the build | +| `time` | string | device time, `yyyy-MM-dd HH:mm:ss` | +| `uptime_s` | number | seconds since the device booted | +| `device` | string | manufacturer, model, SoC (Android 12+; else `Build.HARDWARE`) and board | +| `android` | string | release and SDK level, e.g. `10 (SDK 29)` | +| `screen` | string | display size in pixels, `WxH` | +| `addresses` | list | `http://:8085/` for every address of the device | +| `server` | string | `listening`, or why port 8085 could not be opened | +| `decoders` | list or `"loading"` | the device's video decoders, see [decoders](#decoders) | +| `ui_thread` | string | `ok`, or `not responding within 1000 ms` (then only the fields above are present) | +| `mode` | string | `single`, `multi-view`, `slides` (multi-view with Slides on), `udp` (U cell full-screen), `paused` (app in background); `no activity` when no activity is attached | +| `active_slot` | number | active slot, 1–8 | +| `decoder` | object | multi-view only: `capacity` (macroblocks/s), `learnt` (bool), `failed_load` (macroblocks/s, 0: none), see [architecture.md](architecture.md#decode-planner) | +| `decoder_stuck` | bool | multi-view only: frames arrive in at least two cells and none is decoded for 20 s | +| `udp` | object | U view only: `port` (5600) and `addresses` (the device's IPs to send to) | +| `single` | object | single view only, see [single](#single) | +| `cells` | list | multi-view and U view, see [cells](#cells) | +| `slots` | list | single view and U view: every grid cell with the stream checks, see [slots](#slots) | ### decoders One entry per video decoder and MIME type (encoders are left out). -| Field | Meaning | -|---|---| -| `name` | codec name as Android lists it | -| `type` | MIME type, e.g. `video/hevc` | -| `hardware` | hardware-accelerated (Android 10+) | -| `adaptive` | supports adaptive playback | -| `max_instances` | instances the codec declares (Android 6+) | -| `fps_1080p` | highest declared frame rate at 1920×1080, when 1080p is supported | +| Field | Meaning | +|-----------------|-------------------------------------------------------------------| +| `name` | codec name as Android lists it | +| `type` | MIME type, e.g. `video/hevc` | +| `hardware` | hardware-accelerated (Android 10+) | +| `adaptive` | supports adaptive playback | +| `max_instances` | instances the codec declares (Android 6+) | +| `fps_1080p` | highest declared frame rate at 1920×1080, when 1080p is supported | ### single -| Field | Meaning | -|---|---| -| `url` | the slot URL, redacted | -| `transport` | in use, as in [cells](#cells) | -| `transport_mode` | the slot's setting: `TCP`, `UDP`, `AUTO`, `MULTICAST` | -| `video` | a frame was shown within the last 2 s | -| `status` | the status text on screen, or the session state | -| `size` | `WxH`, once known | -| `codec` | `H.264`, `H.265` or `MJPEG` | -| `audio` | audio codec: `NONE`, `AAC`, `LATM`, `OPUS`, `PCMU`, `PCMA`, `PCM` | -| `out_fps` | frames shown in the last second | -| `decode` | `full`, or `key frames` with Slides | +| Field | Meaning | +|------------------|-------------------------------------------------------------------| +| `url` | the slot URL, redacted | +| `transport` | in use, as in [cells](#cells) | +| `transport_mode` | the slot's setting: `TCP`, `UDP`, `AUTO`, `MULTICAST` | +| `video` | a frame was shown within the last 2 s | +| `status` | the status text on screen, or the session state | +| `size` | `WxH`, once known | +| `codec` | `H.264`, `H.265` or `MJPEG` | +| `audio` | audio codec: `NONE`, `AAC`, `LATM`, `OPUS`, `PCMU`, `PCMA`, `PCM` | +| `out_fps` | frames shown in the last second | +| `decode` | `full`, or `key frames` with Slides | ### cells One entry per grid cell; `slot` is 1–8 or `U`. An empty cell has only `slot`, `video: false` and `text: "not configured"`. -| Field | Meaning | -|---|---| -| `slot` | 1–8, or `U` | -| `video` | the cell showed a frame within the last 2 s | -| `text` | the corner text of the cell, see [below](#corner-text-in-multi-view) | -| `url` | the slot URL, redacted (`udp://@:5600` for U) | -| `transport` | in use: `TCP`, `UDP`, `multicast`, `HTTP`, `UDP push`, `multicast push`; after a fallback `TCP (auto)` or `TCP (multicast)`; the mode alone before anything plays | -| `transport_mode` | the slot's setting (`UDP` for U) | -| `out_fps` | frames decoded per second | -| `in_fps` | frames received per second | -| `lost_fps` | frames per second the decoder lost (queue full, no input buffer); network losses do not count | -| `nal_dropped` | NAL units dropped because one of their packets was lost | -| `nal_damaged` | slices passed on cut short at a lost piece; absent while damaged decoding is off | -| `decode` | `full` or `key frames` | -| `load` | macroblocks/s the cell costs in its current mode, once size and rate are known | -| `paused` | paused by the decode planner (`Paused: decoder overloaded`) | -| `audio` | this cell's sound plays | +| Field | Meaning | +|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `slot` | 1–8, or `U` | +| `video` | the cell showed a frame within the last 2 s | +| `text` | the corner text of the cell, see [below](#corner-text-in-multi-view) | +| `url` | the slot URL, redacted (`udp://@:5600` for U) | +| `transport` | in use: `TCP`, `UDP`, `multicast`, `HTTP`, `UDP push`, `multicast push`; after a fallback `TCP (auto)` or `TCP (multicast)`; the mode alone before anything plays | +| `transport_mode` | the slot's setting (`UDP` for U) | +| `out_fps` | frames decoded per second | +| `in_fps` | frames received per second | +| `lost_fps` | frames per second the decoder lost (queue full, no input buffer); network losses do not count | +| `nal_dropped` | NAL units dropped because one of their packets was lost | +| `nal_damaged` | slices passed on cut short at a lost piece; absent while damaged decoding is off | +| `decode` | `full` or `key frames` | +| `load` | macroblocks/s the cell costs in its current mode, once size and rate are known | +| `paused` | paused by the decode planner (`Paused: decoder overloaded`) | +| `audio` | this cell's sound plays | ### slots Single view and U view: every cell of the grid as the debug page shows it (3×3 in the U view or when the active slot is 5–8). -| Field | Meaning | -|---|---| -| `slot` | 1–8, or `U` | -| `on_screen` | this is the camera on screen | -| `url`, `transport_mode` | as in [cells](#cells), for configured slots only | -| `transport` | on screen: in use; otherwise what a check goes over (`check over TCP` for TCP and Auto slots, `check over UDP`, `check over multicast`, `UDP listen`, `HTTP`, `UDP push` for U) | -| `video`, `text` | on screen: as it plays; otherwise `false` and the check result or why there is none | -| `probe` | not on screen: the check, see below | - -| `probe` field | Meaning | -|---|---| -| `state` | `ok`, `error`, `checking`, `waiting`, `not checked` | -| `checked_s_ago` | seconds since the last check began | -| `summary` | result text | -| `codec`, `size` | when found | -| `fps`, `kbps`, `key_frames` | measured during the check | - -The FLOW-UFO test build ([toys.md](toys.md#flow-ufo)) adds a `flowufo` object per cell and in `single`; it is not in the releases. +| Field | Meaning | +|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `slot` | 1–8, or `U` | +| `on_screen` | this is the camera on screen | +| `url`, `transport_mode` | as in [cells](#cells), for configured slots only | +| `transport` | on screen: in use; otherwise what a check goes over (`check over TCP` for TCP and Auto slots, `check over UDP`, `check over multicast`, `UDP listen`, `HTTP`, `UDP push` for U) | +| `video`, `text` | on screen: as it plays; otherwise `false` and the check result or why there is none | +| `probe` | not on screen: the check, see below | + +| `probe` field | Meaning | +|-----------------------------|-----------------------------------------------------| +| `state` | `ok`, `error`, `checking`, `waiting`, `not checked` | +| `checked_s_ago` | seconds since the last check began | +| `summary` | result text | +| `codec`, `size` | when found | +| `fps`, `kbps`, `key_frames` | measured during the check | + +Toy drone slots ([toys.md](toys.md#flow-ufo)) add a `flowufo` object per cell and in `single` (`jieli` for `jieli://` slots, [toys.md](toys.md#hf--jieli-toys)), and a `heartbeat` object for RTSP slots that send the KY UFO heartbeat, and `rotate` (degrees counter-clockwise) where MJPEG frames are turned ([toys.md](toys.md#ky-ufo-rtsp-toys)). ## Log @@ -206,16 +207,16 @@ Uptime: 1h 2m 3s Examples: `1920×1080 H.265 25/25 fps` at full rate, `1920×1080 H.265 key frames · 0.5 fps` on key frames only. -| Part | Meaning | -|---|---| -| `1920×1080 H.265` | stream size and codec (`H.264`, `H.265`, `MJPEG`) | -| `25/25 fps` | full rate: decoded / received frames per second | -| `key frames · 0.5 fps` | key frames only, with the frames shown per second (MJPEG: `reduced`, one frame per second) | -| `· software` | key frames on a software decoder, beyond the hardware instance limit (MJPEG: BitmapFactory) | -| `· damaged` | frames with lost pieces are decoded with the damage concealed | -| `Connecting…`, `Buffering…`, `Disconnected` | the session state before video arrives | -| `Decoder limit: N × H.265` | no decoder instance left, not even a software one | -| `Paused: decoder overloaded` | paused by the decode planner, tried again later | +| Part | Meaning | +|---------------------------------------------|---------------------------------------------------------------------------------------------| +| `1920×1080 H.265` | stream size and codec (`H.264`, `H.265`, `MJPEG`) | +| `25/25 fps` | full rate: decoded / received frames per second | +| `key frames · 0.5 fps` | key frames only, with the frames shown per second (MJPEG: `reduced`, one frame per second) | +| `· software` | key frames on a software decoder, beyond the hardware instance limit (MJPEG: BitmapFactory) | +| `· damaged` | frames with lost pieces are decoded with the damage concealed | +| `Connecting…`, `Buffering…`, `Disconnected` | the session state before video arrives | +| `Decoder limit: N × H.265` | no decoder instance left, not even a software one | +| `Paused: decoder overloaded` | paused by the decode planner, tried again later | ## Bug reports diff --git a/docs/toys.md b/docs/toys.md index 582a3057..bc62e888 100644 --- a/docs/toys.md +++ b/docs/toys.md @@ -2,7 +2,11 @@ - [How such toys stream](#how-such-toys-stream) - [Connecting](#connecting) +- [Toy families by Wi‑Fi name](#toy-families-by-wifi-name) - [FLOW-UFO](#flow-ufo) +- [KY UFO RTSP toys](#ky-ufo-rtsp-toys) +- [HF / Jieli toys](#hf--jieli-toys) +- [Credits](#credits) - [An unknown toy](#an-unknown-toy) - [What to send in a report](#what-to-send-in-a-report) - [Tested toys](#tested-toys) @@ -22,7 +26,7 @@ Cheap drones, cars and other toys with a Wi‑Fi camera usually work the same wa - The video is often **not a standard stream**: no RTSP, no plain RTP. Many toys send JPEG frames over UDP in their own framing and start only after the client has sent a request; some also expect regular replies while they stream. A standard player then sees nothing, and this app stays in *Buffering*. - Flight or drive commands go over the same network. **This app never sends any**: it only asks for and receives video. -Whether a toy can be played here depends on its protocol. Toys that do offer RTSP or HTTP MJPEG play like any camera ([transports.md](transports.md)); others need a transport of their own, like [FLOW-UFO](#flow-ufo). +Whether a toy can be played here depends on its protocol. Toys that do offer RTSP or HTTP MJPEG play like any camera ([transports.md](transports.md)); others need a transport of their own, like [FLOW-UFO](#flow-ufo) or [HF / Jieli](#hf--jieli-toys). ## Connecting @@ -32,39 +36,113 @@ Whether a toy can be played here depends on its protocol. Toys that do offer RTS 4. Close the vendor app: the toy usually streams to one client only. 5. While the phone is only on the toy's network, open the [debug pages](debug.md) on the phone itself: `http://127.0.0.1:8085/stats.json` and `http://127.0.0.1:8085/log`. Simpler: a **long press on Debug** in the menu [saves a report](debug.md#save-report) (stats and log in one file) to `Download/OpenIPC`; send it later, when the phone has internet again. +## Toy families by Wi‑Fi name + +The Wi‑Fi name and the address the phone gets usually tell the family. The toy is almost always the `.1` address of that network, **not** the phone's own address. + +| Wi‑Fi name | Phone gets | Family | Slot URL | Status | +|----------------------------------------------------------------|-----------------|----------------------------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | `192.168.169.x` | FLOW-UFO (BL60x camera; turbodrone calls it `wifi_uav`) | `flowufo://192.168.169.1` (UDP 8800 and 8801) | works since 2.15, checked with a D16 ([#187](https://github.com/OpenIPC/decoder/issues/187)) | +| `FLY-UFO-xxxxxx` and other cooingdv toys (KY UFO, RC UFO apps) | `192.168.1.x` | KY UFO RTSP | `rtsp://192.168.1.1:7070/webcam` (with the heartbeat to UDP 7099) | works since 2.15, checked with a K200; the picture is turned upright ([#187](https://github.com/OpenIPC/decoder/issues/187), [#200](https://github.com/OpenIPC/decoder/issues/200)) | +| `HF_xxxxxxxx` | `192.168.80.x` | Jieli / HF (Jieli AC79 camera, vendor app *DVRunning 2*) | `jieli://192.168.80.1` (CTP on TCP 3333, video to the phone's UDP 2224 or 6666) | **experimental** (2.15): YL3 video on UDP 2224 arrives and plays, picture quality still being checked ([#187](https://github.com/OpenIPC/decoder/issues/187)) | + +When a `flowufo://` or `jieli://` slot points at the phone itself, or at the network of another family, its status says so (for example *192.168.1.100 is this phone; the toy is usually 192.168.1.1*, *This network looks like a KY UFO RTSP toy: try rtsp://192.168.1.1:7070/webcam* or, on a `192.168.80.x` network, *try jieli://192.168.80.1*). + ## FLOW-UFO -| | | -|---|---| -| Wi‑Fi name | `FLOW-UFO-xxxxxx` | -| Camera | BL60x Wi‑Fi camera module | -| Toy address | `192.168.169.1`, UDP port 8800 | -| Slot URL | `flowufo://192.168.169.1:8800` (or `flowufo://` alone) | -| Status | **in testing**, not in the releases: issue [#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188) | -| Test build | [toys-187-test2](https://github.com/OpenIPC/decoder/releases/tag/toys-187-test2) (2.11 with the FLOW-UFO, Jieli and KY UFO changes; installs over the release and keeps the settings) | +| | | +|-------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Wi‑Fi name | `FLOW-UFO-xxxxxx`, `FLOW_xxxxxx` | +| Camera | BL60x Wi‑Fi camera module (BL608 or BL618 engine) | +| Toy address | `192.168.169.1`, UDP port 8800 (BL608) or 8801 (BL618); the toy answers from port 1234 | +| Slot URL | `flowufo://192.168.169.1:8800` (or `flowufo://` alone) | +| Status | in the releases since 2.15; checked with a D16 ([#187](https://github.com/OpenIPC/decoder/issues/187), PR [#215](https://github.com/OpenIPC/decoder/pull/215)) | What the toy does, as far as is known: -- It sends nothing until a client sends it a short **probe** on UDP port 8800. The app repeats the probe until video arrives, and starts probing again when the toy goes quiet for 3 s. +- It sends nothing until a client sends it a short **probe**. The vendor library starts two engines, BL608 on UDP port 8800 and BL618 on 8801, and keeps the one that delivers. The app does the same: a slot on 8800 (or without a port) probes **both 8800 and 8801** at once, each from a socket of its own, and locks onto the port whose packets arrive first; the acknowledgements then go there. A slot with another port, `8801` included, probes only that port. The probes repeat until video arrives (every 100 ms; 200 ms on 8801), and start again on both ports when the toy goes quiet for 3 s. - The video is **MJPEG**: each frame is cut into fragments of about 1 KB. The fragments carry the compressed picture without JPEG headers; the app adds standard headers and shows the frames through its normal MJPEG path. -- The client **acknowledges** regularly which fragments it has, and can ask for a frame again. The app does this; whether the toy really needs it is one of the open questions of the test. -- Video only, no sound. No flight command is ever sent. +- The client **acknowledges** regularly which fragments of the last frames it has (every 25 ms; on 8801 every 50 ms and only when there is something to acknowledge), and asks for the next frame when the stream pauses. Frames 5 or more behind the newest are left out of the acknowledgement. +- Bytes 0x34/0x35 of each fragment are the main and flow camera status; the stats show them as `camera_status`. +- Video only, no sound. **No flight command is ever sent**: the vendor app and turbodrone put a (neutral) stick command into every acknowledgement; this app leaves that field empty. + +URL options, for comparing with what the toy expects (as with `udp://`, separated by `?` and `&`): + +| Option | Values | Default | What it does | +|------------------|---------------------------|------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `qpara` (or `q`) | 8 hex digits | `324B142D` | the four quality parameters in each acknowledgement; `324B142D` is what turbodrone sends to working drones, `19322348` an older reading of the vendor app | +| `dqt` | `zigzag`, `raster`, `q50` | `zigzag` | the quantization tables in the JPEG header the app puts in front of each frame: `zigzag` = standard tables scaled to the quality byte of the fragment, in zig-zag order as the JPEG standard has it; `raster` = unscaled standard tables in raster order, quality ignored (what turbodrone writes, and it shows real drones' frames); `q50` = unscaled, zig-zag. If the picture is there but blocky, too dark/bright or noisy, try the others | + +Examples: `flowufo://192.168.169.1:8800`, `flowufo://192.168.169.1:8800?dqt=raster`, `flowufo://192.168.169.1:8800?q=19322348`, `flowufo://192.168.169.1:8801?dqt=q50`. + +The implementation was written from a description of the protocol and checked with a D16. Status texts while it connects: *Waiting for the toy: probing 192.168.169.1:8800*, *Receiving from the toy at …*, *The toy went quiet: probing … again*. The app adds a `flowufo` object to `/stats.json`: state, the ports probed (`target`), the port and engine that answered (`port`, `engine`), network, `qpara`, `dqt` (e.g. `zigzag Q50` or `raster (unscaled)`), `camera_status`, frame and fragment counters, losses, `lagging_left_out`, resets, errors, and a `hint` when the address looks wrong. + +To use it: join the `FLOW-UFO-…` Wi‑Fi, [stay connected](#connecting), put `flowufo://192.168.169.1:8800` into a slot (**Settings**, or share the text to the app). If there is no picture, or it looks wrong, wait about 30 s and [send a report](#what-to-send-in-a-report) in [#187](https://github.com/OpenIPC/decoder/issues/187). + +## KY UFO RTSP toys + +Toys of the cooingdv apps (*KY UFO*, *RC UFO* and others), often with a `FLY-UFO-…` Wi‑Fi, give the phone an address in `192.168.1.x` and stream plain RTSP: + +``` +rtsp://192.168.1.1:7070/webcam +``` + +Their apps send a heartbeat, the two bytes `01 01`, to UDP port 7099 of the toy every second while the video plays. For an RTSP slot with port 7070 and path `/webcam` (any host) the app does the same: from the address of the RTSP connection, once a second, until the slot stops. Nothing else is sent to port 7099 (no flight command, no "stop" command). Other RTSP slots send no heartbeat unless asked with `?heartbeat=`, e.g. `rtsp://192.168.1.1:554/live?heartbeat=7099`; `?heartbeat=off` turns it off for a `:7070/webcam` slot. The option is the app's own and is not sent to the camera. The log has one line when it starts (*heartbeat 01 01 every 1000 ms to …*), and `/stats.json` a `heartbeat` object (`target`, `running`, `sent`) for such a slot. + +The camera of some of these toys (the K200) is mounted sideways: its frames come turned, a mug lies on its side as a portrait strip in a landscape screen. For a `:7070/webcam` slot the app therefore turns the MJPEG frames 270° counter-clockwise (90° clockwise) when drawing them ([#200](https://github.com/OpenIPC/decoder/issues/200)), and fits the turned frame (a 480×640 frame shows as 640×480 and fills a landscape screen). `?rotate=0|90|180|270` (degrees counter-clockwise) overrides it, e.g. `rtsp://192.168.1.1:7070/webcam?rotate=0` for a unit mounted upright, and also turns other RTSP MJPEG slots; any other value is an error. The option is the app's own and is not sent to the camera. It turns MJPEG only: an H.264/H.265 stream is shown as it comes and the log says *rotate= only applies to MJPEG streams for now (#201)*. The turn is in the picture itself, so zoom, the long-press screenshot and the debug snapshot show it turned too. The log has one line when it is active (*MJPEG frames turned 270° counter-clockwise*), and `/stats.json` a `rotate` (degrees) for such a slot; its `size` is the turned one. + +Some of these toys use the FLOW-UFO protocol instead (the KY UFO app has both); if RTSP shows nothing, try `flowufo://192.168.1.1`. + +## HF / Jieli toys + +| | | +|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Wi‑Fi name | `HF_xxxxxxxx` | +| Camera | Jieli AC79 (`AC79X_wifi_car_camera`); vendor app *DVRunning 2* (also *HF UFO*) | +| Toy address | `192.168.80.1`: control (CTP) on TCP port 3333; the video goes to UDP port 2224 (or 6666) **of the phone** | +| Slot URL | `jieli://192.168.80.1` (or `jieli://` alone) | +| Status | **experimental** since 2.15: the control part follows a capture of the vendor app; YL3 video arrives and plays, the picture quality is still being checked in [#187](https://github.com/OpenIPC/decoder/issues/187) | + +What the app does, as a capture of the vendor app shows it ([#196](https://github.com/OpenIPC/decoder/issues/196)): + +- It connects to TCP port 3333 and talks **CTP**: each message is `CTP:`, the length of a topic name, the topic (e.g. `OPEN_RT_STREAM`), the length of a JSON text and the JSON. +- `APP_ACCESS` first; the toy answers with `KEEP_ALIVE_INTERVAL` (a timeout, 60 s), its battery and SD card state and `VIDEO_PARAM` (the size, e.g. 640×480 at 25 fps). Then `DATE_TIME` (the phone's time), a `CTP_KEEP_ALIVE` every 7.5 s (a quarter of a shorter timeout), and `OPEN_RT_STREAM` with the size from `VIDEO_PARAM` (640×480 at 25 if the toy sent none). When the slot stops: `CLOSE_RT_STREAM`. +- The video was **not** in the capture (PCAPdroid without root cannot see it). After turbodrone's research on the same Jieli SDK it is **RTP/JPEG** (payload type 26) or H.264 (96) to the phone's **UDP port 6666**. The app listens there and plays payload type 26 through its RTP/JPEG path and 96 through its H.264 path; a whole JPEG in a datagram without RTP is shown too. +- A YL3 sent nothing to 6666 but its own stream to the phone's **UDP port 2224**: per datagram a 20-byte header, then a piece of a JPEG. Header, little-endian: byte 0 the type (2 for JPEG, bit 7 set on the last piece of a frame), bytes 2–3 the length of this piece, 4–7 a packet counter, 8–11 the length of the JPEG, 12–15 the offset of the piece in it. The datagram can be longer than the piece; what follows is left over from earlier frames and is ignored (an earlier test build took it, which broke the lower half of the picture). Its frames are portrait (240×320 or 320×240 rotated): the camera is mounted sideways, so `jieli://` turns them 90° counter-clockwise; 270° showed them upside down; `?rotate=` overrides it. The app listens on 2224 as well as 6666; the first five datagrams go to the log with the whole header, `/stats.json` has a `stream` object (`packets`, `frames`, `dropped`, `last_drop`) and `video_port`; why a frame was dropped is in the log for the first three. The debug page serves the last frame as it came, `/jieli-frame.jpg`, and the last dropped one, `/jieli-dropped.bin`, also after the session ended (the app pauses while the phone's browser is open). +- If nothing arrives on 6666 or 2224 for 5 s after `OPEN_RT_STREAM`, the status says so and the app also listens on UDP 1234 (audio in the vendor SDK), **only to log** what arrives there, for 30 s or 20 packets, then frees the port; it asks for the stream again every 10 s without video. +- **No flight command is ever sent**: no `FLIGHT_CONTROL` over CTP, nothing to the toy's UDP port 3456 (where the vendor app sends the sticks). +- Ports 6666 and 2224 can be used by one slot at a time: one `jieli://` slot in a grid. + +URL options (as with `udp://`, separated by `?` and `&`): + +| Option | Values | Default | What it does | +|-----------------|-------------------------|---------------------------------------|---------------------------------------------------------------------------------------------| +| `w`, `h`, `fps` | numbers | from `VIDEO_PARAM`, else 640, 480, 25 | the stream asked for in `OPEN_RT_STREAM` | +| `rtp` | a UDP port | `6666` | the local port the video is expected on | +| `format` | `0`, `1` | `0` | `0` asks for JPEG (as the vendor app does), `1` for H.264 | +| `rotate` | `0`, `90`, `180`, `270` | `90` | degrees counter-clockwise the JPEG frames are turned (the YL3's camera is mounted sideways) | + +Examples: `jieli://192.168.80.1`, `jieli://192.168.80.1?w=1280&h=720`, `jieli://192.168.80.1?format=1`. + +Status texts while it connects: *Connecting to the toy (CTP 192.168.80.1:3333)*, *Stream requested, waiting for video on UDP 6666 and 2224*, *No video on UDP 6666 and 2224 yet: the toy accepted the request; see /log*, *Receiving from the toy on UDP 2224…* (the port the video came on), *The toy's video stopped: waiting on UDP 6666 and 2224*. The app adds a `jieli` object to `/stats.json`: `state` (`connecting`, `ctp`, `requested`, `playing`, `quiet`), `target`, `network`, `ctp` (messages sent and received, keepalives and replies, the keepalive interval, the topics seen, the last 8 messages from the toy), `video_param`, `requested`, `rtp` (port, packets, video packets, frames, payload types, packets that are not RTP, the first packet in hex), `sources` (sender address:port → packets) and `diagnostic_ports`. The log has the CTP messages (JSON shortened) and the first packets on each port with sender, payload type and first bytes: exactly what a report needs. + +To use it: join the `HF_…` Wi‑Fi, [stay connected](#connecting), close the vendor app, put `jieli://192.168.80.1` into a slot. Wait about 30 s and [send a report](#what-to-send-in-a-report) in [#187](https://github.com/OpenIPC/decoder/issues/187), with or without a picture: `/stats.json` and `/log` tell what the toy really sends. -The implementation was written from a description of the protocol, not tested on a real toy yet. Status texts while it connects: *Waiting for the toy: probing 192.168.169.1:8800*, *Receiving from the toy at …*, *The toy went quiet: probing … again*. The test build adds a `flowufo` object to `/stats.json` (state, network, frame and fragment counters, losses, resets, errors). +## Credits -To test: install the test build, join the `FLOW-UFO-…` Wi‑Fi, [stay connected](#connecting), put `flowufo://192.168.169.1:8800` into a slot (**Settings**, or share the text to the app). If there is no picture, or it looks wrong, wait about 30 s and [send a report](#what-to-send-in-a-report) in [#187](https://github.com/OpenIPC/decoder/issues/187). +What the app does for FLOW-UFO and KY UFO toys was checked against [turbodrone](https://github.com/marshallrichards/turbodrone) (Apache-2.0), which flies these drones from a computer: its `wifi_uav` backend is the FLOW-UFO protocol and its `cooingdv` backend the KY UFO RTSP toys. The default quality parameters, the second port 8801 and its timing, the camera status bytes, leaving lagging frames out of the acknowledgement, the raster tables of `dqt=raster` and the 7099 heartbeat come from its code and research notes; no code of it is used. For the HF / Jieli toys its research notes on the same Jieli SDK (`rxdrone`, `cooingdv_jieli`: RTP/JPEG to UDP 6666, audio on 1234, the stream ports) describe the video part; the control part follows a PCAPdroid capture of the vendor app posted in [#196](https://github.com/OpenIPC/decoder/issues/196). More open-source work on such toys is collected in [#196](https://github.com/OpenIPC/decoder/issues/196). ## An unknown toy First find the toy's address: on Android it is usually the gateway of the Wi‑Fi (*Wi‑Fi → the network → Details*; often `192.168.0.1`, `192.168.1.1`, `192.168.4.1`, `192.168.169.1`, `172.16.10.1` or `192.168.25.1`). Then try, in a slot: -| Try | Why | -|---|---| -| `rtsp://:554/` | some toys run a plain RTSP server; try the paths `/live`, `/stream`, `/11`, `/h264` too | -| `rtsp://:8554/` | another common RTSP port | -| `http://:8080/?action=stream` | MJPEG streamers (mjpg-streamer style) | -| `http:///` and `http://:81/stream` | MJPEG on the web port; open them in a browser first to see what answers | -| `udp://@:` | only if the toy pushes RTP by itself to a port of the phone, which is rare | +| Try | Why | +|----------------------------------------------|-----------------------------------------------------------------------------------------| +| `rtsp://:554/` | some toys run a plain RTSP server; try the paths `/live`, `/stream`, `/11`, `/h264` too | +| `rtsp://:8554/` | another common RTSP port | +| `http://:8080/?action=stream` | MJPEG streamers (mjpg-streamer style) | +| `http:///` and `http://:81/stream` | MJPEG on the web port; open them in a browser first to see what answers | +| `udp://@:` | only if the toy pushes RTP by itself to a port of the phone, which is rare | If nothing plays, the toy most likely uses a protocol of its own, which needs a [report](#what-to-send-in-a-report) and a transport written for it. @@ -85,9 +163,9 @@ Do not send the vendor app itself or parts of it. The capture contains only the The drones a tester checked in [#187](https://github.com/OpenIPC/decoder/issues/187): -| Toy | Wi‑Fi | Family | URL | Status | -|---|---|---|---|---| -| D16 | `FLOW_…` | FLOW-UFO, WIFI UAV app | `flowufo://192.168.169.1:8800` | video works | -| K200 | `FLY-UFO-…` | KY UFO, RTSP | `rtsp://192.168.1.1:7070/webcam` | video works; the picture comes sideways ([#200](https://github.com/OpenIPC/decoder/issues/200)) | -| Drone YL3 | `HF_…` | Jieli AC79, DVRunning 2 app | `jieli://192.168.80.1` (test build) | in testing | +| Toy | Wi‑Fi | Family | URL | Status | +|-----------|-------------|-----------------------------|----------------------------------|-------------------------------------------------------------------------------------------------------------| +| D16 | `FLOW_…` | FLOW-UFO, WIFI UAV app | `flowufo://192.168.169.1:8800` | video works | +| K200 | `FLY-UFO-…` | KY UFO, RTSP | `rtsp://192.168.1.1:7070/webcam` | video works; the sideways picture is turned upright ([#200](https://github.com/OpenIPC/decoder/issues/200)) | +| Drone YL3 | `HF_…` | Jieli AC79, DVRunning 2 app | `jieli://192.168.80.1` | video plays, picture quality being checked | diff --git a/docs/transports.md b/docs/transports.md index 7d0345b2..5c61f329 100644 --- a/docs/transports.md +++ b/docs/transports.md @@ -14,12 +14,12 @@ A slot holds one URL. Its scheme picks the path the video takes; for `rtsp://` the slot's transport mode (chosen under the URL in **Settings**, or with the `transport` extra of a [shared link](usage.md#opening-a-camera-from-another-app)) picks how RTP travels. -| URL | Path | Single view | Multi-view | Audio | -|---|---|---|---|---| -| `rtsp://` | RTSP session, RTP over TCP, UDP or multicast | yes | yes | yes | -| `udp://`, `rtp://` | plain RTP pushed by the camera, no RTSP | yes | yes | no | -| `http://` | MJPEG, `multipart/x-mixed-replace` | yes | yes | no | -| `flowufo://` | toy drone protocol, **test build only** ([#187](https://github.com/OpenIPC/decoder/issues/187)) | yes | yes | no | +| URL | Path | Single view | Multi-view | Audio | +|--------------------|----------------------------------------------------------------------------|-------------|------------|-------| +| `rtsp://` | RTSP session, RTP over TCP, UDP or multicast | yes | yes | yes | +| `udp://`, `rtp://` | plain RTP pushed by the camera, no RTSP | yes | yes | no | +| `http://` | MJPEG, `multipart/x-mixed-replace` | yes | yes | no | +| `flowufo://` | toy drone protocol ([#187](https://github.com/OpenIPC/decoder/issues/187)) | yes | yes | no | The transport in use is shown when a stream starts in the single view (`Connected • UDP • H.265 …`), on the [debug page](debug.md) and in `/stats.json` (`transport`; the slot's setting is `transport_mode`). After a fallback it reads, for example, `TCP (auto)`. @@ -42,12 +42,12 @@ rtsp://[user:password@]host[:port]/path ## RTSP transport modes -| Mode | How the video arrives | Fallback | -|---|---|---| -| **TCP** | RTP interleaved in the RTSP connection | none needed; works through NAT and firewalls | -| **UDP** | RTP over UDP, a port pair per track | never falls back. Lost packets are skipped, not resent | -| **Auto** (default for a new slot) | UDP, else TCP | to TCP when the camera refuses UDP in `SETUP`, when no video arrives within 3 s, or when packets arrive but key frames keep losing a piece | -| **Multicast** | RTSP multicast (Android 8.0+) | to TCP in the same connection when the server refuses it or answers with unicast | +| Mode | How the video arrives | Fallback | +|-----------------------------------|----------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------| +| **TCP** | RTP interleaved in the RTSP connection | none needed; works through NAT and firewalls | +| **UDP** | RTP over UDP, a port pair per track | never falls back. Lost packets are skipped, not resent | +| **Auto** (default for a new slot) | UDP, else TCP | to TCP when the camera refuses UDP in `SETUP`, when no video arrives within 3 s, or when packets arrive but key frames keep losing a piece | +| **Multicast** | RTSP multicast (Android 8.0+) | to TCP in the same connection when the server refuses it or answers with unicast | - After a fallback the URL is remembered as failed for that mode for 10 minutes while the app runs: reconnects, a resume or a rebuilt grid go straight to TCP; after that UDP (or multicast) is tried again. - If the server answers a UDP request with interleaved TCP, the app takes what the server chose. @@ -62,17 +62,17 @@ A slot can receive RTP that the camera pushes by itself, without RTSP. Put a `ud udp://[source@][address]:port[?codec=h264|h265][&pt=N] ``` -| URL | Meaning | -|---|---| -| `udp://@:5600` | Listen on port 5600 | -| `udp://@239.1.2.3:5600` | Join the multicast group 239.1.2.3 on port 5600 | +| URL | Meaning | +|-------------------------------------|----------------------------------------------------| +| `udp://@:5600` | Listen on port 5600 | +| `udp://@239.1.2.3:5600` | Join the multicast group 239.1.2.3 on port 5600 | | `udp://192.168.1.10@239.1.2.3:5600` | The same, but accept packets from this sender only | -| `udp://@:5600?codec=h265&pt=97` | Force the codec or the video payload type | +| `udp://@:5600?codec=h265&pt=97` | Force the codec or the video payload type | -| Option | Values | Default | -|---|---|---| +| Option | Values | Default | +|---------|---------------------------------|-----------------------------------------------| | `codec` | `h264` / `avc`, `h265` / `hevc` | from the payload type: 96 → H.264, 97 → H.265 | -| `pt` | 0–127 | 96 or 97 | +| `pt` | 0–127 | 96 or 97 | - Addresses must be numeric (IPv4 or IPv6 in brackets); no name lookup. A port is required. - A codec change on the camera is followed while playing. Video only. @@ -100,16 +100,20 @@ For example Majestic's `http://user:password@/mjpeg`: a `multipart/x-mix ## Toys: FLOW-UFO -`flowufo://192.168.169.1:8800` (or `flowufo://` alone) plays the video of toy drones with a `FLOW-UFO-xxxxxx` Wi‑Fi access point. It exists only in a test build so far ([#187](https://github.com/OpenIPC/decoder/issues/187), PR [#188](https://github.com/OpenIPC/decoder/pull/188)), not in the releases. Setup and status: [toys.md](toys.md#flow-ufo). +`flowufo://192.168.169.1:8800` (or `flowufo://` alone) plays the video of toy drones with a `FLOW-UFO-xxxxxx` Wi‑Fi access point. Since 2.15 ([#187](https://github.com/OpenIPC/decoder/issues/187), PR [#215](https://github.com/OpenIPC/decoder/pull/215)). Port 8800 (the default) probes 8801 as well and keeps the port that answers. Options: `?qpara=<8 hex digits>` and `?dqt=zigzag|raster|q50`. Setup, options and status: [toys.md](toys.md#flow-ufo). + +`jieli://192.168.80.1` (or `jieli://` alone) plays toy drones with an `HF_xxxxxxxx` Wi‑Fi (Jieli AC79 camera): CTP commands on TCP 3333 ask for the stream, which arrives as JPEG frames behind a 20-byte header on the phone's UDP port 2224 (seen from a YL3) or as RTP/JPEG on UDP 6666; experimental since 2.15. Options: `?w=`, `?h=`, `?fps=`, `?rtp=`, `?format=0|1`, `?rotate=`. See [toys.md](toys.md#hf--jieli-toys). + +RTSP slots on `rtsp://:7070/webcam` (KY UFO and other cooingdv toys) also send the toy's UDP heartbeat `01 01` to port 7099 once a second; `?heartbeat=` turns it on for other RTSP URLs, `?heartbeat=off` off. Their MJPEG frames are drawn turned 270° counter-clockwise (90° clockwise) (the K200's camera is mounted sideways, [#200](https://github.com/OpenIPC/decoder/issues/200)); `?rotate=0|90|180|270` (degrees counter-clockwise, any RTSP MJPEG slot) overrides it and is not sent to the camera either. See [toys.md](toys.md#ky-ufo-rtsp-toys). ## Codecs -| Kind | Codecs | Decoding | -|---|---|---| -| Video | H.264, H.265 | MediaCodec (hardware); a software decoder for key-frame cells beyond the hardware instance limit | -| Video | MJPEG (RTP/JPEG over RTSP, HTTP MJPEG) | the device's JPEG decoder (`video/jpeg`) where it can be created, otherwise BitmapFactory | -| Audio | AAC (RFC 3640 `mpeg4-generic`, ADTS, MP4A-LATM with out-of-band config), Opus | MediaCodec | -| Audio | G.711 μ-law / A-law, L16 PCM (mono or stereo) | converted in the app | +| Kind | Codecs | Decoding | +|-------|-------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| +| Video | H.264, H.265 | MediaCodec (hardware); a software decoder for key-frame cells beyond the hardware instance limit | +| Video | MJPEG (RTP/JPEG over RTSP, HTTP MJPEG) | the device's JPEG decoder (`video/jpeg`) where it can be created, otherwise BitmapFactory | +| Audio | AAC (RFC 3640 `mpeg4-generic`, ADTS, MP4A-LATM with out-of-band config), Opus | MediaCodec | +| Audio | G.711 μ-law / A-law, L16 PCM (mono or stereo) | converted in the app | Not supported: G.726, MP4A-LATM with in-band configuration (`cpresent=1`) ([#14](https://github.com/OpenIPC/decoder/issues/14)). An unsupported audio track is not set up; the video plays without sound. @@ -123,13 +127,13 @@ Not supported: G.726, MP4A-LATM with in-band configuration (`cpresent=1`) ([#14] Measured on the lab cameras. -| Feature | OpenIPC Majestic | XM (`H264DVR 1.0`) | -|---|---|---| -| RTP over TCP (interleaved) | yes | yes | -| RTP over UDP | yes | yes (reply without `/UDP`) | -| RTSP multicast | no: `400` to a multicast request, silent unicast when the request also has `client_port` | no: `451`, silent unicast likewise | -| Plain RTP push (`outgoing`), unicast or multicast | yes | not known | -| HTTP MJPEG | yes (`/mjpeg`) | not tested | -| Audio | Opus, AAC, L16 (G.711 not testable on the lab build) | none (video H.264 only) | +| Feature | OpenIPC Majestic | XM (`H264DVR 1.0`) | +|---------------------------------------------------|------------------------------------------------------------------------------------------|------------------------------------| +| RTP over TCP (interleaved) | yes | yes | +| RTP over UDP | yes | yes (reply without `/UDP`) | +| RTSP multicast | no: `400` to a multicast request, silent unicast when the request also has `client_port` | no: `451`, silent unicast likewise | +| Plain RTP push (`outgoing`), unicast or multicast | yes | not known | +| HTTP MJPEG | yes (`/mjpeg`) | not tested | +| Audio | Opus, AAC, L16 (G.711 not testable on the lab build) | none (video H.264 only) | With a Multicast slot both play over TCP in the same connection. Multicast from OpenIPC is available only as plain RTP via `outgoing`. A server that does RTSP multicast (for example mediamtx) is needed to use the Multicast mode ([#69](https://github.com/OpenIPC/decoder/issues/69)).