Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion android/app/src/main/java/com/openipc/decoder/Decoder.java
Original file line number Diff line number Diff line change
Expand Up @@ -1402,6 +1402,10 @@ private Map<String, Object> debugState() {
c.put("lost_fps", cell.lostFps());
// NAL units dropped because one of their packets was lost (#113)
c.put("nal_dropped", cell.nalDropped());
// Slices passed on cut short at a lost piece (some may still be dropped by
// a full queue); absent while that is off (#133)
int salvaged = cell.nalSalvaged();
if (salvaged >= 0) c.put("nal_damaged", salvaged);
// Every frame, or key frames only while the decoder has no room (#85)
c.put("decode", cell.isKeyFramesOnly() ? "key frames" : "full");
if (cell.knowsLoad()) c.put("load", cell.isKeyFramesOnly()
Expand Down Expand Up @@ -1715,13 +1719,20 @@ public void run() {
int frames = videoDecoder.framesOut() + jpegFramesOut;
long bytes = client != null ? client.getStreamBytes() : 0;
boolean stalled = !quadEnabled && !udpViewShown && client != null
&& client.playsOverUdp() && "connected".equals(singleState)
&& (client.playsOverUdp() || client.listens())
&& "connected".equals(singleState)
&& bytes - lastSingleBytes > 10_000 && frames == lastSingleFrames;
lastSingleFrames = frames;
lastSingleBytes = bytes;
singleStallSeconds = stalled ? singleStallSeconds + 1 : 0;
if (singleStallSeconds >= UDP_STALL_SECONDS) {
singleStallSeconds = 0;
// Every key frame loses a piece: decode them damaged rather than not at all;
// a pushed stream has no TCP to go to (#133)
if (!nalAssembler.salvaging()) {
nalAssembler.setSalvage(true);
Log.w(TAG, "No complete key frame over UDP, decoding damaged ones");
}
client.reportUdpBroken("packets but no decoded frame for " + UDP_STALL_SECONDS
+ " s");
}
Expand Down
44 changes: 41 additions & 3 deletions android/app/src/main/java/com/openipc/decoder/NalAssembler.java
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,13 @@ interface Sink {
private int dropped;
/** Of those, dropped because a packet of them was lost (not for size) (#113). */
private int lostDropped;
/**
* Pass a NAL unit with a lost piece on cut short at the gap instead of dropping it (#133):
* for a stream that loses a piece of every key frame, where dropping them leaves nothing to
* decode. The decoder conceals the missing part; a damaged picture beats a black one.
*/
private volatile boolean salvage;
private int salvaged;

NalAssembler(int bufferSize, Runnable onCodecSwitch, FramePool framePool) {
this.nalBuffer = new byte[bufferSize];
Expand All @@ -63,6 +70,11 @@ void reset() {
/** NAL units dropped because one of their packets was lost (#113). */
int lossCount() { return lostDropped; }

void setSalvage(boolean on) { salvage = on; }
boolean salvaging() { return salvage; }
/** Slices passed on to the sink cut short at a lost piece (#133); the queue may drop some. */
int salvagedCount() { return salvaged; }

/**
* Process one RTP packet ({@code length} bytes at {@code offset}, header included).
* Returns false if the RTP header is malformed.
Expand All @@ -89,7 +101,7 @@ boolean assemble(byte[] buf, int offset, int length, boolean codecH265, Sink sin
lastSeq = seq;
if (gap && inFragment) {
lostDropped++;
drop("packet loss inside a fragmented NAL unit");
dropOrSalvage("packet loss inside a fragmented NAL unit", sink);

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remediation recommended

2. Recovered slices still count as dropped 🐞 Bug ◔ Observability

NalAssembler.assemble() increments lostDropped before dropOrSalvage() can pass the incomplete
slice to the sink. Once salvage is active, each qualifying gap therefore increases both the reported
nal_dropped count and nal_damaged, even though the slice was passed on rather than dropped.
Agent Prompt
## Issue description
Salvaged slices are included in `nal_dropped` because the packet-loss counter increments before the assembler decides whether to emit or drop the slice.

## Fix Focus Areas
- android/app/src/main/java/com/openipc/decoder/NalAssembler.java[101-105]
- android/app/src/main/java/com/openipc/decoder/NalAssembler.java[216-233]
- android/app/src/main/java/com/openipc/decoder/QuadCell.java[423-445]

## Recommended Fix
Count a NAL as dropped only when it is actually discarded. Preserve a separate packet-loss signal for the quad stall detector so changing the diagnostic count does not prevent stall detection.

ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools

orphan = true; // its remaining pieces belong to the unit just counted
}

Expand All @@ -104,7 +116,7 @@ boolean assemble(byte[] buf, int offset, int length, boolean codecH265, Sink sin
// Any other packet ends a fragmented NAL unit that did not finish; its remaining
// pieces, if any arrive, belong to the unit counted here
if (inFragment) {
drop("fragmented NAL unit interrupted by another packet");
dropOrSalvage("fragmented NAL unit interrupted by another packet", sink);
orphan = true;
} else {
orphan = false;
Expand All @@ -131,7 +143,7 @@ private void fragmentUnit(byte[] buf, int start, int end, boolean h265, Sink sin
if (first) {
if (inFragment) {
lostDropped++;
drop("fragmented NAL unit without its end");
dropOrSalvage("fragmented NAL unit without its end", sink);
}
orphan = false;
nalSize = 0;
Expand Down Expand Up @@ -201,6 +213,32 @@ private void emit(byte[] buf, int pos, int size, Sink sink) {
sink.onNal(out);
}

/** The unit collected so far, cut short, when salvaging (#133); dropped otherwise. */
private void dropOrSalvage(String reason, Sink sink) {
// Start code and NAL header plus some slice data, or there is nothing to decode; only
// slices: a cut parameter set would be taken for a valid configuration (Qodo on #146)
if (!salvage || nalSize < 16 || !isSlice(nalBuffer[4], lastCodec == 1)) {
drop(reason);
return;
}
Frame out = framePool.obtain(nalSize);
System.arraycopy(nalBuffer, 0, out.data(), 0, nalSize);
out.setLength(nalSize);
nalSize = 0;
inFragment = false;
if (salvaged++ % 100 == 0) {
Log.w(TAG, "NAL unit passed on cut short (" + salvaged + "): " + reason);
}
sink.onNal(out);
}

/** A coded slice (VCL NAL unit): H.264 types 1–5, H.265 types 0–31. */
static boolean isSlice(byte nalHeader, boolean h265) {
if (h265) return ((nalHeader >> 1) & 0x3F) <= 31;
int type = nalHeader & 0x1F;
return type >= 1 && type <= 5;
}

private void drop(String reason) {
nalSize = 0;
inFragment = false;
Expand Down
28 changes: 25 additions & 3 deletions android/app/src/main/java/com/openipc/decoder/QuadCell.java
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,9 @@ void stop() {
public void onStatus(String status) {
// A new connection (or a lost one) starts the UDP stall count afresh (Qodo on #125)
if (!status.equals(state)) udpStallSamples = 0;
// A pushed stream that went quiet may come back whole: no damaged decoding until it
// stalls again (Qodo on #146)
if ("disconnected".equals(status)) nalAssembler.setSalvage(false);
state = status;
// A connection that works again clears the last connection error
if ("connected".equals(status) && problem != null && !problem.startsWith("Decoder")) {
Expand Down Expand Up @@ -346,9 +349,15 @@ String sample(long nowMs) {
// MJPEG has no key frames: its cheap mode shows one frame per second (#132)
return size + codec + (jpeg ? " reduced" : " key frames") + " · " + rate + " fps"
+ ((jpeg ? jpegRenderer.isSoftware() : decoder.isSoftware())
? " · software" : "");
? " · software" : "") + damaged();
}
return size + codec + " " + Math.round(outFps) + "/" + Math.round(inFps) + " fps";
return size + codec + " " + Math.round(outFps) + "/" + Math.round(inFps) + " fps"
+ damaged();
}

/** Corner note while damaged key frames are decoded (#133). */
private String damaged() {
return !jpeg && nalAssembler.salvaging() ? " · damaged" : "";
}

boolean isKeyFramesOnly() { return keyFramesOnly; }
Expand Down Expand Up @@ -421,7 +430,9 @@ private void checkUdpStall() {
lastVideoPackets = packets;
lastLossCount = loss;
Surface s = surface;
boolean stalled = session.playsOverUdp() && "connected".equals(state)
// A pushed stream (udp:// slot, the U cell) counts too: no TCP to fall back to (#133)
boolean stalled = (session.playsOverUdp() || session.listens())
&& "connected".equals(state)
&& pausedReason == null && problem == null && s != null && s.isValid()
&& arriving && outFps == 0;
if (!stalled) {
Expand All @@ -433,6 +444,12 @@ private void checkUdpStall() {
if (losing || udpStallSamples > 0) udpStallSamples++;
if (udpStallSamples >= UDP_STALL_SAMPLES) {
udpStallSamples = 0;
// Every key frame loses a piece: decode them damaged rather than not at all. A
// pushed stream has no TCP to go to; over RTSP, Auto goes to TCP anyway (#133)
if (!jpeg && !nalAssembler.salvaging()) {
nalAssembler.setSalvage(true);
Log.w(TAG, tag + ": no complete key frame over UDP, decoding damaged ones");
}
session.reportUdpBroken("packets but no decoded frame for " + UDP_STALL_SAMPLES
+ " s (" + loss + (jpeg ? " JPEG frames" : " NAL units") + " lost)");
}
Expand All @@ -441,6 +458,9 @@ private void checkUdpStall() {
/** NAL units dropped so far because a packet of them was lost (#113). */
int nalDropped() { return nalAssembler.lossCount(); }

/** Slices passed on cut short at a lost piece, or -1 while that is off (#133). */
int nalSalvaged() { return nalAssembler.salvaging() ? nalAssembler.salvagedCount() : -1; }

float inFps() { return inFps; }
float outFps() { return outFps; }
float lostFps() { return lostFps; }
Expand Down Expand Up @@ -517,6 +537,8 @@ public void onSdp(Sdp sdp) {
lastLossCount = sdp.videoJpeg ? jpegAssembler.droppedCount() : nalAssembler.lossCount();
udpStallSamples = 0;
nalAssembler.reset();
// A new stream gets a chance to be complete again (#133)
nalAssembler.setSalvage(false);
jpegAssembler.reset();
lastUnknownPayload = -1;
videoPt = sdp.videoPt;
Expand Down
6 changes: 6 additions & 0 deletions android/app/src/main/java/com/openipc/decoder/RtspClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ void configure(String host, String userAgent, Transport.Mode transport) {
boolean isActive() { return session.isActive(); }
/** RTP over UDP or multicast at the moment (#128). */
boolean playsOverUdp() { return session.playsOverUdp(); }
/** A udp:// slot listening for a pushed stream (#133). */
boolean listens() { return session.listens(); }
/** Packets arrive but nothing decodes: with Auto the session goes over to TCP. */
void reportUdpBroken(String why) { session.reportUdpBroken(why); }
/** Transport the stream really uses now, or null (#115). */
Expand All @@ -101,13 +103,17 @@ void stop() {

@Override
public void onStatus(String status) {
// A pushed stream that went quiet may come back whole (Qodo on #146)
if ("disconnected".equals(status)) nalAssembler.setSalvage(false);
Listener l = listenerCb;
if (l != null) l.onStatusChanged(status);
}

@Override
public void onSdp(Sdp sdp) {
nalAssembler.reset();
// A new stream gets a chance to be complete again (#133)
nalAssembler.setSalvage(false);
lastUnknownPayload = -1;
lastRtpTimestamp = -1;
lastRtpArrivalNs = -1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,9 @@ private void markUdpFailed(Transport.Mode mode) {
/** RTSP is playing with RTP over UDP or multicast (not TCP, not a udp:// URL). */
boolean playsOverUdp() { return udpPlaying; }

/** A udp:// slot listening for a pushed stream (#133): UDP too, without RTSP. */
boolean listens() { return listening; }

/**
* Packets arrive over UDP but the stream cannot be decoded (#113: key frames keep losing
* a piece). With Auto or Multicast the session goes over to TCP, as when no video comes
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,52 @@ public void lostMiddleFragmentDropsTheNal() {
assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xD1, 0xD2), out.get(0));
}

@Test
public void salvagePassesTheNalOnCutShortAtTheGap() {
nal.setSalvage(true);
// 16 slice bytes before the gap: enough to be worth decoding (#133)
feed(rtp(0x7C, 0x85, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12), false);
feed(rtp(0x7C, 0x05, 13, 14, 15, 16), false);
seq++; // one packet lost
feed(rtp(0x7C, 0x05, 0xB1), false);
assertEquals(1, out.size());
assertArrayEquals(b(0, 0, 0, 1, 0x65, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16), out.get(0));
assertEquals(1, nal.salvagedCount());
assertEquals(1, nal.lossCount());
// The rest of that unit is skipped; the next one is complete again
feed(rtp(0x7C, 0x45, 0xC1), false);
feed(rtp(0x7C, 0x85, 0xD1), false);
feed(rtp(0x7C, 0x45, 0xD2), false);
assertEquals(2, out.size());
assertArrayEquals(b(0, 0, 0, 1, 0x65, 0xD1, 0xD2), out.get(1));
}

@Test
public void salvageNeverPassesACutParameterSet() {
nal.setSalvage(true);
// FU-A of an SPS (type 7), then a lost packet (Qodo on #146)
feed(rtp(0x7C, 0x87, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), false);
seq++;
feed(rtp(0x7C, 0x47, 0xC1), false);
assertEquals(0, out.size());
assertEquals(0, nal.salvagedCount());
assertTrue(NalAssembler.isSlice((byte) 0x65, false));
assertFalse(NalAssembler.isSlice((byte) 0x68, false));
// H.265: IDR_W_RADL (19) is a slice, VPS (32) is not
assertTrue(NalAssembler.isSlice((byte) (19 << 1), true));
assertFalse(NalAssembler.isSlice((byte) (32 << 1), true));
}

@Test
public void salvageOffDropsAsBefore() {
feed(rtp(0x7C, 0x85, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), false);
seq++;
feed(rtp(0x7C, 0x45, 0xC1), false);
assertEquals(0, out.size());
assertEquals(0, nal.salvagedCount());
}

@Test
public void continuationWithoutStartIsIgnoredAndCountedOnce() {
feed(rtp(0x7C, 0x05, 0xB1), false);
Expand Down
Loading