From 3a662cafe30e319faaa8e52ef840cf217bc6f1c8 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Fri, 26 Jun 2026 22:27:30 +0200 Subject: [PATCH] Fix zstd decompression of replays with trailing bytes Some (older) replay files contain a stray trailing byte (a newline) after the zstd frame. zstd-jni <= 1.5.2 silently ignored bytes after a finished frame, but newer libzstd treats any post-frame bytes as the start of a second frame and fails with "Unknown frame descriptor". This is a streaming decoder behavior change in upstream libzstd; the zstd-jni Java wrapper is unchanged and one-shot/CLI decoding always rejected trailing data. Read exactly the frame's advertised content size so the decompressor stops at the frame boundary and never touches the trailing bytes. Falls back to a full read when the content size is not stored in the header (e.g. streamed files, which carry no trailing junk). This unblocks updating zstd-jni past 1.5.2; bumped to 1.5.6-10. Refs: https://github.com/luben/zstd-jni/issues/301 Co-Authored-By: Claude Opus 4.8 --- .../faforever/commons/replay/ReplayDataParser.java | 13 ++++++++++++- .../com/faforever/commons/replay/ReplayLoader.java | 13 ++++++++++++- gradle/libs.versions.toml | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/data/src/main/java/com/faforever/commons/replay/ReplayDataParser.java b/data/src/main/java/com/faforever/commons/replay/ReplayDataParser.java index 9b1be4a0..22907aa4 100644 --- a/data/src/main/java/com/faforever/commons/replay/ReplayDataParser.java +++ b/data/src/main/java/com/faforever/commons/replay/ReplayDataParser.java @@ -7,6 +7,7 @@ import com.faforever.commons.replay.shared.LoadUtils; import com.faforever.commons.replay.shared.LuaData; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.luben.zstd.Zstd; import com.google.common.annotations.VisibleForTesting; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -161,7 +162,17 @@ private ByteBuffer decompress(ByteBuffer inputBuffer, @NotNull ReplayMetadata me CompressorInputStream compressorInputStream = new CompressorStreamFactory().createCompressorInputStream(arrayInputStream); ByteArrayOutputStream out = new ByteArrayOutputStream(); - IOUtils.copy(compressorInputStream, out); + // Some (older) replay files contain stray trailing bytes after the zstd frame, e.g. a + // trailing newline. zstd-jni <= 1.5.2 silently ignored them, but newer libzstd treats any + // post-frame bytes as the start of a second frame and fails with "Unknown frame descriptor". + // When the frame advertises its content size we read exactly that many bytes so the + // decompressor never touches the trailing data; otherwise we fall back to reading it fully. + long contentSize = Zstd.getFrameContentSize(inputArray); + if (contentSize > 0) { + IOUtils.copyLarge(compressorInputStream, out, 0, contentSize); + } else { + IOUtils.copy(compressorInputStream, out); + } return ByteBuffer.wrap(out.toByteArray()); } case UNKNOWN: diff --git a/data/src/main/java/com/faforever/commons/replay/ReplayLoader.java b/data/src/main/java/com/faforever/commons/replay/ReplayLoader.java index 79d062a3..e15ef5b6 100644 --- a/data/src/main/java/com/faforever/commons/replay/ReplayLoader.java +++ b/data/src/main/java/com/faforever/commons/replay/ReplayLoader.java @@ -9,6 +9,7 @@ import com.faforever.commons.replay.header.Source; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.luben.zstd.Zstd; import org.apache.commons.compress.compressors.CompressorException; import org.apache.commons.compress.compressors.CompressorInputStream; import org.apache.commons.compress.compressors.CompressorStreamFactory; @@ -119,7 +120,17 @@ private static ByteBuffer decompress(ByteBuffer inputBuffer, @NotNull ReplayMeta CompressorInputStream compressorInputStream = new CompressorStreamFactory().createCompressorInputStream(arrayInputStream); ByteArrayOutputStream out = new ByteArrayOutputStream(); - IOUtils.copy(compressorInputStream, out); + // Some (older) replay files contain stray trailing bytes after the zstd frame, e.g. a + // trailing newline. zstd-jni <= 1.5.2 silently ignored them, but newer libzstd treats any + // post-frame bytes as the start of a second frame and fails with "Unknown frame descriptor". + // When the frame advertises its content size we read exactly that many bytes so the + // decompressor never touches the trailing data; otherwise we fall back to reading it fully. + long contentSize = Zstd.getFrameContentSize(inputArray); + if (contentSize > 0) { + IOUtils.copyLarge(compressorInputStream, out, 0, contentSize); + } else { + IOUtils.copy(compressorInputStream, out); + } return ByteBuffer.wrap(out.toByteArray()); } case UNKNOWN: diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 57121a5a..724ca384 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -18,7 +18,7 @@ guava = { module = "com.google.guava:guava", version = "33.5.0-jre" } reactor-core = { module = "io.projectreactor:reactor-core", version.ref = "reactor" } reactor-netty = { module = "io.projectreactor.netty:reactor-netty", version = "1.3.5" } luaj-jse = { module = "org.luaj:luaj-jse", version = "3.0.1" } -zstd-jni = { module = "com.github.luben:zstd-jni", version = "1.5.2-5" } # >=1.5.4-1 fails +zstd-jni = { module = "com.github.luben:zstd-jni", version = "1.5.6-10" } commons-compress = { module = "org.apache.commons:commons-compress", version = "1.28.0" } jsonapi-converter = { module = "com.github.jasminb:jsonapi-converter", version = "0.15" } q-builders = { module = "com.github.rutledgepaulv:q-builders", version = "1.6" }