From 70af46ee6f1ad200af0370cb274b1f76702de605 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 29 Oct 2026 19:48:47 +0100 Subject: [PATCH 1/2] ChunkerFixed: do not take a short read for EOF chunkify() stopped reading the whole input as soon as a read returned less than the requested block size. A short read is not EOF though, e.g. when reading from a pipe: "borg create --read-special" with the fixed chunker archived only the first 64 KiB of a FIFO's content (seen on macOS) and silently dropped the rest. Now keep reading until the block is complete or a read returns nothing. Filling the block also keeps the chunk boundaries at fixed offsets. Port of #10563 from 1.4-maint. master is not affected any more, see #10166. Co-Authored-By: Claude Opus 5.5 --- src/borg/chunker.pyx | 8 ++++++++ src/borg/testsuite/chunker_pytest.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/borg/chunker.pyx b/src/borg/chunker.pyx index ba00c3c24c..6dc108afb1 100644 --- a/src/borg/chunker.pyx +++ b/src/borg/chunker.pyx @@ -205,6 +205,14 @@ class ChunkerFixed: # read block from the range data = dread(offset, wanted, fd, fh) got = len(data) + while 0 < got < wanted: + # a short read is not EOF (e.g. reading from a pipe), try to read the rest of the block. + # filling the block also keeps the chunk boundaries at fixed offsets. + more = dread(offset + got, wanted - got, fd, fh) + if not more: + break # EOF + data += more + got = len(data) if zeros.startswith(data): data = None allocation = CH_ALLOC diff --git a/src/borg/testsuite/chunker_pytest.py b/src/borg/testsuite/chunker_pytest.py index 7ac3192456..5e48fadf75 100644 --- a/src/borg/testsuite/chunker_pytest.py +++ b/src/borg/testsuite/chunker_pytest.py @@ -138,6 +138,28 @@ def get_chunks(fname, sparse, header_size): get_chunks(fn, sparse=sparse, header_size=header_size) == make_content(sparse_map, header_size=header_size) +class ShortReadsFile(BytesIO): + """A file object returning at most bytes per read() call, like a pipe.""" + + def __init__(self, data, limit): + super().__init__(data) + self.limit = limit + + def read(self, size=-1): + return super().read(min(size, self.limit) if size >= 0 else self.limit) + + +@pytest.mark.parametrize('header_size', [0, 1000]) +def test_chunkify_short_reads(header_size): + # short reads must not be taken for EOF and must not change the fixed block boundaries. + data = os.urandom(header_size + 3 * 4096 + 123) + chunker = ChunkerFixed(4096, header_size=header_size, sparse=False) + chunks = cf(chunker.chunkify(ShortReadsFile(data, 100))) + assert b''.join(chunks) == data + expected_sizes = ([header_size] if header_size else []) + [4096, 4096, 4096, 123] + assert [len(c) for c in chunks] == expected_sizes + + def test_buzhash_chunksize_distribution(): data = os.urandom(1048576) min_exp, max_exp, mask = 10, 16, 14 # chunk size target 16kiB, clip at 1kiB and 64kiB From d852d11966ca76335e567a7b89596aec09508003 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 29 Oct 2026 20:52:39 +0100 Subject: [PATCH 2/2] ChunkerFixed: fold the first read into the fill loop Same behaviour and same read calls, just one loop instead of a first read followed by a loop for the rest of the block. Co-Authored-By: Claude Opus 5.5 --- src/borg/chunker.pyx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/borg/chunker.pyx b/src/borg/chunker.pyx index 6dc108afb1..e0caafbd27 100644 --- a/src/borg/chunker.pyx +++ b/src/borg/chunker.pyx @@ -202,17 +202,16 @@ class ChunkerFixed: while range_size: wanted = min(range_size, self.block_size) if is_data: - # read block from the range - data = dread(offset, wanted, fd, fh) - got = len(data) - while 0 < got < wanted: - # a short read is not EOF (e.g. reading from a pipe), try to read the rest of the block. - # filling the block also keeps the chunk boundaries at fixed offsets. - more = dread(offset + got, wanted - got, fd, fh) + # read block from the range. + # a short read is not EOF (e.g. reading from a pipe), so keep reading until the block + # is complete or EOF is reached. filling the block keeps the chunk boundaries at fixed offsets. + data = b'' + while len(data) < wanted: + more = dread(offset + len(data), wanted - len(data), fd, fh) if not more: break # EOF data += more - got = len(data) + got = len(data) if zeros.startswith(data): data = None allocation = CH_ALLOC