ChunkerFixed: do not take a short read for EOF (1.4-maint)

src/borg/chunker.pyx

@@ -202,8 +202,15 @@ class ChunkerFixed:
             while range_size:
                 wanted = min(range_size, self.block_size)
                 if is_data:
-                    # read block from the range
-                    data = dread(offset, wanted, fd, fh)
+                    # read block from the range.
+                    # a short read is not EOF (e.g. reading from a pipe), so keep reading until the block
+                    # is complete or EOF is reached. filling the block keeps the chunk boundaries at fixed offsets.
+                    data = b''
+                    while len(data) < wanted:
+                        more = dread(offset + len(data), wanted - len(data), fd, fh)
+                        if not more:
+                            break  # EOF
+                        data += more
                     got = len(data)
                     if zeros.startswith(data):
                         data = None

src/borg/testsuite/chunker_pytest.py

@@ -138,6 +138,28 @@ def get_chunks(fname, sparse, header_size):
     get_chunks(fn, sparse=sparse, header_size=header_size) == make_content(sparse_map, header_size=header_size)
 
 
+class ShortReadsFile(BytesIO):
+    """A file object returning at most <limit> bytes per read() call, like a pipe."""
+
+    def __init__(self, data, limit):
+        super().__init__(data)
+        self.limit = limit
+
+    def read(self, size=-1):
+        return super().read(min(size, self.limit) if size >= 0 else self.limit)
+
+
+@pytest.mark.parametrize('header_size', [0, 1000])
+def test_chunkify_short_reads(header_size):
+    # short reads must not be taken for EOF and must not change the fixed block boundaries.
+    data = os.urandom(header_size + 3 * 4096 + 123)
+    chunker = ChunkerFixed(4096, header_size=header_size, sparse=False)
+    chunks = cf(chunker.chunkify(ShortReadsFile(data, 100)))
+    assert b''.join(chunks) == data
+    expected_sizes = ([header_size] if header_size else []) + [4096, 4096, 4096, 123]
+    assert [len(c) for c in chunks] == expected_sizes
+
+
 def test_buzhash_chunksize_distribution():
     data = os.urandom(1048576)
     min_exp, max_exp, mask = 10, 16, 14  # chunk size target 16 KiB, clip at 1 KiB and 64 KiB