Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/borg/chunker.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,12 @@ def dread(offset, size, fd=None, fh=-1):
# UNIX-only and, in case of block sizes that are not a multiple of the
# system's page size, it is better used with a bug-fixed Linux kernel > 4.6.0,
# see comment/workaround in _chunker.c and borgbackup issue #907.
os.posix_fadvise(fh, offset, len(data), os.POSIX_FADV_DONTNEED)
try:
os.posix_fadvise(fh, offset, len(data), os.POSIX_FADV_DONTNEED)
except OSError:
# e.g. ESPIPE when reading from a pipe / FIFO. this is just an optimization of the
# cache usage, not critical for correct function, so we ignore these errors.
pass
return data
else:
return fd.read(size)
Expand Down
27 changes: 27 additions & 0 deletions src/borg/testsuite/chunker_pytest.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,33 @@ def test_chunkify_short_reads(header_size):
assert [len(c) for c in chunks] == expected_sizes


def test_chunkify_pipe():
# reading from a pipe via the OS-level file handle: reads come up short and posix_fadvise()
# fails with ESPIPE (e.g. on Linux) - neither must make the chunker stop or fail.
import threading
import time
data = os.urandom(3 * 65536 + 123)
r, w = os.pipe()

def writer():
with os.fdopen(w, 'wb') as f:
for start in range(0, len(data), 65536):
f.write(data[start:start + 65536])
f.flush()
time.sleep(0.05) # writer pause: reads on the other side come up short

t = threading.Thread(target=writer)
t.start()
try:
chunker = ChunkerFixed(131072, sparse=False)
chunks = cf(chunker.chunkify(None, r))
finally:
os.close(r)
t.join()
assert b''.join(chunks) == data
assert [len(c) for c in chunks] == [131072, 65536 + 123]


def test_buzhash_chunksize_distribution():
data = os.urandom(1048576)
min_exp, max_exp, mask = 10, 16, 14 # chunk size target 16 KiB, clip at 1 KiB and 64 KiB
Expand Down
Loading