diff --git a/src/borg/chunker.pyx b/src/borg/chunker.pyx index e0caafbd27..5564d5dbe2 100644 --- a/src/borg/chunker.pyx +++ b/src/borg/chunker.pyx @@ -60,7 +60,12 @@ def dread(offset, size, fd=None, fh=-1): # UNIX only and, in case of block sizes that are not a multiple of the # system's page size, better be used with a bug fixed linux kernel > 4.6.0, # see comment/workaround in _chunker.c and borgbackup issue #907. - os.posix_fadvise(fh, offset, len(data), os.POSIX_FADV_DONTNEED) + try: + os.posix_fadvise(fh, offset, len(data), os.POSIX_FADV_DONTNEED) + except OSError: + # e.g. ESPIPE when reading from a pipe / FIFO. this is just an optimization of the + # cache usage, not critical for correct function, so we ignore these errors. + pass return data else: return fd.read(size) diff --git a/src/borg/testsuite/chunker_pytest.py b/src/borg/testsuite/chunker_pytest.py index 5e48fadf75..70bf027656 100644 --- a/src/borg/testsuite/chunker_pytest.py +++ b/src/borg/testsuite/chunker_pytest.py @@ -160,6 +160,33 @@ def test_chunkify_short_reads(header_size): assert [len(c) for c in chunks] == expected_sizes +def test_chunkify_pipe(): + # reading from a pipe via the OS-level file handle: reads come up short and posix_fadvise() + # fails with ESPIPE (e.g. on Linux) - neither must make the chunker stop or fail. + import threading + import time + data = os.urandom(3 * 65536 + 123) + r, w = os.pipe() + + def writer(): + with os.fdopen(w, 'wb') as f: + for start in range(0, len(data), 65536): + f.write(data[start:start + 65536]) + f.flush() + time.sleep(0.05) # writer pause: reads on the other side come up short + + t = threading.Thread(target=writer) + t.start() + try: + chunker = ChunkerFixed(131072, sparse=False) + chunks = cf(chunker.chunkify(None, r)) + finally: + os.close(r) + t.join() + assert b''.join(chunks) == data + assert [len(c) for c in chunks] == [131072, 65536 + 123] + + def test_buzhash_chunksize_distribution(): data = os.urandom(1048576) min_exp, max_exp, mask = 10, 16, 14 # chunk size target 16kiB, clip at 1kiB and 64kiB