From b2add428a4b0e92979629b65260b7aabe0235a9d Mon Sep 17 00:00:00 2001 From: Bharat Kaurav <117659673+TheNetherWatcher@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:13:28 +0530 Subject: [PATCH] Use larger requests for sequential reads of remote files soundfile reads FLAC files 8 kB at a time, so reading a remote FLAC file took one HTTP request per 32 kB buffer: 229 requests for a 7.4 MB MIMIC-IV signal file, each waiting a second or more for PhysioNet to respond. NetFile now doubles the size of the next request (up to 8 MB) when a read continues where the buffer ends, so the same file takes 8 requests. Reads elsewhere in the file still start with a small request. Fixes #426. --- docs/changes.rst | 3 +++ src/wfdb/io/_url.py | 15 ++++++++++++++- tests/test_url.py | 26 ++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/changes.rst b/docs/changes.rst index db184f8d..eace8376 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -20,6 +20,9 @@ Unreleased **Add timeouts to requests for remote files** Requests for files on PhysioNet (and other web servers) now time out if the server does not respond (10 seconds to connect, and 60 seconds to wait for data), instead of waiting indefinitely. A ``requests.exceptions.Timeout`` (``ConnectTimeout`` or ``ReadTimeout``) is raised in this case. +**Read remote FLAC files with fewer requests** + Remote files read in small pieces, such as FLAC signal files on PhysioNet, now use larger requests as reading goes on (up to 8 MB) instead of one request for every 32 kB. Reading a 7.4 MB MIMIC-IV signal file now takes 8 requests instead of 229, which saves several minutes per file. + **Fix smoothing of frames with invalid samples** When a signal with more than one sample per frame is read with ``smooth_frames=True`` (the default), a frame that contains an invalid sample now gives an invalid value (NaN in physical units), as in the WFDB library. The invalid sample value used to be averaged with the other samples, which gave a large but valid-looking value. diff --git a/src/wfdb/io/_url.py b/src/wfdb/io/_url.py index c0970119..27920ca9 100644 --- a/src/wfdb/io/_url.py +++ b/src/wfdb/io/_url.py @@ -16,6 +16,11 @@ # Default buffer size for remote files. DEFAULT_BUFFER_SIZE = 32768 +# Largest buffer size for sequential reads. Each read that continues where +# the buffer ends doubles the size of the next request, up to this limit, +# so reading a file in small pieces doesn't take one request per piece. +MAX_READAHEAD_SIZE = 8 * 1024 * 1024 + # Timeouts in seconds (connect, read) for requests. The read timeout is the # maximum time to wait for the server to send data (not the total time of a # transfer). Without a timeout, a server that accepts a connection but does @@ -368,7 +373,9 @@ class NetFile(io.BufferedIOBase): request to the server. If buffering = -2, the entire file is downloaded in a single request. If buffering > 0, it specifies the minimum size of the internal buffer. If - buffering = -1, the default buffer size is used. + buffering = -1, the default buffer size is used. When + buffering is enabled, reads that continue where the previous + request ended use larger requests, up to MAX_READAHEAD_SIZE. """ @@ -381,6 +388,7 @@ def __init__(self, url, buffering=-1): self._buffer = b"" self._buffer_start = 0 self._buffer_end = 0 + self._readahead = 0 self._current_url = self.url def _read_buffered_range(self, start, end): @@ -459,6 +467,11 @@ def _read_range(self, start, end): buffer_size = self.buffering if buffer_size < 0: buffer_size = DEFAULT_BUFFER_SIZE + if self._buffer_end > self._buffer_start and start == self._buffer_end: + buffer_size = max( + buffer_size, min(2 * self._readahead, MAX_READAHEAD_SIZE) + ) + self._readahead = buffer_size req_start = start req_end = end if req_end < req_start + buffer_size: diff --git a/tests/test_url.py b/tests/test_url.py index 5fda1cc1..a3f0bb94 100644 --- a/tests/test_url.py +++ b/tests/test_url.py @@ -204,6 +204,27 @@ def _test_binary(self, url, content, buffering): assert result == content assert bf.tell() == len(content) + def test_sequential_reads(self): + """Reading a file in small pieces doesn't take one request per piece.""" + content = bytes(range(256)) * 8192 + with DummyHTTPServer({"/foo.dat": content}, allow_gzip=False) as server: + with wfdb.io._url.openurl(server.url("/foo.dat"), "rb") as f: + pieces = iter(lambda: f.read(8192), b"") + assert b"".join(pieces) == content + # 2 MB in 8 KB pieces: 32 KB, 64 KB, ..., 1 MB, then the rest + assert len(server.ranges) <= 8 + + def test_readahead_reset(self): + """A read elsewhere in the file starts again with a small request.""" + content = bytes(range(256)) * 8192 + with DummyHTTPServer({"/foo.dat": content}, allow_gzip=False) as server: + with wfdb.io._url.openurl(server.url("/foo.dat"), "rb") as f: + for _ in range(100): + f.read(8192) + f.seek(1500000) + assert f.read(100) == content[1500000:1500100] + assert server.ranges[-1] == "bytes=1500000-1532767" + class TestRemoteFLACFiles: """Test reading FLAC files over HTTP.""" @@ -282,6 +303,8 @@ class DummyHTTPServer(http.server.HTTPServer): delay : float, optional Number of seconds that the server waits before it responds to a request (to simulate a server that does not respond). + ranges : list + The Range header (or None) of each GET request the server received. server_address : tuple (str, int), optional A tuple specifying the address and port number where the server should listen for connections. If the port is 0, an @@ -303,6 +326,7 @@ def __init__( self.allow_gzip = allow_gzip self.allow_range = allow_range self.delay = delay + self.ranges = [] def url(self, path="/"): """Generate a URL that points to a file on this server. @@ -358,6 +382,8 @@ def send_head(self): headers = {"Content-Type": "text/plain"} status = 200 + if self.command == "GET": + self.server.ranges.append(self.headers.get("Range")) if self.server.allow_gzip: headers["Vary"] = "Accept-Encoding"