Read remote FLAC files with fewer requests

docs/changes.rst

@@ -20,6 +20,9 @@ Unreleased
 **Add timeouts to requests for remote files**
   Requests for files on PhysioNet (and other web servers) now time out if the server does not respond (10 seconds to connect, and 60 seconds to wait for data), instead of waiting indefinitely. A ``requests.exceptions.Timeout`` (``ConnectTimeout`` or ``ReadTimeout``) is raised in this case.
 
+**Read remote FLAC files with fewer requests**
+  Remote files read in small pieces, such as FLAC signal files on PhysioNet, now use larger requests as reading goes on (up to 8 MB) instead of one request for every 32 kB. Reading a 7.4 MB MIMIC-IV signal file now takes 8 requests instead of 229, which saves several minutes per file.
+
 **Fix smoothing of frames with invalid samples**
   When a signal with more than one sample per frame is read with ``smooth_frames=True`` (the default), a frame that contains an invalid sample now gives an invalid value (NaN in physical units), as in the WFDB library. The invalid sample value used to be averaged with the other samples, which gave a large but valid-looking value.
 

src/wfdb/io/_url.py

@@ -16,6 +16,11 @@
 # Default buffer size for remote files.
 DEFAULT_BUFFER_SIZE = 32768
 
+# Largest buffer size for sequential reads. Each read that continues where
+# the buffer ends doubles the size of the next request, up to this limit,
+# so reading a file in small pieces doesn't take one request per piece.
+MAX_READAHEAD_SIZE = 8 * 1024 * 1024
+
 # Timeouts in seconds (connect, read) for requests. The read timeout is the
 # maximum time to wait for the server to send data (not the total time of a
 # transfer). Without a timeout, a server that accepts a connection but does
@@ -368,7 +373,9 @@ class NetFile(io.BufferedIOBase):
         request to the server.  If buffering = -2, the entire file is
         downloaded in a single request.  If buffering > 0, it
         specifies the minimum size of the internal buffer.  If
-        buffering = -1, the default buffer size is used.
+        buffering = -1, the default buffer size is used.  When
+        buffering is enabled, reads that continue where the previous
+        request ended use larger requests, up to MAX_READAHEAD_SIZE.
 
     """
 
@@ -381,6 +388,7 @@ def __init__(self, url, buffering=-1):
         self._buffer = b""
         self._buffer_start = 0
         self._buffer_end = 0
+        self._readahead = 0
         self._current_url = self.url
 
     def _read_buffered_range(self, start, end):
@@ -459,6 +467,11 @@ def _read_range(self, start, end):
             buffer_size = self.buffering
             if buffer_size < 0:
                 buffer_size = DEFAULT_BUFFER_SIZE
+            if self._buffer_end > self._buffer_start and start == self._buffer_end:
+                buffer_size = max(
+                    buffer_size, min(2 * self._readahead, MAX_READAHEAD_SIZE)
+                )
+            self._readahead = buffer_size
             req_start = start
             req_end = end
             if req_end < req_start + buffer_size:

tests/test_url.py

@@ -204,6 +204,27 @@ def _test_binary(self, url, content, buffering):
             assert result == content
             assert bf.tell() == len(content)
 
+    def test_sequential_reads(self):
+        """Reading a file in small pieces doesn't take one request per piece."""
+        content = bytes(range(256)) * 8192
+        with DummyHTTPServer({"/foo.dat": content}, allow_gzip=False) as server:
+            with wfdb.io._url.openurl(server.url("/foo.dat"), "rb") as f:
+                pieces = iter(lambda: f.read(8192), b"")
+                assert b"".join(pieces) == content
+            # 2 MB in 8 KB pieces: 32 KB, 64 KB, ..., 1 MB, then the rest
+            assert len(server.ranges) <= 8
+
+    def test_readahead_reset(self):
+        """A read elsewhere in the file starts again with a small request."""
+        content = bytes(range(256)) * 8192
+        with DummyHTTPServer({"/foo.dat": content}, allow_gzip=False) as server:
+            with wfdb.io._url.openurl(server.url("/foo.dat"), "rb") as f:
+                for _ in range(100):
+                    f.read(8192)
+                f.seek(1500000)
+                assert f.read(100) == content[1500000:1500100]
+            assert server.ranges[-1] == "bytes=1500000-1532767"
+
 
 class TestRemoteFLACFiles:
     """Test reading FLAC files over HTTP."""
@@ -282,6 +303,8 @@ class DummyHTTPServer(http.server.HTTPServer):
     delay : float, optional
         Number of seconds that the server waits before it responds to a
         request (to simulate a server that does not respond).
+    ranges : list
+        The Range header (or None) of each GET request the server received.
     server_address : tuple (str, int), optional
         A tuple specifying the address and port number where the
         server should listen for connections.  If the port is 0, an
@@ -303,6 +326,7 @@ def __init__(
         self.allow_gzip = allow_gzip
         self.allow_range = allow_range
         self.delay = delay
+        self.ranges = []
 
     def url(self, path="/"):
         """Generate a URL that points to a file on this server.
@@ -358,6 +382,8 @@ def send_head(self):
 
         headers = {"Content-Type": "text/plain"}
         status = 200
+        if self.command == "GET":
+            self.server.ranges.append(self.headers.get("Range"))
 
         if self.server.allow_gzip:
             headers["Vary"] = "Accept-Encoding"