From d097c91347cc20cdeba59042d08f1e5fb8ea5cfb Mon Sep 17 00:00:00 2001 From: Octavio Galland Date: Thu, 27 Aug 2026 16:36:12 -0300 Subject: [PATCH 1/2] don't omit CRLF when those are the only bytes the in the chunk --- cachecontrol/adapter.py | 4 ++++ cachecontrol/filewrapper.py | 9 ++++++--- tests/conftest.py | 8 ++++++++ tests/test_chunked_response.py | 8 ++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/cachecontrol/adapter.py b/cachecontrol/adapter.py index 4f4c185a..d23d74a6 100644 --- a/cachecontrol/adapter.py +++ b/cachecontrol/adapter.py @@ -144,6 +144,10 @@ def _update_chunk_length( super_update_chunk_length(self) if self.chunk_left == 0: self._fp._close() # type: ignore[union-attr] + elif self.chunk_left is not None: + self._fp._set_chunk_bytes_remaining( # type: ignore[union-attr] + self.chunk_left + ) response._update_chunk_length = functools.partial( # type: ignore[method-assign] _update_chunk_length, weakref.ref(response) diff --git a/cachecontrol/filewrapper.py b/cachecontrol/filewrapper.py index 6569fb5c..5185cbb9 100644 --- a/cachecontrol/filewrapper.py +++ b/cachecontrol/filewrapper.py @@ -37,6 +37,7 @@ def __init__( self.__buf = NamedTemporaryFile("rb+", delete=True) self.__fp = fp self.__callback = callback + self.__chunk_bytes_remaining = 0 def __getattr__(self, name: str) -> Any: # The vagaries of garbage collection means that self.__fp is @@ -107,14 +108,16 @@ def read(self, amt: int | None = None) -> bytes: return data + def _set_chunk_bytes_remaining(self, chunk_bytes_remaining: int) -> None: + self.__chunk_bytes_remaining = chunk_bytes_remaining + def _safe_read(self, amt: int) -> bytes: data: bytes = self.__fp._safe_read(amt) # type: ignore[attr-defined] - if amt == 2 and data == b"\r\n": - # urllib executes this read to toss the CRLF at the end - # of the chunk. + if self.__chunk_bytes_remaining == 0 and amt == 2 and data == b"\r\n": return data self.__buf.write(data) + self.__chunk_bytes_remaining -= len(data) if self.__is_fp_closed(): self._close() diff --git a/tests/conftest.py b/tests/conftest.py index 00f4da93..de55a337 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -101,6 +101,14 @@ def stream(self, env, start_response): for i in range(10): yield pformat(i).encode("utf8") + def stream_with_crlf(self, env, start_response): + headers = [("Content-Type", "text/plain"), ("Cache-Control", "max-age=5000")] + start_response("200 OK", headers) + + yield b"AA" + yield b"\r\n" + yield b"BB" + def fixed_length(self, env, start_response): body = b"0123456789" headers = [ diff --git a/tests/test_chunked_response.py b/tests/test_chunked_response.py index 8cc44964..a79a713f 100644 --- a/tests/test_chunked_response.py +++ b/tests/test_chunked_response.py @@ -49,6 +49,14 @@ def test_stream_is_cached(self, url, sess): assert resp_2.from_cache assert content_1 == content_2 + def test_stream_with_crlf_chunk_is_cached_without_corruption(self, url, sess): + resp_1 = sess.get(url + "stream_with_crlf") + resp_2 = sess.get(url + "stream_with_crlf") + + assert resp_1.content == b"AA\r\nBB" + assert resp_2.from_cache + assert resp_2.content == resp_1.content + def test_stream_is_not_cached_when_content_is_not_read(self, url, sess): sess.get(url + "stream", stream=True) resp = sess.get(url + "stream", stream=True) From d6f45232a2e634d7c0d3834fe7cd27ccd6cb7b63 Mon Sep 17 00:00:00 2001 From: Octavio Galland Date: Fri, 11 Sep 2026 14:31:26 -0300 Subject: [PATCH 2/2] ignore trailing content after separator --- cachecontrol/filewrapper.py | 2 +- tests/conftest.py | 16 ++++++++++++++++ tests/test_chunked_response.py | 12 +++++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/cachecontrol/filewrapper.py b/cachecontrol/filewrapper.py index 5185cbb9..74e4fb0f 100644 --- a/cachecontrol/filewrapper.py +++ b/cachecontrol/filewrapper.py @@ -113,7 +113,7 @@ def _set_chunk_bytes_remaining(self, chunk_bytes_remaining: int) -> None: def _safe_read(self, amt: int) -> bytes: data: bytes = self.__fp._safe_read(amt) # type: ignore[attr-defined] - if self.__chunk_bytes_remaining == 0 and amt == 2 and data == b"\r\n": + if self.__chunk_bytes_remaining == 0 and amt == 2: return data self.__buf.write(data) diff --git a/tests/conftest.py b/tests/conftest.py index de55a337..27685330 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ import cherrypy import pytest +from cheroot.server import HTTPRequest class SimpleApp: @@ -155,6 +156,21 @@ def url(server): return "http://%s:%s/" % server.bind_addr +@pytest.fixture() +def malformed_chunk_delimiters(monkeypatch): + """Make the test server send XX instead of each chunk's trailing CRLF.""" + write = HTTPRequest.write + + def write_malformed_chunk(request, chunk): + if request.chunked_write and chunk: + data = f"{len(chunk):x}\r\n".encode() + chunk + b"XX" + request.conn.wfile.write(data) + else: + write(request, chunk) + + monkeypatch.setattr(HTTPRequest, "write", write_malformed_chunk) + + def get_free_port(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("", 0)) diff --git a/tests/test_chunked_response.py b/tests/test_chunked_response.py index a79a713f..8212df8f 100644 --- a/tests/test_chunked_response.py +++ b/tests/test_chunked_response.py @@ -43,7 +43,7 @@ def test_stream_is_cached(self, url, sess): content_1 = resp_1.content resp_2 = sess.get(url + "stream") - content_2 = resp_1.content + content_2 = resp_2.content assert not resp_1.from_cache assert resp_2.from_cache @@ -62,3 +62,13 @@ def test_stream_is_not_cached_when_content_is_not_read(self, url, sess): resp = sess.get(url + "stream", stream=True) assert not resp.from_cache + + def test_stream_with_malformed_delimiters_is_cached_without_corruption( + self, url, sess, malformed_chunk_delimiters + ): + resp_1 = sess.get(url + "stream_with_crlf") + resp_2 = sess.get(url + "stream_with_crlf") + + assert resp_1.content == b"AA\r\nBB" + assert resp_2.from_cache + assert resp_2.content == resp_1.content