From 2ce6988b6b6915b171c4c124d4f317661a896ee0 Mon Sep 17 00:00:00 2001 From: Piotr Roszatycki Date: Thu, 12 Nov 2026 18:01:52 +0100 Subject: [PATCH 1/3] Pyjo.App.WSGI: the call of the application, its iteration and its close in one context of contextvars A WSGI server gives them one thread, where the callbacks of the event loop of asyncio each run in a context of their own: the iterator of stream_with_context of Flask resets its context variables at its end with the tokens of its start, which raised ValueError in another context, so that the connection of a streamed response closed before its last chunk, which the user agent took for the end of the response until Mojolicious 9.54. Co-Authored-By: Claude Opus 5.5 --- Pyjo/App/WSGI.py | 11 ++++++++--- t/Pyjo/App_WSGI.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/Pyjo/App/WSGI.py b/Pyjo/App/WSGI.py index f22bb5d8..db41803b 100644 --- a/Pyjo/App/WSGI.py +++ b/Pyjo/App/WSGI.py @@ -86,6 +86,7 @@ def hello(environ, start_response): import Pyjo.Server.Prethread import Pyjo.Transaction +import contextvars import io import sys import traceback @@ -184,6 +185,10 @@ def __init__(self, app: Pyjo_App_WSGI, application: WSGIApplication, tx: Any, c: self.c = c self.tx = tx self.response = _Response() + # The context of contextvars of the call, its iteration and its close, which a WSGI server gives them in one + # thread, where the callbacks of an event loop of asyncio each run in a context of their own, as the iterator of + # stream_with_context of Flask resets its context variables at its end with the tokens of its start + self.context = contextvars.copy_context() self.busy = False self.chunked = False self.closed = False @@ -193,14 +198,14 @@ def start(self) -> None: self.tx.on('finish', self._finish) try: environ = _environ(self.tx, self.c, self.app._model(self.tx)) # pyright: ignore[reportPrivateUsage] - self._step(Pyjo.IOLoop.blocking(self.response.start, self.application, environ)) + self._step(Pyjo.IOLoop.blocking(self.context.run, self.response.start, self.application, environ)) except Exception as e: self._error(e) def _close(self) -> None: self.done = True try: - Pyjo.IOLoop.blocking(self.response.close) + Pyjo.IOLoop.blocking(self.context.run, self.response.close) except Exception as e: self.app.log.error(f'WSGI application failed to close: {_message(e)}') @@ -308,7 +313,7 @@ def _tick(self, *args: Any) -> None: self._close() return try: - self._step(Pyjo.IOLoop.blocking(self.response.next)) + self._step(Pyjo.IOLoop.blocking(self.context.run, self.response.next)) except Exception as e: self._error(e) diff --git a/t/Pyjo/App_WSGI.py b/t/Pyjo/App_WSGI.py index 5fdf386b..735f417e 100644 --- a/t/Pyjo/App_WSGI.py +++ b/t/Pyjo/App_WSGI.py @@ -16,6 +16,7 @@ import asyncio import concurrent.futures +import contextvars import http.client import os import re @@ -311,6 +312,34 @@ def write(n): t.get_ok('/prefix/missing').status_is(404) +# An iterator that sets a context variable at its start and resets it with its token at its end, as the one of +# stream_with_context of Flask, which needs one context for the call, the iteration and the close, as a WSGI server gives +variable = contextvars.ContextVar('variable') + + +@validator +def contextual(environ, start_response): + def body(): + token = variable.set('set') + try: + yield b'one' + yield b'two' + finally: + variable.reset(token) + + start_response('200 OK', [('Content-Type', 'text/plain')]) + return body() + + +with subtest('Context of the call'): + with Pyjo.App.WSGI.new(app=contextual) as app: + t = Pyjo.Test.Pyjo.new(app) + with app.log.capture('error') as logs: + t.get_ok('/').status_is(200).content_is('onetwo') + ok(not t.tx.error, 'whole response') + is_deeply_ok(list(logs), [], 'no error of the application') + + # A list, without the validator, whose iterator would hide it def whole(environ, start_response): start_response('201 Created', [('Content-Type', 'text/plain'), ('X-Foo', 'a'), ('X-Foo', 'b')]) From d46c1a97a1cd510cb5de48c3952ad66f15c22380 Mon Sep 17 00:00:00 2001 From: Piotr Roszatycki Date: Thu, 12 Nov 2026 18:01:55 +0100 Subject: [PATCH 2/3] Pyjo.App.WSGI: the steps of a streamed response on the event loop of its connection The event loop of the server that runs the application, which is the singleton, or one of its own, as of the server of a blocking request of a user agent, where a step on the singleton, which waits for the blocking request, never came, so that the response ended without its last chunk. Co-Authored-By: Claude Opus 5.5 --- Pyjo/App/WSGI.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Pyjo/App/WSGI.py b/Pyjo/App/WSGI.py index db41803b..4079dcfa 100644 --- a/Pyjo/App/WSGI.py +++ b/Pyjo/App/WSGI.py @@ -224,7 +224,7 @@ def _error(self, e: BaseException) -> None: if self.response.started: self.app.log.error(f'WSGI application failed: {message}') if (connection := self.tx.connection) is not None and ( - stream := Pyjo.IOLoop.stream(connection) + stream := self._loop().stream(connection) ) is not None: stream.close() return @@ -277,13 +277,23 @@ def _headers(self, chunks: list[bytes] | None) -> None: # A body without a chunk yet tries again after the events of the next cycle of the event loop def _later(self) -> None: self.busy = True - Pyjo.IOLoop.timer(0, self._tick) + self._loop().timer(0, self._tick) + + # The event loop of the connection, which is the one of the server that runs the application: the singleton, or one of + # its own, as of the server of a blocking request of a user agent, where a step on the singleton, which waits for the + # blocking request, would never come + def _loop(self) -> Pyjo.IOLoop.object: + if (connection := self.tx.connection) is not None: + for loop in (Pyjo.IOLoop.singleton, *Pyjo.IOLoop._LOOPS): # pyright: ignore[reportPrivateUsage] + if loop.stream(connection) is not None: + return loop + return Pyjo.IOLoop.singleton # The next chunks of the body once the chunk before has been written, in the next tick of the event loop, so that the # application is not called in the middle of what the server does def _next(self, *args: Any) -> None: self.busy = True - Pyjo.IOLoop.next_tick(self._tick) + self._loop().next_tick(self._tick) def _rendered(self) -> None: if (c := self.c) is not None: From d3451790ff1048f921e0083aca970d13a72f9497 Mon Sep 17 00:00:00 2001 From: Piotr Roszatycki Date: Thu, 12 Nov 2026 18:02:05 +0100 Subject: [PATCH 3/3] Pyjo.UserAgent, Pyjo.Server.Daemon, Pyjo.Message.Request: Mojolicious 9.54 - Pyjo.UserAgent: a response that is not complete when its connection closes is a premature connection close, also after its start-line, unless its body is delimited by the close of the connection, as RFC 9112 says, or the user agent closes it itself, and the finish events see the error (e6d5a54f0). - Pyjo.Server.Daemon: the cleanup of the server with active connections emits the finish events of their transactions (60624d154). - Pyjo.Message.Request: a method of the environment that is defined, also an empty one, as a method and a query of "0" are true in Python already (8b0dbdf72). Part of #408 Co-Authored-By: Claude Opus 5.5 --- Pyjo/Message/Request.py | 6 +-- Pyjo/Server/Daemon.py | 8 ++-- Pyjo/UserAgent/__init__.py | 20 ++++++---- t/Pyjo/Message_Request_CGI.py | 12 ++++++ t/Pyjo/Server_Daemon.py | 3 ++ t/Pyjo/UserAgent.py | 70 +++++++++++++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 13 deletions(-) diff --git a/Pyjo/Message/Request.py b/Pyjo/Message/Request.py index 6d6465b5..0f5e7106 100644 --- a/Pyjo/Message/Request.py +++ b/Pyjo/Message/Request.py @@ -51,8 +51,8 @@ :mod:`Pyjoyment`, https://mojolicious.org. -Ported from `Mojo::Message::Request `_ -of Mojolicious 9.51. +Ported from `Mojo::Message::Request `_ +of Mojolicious 9.54. """ import Pyjo.Cookie.Request @@ -635,7 +635,7 @@ def _parse_env(self, env: Mapping[str, Any]) -> None: url.query.parse(str(env['QUERY_STRING'])) # Method - if env.get('REQUEST_METHOD'): + if env.get('REQUEST_METHOD') is not None: self.method = str(env['REQUEST_METHOD']) # Scheme/Version diff --git a/Pyjo/Server/Daemon.py b/Pyjo/Server/Daemon.py index f73dc57f..03acf539 100644 --- a/Pyjo/Server/Daemon.py +++ b/Pyjo/Server/Daemon.py @@ -109,8 +109,8 @@ def request(daemon, tx): :mod:`Pyjoyment`, https://mojolicious.org. -Ported from `Mojo::Server::Daemon `_ -of Mojolicious 9.52. +Ported from `Mojo::Server::Daemon `_ +of Mojolicious 9.54. Classes ------- @@ -724,8 +724,10 @@ def resume(tx: Pyjo.Transaction.HTTP.object) -> None: return tx def _cleanup(self) -> None: + for id in list(self._connections): + self._remove(id) loop = self.ioloop - for id in [*self._connections, *self.acceptors]: + for id in self.acceptors: loop.remove(id) self.acceptors = [] # Not in Mojolicious: the listen sockets of a stopped daemon, which Perl closes as it frees them diff --git a/Pyjo/UserAgent/__init__.py b/Pyjo/UserAgent/__init__.py index 5a86b19c..f5a7409b 100644 --- a/Pyjo/UserAgent/__init__.py +++ b/Pyjo/UserAgent/__init__.py @@ -178,8 +178,8 @@ def start(ua, tx): :mod:`Pyjoyment`, https://mojolicious.org. -Ported from `Mojo::UserAgent `_ -of Mojolicious 9.53. +Ported from `Mojo::UserAgent `_ +of Mojolicious 9.54. Classes and functions --------------------- @@ -964,7 +964,7 @@ def message(tx, msg): def _cleanup(self) -> Self: self._pid = None for id in list(self._connections): - self._finish(id, True) + self._finish(id, True, True) return self def _connect(self, loop: Pyjo.IOLoop.object, tx: Pyjo.Transaction.object, handle: Any = None) -> str: @@ -1122,7 +1122,7 @@ def _error(self, id: str, err: Any) -> None: tx.res.error = {'message': str(err)} self._finish(id, True) - def _finish(self, id: str, close: bool = False) -> None: + def _finish(self, id: str, close: bool = False, local: bool = False) -> None: # Remove request timeout and finish transaction if (c := self._connections.get(id)) is None: return None @@ -1131,11 +1131,17 @@ def _finish(self, id: str, close: bool = False) -> None: if (old := c.get('tx')) is None: return self._reuse(id, close) - # Premature connection close - res = old.closed().res.finish() - if close and not res.code and not res.error: + # Premature connection close (RFC 9112 8: only close-delimited bodies may end with the server closing the connection) + res = old.res + content = res.content + premature = not res.code or (not res.is_finished and (local or content.is_chunked or not content.relaxed)) + if close and premature and not res.error: res.error = {'message': 'Premature connection close'} + # Finish events need to see the error + res.finish() + old.closed() + # Always remove connection for WebSockets if old.is_websocket: return self._remove(id) diff --git a/t/Pyjo/Message_Request_CGI.py b/t/Pyjo/Message_Request_CGI.py index 3fc6002a..079c9d4a 100644 --- a/t/Pyjo/Message_Request_CGI.py +++ b/t/Pyjo/Message_Request_CGI.py @@ -624,4 +624,16 @@ def progress_cb(req, *args): is_ok(req.version, '1.1', 'right version') ok(not req.is_secure, 'not secure') +# Not in Mojolicious: the method and the query of "0", which Mojolicious 9.54 takes from the environment without a test, +# and an empty method, which is defined +with subtest('Parse CGI environment variables with a method and a query of 0'): + req = Pyjo.Message.Request.new() + req.parse({'PATH_INFO': '/', 'QUERY_STRING': '0', 'REQUEST_METHOD': '0', 'HTTP_HOST': 'localhost'}) + is_ok(req.method, '0', 'right method') + is_ok(req.url.query.to_str(), '0', 'right query') + + req = Pyjo.Message.Request.new() + req.parse({'PATH_INFO': '/', 'REQUEST_METHOD': '', 'HTTP_HOST': 'localhost'}) + is_ok(req.method, '', 'empty method') + done_testing() diff --git a/t/Pyjo/Server_Daemon.py b/t/Pyjo/Server_Daemon.py index d471091a..dcb0dbcb 100644 --- a/t/Pyjo/Server_Daemon.py +++ b/t/Pyjo/Server_Daemon.py @@ -425,6 +425,8 @@ def tick(loop): app_ = Pyjoyment.new() app_.log.level = 'fatal' got = {} + finished = [] + app_.hook('after_build_tx', lambda tx, app: tx.on('finish', lambda tx: finished.append(True))) @app_.routes.get('/') def destroy(c): @@ -453,6 +455,7 @@ def close(stream): loop.start() ok(closed, 'connection has been closed') ok(not timeout, 'no timeout') + ok(finished, 'transaction has been finished') is_deeply_ok([e for e in errors if 'I/O watcher failed' in e], [], 'no exceptions') # Not in Mojolicious: the rest of a request that has its response already, which the server reads before it closes the diff --git a/t/Pyjo/UserAgent.py b/t/Pyjo/UserAgent.py index 891db038..ddd10870 100644 --- a/t/Pyjo/UserAgent.py +++ b/t/Pyjo/UserAgent.py @@ -764,6 +764,76 @@ def read(stream, chunk): is_ok(tx.res.code, 200, 'right status') is_ok(tx.res.body, b'Hi!', 'right content') +with subtest('Premature connection close after start-line'): + response, close = [None], [True] + + @Pyjo.IOLoop.server(address='127.0.0.1') + def premature(loop, stream, id): + req = Pyjo.Message.Request.new() + + @stream.on('read') + def read(stream, chunk): + if req.parse(chunk).is_finished: + stream.write(response[0], lambda stream: stream.close_gracefully() if close[0] else None) + + port = Pyjo.IOLoop.acceptor(premature).port + + with subtest('Content-Length not reached'): + response[0] = b'HTTP/1.1 200 OK\x0d\x0aContent-Length: 10\x0d\x0a\x0d\x0aHi!' + tx = ua.build_tx('GET', f'http://127.0.0.1:{port}/') + tx_error, res_error = [], [] + tx.on('finish', lambda tx: tx_error.append(tx.error)) + tx.res.on('finish', lambda res: res_error.append(res.error)) + tx = ua.start(tx) + is_ok(tx.error['message'], 'Premature connection close', 'right error') + is_ok(tx_error[0]['message'], 'Premature connection close', 'error is visible in transaction finish event') + is_ok(res_error[0]['message'], 'Premature connection close', 'error is visible in response finish event') + is_ok(tx.res.code, 200, 'right status') + is_ok(tx.res.body, b'Hi!', 'right content') + + with subtest('Chunked without last chunk'): + response[0] = b'HTTP/1.1 200 OK\x0d\x0aTransfer-Encoding: chunked\x0d\x0a\x0d\x0a3\x0d\x0aHi!\x0d\x0a' + tx = ua.get(f'http://127.0.0.1:{port}/') + is_ok(tx.error['message'], 'Premature connection close', 'right error') + is_ok(tx.res.code, 200, 'right status') + is_ok(tx.res.body, b'Hi!', 'right content') + + with subtest('Incomplete headers'): + response[0] = b'HTTP/1.1 200 OK\x0d\x0aContent-Le' + tx = ua.get(f'http://127.0.0.1:{port}/') + is_ok(tx.error['message'], 'Premature connection close', 'right error') + is_ok(tx.res.code, 200, 'right status') + is_ok(tx.res.body, b'', 'no content') + + with subtest('Body delimited by connection close'): + response[0] = b'HTTP/1.1 200 OK\x0d\x0aConnection: close\x0d\x0a\x0d\x0aHi!' + tx = ua.get(f'http://127.0.0.1:{port}/') + ok(not tx.error, 'no error') + is_ok(tx.res.code, 200, 'right status') + is_ok(tx.res.body, b'Hi!', 'right content') + + with subtest('User agent destroyed during body delimited by connection close'): + response[0] = b'HTTP/1.1 200 OK\x0d\x0aConnection: close\x0d\x0a\x0d\x0aHi!' + close[0] = False + destroyed = Pyjo.UserAgent.new() + tx = destroyed.build_tx('GET', f'http://127.0.0.1:{port}/') + # close() releases the connections, as Perl does in DESTROY of the user agent + tx.res.content.once('read', lambda content, chunk: Pyjo.IOLoop.next_tick(lambda loop: destroyed.close())) + error = [] + + def destroyed_cb(ua, tx): + error.append(tx.error) + Pyjo.IOLoop.stop() + + destroyed.start(tx, destroyed_cb) + Pyjo.IOLoop.start() + close[0] = True + is_ok(error[0]['message'], 'Premature connection close', 'right error') + is_ok(tx.res.body, b'Hi!', 'right content') + + # Not in Mojolicious: the server of the subtest, which the rest of the test does not need + Pyjo.IOLoop.remove(premature) + # Connection limit ua = Pyjo.UserAgent.new(max_connections=2) result = None