diff --git a/Pyjo/App/WSGI.py b/Pyjo/App/WSGI.py index f22bb5d8..4079dcfa 100644 --- a/Pyjo/App/WSGI.py +++ b/Pyjo/App/WSGI.py @@ -86,6 +86,7 @@ def hello(environ, start_response): import Pyjo.Server.Prethread import Pyjo.Transaction +import contextvars import io import sys import traceback @@ -184,6 +185,10 @@ def __init__(self, app: Pyjo_App_WSGI, application: WSGIApplication, tx: Any, c: self.c = c self.tx = tx self.response = _Response() + # The context of contextvars of the call, its iteration and its close, which a WSGI server gives them in one + # thread, where the callbacks of an event loop of asyncio each run in a context of their own, as the iterator of + # stream_with_context of Flask resets its context variables at its end with the tokens of its start + self.context = contextvars.copy_context() self.busy = False self.chunked = False self.closed = False @@ -193,14 +198,14 @@ def start(self) -> None: self.tx.on('finish', self._finish) try: environ = _environ(self.tx, self.c, self.app._model(self.tx)) # pyright: ignore[reportPrivateUsage] - self._step(Pyjo.IOLoop.blocking(self.response.start, self.application, environ)) + self._step(Pyjo.IOLoop.blocking(self.context.run, self.response.start, self.application, environ)) except Exception as e: self._error(e) def _close(self) -> None: self.done = True try: - Pyjo.IOLoop.blocking(self.response.close) + Pyjo.IOLoop.blocking(self.context.run, self.response.close) except Exception as e: self.app.log.error(f'WSGI application failed to close: {_message(e)}') @@ -219,7 +224,7 @@ def _error(self, e: BaseException) -> None: if self.response.started: self.app.log.error(f'WSGI application failed: {message}') if (connection := self.tx.connection) is not None and ( - stream := Pyjo.IOLoop.stream(connection) + stream := self._loop().stream(connection) ) is not None: stream.close() return @@ -272,13 +277,23 @@ def _headers(self, chunks: list[bytes] | None) -> None: # A body without a chunk yet tries again after the events of the next cycle of the event loop def _later(self) -> None: self.busy = True - Pyjo.IOLoop.timer(0, self._tick) + self._loop().timer(0, self._tick) + + # The event loop of the connection, which is the one of the server that runs the application: the singleton, or one of + # its own, as of the server of a blocking request of a user agent, where a step on the singleton, which waits for the + # blocking request, would never come + def _loop(self) -> Pyjo.IOLoop.object: + if (connection := self.tx.connection) is not None: + for loop in (Pyjo.IOLoop.singleton, *Pyjo.IOLoop._LOOPS): # pyright: ignore[reportPrivateUsage] + if loop.stream(connection) is not None: + return loop + return Pyjo.IOLoop.singleton # The next chunks of the body once the chunk before has been written, in the next tick of the event loop, so that the # application is not called in the middle of what the server does def _next(self, *args: Any) -> None: self.busy = True - Pyjo.IOLoop.next_tick(self._tick) + self._loop().next_tick(self._tick) def _rendered(self) -> None: if (c := self.c) is not None: @@ -308,7 +323,7 @@ def _tick(self, *args: Any) -> None: self._close() return try: - self._step(Pyjo.IOLoop.blocking(self.response.next)) + self._step(Pyjo.IOLoop.blocking(self.context.run, self.response.next)) except Exception as e: self._error(e) diff --git a/Pyjo/Message/Request.py b/Pyjo/Message/Request.py index 6d6465b5..0f5e7106 100644 --- a/Pyjo/Message/Request.py +++ b/Pyjo/Message/Request.py @@ -51,8 +51,8 @@ :mod:`Pyjoyment`, https://mojolicious.org. -Ported from `Mojo::Message::Request `_ -of Mojolicious 9.51. +Ported from `Mojo::Message::Request `_ +of Mojolicious 9.54. """ import Pyjo.Cookie.Request @@ -635,7 +635,7 @@ def _parse_env(self, env: Mapping[str, Any]) -> None: url.query.parse(str(env['QUERY_STRING'])) # Method - if env.get('REQUEST_METHOD'): + if env.get('REQUEST_METHOD') is not None: self.method = str(env['REQUEST_METHOD']) # Scheme/Version diff --git a/Pyjo/Server/Daemon.py b/Pyjo/Server/Daemon.py index f73dc57f..03acf539 100644 --- a/Pyjo/Server/Daemon.py +++ b/Pyjo/Server/Daemon.py @@ -109,8 +109,8 @@ def request(daemon, tx): :mod:`Pyjoyment`, https://mojolicious.org. -Ported from `Mojo::Server::Daemon `_ -of Mojolicious 9.52. +Ported from `Mojo::Server::Daemon `_ +of Mojolicious 9.54. Classes ------- @@ -724,8 +724,10 @@ def resume(tx: Pyjo.Transaction.HTTP.object) -> None: return tx def _cleanup(self) -> None: + for id in list(self._connections): + self._remove(id) loop = self.ioloop - for id in [*self._connections, *self.acceptors]: + for id in self.acceptors: loop.remove(id) self.acceptors = [] # Not in Mojolicious: the listen sockets of a stopped daemon, which Perl closes as it frees them diff --git a/Pyjo/UserAgent/__init__.py b/Pyjo/UserAgent/__init__.py index 5a86b19c..f5a7409b 100644 --- a/Pyjo/UserAgent/__init__.py +++ b/Pyjo/UserAgent/__init__.py @@ -178,8 +178,8 @@ def start(ua, tx): :mod:`Pyjoyment`, https://mojolicious.org. -Ported from `Mojo::UserAgent `_ -of Mojolicious 9.53. +Ported from `Mojo::UserAgent `_ +of Mojolicious 9.54. Classes and functions --------------------- @@ -964,7 +964,7 @@ def message(tx, msg): def _cleanup(self) -> Self: self._pid = None for id in list(self._connections): - self._finish(id, True) + self._finish(id, True, True) return self def _connect(self, loop: Pyjo.IOLoop.object, tx: Pyjo.Transaction.object, handle: Any = None) -> str: @@ -1122,7 +1122,7 @@ def _error(self, id: str, err: Any) -> None: tx.res.error = {'message': str(err)} self._finish(id, True) - def _finish(self, id: str, close: bool = False) -> None: + def _finish(self, id: str, close: bool = False, local: bool = False) -> None: # Remove request timeout and finish transaction if (c := self._connections.get(id)) is None: return None @@ -1131,11 +1131,17 @@ def _finish(self, id: str, close: bool = False) -> None: if (old := c.get('tx')) is None: return self._reuse(id, close) - # Premature connection close - res = old.closed().res.finish() - if close and not res.code and not res.error: + # Premature connection close (RFC 9112 8: only close-delimited bodies may end with the server closing the connection) + res = old.res + content = res.content + premature = not res.code or (not res.is_finished and (local or content.is_chunked or not content.relaxed)) + if close and premature and not res.error: res.error = {'message': 'Premature connection close'} + # Finish events need to see the error + res.finish() + old.closed() + # Always remove connection for WebSockets if old.is_websocket: return self._remove(id) diff --git a/t/Pyjo/App_WSGI.py b/t/Pyjo/App_WSGI.py index 5fdf386b..735f417e 100644 --- a/t/Pyjo/App_WSGI.py +++ b/t/Pyjo/App_WSGI.py @@ -16,6 +16,7 @@ import asyncio import concurrent.futures +import contextvars import http.client import os import re @@ -311,6 +312,34 @@ def write(n): t.get_ok('/prefix/missing').status_is(404) +# An iterator that sets a context variable at its start and resets it with its token at its end, as the one of +# stream_with_context of Flask, which needs one context for the call, the iteration and the close, as a WSGI server gives +variable = contextvars.ContextVar('variable') + + +@validator +def contextual(environ, start_response): + def body(): + token = variable.set('set') + try: + yield b'one' + yield b'two' + finally: + variable.reset(token) + + start_response('200 OK', [('Content-Type', 'text/plain')]) + return body() + + +with subtest('Context of the call'): + with Pyjo.App.WSGI.new(app=contextual) as app: + t = Pyjo.Test.Pyjo.new(app) + with app.log.capture('error') as logs: + t.get_ok('/').status_is(200).content_is('onetwo') + ok(not t.tx.error, 'whole response') + is_deeply_ok(list(logs), [], 'no error of the application') + + # A list, without the validator, whose iterator would hide it def whole(environ, start_response): start_response('201 Created', [('Content-Type', 'text/plain'), ('X-Foo', 'a'), ('X-Foo', 'b')]) diff --git a/t/Pyjo/Message_Request_CGI.py b/t/Pyjo/Message_Request_CGI.py index 3fc6002a..079c9d4a 100644 --- a/t/Pyjo/Message_Request_CGI.py +++ b/t/Pyjo/Message_Request_CGI.py @@ -624,4 +624,16 @@ def progress_cb(req, *args): is_ok(req.version, '1.1', 'right version') ok(not req.is_secure, 'not secure') +# Not in Mojolicious: the method and the query of "0", which Mojolicious 9.54 takes from the environment without a test, +# and an empty method, which is defined +with subtest('Parse CGI environment variables with a method and a query of 0'): + req = Pyjo.Message.Request.new() + req.parse({'PATH_INFO': '/', 'QUERY_STRING': '0', 'REQUEST_METHOD': '0', 'HTTP_HOST': 'localhost'}) + is_ok(req.method, '0', 'right method') + is_ok(req.url.query.to_str(), '0', 'right query') + + req = Pyjo.Message.Request.new() + req.parse({'PATH_INFO': '/', 'REQUEST_METHOD': '', 'HTTP_HOST': 'localhost'}) + is_ok(req.method, '', 'empty method') + done_testing() diff --git a/t/Pyjo/Server_Daemon.py b/t/Pyjo/Server_Daemon.py index d471091a..dcb0dbcb 100644 --- a/t/Pyjo/Server_Daemon.py +++ b/t/Pyjo/Server_Daemon.py @@ -425,6 +425,8 @@ def tick(loop): app_ = Pyjoyment.new() app_.log.level = 'fatal' got = {} + finished = [] + app_.hook('after_build_tx', lambda tx, app: tx.on('finish', lambda tx: finished.append(True))) @app_.routes.get('/') def destroy(c): @@ -453,6 +455,7 @@ def close(stream): loop.start() ok(closed, 'connection has been closed') ok(not timeout, 'no timeout') + ok(finished, 'transaction has been finished') is_deeply_ok([e for e in errors if 'I/O watcher failed' in e], [], 'no exceptions') # Not in Mojolicious: the rest of a request that has its response already, which the server reads before it closes the diff --git a/t/Pyjo/UserAgent.py b/t/Pyjo/UserAgent.py index 891db038..ddd10870 100644 --- a/t/Pyjo/UserAgent.py +++ b/t/Pyjo/UserAgent.py @@ -764,6 +764,76 @@ def read(stream, chunk): is_ok(tx.res.code, 200, 'right status') is_ok(tx.res.body, b'Hi!', 'right content') +with subtest('Premature connection close after start-line'): + response, close = [None], [True] + + @Pyjo.IOLoop.server(address='127.0.0.1') + def premature(loop, stream, id): + req = Pyjo.Message.Request.new() + + @stream.on('read') + def read(stream, chunk): + if req.parse(chunk).is_finished: + stream.write(response[0], lambda stream: stream.close_gracefully() if close[0] else None) + + port = Pyjo.IOLoop.acceptor(premature).port + + with subtest('Content-Length not reached'): + response[0] = b'HTTP/1.1 200 OK\x0d\x0aContent-Length: 10\x0d\x0a\x0d\x0aHi!' + tx = ua.build_tx('GET', f'http://127.0.0.1:{port}/') + tx_error, res_error = [], [] + tx.on('finish', lambda tx: tx_error.append(tx.error)) + tx.res.on('finish', lambda res: res_error.append(res.error)) + tx = ua.start(tx) + is_ok(tx.error['message'], 'Premature connection close', 'right error') + is_ok(tx_error[0]['message'], 'Premature connection close', 'error is visible in transaction finish event') + is_ok(res_error[0]['message'], 'Premature connection close', 'error is visible in response finish event') + is_ok(tx.res.code, 200, 'right status') + is_ok(tx.res.body, b'Hi!', 'right content') + + with subtest('Chunked without last chunk'): + response[0] = b'HTTP/1.1 200 OK\x0d\x0aTransfer-Encoding: chunked\x0d\x0a\x0d\x0a3\x0d\x0aHi!\x0d\x0a' + tx = ua.get(f'http://127.0.0.1:{port}/') + is_ok(tx.error['message'], 'Premature connection close', 'right error') + is_ok(tx.res.code, 200, 'right status') + is_ok(tx.res.body, b'Hi!', 'right content') + + with subtest('Incomplete headers'): + response[0] = b'HTTP/1.1 200 OK\x0d\x0aContent-Le' + tx = ua.get(f'http://127.0.0.1:{port}/') + is_ok(tx.error['message'], 'Premature connection close', 'right error') + is_ok(tx.res.code, 200, 'right status') + is_ok(tx.res.body, b'', 'no content') + + with subtest('Body delimited by connection close'): + response[0] = b'HTTP/1.1 200 OK\x0d\x0aConnection: close\x0d\x0a\x0d\x0aHi!' + tx = ua.get(f'http://127.0.0.1:{port}/') + ok(not tx.error, 'no error') + is_ok(tx.res.code, 200, 'right status') + is_ok(tx.res.body, b'Hi!', 'right content') + + with subtest('User agent destroyed during body delimited by connection close'): + response[0] = b'HTTP/1.1 200 OK\x0d\x0aConnection: close\x0d\x0a\x0d\x0aHi!' + close[0] = False + destroyed = Pyjo.UserAgent.new() + tx = destroyed.build_tx('GET', f'http://127.0.0.1:{port}/') + # close() releases the connections, as Perl does in DESTROY of the user agent + tx.res.content.once('read', lambda content, chunk: Pyjo.IOLoop.next_tick(lambda loop: destroyed.close())) + error = [] + + def destroyed_cb(ua, tx): + error.append(tx.error) + Pyjo.IOLoop.stop() + + destroyed.start(tx, destroyed_cb) + Pyjo.IOLoop.start() + close[0] = True + is_ok(error[0]['message'], 'Premature connection close', 'right error') + is_ok(tx.res.body, b'Hi!', 'right content') + + # Not in Mojolicious: the server of the subtest, which the rest of the test does not need + Pyjo.IOLoop.remove(premature) + # Connection limit ua = Pyjo.UserAgent.new(max_connections=2) result = None