diff --git a/AGENTS.md b/AGENTS.md index e7a5b1e..cf4d1f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,6 +85,34 @@ Pyjo-Reactor-Uvloop, whose workflows and linters it shares, and whose error lasts, as with the loop of asyncio. It stops watching only on an error of the poll itself, as of a socket that is closed, and `select.poll`, with which uvloop checks `POLLERR` on Linux, does not exist on Windows. +- A socket that loops of the process watch, as the workers of + `Pyjo.Server.Prethread` watch the listen sockets of the manager on Windows, + the same sockets in every thread, is polled by one loop at a time. libuv of + Windows submits every poll of AFD with `Exclusive = TRUE`, which ends the + poll of the same socket of any other loop, also of a copy of the socket, as + AFD is exclusive for the socket and not for its handle; that loop polls it + again, and so on, in a busy loop of C with no event at all, which the + documentation of libuv warns of: two idle loops took 70-80% of a processor, + four took 240%, with or without the GIL, and starved the process, as + `t/Pyjo/Server_Prethread.py` of Pyjoyment, and the tests next to it. `_Loop` + keeps the readers and the writers of each socket, and a registry of the + process, `_SOCKETS`, has the loops that watch it, the owner that libuv polls + it for, and the loops that run and wait for it: the owner hands it over to + the next one after each of its events, so the connections go to every worker + in turn, and at the end of its run, and a loop that starts to run takes it + from an owner that does not run. A loop changes the polls of libuv only in + its thread, with `call_soon_threadsafe` when another loop hands a socket over + to it. A socket that one loop watches goes to libuv as before, and the run + of a loop looks at its sockets only while `_shared` counts sockets that more + than one loop watches. +- libuv counts a timer in whole milliseconds from `uv_now`, its clock rounded + down, and Winloop rounds the delay to them, as uvloop does, so a timer, as + `asyncio.sleep`, was due up to a millisecond before its time by + `time.monotonic`, where the loop of asyncio is never early: a sleep of 2 ms + took 1.5 ms. `call_later` of `_Loop`, which `call_at` of Winloop calls too, + rounds the delay up to a millisecond and adds the one that the clock loses, + as `_Loop` of Pyjo.Reactor.Uvloop does, after MagicStack/uvloop#755; a delay + of no time stays a `call_soon` of Winloop. - `PYJO_ASYNCIO_SLOW_CALLBACK` sets the `slow_callback_duration` of a loop of its own, as for `Pyjo.Reactor.Asyncio`, which the loop of Winloop has too. - `Pyjo.Reactor._finalize` defers the cleanup of a finalizer that runs below a @@ -224,8 +252,10 @@ mise exec -- uv run --group docs --with-editable ../Pyjoyment sphinx-build -W -- loop of Winloop: a loop of its own, a given loop, a loop that is running, of asyncio, a `SelectorEventLoop`, and of Winloop, a coroutine, `PYJO_ASYNCIO_SLOW_CALLBACK`, a signal while the loop waits, the finalizer of -an async generator in the debug mode, a stop in a callback of I/O and the -writer of a socket whose peer has closed it. The subtests of Uvloop for the +an async generator in the debug mode, a stop in a callback of I/O, the +writer of a socket whose peer has closed it, a socket that reactors in +threads watch and a timer that is not due before its time. The subtests of +Uvloop for the fork, the wakeup fd of a program and the blocking mode of a pipe are left out, as they do not apply on Windows. `t/examples.py` stops the server of `examples/hello_world_winloop.py` with diff --git a/Pyjo/Reactor/Winloop.py b/Pyjo/Reactor/Winloop.py index af0a599..dd73ff2 100644 --- a/Pyjo/Reactor/Winloop.py +++ b/Pyjo/Reactor/Winloop.py @@ -70,7 +70,16 @@ def timeout(reactor): A signal handler of Python runs while a loop of its own waits, as on the Ctrl-C or Ctrl-Break that stops a server, but the wakeup fd that a program sets with :func:`signal.set_wakeup_fd` is not written to then, as Winloop -sets one of its own while it runs, in the main thread. +sets one of its own while it runs, in the main thread. A timer of a loop of +its own, as :func:`asyncio.sleep`, is never due before its time, as with the +loop of :mod:`asyncio`, although libuv counts it in whole milliseconds, so it +may be up to two milliseconds late. + +Reactors in threads of a process that watch the same socket, as the workers +of :mod:`Pyjo.Server.Prethread` watch the listen sockets of the manager, take +turns: libuv of Windows cannot poll a socket for more than one loop at a time, +so the loop of one of them watches it, and hands it over to the next one after +each of its events, and when it stops running. Winloop runs on Windows, and on CPython, not on PyPy. The ``PYJO_ASYNCIO_SLOW_CALLBACK`` environment variable sets the @@ -94,11 +103,14 @@ def timeout(reactor): import Pyjo.Reactor.Asyncio import asyncio +import collections import importlib.metadata +import math import os +import threading import weakref -from collections.abc import Callable +from collections.abc import Callable, Sequence from typing import Any # Winloop is installed on Windows only, and pyright of Linux and macOS takes the stub of typings/winloop @@ -125,9 +137,39 @@ def _own_loop(self) -> asyncio.AbstractEventLoop: return loop +# A socket that loops of Winloop of the process watch, as the workers of Pyjo.Server.Prethread watch the listen sockets +# of the manager on Windows: libuv of Windows polls a socket with a request of AFD that is exclusive, which ends the +# request of any other loop for the same socket, also of a copy of it, and that loop polls it again, which ends the +# request of the first one, and so on, in a busy loop of C, with no event at all. One loop polls it at a time, its owner, +# and the others that run wait for it in turn: the owner hands it over to the next one after each of its events, so that +# the connections of a listen socket go to every loop, and when it stops running. A loop that does not run takes it from +# an owner that does not run either, once it runs +class _Socket: + __slots__ = ('loops', 'owner', 'waiting') + + def __init__(self) -> None: + self.loops: set[_Loop] = set() + self.owner: _Loop | None = None + self.waiting: collections.deque[_Loop] = collections.deque() + + +# The sockets by their numbers, which the lock guards, and how many of them more than one loop watches +_LOCK = threading.Lock() +_SOCKETS: dict[int, _Socket] = {} +_shared = 0 + + class _Loop(winloop.Loop): _halt: asyncio.Handle | None = None _halted = False + _active = False + + def __init__(self) -> None: + super().__init__() + # The handle, the reader and the writer of each socket that the loop watches, and the handles of the ones that + # libuv polls for it, which only the thread of the loop changes + self._watched: dict[int, list[Any]] = {} + self._polled: dict[int, Any] = {} # After stop, Winloop ends the iteration of libuv, and the iteration that a callback of I/O stops goes on to the poll # of the next one, where libuv of Windows takes the completions of its I/O completion port and calls the readers and @@ -137,7 +179,12 @@ class _Loop(winloop.Loop): # ready in the next run still. A stop in a ready callback does not halt the poll of its own iteration, as the loop of # asyncio runs the callbacks of its select along with it, which would starve the I/O of one_tick otherwise def run_forever(self) -> None: + if self.is_running(): + return super().run_forever() self._halted = False + self._active = True + if _shared: + self._claim() try: return super().run_forever() finally: @@ -145,6 +192,9 @@ def run_forever(self) -> None: if (halt := self._halt) is not None: self._halt = None halt.cancel() + self._active = False + if _shared: + self._yield() def stop(self) -> None: if self._halt is None and self.is_running(): @@ -155,15 +205,184 @@ def _halting(self) -> None: self._halt = None self._halted = True + # libuv counts a timer in whole milliseconds from its clock, which it rounds down, and Winloop rounds the delay, so + # that a timer, as asyncio.sleep, is due up to a millisecond before its time by time.monotonic, where the loop of + # asyncio is never early; the delay is rounded up, with the millisecond that the clock loses, as Pyjo.Reactor.Uvloop + # does (MagicStack/uvloop#755). call_at of Winloop calls it too + def call_later( + self, delay: float, callback: Callable[..., Any], *args: Any, context: Any = None + ) -> asyncio.TimerHandle: + if 0 < delay < math.inf: + delay = (math.ceil(delay * 1000) + 1) / 1000 + return super().call_later(delay, callback, *args, context=context) + def add_reader(self, fd: Any, callback: Callable[..., Any], *args: Any) -> None: - super().add_reader(fd, self._ready, callback, *args) + self._watch(fd, 1, (callback, args)) def add_writer(self, fd: Any, callback: Callable[..., Any], *args: Any) -> None: - super().add_writer(fd, self._ready, callback, *args) + self._watch(fd, 2, (callback, args)) + + def remove_reader(self, fd: Any) -> bool: + return self._unwatch(fd, 1) + + def remove_writer(self, fd: Any) -> bool: + return self._unwatch(fd, 2) + + def close(self) -> None: + super().close() + with _LOCK: + handovers = [self._leave(key) for key in self._watched] + self._watched.clear() + self._polled.clear() + _Loop._hand_over(handovers) - def _ready(self, callback: Callable[..., Any], *args: Any) -> None: - if not self._halted: + def _ready(self, key: int, callback: Callable[..., Any], *args: Any) -> None: + if self._halted: + return + try: callback(*args) + finally: + # The next loop that waits for the socket takes it, and this one waits for it again + if _shared and (sock := _SOCKETS.get(key)) is not None and sock.waiting and sock.owner is self: + with _LOCK: + loop = _Loop._next(sock) if sock.owner is self and key in self._watched else None + if loop is not None: + sock.owner = loop + sock.waiting.append(self) + if loop is not None: + self._sync(key) + _Loop._hand_over([(loop, key)]) + + def _watch(self, fd: Any, index: int, cb: tuple[Callable[..., Any], tuple[Any, ...]]) -> None: + global _shared + key = _number(fd) + with _LOCK: + if (watched := self._watched.get(key)) is None: + watched = self._watched[key] = [fd, None, None] + if (sock := _SOCKETS.get(key)) is None: + sock = _SOCKETS[key] = _Socket() + sock.loops.add(self) + if len(sock.loops) == 2: + _shared += 1 + else: + sock = _SOCKETS[key] + watched[index] = cb + self._contend(sock) + self._sync(key, index) + + def _unwatch(self, fd: Any, index: int) -> bool: + key = _number(fd) + handover = None + with _LOCK: + if (watched := self._watched.get(key)) is None or watched[index] is None: + return False + watched[index] = None + if watched[1] is None and watched[2] is None: + del self._watched[key] + handover = self._leave(key) + self._sync(key, index) + _Loop._hand_over([handover]) + return True + + # The loop owns the socket when nobody does, or when it runs and the owner does not, or else waits for it in turn + # while it runs; with the lock + def _contend(self, sock: _Socket) -> None: + owner = sock.owner + if owner is None or (self._active and owner is not self and not owner._active): + sock.owner = self + elif owner is not self and self._active and self not in sock.waiting: + sock.waiting.append(self) + + # The loop stops watching the socket, which goes to the next loop that waits for it, or else to one that does not + # run, which polls it once it runs; with the lock + def _leave(self, key: int) -> 'tuple[_Loop, int] | None': + global _shared + sock = _SOCKETS[key] + sock.loops.discard(self) + if len(sock.loops) == 1: + _shared -= 1 + if self in sock.waiting: + sock.waiting.remove(self) + if not sock.loops: + del _SOCKETS[key] + return None + if sock.owner is not self: + return None + sock.owner = _Loop._next(sock) or next(iter(sock.loops)) + return sock.owner, key + + # At the start of a run, the loop takes the sockets of owners that do not run, waits for the others, and stops + # polling the ones that another loop took while it did not run + def _claim(self) -> None: + with _LOCK: + for key in self._watched: + self._contend(_SOCKETS[key]) + keys = [key for key in self._watched if (_SOCKETS[key].owner is self) != (key in self._polled)] + for key in keys: + self._sync(key) + + # At the end of a run, the sockets that other loops wait for go to them + def _yield(self) -> None: + handovers: list[tuple[_Loop, int]] = [] + with _LOCK: + for key in self._watched: + sock = _SOCKETS[key] + if self in sock.waiting: + sock.waiting.remove(self) + if sock.owner is self and (loop := _Loop._next(sock)) is not None: + sock.owner = loop + handovers.append((loop, key)) + for _, key in handovers: + self._sync(key) + _Loop._hand_over(handovers) + + # libuv polls the socket for the loop as long as it owns it, with its reader and its writer, or only the one that + # changed of a socket that it polls already; in the thread of the loop + def _sync(self, key: int, index: int = 0) -> None: + with _LOCK: + sock = _SOCKETS.get(key) + watched = self._watched.get(key) if sock is not None and sock.owner is self else None + watched = list(watched) if watched is not None else None + if self.is_closed(): + return + if watched is not None: + fd = watched[0] + if key not in self._polled: + index = 0 + self._polled[key] = fd + if index != 2: + if (reader := watched[1]) is not None: + super().add_reader(fd, self._ready, key, reader[0], *reader[1]) + else: + super().remove_reader(fd) + if index != 1: + if (writer := watched[2]) is not None: + super().add_writer(fd, self._ready, key, writer[0], *writer[1]) + else: + super().remove_writer(fd) + elif (fd := self._polled.pop(key, None)) is not None: + super().remove_reader(fd) + super().remove_writer(fd) + + # The next loop that runs and waits for the socket; with the lock + @staticmethod + def _next(sock: _Socket) -> '_Loop | None': + while sock.waiting: + loop = sock.waiting.popleft() + if loop._active and loop in sock.loops: + return loop + return None + + # A loop that takes a socket polls it in its thread; a loop that is closed has left it to the next one already + @staticmethod + def _hand_over(handovers: 'Sequence[tuple[_Loop, int] | None]') -> None: + for handover in handovers: + if handover is not None: + loop, key = handover + try: + loop.call_soon_threadsafe(loop._sync, key) + except RuntimeError: + pass # The finalizer of an async generator, which a dealloc calls while CPython clears a frame, in Python, as the one of # asyncio: the handle of Winloop takes the stack in the debug mode with the current frame of the thread, which @@ -173,6 +392,10 @@ def _asyncgen_finalizer_hook(self, agen: Any) -> None: super()._asyncgen_finalizer_hook(agen) +def _number(fd: Any) -> int: + return fd if isinstance(fd, int) else int(fd.fileno()) + + # The threshold of a slow callback of PYJO_ASYNCIO_SLOW_CALLBACK, as for a loop of its own of Pyjo.Reactor.Asyncio def _new_event_loop() -> asyncio.AbstractEventLoop: loop = _Loop() diff --git a/t/Reactor_Winloop.py b/t/Reactor_Winloop.py index bb57c43..bb651a7 100644 --- a/t/Reactor_Winloop.py +++ b/t/Reactor_Winloop.py @@ -624,4 +624,104 @@ def writer(reactor, writable): reactor.remove(a) a.close() +# Not in Mojolicious: libuv of Windows polls a socket with an exclusive request of AFD, which ends the request of another +# loop for the same socket, which polls it again, in a busy loop; the loops of reactors in threads that watch one socket, +# as the workers of Pyjo.Server.Prethread watch the listen sockets of the manager, poll it one at a time, in turn +with subtest('A socket that reactors in threads watch'): + server = socket.create_server(('127.0.0.1', 0)) + server.setblocking(False) + port = server.getsockname()[1] + accepted = [0, 0, 0, 0] + reactors = [] + ready = threading.Barrier(5) + + def serve(n): + with Pyjo.Reactor.Winloop.new() as reactor: + + def accept(reactor, writable): + try: + conn, _ = server.accept() + except BlockingIOError: + return + accepted[n] += 1 + conn.close() + + reactor.io(server, accept).watch(server, True, False) + reactors.append(reactor) + reactor.next_tick(lambda reactor: ready.wait()) + reactor.start() + reactor.remove(server) + + threads = [threading.Thread(target=serve, args=(n,)) for n in range(4)] + for thread in threads: + thread.start() + ready.wait() + cpu, started = time.process_time(), time.monotonic() + time.sleep(1) + cpu, elapsed = time.process_time() - cpu, time.monotonic() - started + ok(cpu < elapsed / 2, 'idle loops do not poll the socket in a busy loop') + for _ in range(40): + with socket.create_connection(('127.0.0.1', port)) as client: + client.recv(1) + is_ok(sum(accepted), 40, 'all connections accepted') + ok(all(accepted), 'by every reactor in turn') + for reactor in reactors: + reactor.loop.call_soon_threadsafe(reactor.stop) + for thread in threads: + thread.join(10) + ok(not any(thread.is_alive() for thread in threads), 'reactors stopped') + server.close() + +# Not in Mojolicious: libuv counts a timer in whole milliseconds of a clock that it rounds down, so that a timer of +# Winloop is due up to a millisecond before its time, where the loop of asyncio is never early; by the clock of +# time.perf_counter, which time.monotonic is on Windows since Python 3.13, and which counted in the ticks of the system +# of 15.6 ms before +with subtest('Timer not due before its time'): + with Pyjo.Reactor.Winloop.new() as reactor: + early = [] + + def timers(after, schedule, count=150): + # A start anywhere within a millisecond, which a start right after a timer is not + deadline = time.perf_counter() + (count % 10) / 10000 + while time.perf_counter() < deadline: + pass + start = time.perf_counter() + + def due(*args): + if (elapsed := time.perf_counter() - start) < after: + early.append(elapsed) + if count > 1: + timers(after, schedule, count - 1) + else: + reactor.stop() + + schedule(after, due) + + timers(0.002, lambda after, due: reactor.timer(after, due)) + reactor.start() + is_deeply_ok(early, [], 'timer of the reactor') + early.clear() + timers(0.002, lambda after, due: reactor.loop.call_later(after, due)) + reactor.start() + is_deeply_ok(early, [], 'call_later') + early.clear() + timers(0.002, lambda after, due: reactor.loop.call_at(reactor.loop.time() + after, due)) + reactor.start() + is_deeply_ok(early, [], 'call_at') + early.clear() + + async def sleeps(): + for i in range(150): + deadline = time.perf_counter() + (i % 10) / 10000 + while time.perf_counter() < deadline: + pass + start = time.perf_counter() + await asyncio.sleep(0.002) + if (elapsed := time.perf_counter() - start) < 0.002: + early.append(elapsed) + + reactor.run_coroutine(sleeps(), lambda value: reactor.stop(), lambda err: (early.append(err), reactor.stop())) + reactor.start() + is_deeply_ok(early, [], 'asyncio.sleep') + done_testing() diff --git a/typings/winloop/__init__.pyi b/typings/winloop/__init__.pyi index 010b97c..a3bb004 100644 --- a/typings/winloop/__init__.pyi +++ b/typings/winloop/__init__.pyi @@ -11,9 +11,19 @@ class Loop(asyncio.AbstractEventLoop): def run_forever(self) -> None: ... def stop(self) -> None: ... def is_running(self) -> bool: ... + def is_closed(self) -> bool: ... + def close(self) -> None: ... def call_soon(self, callback: Callable[..., object], *args: Any, context: Any = None) -> asyncio.Handle: ... + def call_later( + self, delay: float, callback: Callable[..., object], *args: Any, context: Any = None + ) -> asyncio.TimerHandle: ... + def call_soon_threadsafe( + self, callback: Callable[..., object], *args: Any, context: Any = None + ) -> asyncio.Handle: ... def add_reader(self, fd: Any, callback: Callable[..., Any], *args: Any) -> None: ... def add_writer(self, fd: Any, callback: Callable[..., Any], *args: Any) -> None: ... + def remove_reader(self, fd: Any) -> bool: ... + def remove_writer(self, fd: Any) -> bool: ... def _asyncgen_finalizer_hook(self, agen: Any) -> None: ... def new_event_loop() -> Loop: ... diff --git a/uv.lock b/uv.lock index e0c1e3a..8141561 100644 --- a/uv.lock +++ b/uv.lock @@ -372,7 +372,7 @@ docs = [{ name = "sphinx", marker = "python_full_version >= '3.14'", specifier = [[package]] name = "pyjoyment" version = "0.2.1.dev1" -source = { git = "https://github.com/pyjoyment/Pyjoyment?branch=main#2cc52c705d7932ea98d01f72e01a2d8d90ed1836" } +source = { git = "https://github.com/pyjoyment/Pyjoyment?branch=main#7b09754b6f32954ce4cf4ffa8bcd4143f54dfcf6" } [[package]] name = "requests"