diff --git a/Pyjoyment/Command/Author/generate/kubernetes.py b/Pyjoyment/Command/Author/generate/kubernetes.py index 8da92af3..9ceba1f6 100644 --- a/Pyjoyment/Command/Author/generate/kubernetes.py +++ b/Pyjoyment/Command/Author/generate/kubernetes.py @@ -10,6 +10,8 @@ python -m MyApp generate kubernetes --image ghcr.io/me/myapp:1.0 python -m MyApp generate kubernetes --host-path python -m MyApp generate kubernetes --graceful-timeout 25 + python -m MyApp generate kubernetes --grace-period 30 + python -m MyApp generate kubernetes --liveness-probe 3000 python -m MyApp generate kubernetes --shutdown-delay 10 python -m MyApp generate kubernetes --no-dev python -m MyApp generate kubernetes --server prefork -- --workers 2 @@ -19,6 +21,9 @@ the project in the pod of "dev", which --no-dev leaves out, defaults to installing it + --grace-period terminationGracePeriodSeconds of "prod", + whose graceful timeout is what the + shutdown delay and 5 seconds leave -G, --graceful-timeout Graceful timeout of the workers of the server of "prod", defaults to the one of the server @@ -29,6 +34,9 @@ -i, --image Image of "prod", the one of "generate dockerfile", defaults to the name of the application + --liveness-probe Liveness probe on the port, or of the + path, defaults to the liveness of the + plugin Health, or the port without it -p, --port Port that the server listens on, defaults to 3000 -S, --shutdown-delay Time that the server of "prod" keeps @@ -68,6 +76,16 @@ it has started. The probes are a patch of their own, ``base/probes.yaml``, as the application gets the plugin or loses it. +``--liveness-probe`` gives the liveness probe a path of its own, or a port, +which the kubelet connects to with ``tcpSocket``. A server whose workers +serve one connection at a time, as ``prefork -c 1`` of an application that +blocks, keeps a request of a probe in the backlog while every worker is busy, +so that a liveness probe of HTTP restarts a pod that is only busy, where the +kernel accepts the connection of ``tcpSocket`` into the backlog. The startup +probe stays the one of the liveness of the application, as a pod gets no +requests before it has started, and the readiness probe the one of its +readiness. + The overlay ``prod`` runs the image of ``--image``, which :mod:`Pyjoyment.Command.Author.generate.dockerfile` builds, with the pre-forking server of the image, in the operating mode ``production``. @@ -78,7 +96,11 @@ seconds above the delay and the 25 seconds that :attr:`Pyjo.Server.Prefork.Pyjo_Server_Prefork.graceful_timeout` gives a worker, or ``--graceful-timeout``, which sets ``PYJO_GRACEFUL_TIMEOUT``, so -that the server stops by itself before the kill of Kubernetes. Without +that the server stops by itself before the kill of Kubernetes. +``--grace-period`` gives ``terminationGracePeriodSeconds`` instead, as the +30 seconds that Kubernetes gives a pod by default, and the graceful timeout +is what the delay and the 5 seconds leave of it, 20 seconds with the delay of +5 seconds. Without ``--image``, the image is the one of the name of the application, as ``docker build -t myapp .`` tags it, which a cluster of Docker Desktop or OrbStack takes from Docker. @@ -187,8 +209,15 @@ def run(self, *args: str) -> None: Run this command. """ arguments = list(args) - spec = {'dev!': None, 'G|graceful-timeout=i': None, 'host-path': None, 'i|image=s': None, 'p|port=i': None} - if not (opts := getopt(arguments, {**spec, 'S|shutdown-delay=i': None, 's|server=s': None})): + spec = { + 'dev!': None, + 'grace-period=i': None, + 'G|graceful-timeout=i': None, + 'host-path': None, + 'i|image=s': None, + } + more = {'liveness-probe=s': None, 'p|port=i': None, 'S|shutdown-delay=i': None, 's|server=s': None} + if not (opts := getopt(arguments, {**spec, **more})): raise Pyjo.Exception.new(self.usage) # The server of dev, which reloads the application of the directory of the project, where its files change, @@ -208,6 +237,12 @@ def run(self, *args: str) -> None: health = app.renderer.get_helper('health.path') is not None probes = {'livez': app.health.path('livez'), 'readyz': app.health.path('readyz')} if health else None + # The liveness probe of a path of its own, or of a port, which the kernel accepts a connection of into the + # backlog, where a request waits behind the workers that are busy + liveness: str | None = opts.liveness_probe + if liveness is not None and not re.fullmatch(r'/\S*|[1-9][0-9]{0,4}', liveness): + raise Pyjo.Exception.new(f'Unknown liveness probe: {liveness}') + # The script of Pyjoyment.Lite or the package of the application, and the dependencies of pyproject.toml, or of # the inline metadata of PEP 723 of a script without it, as generate dockerfile has them exe = os.environ.get('PYJO_EXE') @@ -219,7 +254,7 @@ def run(self, *args: str) -> None: lock = 'uv.lock' if project else f'{script}.lock' port = opts.port or 3000 - base = {'moniker': moniker, 'name': name, 'port': port, 'probes': probes} + base = {'liveness': liveness, 'moniker': moniker, 'name': name, 'port': port, 'probes': probes} self.render_to_rel_file('base_kustomization', 'kubernetes/base/kustomization.yaml', base) self.render_to_rel_file('base_deployment', 'kubernetes/base/deployment.yaml', base) self.render_to_rel_file('base_service', 'kubernetes/base/service.yaml', base) @@ -228,6 +263,17 @@ def run(self, *args: str) -> None: graceful: int | None = opts.graceful_timeout delay: int = 5 if opts.shutdown_delay is None else opts.shutdown_delay + + # The graceful timeout of the time that Kubernetes gives the pod, after the delay and the 5 seconds of the + # server to stop by itself + if (period := opts.grace_period) is not None: + if graceful is not None: + raise Pyjo.Exception.new('The grace period gives the graceful timeout, without --graceful-timeout') + if (graceful := period - delay - 5) < 1: + raise Pyjo.Exception.new( + f'The grace period of {period} seconds leaves no graceful timeout after the shutdown delay of ' + f'{delay} seconds and 5 seconds' + ) prod = {'delay': delay, 'graceful_timeout': graceful, 'image': opts.image, 'moniker': moniker, 'name': name} self.render_to_rel_file('prod_kustomization', 'kubernetes/prod/kustomization.yaml', prod) self.render_to_rel_file('prod_deployment', 'kubernetes/prod/deployment.yaml', prod) @@ -379,7 +425,14 @@ def _key(path: str, keys: set[str]) -> str: periodSeconds: 1 failureThreshold: 30 livenessProbe: -% if probes: +% if liveness is not None and liveness.startswith('/'): + httpGet: + path: <%= liveness %> + port: http +% elif liveness is not None: + tcpSocket: + port: <%= liveness %> +% elif probes: httpGet: path: <%= probes['livez'] %> port: http diff --git a/Pyjoyment/Plugin/Health.py b/Pyjoyment/Plugin/Health.py index d34e8d6f..2f3b55b8 100644 --- a/Pyjoyment/Plugin/Health.py +++ b/Pyjoyment/Plugin/Health.py @@ -42,12 +42,17 @@ Pyjoyment, and its workers, tells: a client whose connection is kept alive gets ``503`` with its last response. -The checks run at the same time, each with a timeout, and a check passes -unless it raises an exception, returns ``False``, or returns a promise, a -coroutine or another awaitable that is rejected, is fulfilled with ``False``, -or does not settle within its timeout. The reason of a failure goes to the -log, as the ``warn`` message ``Health check "db" failed: ...``, not to the -response, which withholds it. +The checks run at the same time, and a check passes unless it raises an +exception, returns ``False``, or returns a promise, a coroutine or another +awaitable that is rejected, is fulfilled with ``False``, or does not settle +within its timeout. A check that returns a value has run to its end before its +timeout starts, so the timeout does not bound it: it blocks the event loop +while it runs, as any code that blocks does, and the timeouts of its own +client bound it, as ``connect_timeout`` of a database, and a library that +refuses a running loop of :mod:`asyncio`, as the ORM of Django, runs in +:func:`Pyjo.IOLoop.blocking`. The reason of a failure goes to the log, as the +``warn`` message ``Health check "db" failed: ...``, not to the response, +which withholds it. A query of ``verbose`` lists the checks on success too, ``exclude`` leaves a check out, as many times as there are checks to leave out, and a check of its @@ -74,7 +79,18 @@ hook rejects it, as :mod:`Pyjoyment.Plugin.AllowedHosts` would reject the ``Host`` of the address of a pod, which a probe of Kubernetes sends. A hook of ``around_dispatch`` that the application registers before the plugin wraps -the probes too. +the probes too. So an endpoint takes the path from the routes of the +application, and from an application that :mod:`Pyjoyment.Plugin.Mount` +mounts: one with a ``/healthz`` of its own, which its monitors request, keeps +it with ``healthz=None``. + +A server whose workers serve one connection at a time, as ``prefork -c 1`` +for an application that blocks, keeps a probe in the backlog while every +worker is busy, so that a liveness probe of HTTP restarts a container that is +only busy. Its liveness probe connects to the port, as ``tcpSocket`` of +Kubernetes, which the kernel accepts into the backlog, as +:mod:`Pyjoyment.Command.Author.generate.kubernetes` makes it with +``--liveness-probe``, and ``/readyz`` stays its readiness probe. See :mod:`Pyjoyment.Plugins` for a list of plugins that are available by default. @@ -115,8 +131,9 @@ app.plugin('Health', timeout=0.5) - The time in seconds that a check may take, defaults to ``1``, the - timeout of a probe of Kubernetes. + The time in seconds that a check of a promise, a coroutine or another + awaitable may take, defaults to ``1``, the timeout of a probe of + Kubernetes. Helpers ------- diff --git a/docs/guides/cookbook.rst b/docs/guides/cookbook.rst index 624f5d40..7d4c9be8 100644 --- a/docs/guides/cookbook.rst +++ b/docs/guides/cookbook.rst @@ -526,6 +526,19 @@ waits for the checks of the application, as a database that is not up yet. The server listens before it forks its workers, and a connection waits for a worker in the backlog of the listen socket. +So does a probe: with workers that serve one connection at a time, as +``prefork -c 1`` for an application that blocks, a liveness probe of HTTP +waits behind the busy workers, and restarts a pod that is only busy. Its +liveness probe connects to the port, which the kernel accepts into the +backlog, as ``--liveness-probe`` of +:mod:`Pyjoyment.Command.Author.generate.kubernetes` makes it. + +.. code-block:: yaml + + livenessProbe: + tcpSocket: + port: 3000 + .. mermaid:: sequenceDiagram @@ -572,6 +585,10 @@ of ``PYJO_SHUTDOWN_DELAY``, until they know, and then closes its listen sockets, so a new connection that one of them still opens is refused. ``terminationGracePeriodSeconds`` has room for the delay and for the 25 seconds of :attr:`~Pyjo.Server.Prefork.Pyjo_Server_Prefork.graceful_timeout`. +A deployment that keeps the 30 seconds that Kubernetes gives by default +shortens the graceful timeout instead, as ``PYJO_GRACEFUL_TIMEOUT=20`` with +the delay of 5 seconds, which ``--grace-period 30`` of +:mod:`Pyjoyment.Command.Author.generate.kubernetes` sets. From ``TERM`` on, readiness fails with the check ``shutdown``, so that a load balancer that checks readiness, as one outside of the cluster, takes the instance out of its rotation during the delay, and one that keeps its diff --git a/plugins/pyjoyment/skills/deploy/SKILL.md b/plugins/pyjoyment/skills/deploy/SKILL.md index ef43d94b..febbb88b 100644 --- a/plugins/pyjoyment/skills/deploy/SKILL.md +++ b/plugins/pyjoyment/skills/deploy/SKILL.md @@ -93,9 +93,11 @@ worker model: - A shell in the `CMD` of a container runs the server with `exec`, so that it gets `TERM`. - `-S` (`PYJO_SHUTDOWN_DELAY`) and `-G` go into `terminationGracePeriodSeconds` - of Kubernetes, 30 by default: `-S 5 -G 20` with the default. + of Kubernetes, 30 by default: `-S 5 -G 20` with the default, as + `generate kubernetes --grace-period 30` sets them. - The liveness probe of an application with `-c 1` is on `tcpSocket`, as an - HTTP probe waits behind busy workers and restarts the pod; `/readyz` is the + HTTP probe waits behind busy workers and restarts the pod, as + `generate kubernetes --liveness-probe 8080` makes it; `/readyz` is the readiness probe. ## Access log @@ -153,7 +155,11 @@ kubectl apply -k kubernetes/prod makes an image of AWS Lambda. - `kubernetes/dev` runs the files of the project from a ConfigMap on the image of uv, `kubernetes/prod` the image of `-i`, with the probes of `/livez` and - `/readyz` and the graceful stop of the pods. + `/readyz` and the graceful stop of the pods. `--grace-period 30` keeps the + `terminationGracePeriodSeconds` of Kubernetes, with the graceful timeout of + what the shutdown delay and 5 seconds leave, 20 seconds, and + `--liveness-probe 8080` puts the liveness probe on `tcpSocket` of the port, + for workers of `-c 1`. - On AWS Lambda the script ends with `handler = app.start()`: the runtime of Python calls `app.handler`, and a custom runtime runs the script itself. - The public layers of @@ -238,6 +244,12 @@ PYJO_LISTEN='fcgi://127.0.0.1:9000' python app.py daemon # FastCGI behind ngi deployment: take it from the environment or the configuration. - `/readyz` with the checks of a database in the liveness probe restarts the containers when the database is down: liveness is `/livez`. +- The `timeout` of `Health` bounds a check that returns a promise or a + coroutine; a check that returns a value runs to its end, blocking the event + loop, so the timeouts of its client bound it, as `connect_timeout` of a + database. +- `plugin('Health')` answers `/healthz` before the routes, also of a mounted + application with its own: `healthz=None` leaves it to the application. - A blocking request handler under `daemon` blocks every connection: more workers of `prefork`, `-c 1`, or code that does not block. - The log goes to the standard error, which a container, Kubernetes and AWS diff --git a/plugins/pyjoyment/skills/mount/SKILL.md b/plugins/pyjoyment/skills/mount/SKILL.md index ed03861e..79388273 100644 --- a/plugins/pyjoyment/skills/mount/SKILL.md +++ b/plugins/pyjoyment/skills/mount/SKILL.md @@ -132,6 +132,9 @@ plugin('Mount', {re.compile(r'/(?P\w+)/blog(?P/.*)?'): flask_app}) the request is the 404 of the application that mounts it; a regular expression of the whole path, with its group `path`, is exact. - `PYJO_REACTOR=Pyjo.Reactor.Poll` cannot run an ASGI application. +- `plugin('Health')` answers `/livez`, `/readyz` and `/healthz` before the + routes, so a mounted application with a `/healthz` of its own, which its + monitors request, loses it: `plugin('Health', healthz=None)`. - The `SCRIPT_NAME` of WSGI and the `root_path` of ASGI carry the prefix: an application that builds its URLs from them, as Flask with `url_for`, links under the prefix. diff --git a/t/Pyjoyment/health_lite_app.py b/t/Pyjoyment/health_lite_app.py index feabdeb8..62d71738 100644 --- a/t/Pyjoyment/health_lite_app.py +++ b/t/Pyjoyment/health_lite_app.py @@ -17,6 +17,7 @@ import http.client import signal import socket +import time # Not in Mojolicious: Pyjoyment.Plugin.Health, the liveness and the readiness of containers and orchestrators @@ -154,6 +155,17 @@ def raising(c): warmup.resolve() t2.get_ok('/readyz').status_is(200).content_is('ok') + # A check that returns a value has run to its end before its timeout starts, so the timeout does not bound it + def blocking(c): + time.sleep(0.08) + return True + + other = application({'checks': {'blocking': blocking}, 'timeout': 0.02}) + t2 = Pyjo.Test.Pyjo.new(other) + started = time.monotonic() + t2.get_ok('/readyz').status_is(200).content_is('ok') + cmp_ok(time.monotonic() - started, '>=', 0.08, 'ran to its end') + with subtest('Helpers'): other = application() is_ok(other.health.path('livez'), '/livez', 'right path') diff --git a/t/docs/Pyjoyment_Command_Author_generate_kubernetes.py b/t/docs/Pyjoyment_Command_Author_generate_kubernetes.py index 6d3170bc..7c55a6d1 100644 --- a/t/docs/Pyjoyment_Command_Author_generate_kubernetes.py +++ b/t/docs/Pyjoyment_Command_Author_generate_kubernetes.py @@ -103,6 +103,53 @@ def script(tmp, content=''): 'graceful timeout', ) + # python -m MyApp generate kubernetes --grace-period 30 + tmp = project() + generate(tmp, '--grace-period', '30') + like_ok( + read(tmp, 'prod/deployment.yaml'), r'(?m)^ terminationGracePeriodSeconds: 30$', 'grace period of its own' + ) + like_ok( + read(tmp, 'prod/deployment.yaml'), + r'(?m)^ - name: PYJO_GRACEFUL_TIMEOUT\n value: "20"$', + 'graceful timeout of what the delay and 5 seconds leave', + ) + tmp = project() + generate(tmp, '--grace-period', '60', '--shutdown-delay', '10') + like_ok(read(tmp, 'prod/deployment.yaml'), r'(?m)^ terminationGracePeriodSeconds: 60$', 'with a delay') + like_ok(read(tmp, 'prod/deployment.yaml'), r'PYJO_GRACEFUL_TIMEOUT\n value: "45"', 'and its timeout') + throws_ok( + lambda: generate(project(), '--grace-period', '30', '--graceful-timeout', '20'), + re.compile(r'^The grace period gives the graceful timeout, without --graceful-timeout'), + 'not with a graceful timeout', + ) + throws_ok( + lambda: generate(project(), '--grace-period', '10'), + re.compile(r'^The grace period of 10 seconds leaves no graceful timeout after the shutdown delay of 5 seconds'), + 'too short', + ) + + # python -m MyApp generate kubernetes --liveness-probe 3000 + tmp = project() + generate(tmp, '--liveness-probe', '3000') + like_ok( + read(tmp, 'base/probes.yaml'), + r'livenessProbe:\n tcpSocket:\n port: 3000\n', + 'on the port', + ) + tmp = project() + generate(tmp, '--liveness-probe', '/-/live') + like_ok( + read(tmp, 'base/probes.yaml'), + r'livenessProbe:\n httpGet:\n path: /-/live\n port: http\n', + 'of the path', + ) + throws_ok( + lambda: generate(project(), '--liveness-probe', 'http'), + re.compile(r'^Unknown liveness probe: http'), + 'neither a port nor a path', + ) + # python -m MyApp generate kubernetes --shutdown-delay 10 tmp = project() generate(tmp, '--shutdown-delay', '10') @@ -182,6 +229,21 @@ def script(tmp, content=''): ) like_ok(read(probes, 'dev/deployment.yaml'), r'startupProbe:\n failureThreshold: 300', 'start of dev') + # A liveness probe on the port, with the startup and the readiness of the plugin + busy = project() + generate(busy, '--liveness-probe', '3000') + like_ok( + read(busy, 'base/probes.yaml'), r'startupProbe:\n httpGet:\n path: /livez', 'startup' + ) + like_ok( + read(busy, 'base/probes.yaml'), r'livenessProbe:\n tcpSocket:\n port: 3000', 'liveness' + ) + like_ok( + read(busy, 'base/probes.yaml'), + r'readinessProbe:\n httpGet:\n path: /readyz', + 'readiness', + ) + # prod, in the operating mode production, on the image of the name of the application like_ok(read(tmp, 'base/deployment.yaml'), r'image: pyjoyment\n imagePullPolicy: IfNotPresent', 'image') like_ok(read(tmp, 'prod/deployment.yaml'), r'- name: PYJO_MODE\n value: production', 'production') @@ -276,13 +338,15 @@ def script(tmp, content=''): kubernetes = Pyjoyment.Command.Author.generate.kubernetes.new(app=application) tmp = project() tmp.child('MyApp').make_path().child('__init__.py').spew('pass\n') - generate(tmp, '--image', 'ghcr.io/me/myapp:1.0') + generate(tmp, '--image', 'ghcr.io/me/myapp:1.0', '--liveness-probe', '3000', '--grace-period', '30') for overlay in ('dev', 'prod'): command = [str(kubectl), 'kustomize', tmp.child('kubernetes', overlay).to_str()] result = subprocess.run(command, capture_output=True, text=True) is_ok(result.returncode, 0, f'{overlay} builds') like_ok(result.stdout, r'selector:\n matchLabels:\n app\.kubernetes\.io/name: pyjoyment', 'selector') like_ok(result.stdout, r'image: ghcr\.io/me/myapp:1\.0', 'image of prod') + like_ok(result.stdout, r'livenessProbe:\n tcpSocket:\n port: 3000\n', 'liveness on the port') + like_ok(result.stdout, r'terminationGracePeriodSeconds: 30\n', 'grace period') result = subprocess.run( [str(kubectl), 'kustomize', tmp.child('kubernetes', 'dev').to_str()], capture_output=True, text=True )