From 6c031e3b0b46a4e98b192bbd6bb66470d23609c5 Mon Sep 17 00:00:00 2001 From: Sylvain Hellegouarch Date: Sun, 27 Sep 2026 20:29:19 +0200 Subject: [PATCH 1/2] Do not turn an interruption into a failed Python activity An InterruptExecution raised while a Python activity runs, on SIGTERM for instance, was wrapped into ActivityFailed. The run then carried on as if only that activity had failed and could complete. It now reaches the runner and the run is interrupted. Signed-off-by: Sylvain Hellegouarch --- CHANGELOG.md | 7 ++++ chaoslib/provider/python.py | 11 +++++- tests/fixtures/interrupter.py | 5 +++ tests/test_exit.py | 66 +++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e07dd24..1c1b652 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ [Unreleased]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/1.45.0...HEAD +### Fixed + +* An interruption raised while a Python activity runs, for instance on + SIGTERM, is no longer turned into a failure of that activity. The run is + now interrupted and reported as such, instead of carrying on and possibly + completing + ## [1.45.0][] - 2026-08-08 [1.45.0]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/1.44.0...1.45.0 diff --git a/chaoslib/provider/python.py b/chaoslib/provider/python.py index 01f193d..fe2aba9 100644 --- a/chaoslib/provider/python.py +++ b/chaoslib/provider/python.py @@ -6,7 +6,11 @@ from typing import Any from chaoslib import substitute -from chaoslib.exceptions import ActivityFailed, InvalidActivity +from chaoslib.exceptions import ( + ActivityFailed, + InterruptExecution, + InvalidActivity, +) from chaoslib.types import Activity, Configuration, Secrets __all__ = ["run_python_activity", "validate_python_activity"] @@ -54,6 +58,11 @@ def run_python_activity( try: return func(**arguments) + except InterruptExecution: + # the whole execution is being interrupted, by SIGTERM or a control + # for instance, while this activity ran: that is not a failure of + # the activity and must reach the runner as is + raise except Exception as x: # noqa: BLE001 - wrap any user-code exception raise ActivityFailed( traceback.format_exception_only(type(x), x)[0].strip() diff --git a/tests/fixtures/interrupter.py b/tests/fixtures/interrupter.py index df841ea..5fd257a 100644 --- a/tests/fixtures/interrupter.py +++ b/tests/fixtures/interrupter.py @@ -1,5 +1,6 @@ import time +from chaoslib.exceptions import InterruptExecution from chaoslib.exit import exit_gracefully, exit_ungracefully @@ -11,3 +12,7 @@ def interrupt_gracefully_in(seconds: int = 1.5): def interrupt_ungracefully_in(seconds: int = 1.5): time.sleep(seconds) exit_ungracefully() + + +def raise_interruption() -> None: + raise InterruptExecution("interrupted from an activity") diff --git a/tests/test_exit.py b/tests/test_exit.py index a57eeae..a63ef39 100644 --- a/tests/test_exit.py +++ b/tests/test_exit.py @@ -155,3 +155,69 @@ def _exit_soon(): journal = runner.run(x) assert journal["status"] == "interrupted" assert journal["run"][0]["status"] == "succeeded" + + +def _interrupted_python_experiment(module: str, func: str, arguments=None): + return { + "title": "interrupted while a python activity runs", + "description": "n/a", + "method": [ + { + "type": "action", + "name": "long-python-call", + "provider": { + "type": "python", + "module": module, + "func": func, + "arguments": arguments or {}, + }, + }, + { + "type": "action", + "name": "must-not-run", + "provider": {"type": "process", "path": "echo"}, + }, + ], + "rollbacks": [ + { + "type": "action", + "name": "rollback", + "provider": {"type": "process", "path": "echo"}, + } + ], + } + + +def test_sigterm_during_python_activity_interrupts_the_run(): + import os + import signal + + x = _interrupted_python_experiment( + "fixtures.longpythonfunc", "pause", {"howlong": 5} + ) + timer = threading.Timer(0.5, os.kill, (os.getpid(), signal.SIGTERM)) + with Runner(Strategy.DEFAULT) as runner: + timer.start() + journal = runner.run( + x, settings={"runtime": {"rollbacks": {"strategy": "always"}}} + ) + + assert journal["status"] == "interrupted" + assert [r["activity"]["name"] for r in journal["run"]] == [ + "long-python-call" + ] + assert journal["run"][0].get("status") != "failed" + assert len(journal["rollbacks"]) == 1 + + +def test_interruption_raised_by_python_activity_interrupts_the_run(): + x = _interrupted_python_experiment( + "fixtures.interrupter", "raise_interruption" + ) + with Runner(Strategy.DEFAULT) as runner: + journal = runner.run( + x, settings={"runtime": {"rollbacks": {"strategy": "always"}}} + ) + + assert journal["status"] == "interrupted" + assert len(journal["run"]) == 1 From bb66a4f3ae48f0acbd7cacd4580055cf036cb321 Mon Sep 17 00:00:00 2001 From: Sylvain Hellegouarch Date: Sun, 27 Sep 2026 20:48:00 +0200 Subject: [PATCH 2/2] Relax timing bound of a background activity test on slow runners Signed-off-by: Sylvain Hellegouarch --- tests/test_exit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_exit.py b/tests/test_exit.py index a63ef39..837d68c 100644 --- a/tests/test_exit.py +++ b/tests/test_exit.py @@ -121,7 +121,9 @@ def test_wait_for_background_activity_on_graceful_exit(): journal = runner.run(x) assert journal["status"] == "interrupted" - assert 3.0 < journal["run"][0]["duration"] < 3.2 + # the background activity ran its full 3s: the upper bound only + # tells it apart from the 5s HTTP call, with room for slow runners + assert 3.0 < journal["run"][0]["duration"] < 4.0 finally: stop_http_server(httpd, server)