Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,23 @@

[Unreleased]: https://github.com/chaostoolkit/chaostoolkit-addons/compare/0.11.0...HEAD

### Added

* The safeguards control records the safeguard that interrupted the
execution, and its run, in the journal under `safeguards`

### Changed

* Requires Python 3.12+
* Build against Python 3.12, 3.13 and 3.14 and refresh the lock file
* Pin the linting rules to ruff's former defaults
* Use supported versions of the checkout and artifact GitHub actions

### Fixed

* The safeguards control resets its triggered state when configured so a
previous execution in the same process does not leak into the next one

## [0.11.0][]

[0.11.0]: https://github.com/chaostoolkit/chaostoolkit-addons/compare/0.10.0...0.11.0
Expand Down
30 changes: 28 additions & 2 deletions chaosaddons/controls/safeguards.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,15 @@

If either of them doesn't meet its tolerance, the entire execution will
terminate as soon as possible and leave the status of the experiment to
`interrupted`.
`interrupted`. The journal then records the safeguard that triggered the
interruption, and its run, under `safeguards`:

```json
"safeguards": {
"triggered_by": "safeguard_3",
"run": {"activity": {...}, "status": "succeeded", "output": ...}
}
```

Probes that do not declare the `background` or `frequency` properties are meant
to run before the experiment really starts and will block until they are all
Expand Down Expand Up @@ -103,6 +111,7 @@
Configuration,
Control,
Experiment,
Journal,
Probe,
Run,
Secrets,
Expand Down Expand Up @@ -151,6 +160,12 @@ def prepare(self, probes: List[Probe]) -> None:
Configure the guardian so that it runs with the right amount of
resources.
"""
with self._lock:
self._interrupted = False
self.triggered_by = None
self.triggered_by_run = None
self.was_triggered = False

once_count = 0
repeating_count = 0
now_count = 0
Expand Down Expand Up @@ -326,8 +341,19 @@ def before_experiment_control(
guardian.run(experiment, probes, configuration, secrets, settings)


def after_experiment_control(**kwargs):
def after_experiment_control(
context: Experiment = None, state: Journal = None, **kwargs
) -> None:
"""
Stop the safeguards. When one of them interrupted the execution, record
which one and its run in the journal under `safeguards`.
"""
guardian.terminate()
if guardian.was_triggered and isinstance(state, dict):
state["safeguards"] = {
"triggered_by": guardian.triggered_by,
"run": guardian.triggered_by_run,
}


###############################################################################
Expand Down
20 changes: 20 additions & 0 deletions tests/test_safeguard.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,3 +102,23 @@ def test_fail_when_no_probes_were_given():
}
with pytest.raises(InvalidActivity) as x:
validate_control(invalid_python_func_probe)


def test_journal_records_the_triggering_safeguard():
from chaosaddons.controls import safeguards

guardian = safeguards.guardian
guardian.prepare([])
try:
run = {"activity": {"name": "sg"}, "status": "succeeded", "output": 1}
guardian.interrupt_now("sg", run)
journal = {}
safeguards.after_experiment_control(context={}, state=journal)
assert journal["safeguards"] == {"triggered_by": "sg", "run": run}
finally:
guardian.prepare([])
guardian.terminate()

journal = {}
safeguards.after_experiment_control(context={}, state=journal)
assert "safeguards" not in journal
Loading