From e98a5f28664008d3cac5b6afd022a7581370379a Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Thu, 3 Sep 2026 18:05:56 -0500 Subject: [PATCH] ci(tooling): port the second fault-handler belt, calibrate the gate bound (BACKLOG #1304) The tooling tier carried one of the engine tier's two hang-diagnostic belts. PYTHONFAULTHANDLER=1 plus -o faulthandler_timeout= was absent from `Harness tests (pytest)`. Ported at 150 against that step's own 120, pinned by tests/test_ci_faulthandler_belts.py. All three mutations verified red first. Measured, and it corrects ci.yml's stated reason: pytest-timeout's thread method DOES fire on a wedged subprocess.wait. It dumps from a watchdog thread and calls os._exit(1), so it never needs to interrupt the main thread. The port is a second opinion on an independent mechanism, not the thing that names the failure. GATE_TIMEOUT_S carried no calibration. Worst case 4.6s over n=127 real launches, and calibrating it found a defect: at 60 against addopts' --timeout=60 the diagnostic could never fire locally. Moved to 45, a 9.8x margin. --- .github/workflows/ci.yml | 45 ++++++- docs/BACKLOG.md | 13 ++ tests/test_ci_faulthandler_belts.py | 181 ++++++++++++++++++++++++++++ tests/test_worktree_gate.py | 24 +++- tests/tooling_manifest.txt | 1 + 5 files changed, 262 insertions(+), 2 deletions(-) create mode 100644 tests/test_ci_faulthandler_belts.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad60a6bdc..acc9a4d93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1153,13 +1153,56 @@ jobs: # # `--dist loadfile` for the same reason as the engine step: these tests bind real port windows and # own module-scoped fixtures, so a file's tests must stay in one worker. + # + # THE SECOND FAULTHANDLER BELT, PORTED FROM THE ENGINE STEP (BACKLOG #1304). The engine + # `Tests (pytest)` step carries TWO watchdogs against a hung Windows leg; its derivation is there, + # not repeated here. This tier carried exactly ONE of them. `--timeout-method=thread` arrives + # repo-wide through `addopts` in pyproject.toml, so it was already live here; `PYTHONFAULTHANDLER=1` + # plus `-o faulthandler_timeout=` is set on the engine and web-console test steps and on nine + # further steps in this file, and on this one it was absent. + # + # THIS DOES NOT TURN AN UNNAMED FAILURE INTO A NAMED ONE, AND THE MEASUREMENT SAYS SO. Paired local + # arms on a test that blocks in `subprocess.run` with no `timeout=` of its own -- the shape 164 of + # this tier's 317 `subprocess` calls have, across 57 of the 138 files in tests/tooling_manifest.txt + # (AST count at 46ea10a78 against a control of 153 bounded calls, so a zero was reachable). WITHOUT + # this belt the thread method already fires and already names the frame, down to + # `_winapi.WaitForSingleObject`, and the `+ Timeout +` header appears twice in BOTH arms while the + # faulthandler header appears in ONE. Reading the pinned pytest-timeout on disk + # says why -- `timeout_timer` dumps from a WATCHDOG THREAD and calls `os._exit(1)`, so it never + # needs to interrupt the wedged main thread. Anyone porting this on the belief that the thread + # method cannot fire on a C-level wait is porting it for a reason that does not hold. + # + # WHAT IT ACTUALLY BUYS, which is narrower and still worth the two lines. A SECOND watchdog on an + # INDEPENDENT mechanism (CPython's C-level `dump_traceback_later`, not a Python `threading.Timer`), + # writing down a DIFFERENT path: faulthandler dumps to a raw dup'd stderr fd, while pytest-timeout + # writes through `config.get_terminal_writer()`. That distinction is why it is worth having on the + # one tier that runs `-n 4` -- `_pytest/faulthandler.py` comments that xdist monkeypatches + # `sys.stderr` with a non-file, and whether the terminal-writer dump survives a worker's `os._exit` + # and reaches the controller log is NOT established here (no xdist on the measuring box). + # `PYTHONFAULTHANDLER=1` additionally arms the fatal-signal handler from interpreter start, before + # `pytest_configure` arms pytest's own and after `pytest_unconfigure` disables it, and is inherited + # by Python children -- and this tier spawns more children than any other. faulthandler only DUMPS, + # never kills, so the outcome of every run is unchanged. + # + # 150 AGAINST THE 120 BELOW, and the ordering is the contract, not the numbers: the per-test bound + # must be attributed FIRST, so this sits above it, exactly as the engine legs pair 150/120 and + # 90/60. Both legs here share one `--timeout=120` (unlike the engine matrix, which varies it per + # leg), so one literal covers both and no matrix field is needed. Move one, move the other. + # + # WHAT NEITHER BELT COVERS, stated so nobody reads this as more than it is. Both arm per-ITEM, so a + # wedge during collection or in a session fixture reaches neither. And under `-n` the xdist + # CONTROLLER runs no items, so a controller left polling a dead worker is silent until the step cap + # -- the engine step answers THAT with a third belt (`--max-worker-restart=0`) this tier does not + # carry. It was deliberately not ported blind: it is a behaviour change whose zero cost was + # measured on the engine tier's history and on no sample from this one. See BACKLOG #1304. - name: Harness tests (pytest) timeout-minutes: 30 # see the job cap above for the derivation; must stay under it with setup env: # Same reason as the apt step above: collection imports the Qt modules whether or not this # job executes them, and the shipped ones construct QApplication at import on some paths. QT_QPA_PLATFORM: offscreen - run: pytest -q -n 4 --dist loadfile -m tooling --ignore-glob='*messagefoundry-webconsole*' --timeout=120 --junitxml=tooling-junit.xml + PYTHONFAULTHANDLER: "1" + run: pytest -q -n 4 --dist loadfile -m tooling --ignore-glob='*messagefoundry-webconsole*' -o faulthandler_timeout=150 --timeout=120 --junitxml=tooling-junit.xml # A marker typo, a manifest rename, or a conftest hook that silently stops firing all produce the # same thing: zero selected tests and a GREEN job. Deselection cannot be distinguished from success diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index a19e6b606..c4de0ede6 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -14508,6 +14508,19 @@ point, which are the parts that must survive it.** > **THIS ALSO PUTS A SECOND ARM INTO THIS ITEM'S SIGNATURE, alongside the crash above.** The observable set is now: `subprocess.TimeoutExpired` on a pwsh launch, a crashed xdist worker, and a monotonicity SLO on `test_connscale_smoke`. Whether they share one cause is **still not established** and must not be inferred from their sharing a leg -- the note above says so about the crash and it applies identically here. > **AND THE BASE RATE, WITH ITS DENOMINATOR, because the item argues from a streak and never states one.** Over **100 CI runs on `main`** (2026-08-14T19:25Z to 2026-08-22T04:23Z, the full window the Actions API returns in one page): **68 success, 24 failure, 8 cancelled.** Of the 24 failures, `repo harness tests (windows-2025)` is named in **7**, and in **all 7** `CI gate` -- a REQUIRED context -- fails with it: runs 32549032927, 32497283563, 32483211076, 32378103644, 32293477690, 32268545492, 32206563674. > **THAT IS THE SEVERITY SENTENCE THIS ITEM WAS MISSING.** Not "it blocked an eight-line prose PR twice" but: this leg is the **single largest contributor to a red default branch**, and every one of its failures reddens the roll-up that gates every merge. **DELIBERATELY NOT CLAIMED:** that the other 17 failures share a cause. Four other jobs appear in that list -- `test (windows-2025)` 7 times, `test (windows-2022)` 4, `test (ubuntu-latest)` 3, plus web-console and SQL Server legs -- and **none of them was investigated here**. The 7 is a floor for this item's subject, not a partition of the 24. +> **MORE TEST NAMES, 2026-09-03, AND THIS TIME THE LIST IS LEFT OPEN RATHER THAN RE-COUNTED.** Three occurrences tonight while the Console triaged the queue; two read directly from the job log on PR 763 and PR 771. Same file as one already named above, same bound, two different test functions. +> ``` +> tests/test_worktree_gate_control_plane.py::test_a_bare_config_read_of_a_d0 worker gw1 +> tests/test_worktree_gate_control_plane.py::test_a_bare_config_read_of_a_d3 worker gw0 +> pwsh -NoProfile -NonInteractive -File scripts/hooks/worktree_gate.ps1 -ReposFile /git_repos.txt +> subprocess.TimeoutExpired ... timed out after 60 seconds +> ``` +> **DO NOT RE-COUNT THIS LIST -- IT IS OPEN, AND THE NEXT READER SHOULD STOP COLLECTING NAMES.** The amendment above already recorded that an enumeration nothing re-checks is the SDS-3.6 shape, and this is the second time this one understated; a fresh total would only be the third. **The stable signature is the SCRIPT and the BOUND, not the test function:** grep a CI log for `worktree_gate.ps1` beside `TimeoutExpired`, or for `claim-reconcile.ps1`. Every `TimeoutExpired` occurrence recorded here has been one of those two scripts, while the test function has changed on every sighting. +> **THE ONE CHECKABLE QUESTION THIS ITEM RAISED IS ANSWERED, AND THE ANSWER IS HALF.** The question was whether the tooling tier carries the engine tier's two fault-handler belts. **It carried one.** `--timeout-method=thread` reaches every leg through `addopts` in `pyproject.toml`, so it was already live here. `PYTHONFAULTHANDLER=1` plus `-o faulthandler_timeout=` was set on the engine and web-console test steps and on nine further steps in `ci.yml`, and was absent from `Harness tests (pytest)`. Ported in this change, with the ordering (150 above the step's own 120) and a guard at `tests/test_ci_faulthandler_belts.py` that goes red on each half being removed -- all three mutations verified failing before the fix was called done. +> **AND THE ENGINE NOTE'S MECHANISM SENTENCE IS WRONG, WHICH MATTERS MORE THAN THE PORT.** `ci.yml` says the faulthandler belt "fires even when the thread-timer CANNOT interrupt a main-thread C-level wait", naming `subprocess.wait`. **Measured false**, paired local arms on a test blocking in `subprocess.run` with no `timeout=` of its own: **without** the faulthandler belt the thread method already fires and already names the frame down to `_winapi.WaitForSingleObject`, and the `+ Timeout +` header appears in **both** arms while the faulthandler header appears in **one**. Reading the pinned `pytest_timeout` on disk says why: `timeout_timer` dumps from a watchdog **thread** and calls `os._exit(1)`, so it never needs to interrupt anything. **So this tier was ALREADY instrumented for the observed signature, and the port is a second opinion rather than the thing that names the failure** -- an independent mechanism writing down a different path (a dup'd raw stderr fd, not `config.get_terminal_writer()`), which is worth having on the one tier that runs `-n 4`. The engine comment was left as it stands: its belt is still worth carrying, only its stated reason is wrong, and rewriting a contended region of another tier's derivation is not this change's business. +> **A MECHANISM THE CRASHED-WORKER AMENDMENT ASKED FOR, READ FROM THE PINNED LIBRARY RATHER THAN INFERRED.** That amendment says "a timeout that takes the worker down with it would present exactly this way" and correctly refuses to assume it. `pytest_timeout.timeout_timer` ending in `os._exit(1)` **is** such a path: under `-n`, a worker that `os._exit`s closes its execnet channel with no `workerfinished`, which the controller reports as a crashed worker. **This establishes that the mechanism EXISTS, not that it fired.** Nothing here distinguishes it from a second cause, and the amendment's refusal stands. +> **THE 60s BOUND IS CALIBRATED, AND CALIBRATING IT FOUND A DEFECT.** The surviving half of the not-measured list, closed at `tests/test_worktree_gate.py`. Worst case **4.6s per call** over n=127 real launches across both gate files (p50 2.1s, p99 4.2s), sequential on a box already running peer pytest sessions -- a contended sample, and still not a 4-vCPU runner, so the shape transfers and the numbers do not. **The defect: at 60 the diagnostic could never fire locally.** pytest-timeout arms in `pytest_runtest_protocol`, so its window strictly contains this one and at equal values it expires first -- measured with a paired control, 5s against `--timeout=5` gave pytest the win 2/2, 5s against `--timeout=30` gave the diagnostic the win 2/2. `addopts` carries `--timeout=60`. So the bound was live on CI's tooling leg (which overrides to 120) and **decorative on every local run**, which is exactly the silent one-platform failure its 45s sibling's comment was written to prevent. **Moved to 45**, a 9.8x margin, clearing both bounds. **Failures before this change read `after 60 seconds`.** +> **STILL OPEN, EACH NAMING EXACTLY ONE THING.** (1) **The cause**, unchanged and unmeasurable off the runner -- not touched here. (2) **`--max-worker-restart=0`**, the engine step's third belt, **not ported**: it is a behaviour change (a worker death fails the run instead of being survived) whose zero cost was measured on the engine tier's history and on **no** sample from this one. It is the belt that would answer a controller left polling a dead worker until the 30-minute step cap, so it is the strongest remaining candidate and it needs its own measurement first. (3) **The rerun budget**, still an unwritten norm; a recommendation is in this change's PR body and the decision is the owner's. > Verdict: build > Closing-act: code diff --git a/tests/test_ci_faulthandler_belts.py b/tests/test_ci_faulthandler_belts.py new file mode 100644 index 000000000..ddc8b3dbd --- /dev/null +++ b/tests/test_ci_faulthandler_belts.py @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Organization and contributors +"""The hang-diagnostic belts on CI's two pytest steps, pinned so a silent removal goes red. + +BACKLOG #1304. The ``repo harness tests (windows-2025)`` leg intermittently hangs on a ``pwsh`` +launch. The engine ``Tests (pytest)`` step already carried two watchdogs against a hung Windows +leg; the tooling step carried one, and the second was ported to it. This file stops either half +drifting off either step without a reader. + +**WHAT EACH BELT IS, because they are easy to conflate.** + +* ``--timeout-method=thread`` reaches BOTH steps through ``addopts`` in ``pyproject.toml``, so it is + repo-wide and is not asserted per-step here -- ``test_thread_method_is_repo_wide`` pins it at its + one source instead. +* ``PYTHONFAULTHANDLER=1`` plus ``-o faulthandler_timeout=`` is per-step, and that is the half that + was missing from the tooling step. + +**THIS BELT IS A SECOND OPINION, NOT THE THING THAT NAMES THE FAILURE, and the measurement says so.** +Paired local arms on a test blocking in ``subprocess.run`` with no ``timeout=`` of its own: without +the faulthandler belt the thread method already fires and already names the frame down to +``_winapi.WaitForSingleObject``. Reading the pinned ``pytest_timeout`` on disk says why -- its +``timeout_timer`` dumps from a watchdog THREAD and calls ``os._exit(1)``, so it never needs to +interrupt the wedged main thread. What the belt adds is an independent mechanism (CPython's +``dump_traceback_later``) writing down a different path (a dup'd raw stderr fd, not +``config.get_terminal_writer()``), which matters most on the one tier that runs ``-n 4``. + +**THE ORDERING IS THE CONTRACT, NOT THE NUMBERS.** ``faulthandler_timeout`` must sit ABOVE the +step's own ``--timeout`` so the per-test bound is attributed first and the faulthandler dump stays +the last resort. Asserted numerically where both are literals; the engine step passes both through +env from the matrix, so only presence is checkable there without re-implementing the matrix. +""" + +from __future__ import annotations + +import re +import shlex +import tomllib +from pathlib import Path + +import pytest +import yaml + +_ROOT = Path(__file__).resolve().parents[1] +_CI = _ROOT / ".github" / "workflows" / "ci.yml" +_PYPROJECT = _ROOT / "pyproject.toml" + +#: Both pytest steps, addressed by NAME. A rename must break this file, so whoever renames a step +#: re-points the guard -- the failure mode ``test_ci_engine_step_excludes_webconsole`` records is a +#: locator that quietly binds the wrong step and passes. +_ENGINE_STEP = "Tests (pytest)" +_TOOLING_STEP = "Harness tests (pytest)" + + +def _logical_lines(run: str) -> list[str]: + """The whole ``run:`` block with backslash-continuations joined, so one command is one string.""" + return [line.strip() for line in re.sub(r"\\\n[ \t]*", " ", run).splitlines() if line.strip()] + + +def _is_pytest_command(line: str) -> list[str] | None: + """This line's argv when ``pytest`` appears as a command WORD, else None. + + Normalizing through ``shlex`` is what makes prose a non-answer: a mention inside quotes stays one + argument to whatever quoted it, and a shell comment is dropped before any assertion reads it. + """ + try: + argv = shlex.split(line, comments=True) + except ValueError: + # An untokenizable line cannot be SHOWN to be an invocation, so it is not treated as one. + # This can only lose a candidate, and losing every candidate fails loudly below. + return None + if not any(word == "pytest" or word.endswith("/pytest") for word in argv): + return None + return argv + + +def _step(step_name: str) -> tuple[str, dict[str, object]]: + """The single step with this name, as ``(job id, step)``. Zero or many fails loudly. + + A guard that finds nothing and passes is the defect, one rename away. + """ + workflow = yaml.safe_load(_CI.read_text(encoding="utf-8")) + found = [ + (job_id, step) + for job_id, job in workflow["jobs"].items() + for step in (job.get("steps") or []) + if step.get("name") == step_name + ] + if len(found) != 1: + pytest.fail( + f"expected exactly one step named {step_name!r} in {_CI}; found {len(found)} " + f"({[job for job, _ in found]}). Re-point this guard rather than deleting it." + ) + return found[0] + + +def _invocation(step_name: str) -> tuple[str, list[str]]: + """The step's one pytest argv, as ``(job id, argv)``.""" + job_id, step = _step(step_name) + run = str(step.get("run", "")) + argvs = [argv for line in _logical_lines(run) if (argv := _is_pytest_command(line)) is not None] + if len(argvs) != 1: + pytest.fail( + f"expected exactly one pytest invocation in the {step_name!r} step of job {job_id!r}; " + f"found {len(argvs)}:\n{run}" + ) + return job_id, argvs[0] + + +def _ini_override(argv: list[str], key: str) -> str | None: + """The value of ``-o key=value``, in either the ``-o k=v`` or ``-ok=v`` spelling.""" + for i, word in enumerate(argv): + if word == "-o" and i + 1 < len(argv) and argv[i + 1].startswith(f"{key}="): + return argv[i + 1].split("=", 1)[1] + if word.startswith(f"-o{key}="): + return word.split("=", 1)[1] + return None + + +def _flag_value(argv: list[str], flag: str) -> str | None: + """The value of ``--flag=value`` or ``--flag value``.""" + for i, word in enumerate(argv): + if word.startswith(f"{flag}="): + return word.split("=", 1)[1] + if word == flag and i + 1 < len(argv): + return argv[i + 1] + return None + + +def test_thread_method_is_repo_wide() -> None: + """Belt one lives in ``addopts``, so both steps inherit it. Pin it at its single source.""" + cfg = tomllib.loads(_PYPROJECT.read_text(encoding="utf-8")) + addopts = cfg["tool"]["pytest"]["ini_options"]["addopts"] + assert "--timeout-method=thread" in addopts, ( + "`--timeout-method=thread` is the ONLY timeout method available on Windows (SIGALRM is " + f"POSIX-only) and it reaches every leg through addopts. Got: {addopts!r}" + ) + + +@pytest.mark.parametrize("step_name", [_ENGINE_STEP, _TOOLING_STEP]) +def test_step_exports_pythonfaulthandler(step_name: str) -> None: + """Arms the fatal-signal handler from interpreter start, and is inherited by Python children.""" + job_id, step = _step(step_name) + env = step.get("env") or {} + assert isinstance(env, dict) + assert str(env.get("PYTHONFAULTHANDLER", "")) == "1", ( + f"the {step_name!r} step in job {job_id!r} must export PYTHONFAULTHANDLER=1 (BACKLOG " + f"#1304). Got env: {env!r}" + ) + + +@pytest.mark.parametrize("step_name", [_ENGINE_STEP, _TOOLING_STEP]) +def test_step_arms_faulthandler_timeout(step_name: str) -> None: + """The env var alone arms no watchdog -- the per-test dump needs the ini key as well.""" + job_id, argv = _invocation(step_name) + value = _ini_override(argv, "faulthandler_timeout") + assert value, ( + f"the {step_name!r} step in job {job_id!r} must pass `-o faulthandler_timeout=` (BACKLOG " + "#1304). The plugin exposes this as an ini key, NOT as a `--faulthandler-*` CLI flag, so " + f"`-o` is the only spelling that arms it. Got: {shlex.join(argv)}" + ) + + +def test_tooling_faulthandler_sits_above_its_own_pytest_timeout() -> None: + """Below the per-test bound the last-resort dump fires FIRST and steals the attribution. + + Only the tooling step is checked numerically: it is the one carrying both values as literals. + The engine step passes both through env from the runtime matrix, so a numeric check here would + re-implement that matrix and drift from it. + """ + job_id, argv = _invocation(_TOOLING_STEP) + fault = _ini_override(argv, "faulthandler_timeout") + per_test = _flag_value(argv, "--timeout") + assert fault is not None and per_test is not None, ( + f"the {_TOOLING_STEP!r} step in job {job_id!r} must carry both bounds as literals; got " + f"faulthandler_timeout={fault!r}, --timeout={per_test!r}" + ) + assert float(fault) > float(per_test), ( + f"faulthandler_timeout ({fault}) must sit ABOVE --timeout ({per_test}) so the per-test " + "bound is attributed first and the faulthandler dump stays the last resort. Move one, " + "move the other." + ) diff --git a/tests/test_worktree_gate.py b/tests/test_worktree_gate.py index 1d8a62666..109aadd8f 100644 --- a/tests/test_worktree_gate.py +++ b/tests/test_worktree_gate.py @@ -32,7 +32,29 @@ #: Seconds allowed for ONE ``pwsh`` launch plus the gate's own work. Named so the diagnostic below can #: quote it, rather than repeating the literal in a message that then drifts from the argument. -GATE_TIMEOUT_S = 60 +#: +#: CALIBRATED 2026-09-03 (BACKLOG #1304), and it was 60 until then with no recorded calibration at all -- +#: the surviving half of that item's not-measured list. Same shape as its sibling at +#: ``tests/test_coord_claim_reconcile.py``: this is a DIAGNOSTIC sited deliberately BELOW pytest's own +#: bound, so that when it fires the message names a hung ``pwsh`` spawn instead of pytest's generic +#: timeout. Measured worst case is **4.6s per call** (n=127 real launches across this file and +#: test_worktree_gate_control_plane.py; p50 2.1s, p99 4.2s; one sub-50ms sample excluded as a call that +#: raised before launching). Sequential, on a developer box already running several peer pytest sessions +#: -- so it is a CONTENDED sample, which is the useful direction, but a 4-vCPU hosted runner is still not +#: measurable from here and the absolute numbers do not transfer. 45 against 4.6s is a **9.8x margin**. +#: +#: WHY IT MOVED DOWN FROM 60, and this is the whole reason the calibration was worth doing. pytest-timeout +#: arms in ``pytest_runtest_protocol``, covering setup + call + teardown; this bound starts later, inside +#: the call. So pytest's window strictly CONTAINS this one and at equal values pytest expires first -- +#: measured, not reasoned, with a paired control: at 5s against ``--timeout=5`` pytest won 2/2 and this +#: diagnostic never fired; at 5s against ``--timeout=30`` it won 2/2. ``addopts`` carries +#: ``--timeout=60``, so at 60 this diagnostic could NEVER fire on a bare local ``pytest`` -- decorative +#: locally, live only on CI's tooling leg, which overrides to 120. That is exactly the silent +#: one-platform failure the sibling comment was written to prevent. 45 clears both bounds. +#: +#: RE-DERIVE IF ``addopts`` or the tooling leg's ``--timeout`` moves, or if a call is ever observed above +#: ~15s. NOTE FOR ANYONE GREPPING CI HISTORY: failures before this change read ``after 60 seconds``. +GATE_TIMEOUT_S = 45 def run_gate(payload: dict[str, Any] | str, repos_file: Path) -> dict[str, Any] | None: diff --git a/tests/tooling_manifest.txt b/tests/tooling_manifest.txt index bb5029a6f..8503658f9 100644 --- a/tests/tooling_manifest.txt +++ b/tests/tooling_manifest.txt @@ -43,6 +43,7 @@ tests/test_bash_resolver.py tests/test_blanket_stage_guard.py tests/test_ci_docs_only_detector.py tests/test_ci_engine_step_excludes_webconsole.py +tests/test_ci_faulthandler_belts.py tests/test_ci_leg_data_class.py tests/test_ci_retry_native_crash.py tests/test_ci_step_margin.py