From 9e751d89c81f57ab5447af7ae64accca66c1e67f Mon Sep 17 00:00:00 2001 From: nicklafleur <55208706+nicklafleur@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:49:55 -0400 Subject: [PATCH 1/5] feat(workers): add fork isolation utilities and OrchestratorCrashError Add pipe-based fork isolation helpers (run_in_fork_with_result, run_in_fork) that run functions in forked children so the parent process never imports pytest/conftest and stays fork-safe. This is the foundation for the upcoming hot-fork runner where the parent acts purely as an orchestrator. Also add OrchestratorCrashError, which reports the crashed orchestrator's exit code, the lost in-flight mutants (truncated past 10), an optional crash-log path, and instructions to resume with 'mutmut run'. --- src/mutmut/workers/isolation.py | 138 ++++++++++++++++++++++++ tests/workers/test_isolation.py | 179 ++++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+) create mode 100644 src/mutmut/workers/isolation.py create mode 100644 tests/workers/test_isolation.py diff --git a/src/mutmut/workers/isolation.py b/src/mutmut/workers/isolation.py new file mode 100644 index 00000000..561ea476 --- /dev/null +++ b/src/mutmut/workers/isolation.py @@ -0,0 +1,138 @@ +""" +Fork isolation utilities for keeping the parent process clean. + +The main process must not import pytest/test code directly, because test +conftest.py files may call gevent.monkey.patch_all() or import grpc, which +makes the process fork-unsafe. + +These utilities run operations in forked children so the parent stays clean. +Uses pipe-based IPC for reduced overhead (no temp files, no cleanup needed). +""" + +from __future__ import annotations + +import os +import pickle +from collections.abc import Callable +from typing import Any + + +def run_in_fork_with_result(fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + """Fork a child to run a function, return its result via pipe. + + The parent process stays clean - child's imports don't affect parent. + Uses os.pipe() for IPC - lower overhead than temp files. + + Args: + fn: Function that returns a picklable result. + *args, **kwargs: Arguments to pass to fn. + + Returns: + The return value of fn(*args, **kwargs). + + Raises: + ChildProcessError: If child exits with non-zero status. + """ + # Create pipe for result transfer + read_fd, write_fd = os.pipe() + + pid = os.fork() + if pid == 0: + # Child: close read end, run function, write result + os.close(read_fd) + try: + result = fn(*args, **kwargs) + with os.fdopen(write_fd, "wb") as f: + pickle.dump({"ok": True, "value": result}, f) + os._exit(0) + except Exception as e: + try: + with os.fdopen(write_fd, "wb") as f: + pickle.dump({"ok": False, "error": str(e)}, f) + except Exception: + pass + os._exit(1) + + # Parent: close write end, wait for child, read result + os.close(write_fd) + _, status = os.waitpid(pid, 0) + exit_code = os.waitstatus_to_exitcode(status) + + # Read result from pipe + with os.fdopen(read_fd, "rb") as f: + try: + data = pickle.load(f) + except Exception: + data = None + + if exit_code != 0 or data is None: + error_msg = f"Child exited with code {exit_code}" + if data and not data.get("ok") and "error" in data: + error_msg += f": {data['error']}" + raise ChildProcessError(error_msg) + + if not data.get("ok"): + raise ChildProcessError(f"Child failed: {data.get('error', 'unknown')}") + + return data["value"] + + +def run_in_fork(fn: Callable[..., int], *args: Any, **kwargs: Any) -> int: + """Fork a child to run a function, return its exit code. + + Use for operations that only need pass/fail result (clean test, forced fail). + Parent process stays clean - child's imports don't affect parent. + + Args: + fn: Function that returns an exit code (0-255). + *args, **kwargs: Arguments to pass to fn. + + Returns: + Exit code from the child process. + """ + pid = os.fork() + if pid == 0: + # Child: run function and exit with its return code + try: + exit_code = fn(*args, **kwargs) + os._exit(exit_code if isinstance(exit_code, int) else 0) + except Exception: + os._exit(1) + + # Parent waits for child + _, status = os.waitpid(pid, 0) + return os.waitstatus_to_exitcode(status) + + +class OrchestratorCrashError(Exception): + """Raised when the hot-fork orchestrator crashes unexpectedly. + + The orchestrator manages all mutant test runs. If it crashes, any + in-flight mutants are lost. The user can resume by running + `mutmut run` again - completed results are preserved. + """ + + def __init__(self, exit_code: int, lost_mutants: list[str], crash_log: str | None = None) -> None: + self.exit_code = exit_code + self.lost_mutants = lost_mutants + self.crash_log = crash_log + + # Build detailed message + details = [ + f"Hot-fork orchestrator crashed unexpectedly (exit code: {exit_code})", + f"Lost {len(lost_mutants)} in-flight mutant(s):", + ] + for m in lost_mutants[:10]: + details.append(f" - {m}") + if len(lost_mutants) > 10: + details.append(f" ... and {len(lost_mutants) - 10} more") + + details.append("") + details.append("This usually indicates a bug in pytest or conftest.py.") + if crash_log: + details.append(f"Crash log: {crash_log}") + details.append("") + details.append("To resume: mutmut run") + details.append("(Completed mutants are saved; lost ones will be re-run)") + + super().__init__("\n".join(details)) diff --git a/tests/workers/test_isolation.py b/tests/workers/test_isolation.py new file mode 100644 index 00000000..36e20510 --- /dev/null +++ b/tests/workers/test_isolation.py @@ -0,0 +1,179 @@ +"""Tests for fork isolation utilities.""" + +import os + +import pytest + +from mutmut.workers.isolation import OrchestratorCrashError +from mutmut.workers.isolation import run_in_fork +from mutmut.workers.isolation import run_in_fork_with_result + + +@pytest.mark.skipif(os.name == "nt", reason="Forking not supported on Windows") +class TestRunInForkWithResult: + """Tests for run_in_fork_with_result.""" + + def test_returns_simple_value(self): + """Function return value is passed back to parent.""" + result = run_in_fork_with_result(lambda: 42) + assert result == 42 + + def test_returns_complex_value(self): + """Complex picklable objects are returned correctly.""" + result = run_in_fork_with_result(lambda: {"a": [1, 2, 3], "b": "hello"}) + assert result == {"a": [1, 2, 3], "b": "hello"} + + def test_passes_args_and_kwargs(self): + """Arguments are passed to the function.""" + + def add(a, b, multiplier=1): + return (a + b) * multiplier + + result = run_in_fork_with_result(add, 2, 3, multiplier=10) + assert result == 50 + + def test_child_import_does_not_affect_parent(self): + """Imports in child don't pollute parent's namespace.""" + + def import_and_use(): + import json + + return json.dumps({"test": True}) + + result = run_in_fork_with_result(import_and_use) + assert result == '{"test": true}' + + def test_child_crash_raises_error(self): + """Child process crash raises ChildProcessError.""" + + def crash(): + raise RuntimeError("boom") + + with pytest.raises(ChildProcessError): + run_in_fork_with_result(crash) + + def test_child_crash_includes_error_message(self): + """ChildProcessError includes the original exception message.""" + + def crash(): + raise RuntimeError("specific error message") + + with pytest.raises(ChildProcessError) as exc_info: + run_in_fork_with_result(crash) + + assert "specific error message" in str(exc_info.value) + + def test_no_temp_files_created(self, tmp_path, monkeypatch): + """Pipe-based transport doesn't create temp files.""" + monkeypatch.chdir(tmp_path) + (tmp_path / "mutants").mkdir() + + run_in_fork_with_result(lambda: "test") + + # No pickle files should exist + assert not list(tmp_path.glob("**/*.pickle")) + + +@pytest.mark.skipif(os.name == "nt", reason="Forking not supported on Windows") +class TestRunInFork: + """Tests for run_in_fork (returns exit code).""" + + def test_returns_zero_on_success(self): + """Returns 0 when function returns 0.""" + result = run_in_fork(lambda: 0) + assert result == 0 + + def test_returns_function_exit_code(self): + """Returns the exit code from the function.""" + result = run_in_fork(lambda: 42) + assert result == 42 + + def test_returns_one_on_exception(self): + """Returns 1 when function raises.""" + + def crash(): + raise RuntimeError("boom") + + result = run_in_fork(crash) + assert result == 1 + + def test_side_effects_in_child(self, tmp_path): + """Side effects happen in child (verifiable via file).""" + marker = tmp_path / "marker.txt" + + def create_marker(): + marker.write_text("created") + return 0 + + run_in_fork(create_marker) + + assert marker.read_text() == "created" + + +class TestOrchestratorCrashError: + """Tests for OrchestratorCrashError exception.""" + + def test_error_message_includes_exit_code(self): + """Exit code is included in message.""" + err = OrchestratorCrashError(exit_code=1, lost_mutants=[]) + assert "exit code: 1" in str(err) + + def test_error_message_lists_lost_mutants(self): + """Lost mutants are listed in message.""" + err = OrchestratorCrashError(exit_code=1, lost_mutants=["mutant_1", "mutant_2"]) + assert "mutant_1" in str(err) + assert "mutant_2" in str(err) + assert "2 in-flight mutant(s)" in str(err) + + def test_truncates_long_mutant_list(self): + """Only first 10 mutants shown, rest summarized.""" + mutants = [f"mutant_{i}" for i in range(15)] + err = OrchestratorCrashError(exit_code=1, lost_mutants=mutants) + + assert "mutant_0" in str(err) + assert "mutant_9" in str(err) + assert "mutant_10" not in str(err) + assert "5 more" in str(err) + + def test_includes_resume_instructions(self): + """Message includes how to resume.""" + err = OrchestratorCrashError(exit_code=1, lost_mutants=[]) + assert "mutmut run" in str(err) + + def test_includes_crash_log_path(self): + """Crash log path is shown if provided.""" + err = OrchestratorCrashError(exit_code=1, lost_mutants=[], crash_log="mutants/.orchestrator-crash.log") + assert ".orchestrator-crash.log" in str(err) + + def test_attributes_accessible(self): + """Exception attributes are accessible.""" + err = OrchestratorCrashError(exit_code=42, lost_mutants=["a", "b"], crash_log="/path/to/log") + assert err.exit_code == 42 + assert err.lost_mutants == ["a", "b"] + assert err.crash_log == "/path/to/log" + + def test_empty_lost_mutants(self): + """Works correctly with empty lost mutants list.""" + err = OrchestratorCrashError(exit_code=0, lost_mutants=[]) + assert "0 in-flight mutant(s)" in str(err) + + def test_no_crash_log(self): + """Works correctly without crash log.""" + err = OrchestratorCrashError(exit_code=1, lost_mutants=["m1"]) + # Should not raise and should not include "Crash log:" + msg = str(err) + assert "Crash log:" not in msg + + def test_is_exception(self): + """OrchestratorCrashError is an Exception subclass.""" + err = OrchestratorCrashError(exit_code=1, lost_mutants=[]) + assert isinstance(err, Exception) + + def test_can_be_raised_and_caught(self): + """Exception can be raised and caught properly.""" + with pytest.raises(OrchestratorCrashError) as exc_info: + raise OrchestratorCrashError(exit_code=255, lost_mutants=["test_mutant"], crash_log="/tmp/crash.log") + + assert exc_info.value.exit_code == 255 + assert exc_info.value.lost_mutants == ["test_mutant"] + assert exc_info.value.crash_log == "/tmp/crash.log" From ce0ae92a4b494467f1649bb168c0696e829f9a2c Mon Sep 17 00:00:00 2001 From: nicklafleur <55208706+nicklafleur@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:01:34 -0400 Subject: [PATCH 2/5] feat(runners): extract MutantRunner abstraction with ForkRunner Introduce a MutantRunner ABC that owns the process-isolation strategy for testing mutants and also fronts the surrounding test operations (stats collection, clean tests, forced-fail, test listing), so __main__ no longer drives os.fork() directly. ForkRunner encapsulates the traditional os.fork()-per-mutant loop that lived inline in _run(): submit() forks a child under a CPU/wall timeout, and wait_for_result() reaps one child into a MutantResult. get_mutant_runner() selects the runner from the new process_isolation config (ProcessIsolation enum; only 'fork' is wired up here, 'hot-fork' raises pending a later commit). _run() now drives the runner through submit/has_capacity/wait_for_result/ pending_count/shutdown and registers results by mutant name. The stale-stats protections are preserved verbatim: _check_test_to_mutant_associations() still runs, and collect_or_load_stats() keeps its apply_config_invalidation path (now routed through MutantRunner.collect_stats/list_all_tests). Behavior for the default fork path is unchanged; the full suite (incl. the e2e run) is green. --- src/mutmut/__main__.py | 178 +++++++------------- src/mutmut/configuration.py | 21 +++ src/mutmut/core.py | 4 + src/mutmut/mutation/data.py | 23 --- src/mutmut/mutation/trampoline.py | 4 +- src/mutmut/stats.py | 28 ++++ src/mutmut/workers/isolation.py | 265 +++++++++++++++++++++++++++++- tests/mutation/test_mutation.py | 62 ++++++- tests/test_configuration.py | 2 + tests/workers/test_isolation.py | 60 +++++++ 10 files changed, 491 insertions(+), 156 deletions(-) diff --git a/src/mutmut/__main__.py b/src/mutmut/__main__.py index c8563bd7..0254aef6 100644 --- a/src/mutmut/__main__.py +++ b/src/mutmut/__main__.py @@ -18,11 +18,8 @@ sys.exit(1) import ast import fnmatch -import gc import hashlib -import inspect import json -import resource import shutil import subprocess import warnings @@ -50,6 +47,7 @@ from mutmut.code_coverage import get_excluded_lines_for_file from mutmut.configuration import Config from mutmut.configuration import config +from mutmut.core import MutmutProgrammaticFailException from mutmut.mutation.data import MutantLineSpans from mutmut.mutation.data import SourceFileMutationData from mutmut.mutation.diff_apply import apply_mutant @@ -59,9 +57,9 @@ from mutmut.mutation.file_mutation import MutatedFile from mutmut.mutation.file_mutation import filter_mutants_with_type_checker from mutmut.mutation.file_mutation import mutate_file_contents +from mutmut.mutation.trampoline import set_mutant_under_test from mutmut.runners.harness import CollectTestsFailedException from mutmut.runners.harness import PytestRunner -from mutmut.runners.harness import TestRunner from mutmut.runners.harness import collected_test_names from mutmut.stats import calculate_summary_stats from mutmut.stats import emoji_by_status @@ -76,44 +74,15 @@ from mutmut.utils.file_utils import setup_source_paths from mutmut.utils.file_utils import walk_mutatable_files from mutmut.utils.file_utils import walk_source_files -from mutmut.utils.safe_setproctitle import safe_setproctitle as setproctitle -from mutmut.workers.timeout import register_timeout +from mutmut.workers.isolation import MutantResult +from mutmut.workers.isolation import MutantRunner +from mutmut.workers.isolation import get_mutant_runner # Document: surviving mutants are retested when you ask mutmut to retest them, interactively in the UI or via command line # TODO: pragma no mutate should end up in `skipped` category -def record_trampoline_hit(name: str, caller: str | None = None) -> None: - assert not name.startswith("src."), "Failed trampoline hit. Module name starts with `src.`, which is invalid" - - mutated_source_paths = config().resolved_mutated_source_paths - - if config().max_stack_depth != -1: - f = inspect.currentframe() - c = config().max_stack_depth - while c and f: - filename = f.f_code.co_filename - f = f.f_back - if "pytest" in filename or "hammett" in filename or "unittest" in filename: - break - file_path = Path(filename).resolve(strict=True) - if any(path in file_path.parents for path in mutated_source_paths): - # only include stack frames of user-code; exclude mutmut and 3rd library stack frames - c -= 1 - - if not c: - return - - state()._stats.add(name) - if caller is not None and config().track_dependencies: - state().function_dependencies[name].add(caller) - - -class MutmutProgrammaticFailException(Exception): - pass - - class InvalidGeneratedSyntaxException(Exception): def __init__(self, file: Path | str) -> None: super().__init__( @@ -269,20 +238,8 @@ def write_all_mutants_to_file(*, out: TextIOBase, source: str, filename: Path) - return mutated_file -def _set_mutant_under_test(name: str) -> None: - """Activate a mutant in process-local state and the environment. - - Late-import the trampoline setter: that module imports from this file - at load time. The process-local copy is used when tests scrub - ``os.environ``. See #511. - """ - from mutmut.mutation.trampoline import set_mutant_under_test - - set_mutant_under_test(name) - - -def run_forced_fail_test(runner: TestRunner) -> None: - _set_mutant_under_test("fail") +def run_forced_fail_test(runner: MutantRunner) -> None: + set_mutant_under_test("fail") with CatchOutput(spinner_title="Running forced fail test") as catcher: try: if runner.run_forced_fail() == 0: @@ -291,7 +248,7 @@ def run_forced_fail_test(runner: TestRunner) -> None: raise SystemExit(1) except MutmutProgrammaticFailException: pass - _set_mutant_under_test("") + set_mutant_under_test("") print(" done") @@ -359,18 +316,18 @@ def cli() -> None: pass -def run_stats_collection(runner: TestRunner, tests: Iterable[str] | None = None) -> None: +def run_stats_collection(runner: MutantRunner, tests: Iterable[str] | None = None) -> None: if tests is None: tests = [] # Meaning all... - _set_mutant_under_test("stats") + set_mutant_under_test("stats") os.environ["PY_IGNORE_IMPORTMISMATCH"] = "1" depth = config().dependency_tracking_depth os.environ["MUTMUT_DEPENDENCY_DEPTH"] = str(depth) start_cpu_time = process_time() with CatchOutput(spinner_title="Running stats") as output_catcher: - collect_stats_exit_code = runner.run_stats(tests=tests) + collect_stats_exit_code = runner.collect_stats(tests) if collect_stats_exit_code != 0: output_catcher.dump_output() print(f"failed to collect stats. runner returned {collect_stats_exit_code}") @@ -699,7 +656,7 @@ def _apply_config_change_invalidation(mutants_caught_by_type_checker: dict[str, def collect_or_load_stats( - runner: TestRunner, + runner: MutantRunner, *, mutants_caught_by_type_checker: dict[str, Any] | None = None, apply_config_invalidation: bool = False, @@ -724,7 +681,7 @@ def collect_or_load_stats( # Run incremental stats with CatchOutput(spinner_title="Listing all tests") as output_catcher: - _set_mutant_under_test("list_all_tests") + set_mutant_under_test("list_all_tests") try: all_tests_result = runner.list_all_tests() except CollectTestsFailedException: @@ -918,8 +875,7 @@ def estimated_worst_case_time(mutant_name: str) -> float: def print_time_estimates(mutant_names: tuple[str, ...]) -> None: assert isinstance(mutant_names, (tuple, list)), mutant_names - runner = PytestRunner() - runner.prepare_main_test_run() + runner = get_mutant_runner() collect_or_load_stats(runner) @@ -948,9 +904,20 @@ def tests_for_mutant(mutant_name: str) -> None: print(test) -def stop_all_children(mutants: list[tuple[SourceFileMutationData, str, int | None]]) -> None: - for m, _, _ in mutants: - m.stop_children() +def _register_mutant_result( + result: MutantResult, + mutation_data_by_mutant_name: dict[str, SourceFileMutationData], +) -> None: + """Record a completed mutant's exit code and duration onto its mutation data.""" + mutation_data = mutation_data_by_mutant_name[result.mutant_name] + # Writing a key the meta file does not already know about would silently create a + # phantom mutant instead of recording a verdict, so fail loudly on a name mismatch. + assert result.mutant_name in mutation_data.exit_code_by_key, ( + f"{result.mutant_name} is not a known mutant of {mutation_data.path}" + ) + mutation_data.exit_code_by_key[result.mutant_name] = result.exit_code + mutation_data.durations_by_key[result.mutant_name] = result.duration + mutation_data.save() # Guard against "context has already been set" when mutmut.__main__ is @@ -978,7 +945,7 @@ def run(mutant_names: tuple[str, ...] | list[str], *, max_children: int | None) def _run(mutant_names: tuple[str, ...] | list[str], max_children: int | None) -> None: # TODO: run no-ops once in a while to detect if we get false negatives # TODO: we should be able to get information on which tests killed mutants, which means we can get a list of tests and how many mutants each test kills. Those that kill zero mutants are redundant! - _set_mutant_under_test("mutant_generation") + set_mutant_under_test("mutant_generation") if max_children is None: max_children = os.cpu_count() or 4 @@ -1002,10 +969,8 @@ def _run(mutant_names: tuple[str, ...] | list[str], max_children: int | None) -> with CatchOutput(spinner_title="Filtering mutations with type checker"): mutants_caught_by_type_checker = filter_mutants_with_type_checker() - # TODO: config/option for runner - # runner = HammettRunner() - runner = PytestRunner() - runner.prepare_main_test_run() + # TODO: config/option for the test runner (e.g. HammettRunner) + runner: MutantRunner = get_mutant_runner(max_children) # TODO: run these steps only if we have mutants to test @@ -1019,11 +984,11 @@ def _run(mutant_names: tuple[str, ...] | list[str], max_children: int | None) -> _check_test_to_mutant_associations(source_file_mutation_data_by_path) - _set_mutant_under_test("") + set_mutant_under_test("") with CatchOutput(spinner_title="Running clean tests") as output_catcher: tests = tests_for_mutant_names(mutant_names) - clean_test_exit_code = runner.run_tests(mutant_name=None, tests=tests) + clean_test_exit_code = runner.run_clean_tests(tests=tests) if clean_test_exit_code != 0: output_catcher.dump_output() print("Failed to run clean test") @@ -1033,24 +998,23 @@ def _run(mutant_names: tuple[str, ...] | list[str], max_children: int | None) -> # this can't be the first thing, because it can fail deep inside pytest/django setup and then everything is destroyed run_forced_fail_test(runner) - runner.prepare_main_test_run() + # Maps each submitted mutant to its mutation data, for result registration. + mutation_data_by_mutant_name: dict[str, SourceFileMutationData] = {} + count_tried = 0 - def read_one_child_exit_status() -> None: - pid, wait_status = os.wait() - exit_code = os.waitstatus_to_exitcode(wait_status) + def drain_one_result() -> None: + nonlocal count_tried + result = runner.wait_for_result() if config().debug: - print(" worker exit code", exit_code) - source_file_mutation_data_by_pid[pid].register_result(pid=pid, exit_code=exit_code) - - source_file_mutation_data_by_pid: dict[int, SourceFileMutationData] = {} # many pids map to one MutationData - running_children = 0 - count_tried = 0 + print(" worker exit code", result.exit_code) + _register_mutant_result(result, mutation_data_by_mutant_name) + count_tried += 1 # Run estimated fast mutants first, calculated as the estimated time for a surviving mutant. mutants = sorted(mutants, key=lambda x: estimated_worst_case_time(x[1])) start = datetime.now() + runner.startup() try: - gc.freeze() print("Running mutation testing") # Now do mutation @@ -1078,54 +1042,28 @@ def read_one_child_exit_status() -> None: continue cfg = config() - pid = os.fork() - if pid == 0: - # In the child - _set_mutant_under_test(mutant_name) - setproctitle(f"mutmut: {mutant_name}") - - # Run fast tests first - sorted_tests = sorted(tests, key=lambda test_name: state().duration_by_test[test_name]) - if not sorted_tests: - os._exit(33) - - cpu_time_limit_s = ceil( - (estimated_time_of_tests + cfg.timeout_constant) * cfg.timeout_multiplier * 2 + process_time() - ) - # signal SIGXCPU after . One second later signal SIGKILL if it is still running - resource.setrlimit(resource.RLIMIT_CPU, (cpu_time_limit_s, cpu_time_limit_s + 1)) - - with CatchOutput(): - result = runner.run_tests(mutant_name=mutant_name, tests=sorted_tests) - - if result != 0: - pass - os._exit(result) - else: - # in the parent - wall_time_limit_s = (estimated_time_of_tests + cfg.timeout_constant) * cfg.timeout_multiplier - register_timeout(pid=pid, timeout_s=wall_time_limit_s) - source_file_mutation_data_by_pid[pid] = mutation_data - mutation_data.register_pid(pid=pid, key=mutant_name) - running_children += 1 - - if running_children >= max_children: - read_one_child_exit_status() - count_tried += 1 - running_children -= 1 + # signal SIGXCPU after this many CPU seconds; the runner adds one more before SIGKILL. + cpu_time_limit_s = ceil((estimated_time_of_tests + cfg.timeout_constant) * cfg.timeout_multiplier * 2) + + # Block for a free worker slot before submitting more work. + while not runner.has_capacity(): + drain_one_result() + + mutation_data_by_mutant_name[mutant_name] = mutation_data + runner.submit(mutant_name, list(tests), cpu_time_limit_s, estimated_time_of_tests) + + runner.signal_work_complete() try: - while running_children: - read_one_child_exit_status() - count_tried += 1 - running_children -= 1 + while runner.pending_count() > 0: + drain_one_result() except ChildProcessError: pass except KeyboardInterrupt: print("Stopping...") - stop_all_children(mutants) + runner.stop_all_workers() finally: - gc.unfreeze() + runner.shutdown() elapsed_time = datetime.now() - start diff --git a/src/mutmut/configuration.py b/src/mutmut/configuration.py index e5710dd8..ef5a3cb2 100644 --- a/src/mutmut/configuration.py +++ b/src/mutmut/configuration.py @@ -11,12 +11,24 @@ from configparser import NoOptionError from configparser import NoSectionError from dataclasses import dataclass +from enum import Enum from os.path import isdir from os.path import isfile from pathlib import Path from typing import Any +class ProcessIsolation(str, Enum): + """Valid values for the ``process_isolation`` config. + + Subclassing ``str`` allows direct string comparison while still giving us + validation and IDE support. + """ + + FORK = "fork" # Default: fork the (test-polluted) parent per mutant. + HOT_FORK = "hot-fork" # Fork-safe orchestrator for gevent/grpc/torch. + + def _config_reader() -> Callable[[str, Any], Any]: path = Path("pyproject.toml") if path.exists(): @@ -122,6 +134,13 @@ def _load_config() -> Config: f'The configs only_mutate and do_not_mutate expect glob patterns like "src/api/*" or "src/main.py". Following patterns are likely invalid: {invalid_patterns}' ) + isolation_str = s("process_isolation", "fork") + try: + process_isolation = ProcessIsolation(isolation_str) + except ValueError: + valid = [e.value for e in ProcessIsolation] + raise ValueError(f"Invalid process_isolation value: {isolation_str!r}. Expected one of: {valid}") from None + return Config( only_mutate=only_mutate, do_not_mutate=do_not_mutate, @@ -157,6 +176,7 @@ def _load_config() -> Config: cache_invalidation_exclude=s("cache_invalidation_exclude", []), on_dependency_change=s("on_dependency_change", "warn"), use_git_change_detection=s("use_git_change_detection", True), + process_isolation=process_isolation, ) @@ -186,6 +206,7 @@ class Config: cache_invalidation_exclude: list[str] on_dependency_change: str use_git_change_detection: bool + process_isolation: ProcessIsolation def config_fingerprint(self) -> dict[str, str]: """Hash the config fields that can change cached mutant *results*, grouped so the diff --git a/src/mutmut/core.py b/src/mutmut/core.py index 4e42db7a..53dac93c 100644 --- a/src/mutmut/core.py +++ b/src/mutmut/core.py @@ -19,3 +19,7 @@ def set(cls, value: tuple[str, int]) -> Token[tuple[str | None, int]]: @classmethod def reset(cls, token: Token[tuple[str | None, int]]) -> None: cls._ctx.reset(token) + + +class MutmutProgrammaticFailException(Exception): + """Raised by the trampoline when the ``fail`` mutant is active.""" diff --git a/src/mutmut/mutation/data.py b/src/mutmut/mutation/data.py index 316d9c07..9c806b6f 100644 --- a/src/mutmut/mutation/data.py +++ b/src/mutmut/mutation/data.py @@ -1,10 +1,7 @@ import json -import os -import signal from collections.abc import Mapping from collections.abc import Sequence from dataclasses import dataclass -from datetime import datetime from itertools import islice from pathlib import Path @@ -102,11 +99,9 @@ def __init__(self, *, path: Path | str) -> None: self.estimated_time_of_tests_by_mutant: dict[str, float] = {} self.path = path self.meta_path = Path("mutants") / (str(path) + ".meta") - self.key_by_pid: dict[int, str] = {} self.exit_code_by_key: dict[str, int | None] = {} self.hash_by_function_name: dict[str, str] = {} self.durations_by_key: dict[str, float] = {} - self.start_time_by_pid: dict[int, datetime] = {} self.type_check_error_by_key: dict[str, str | None] = {} def load(self) -> None: @@ -123,24 +118,6 @@ def load(self) -> None: self.estimated_time_of_tests_by_mutant = meta.pop("estimated_durations_by_key") assert not meta, f"Meta file {self.meta_path} constains unexpected keys: {set(meta.keys())}" - def register_pid(self, *, pid: int, key: str) -> None: - self.key_by_pid[pid] = key - self.start_time_by_pid[pid] = datetime.now() - - def register_result(self, *, pid: int, exit_code: int) -> None: - assert self.key_by_pid[pid] in self.exit_code_by_key - key = self.key_by_pid[pid] - self.exit_code_by_key[key] = exit_code - self.durations_by_key[key] = (datetime.now() - self.start_time_by_pid[pid]).total_seconds() - # TODO: maybe rate limit this? Saving on each result can slow down mutation testing a lot if the test run is fast. - del self.key_by_pid[pid] - del self.start_time_by_pid[pid] - self.save() - - def stop_children(self) -> None: - for pid in self.key_by_pid.keys(): - os.kill(pid, signal.SIGTERM) - def save(self) -> None: with open(self.meta_path, "w") as f: json.dump( diff --git a/src/mutmut/mutation/trampoline.py b/src/mutmut/mutation/trampoline.py index 7bfc1718..c8f7e945 100644 --- a/src/mutmut/mutation/trampoline.py +++ b/src/mutmut/mutation/trampoline.py @@ -7,9 +7,9 @@ from typing import ParamSpec from typing import TypeVar -from mutmut.__main__ import MutmutProgrammaticFailException -from mutmut.__main__ import record_trampoline_hit from mutmut.core import MutmutCallStack +from mutmut.core import MutmutProgrammaticFailException +from mutmut.stats import record_trampoline_hit from mutmut.utils.format_utils import mangled_name_from_mutant_name TReturn = TypeVar("TReturn") diff --git a/src/mutmut/stats.py b/src/mutmut/stats.py index bd6e0b28..6277c667 100644 --- a/src/mutmut/stats.py +++ b/src/mutmut/stats.py @@ -1,9 +1,11 @@ from __future__ import annotations +import inspect import json from collections import defaultdict from dataclasses import dataclass from json import JSONDecodeError +from pathlib import Path from mutmut.configuration import config from mutmut.mutation.data import SourceFileMutationData @@ -143,3 +145,29 @@ def save_stats() -> None: f, indent=4, ) + + +def record_trampoline_hit(name: str, caller: str | None = None) -> None: + assert not name.startswith("src."), "Failed trampoline hit. Module name starts with `src.`, which is invalid" + + mutated_source_paths = config().resolved_mutated_source_paths + + if config().max_stack_depth != -1: + f = inspect.currentframe() + c = config().max_stack_depth + while c and f: + filename = f.f_code.co_filename + f = f.f_back + if "pytest" in filename or "hammett" in filename or "unittest" in filename: + break + file_path = Path(filename).resolve(strict=True) + if any(path in file_path.parents for path in mutated_source_paths): + # only include stack frames of user-code; exclude mutmut and 3rd library stack frames + c -= 1 + + if not c: + return + + state()._stats.add(name) + if caller is not None and config().track_dependencies: + state().function_dependencies[name].add(caller) diff --git a/src/mutmut/workers/isolation.py b/src/mutmut/workers/isolation.py index 561ea476..a451eea7 100644 --- a/src/mutmut/workers/isolation.py +++ b/src/mutmut/workers/isolation.py @@ -1,20 +1,44 @@ """ -Fork isolation utilities for keeping the parent process clean. +Fork isolation utilities and mutation-test runners. The main process must not import pytest/test code directly, because test conftest.py files may call gevent.monkey.patch_all() or import grpc, which makes the process fork-unsafe. -These utilities run operations in forked children so the parent stays clean. -Uses pipe-based IPC for reduced overhead (no temp files, no cleanup needed). +The low-level ``run_in_fork*`` helpers run operations in forked children so the +parent stays clean. On top of them, ``MutantRunner`` abstracts the process +isolation strategy used to test each mutant. ``ForkRunner`` is the traditional +os.fork()-per-mutant approach; ``HotForkRunner`` (single orchestrator) will be +layered on later for fork-unsafe libraries. """ from __future__ import annotations +import gc import os import pickle +import resource +import signal +import sys +from abc import ABC +from abc import abstractmethod from collections.abc import Callable +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import datetime +from time import process_time from typing import Any +from typing import NamedTuple + +from mutmut.configuration import ProcessIsolation +from mutmut.configuration import config +from mutmut.mutation.trampoline import set_mutant_under_test +from mutmut.runners.harness import ListAllTestsResult +from mutmut.runners.harness import PytestRunner +from mutmut.runners.harness import TestRunner +from mutmut.state import state +from mutmut.utils.safe_setproctitle import safe_setproctitle as setproctitle +from mutmut.workers.timeout import register_timeout def run_in_fork_with_result(fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: @@ -53,18 +77,21 @@ def run_in_fork_with_result(fn: Callable[..., Any], *args: Any, **kwargs: Any) - pass os._exit(1) - # Parent: close write end, wait for child, read result + # Parent: close write end, read result BEFORE waiting for the child. + # IMPORTANT: read before waitpid to avoid deadlock. If the pickled data + # exceeds the pipe buffer (~64KB) the child blocks on write until the parent + # reads; if the parent waits on the child first that is a deadlock. os.close(write_fd) - _, status = os.waitpid(pid, 0) - exit_code = os.waitstatus_to_exitcode(status) - # Read result from pipe with os.fdopen(read_fd, "rb") as f: try: data = pickle.load(f) except Exception: data = None + _, status = os.waitpid(pid, 0) + exit_code = os.waitstatus_to_exitcode(status) + if exit_code != 0 or data is None: error_msg = f"Child exited with code {exit_code}" if data and not data.get("ok") and "error" in data: @@ -136,3 +163,227 @@ def __init__(self, exit_code: int, lost_mutants: list[str], crash_log: str | Non details.append("(Completed mutants are saved; lost ones will be re-run)") super().__init__("\n".join(details)) + + +@dataclass +class MutantResult: + """Result of testing a single mutant.""" + + mutant_name: str + exit_code: int + duration: float + + +class RunningWorker(NamedTuple): + """Tracks an in-flight mutation-test worker for ForkRunner.""" + + mutant_name: str + start_time: datetime + estimated_time: float + + +class MutantRunner(ABC): + """Abstract base class for mutation-test runners. + + A runner owns the process-isolation strategy for testing mutants and also + exposes the surrounding test operations (stats collection, clean tests, + forced-fail, test listing) so the caller never has to touch a raw test + runner directly. + + Usage:: + + runner = get_mutant_runner(max_children) + runner.startup() + for mutant in mutants: + while not runner.has_capacity(): + register_result(runner.wait_for_result()) + runner.submit(mutant_name, tests, cpu_time_limit, estimated_time) + runner.signal_work_complete() + while runner.pending_count() > 0: + register_result(runner.wait_for_result()) + runner.shutdown() + """ + + @abstractmethod + def startup(self) -> None: + """Called once before mutation testing begins.""" + + @abstractmethod + def submit(self, mutant_name: str, tests: list[str], cpu_time_limit: int, estimated_time: float) -> None: + """Submit a mutant for testing. + + Does not wait for the result, but may block while handing the work off + if the runner's intake is full. + + Args: + mutant_name: The mutant identifier (e.g. 'module.func__mutmut_1'). + tests: Test node ids to run. + cpu_time_limit: CPU-time limit in seconds for the test run. + estimated_time: Estimated test duration, used for timeout tracking. + """ + + @abstractmethod + def has_capacity(self) -> bool: + """True if we can submit more work without exceeding max workers.""" + + @abstractmethod + def wait_for_result(self, timeout: float | None = None) -> MutantResult: + """Block until one result is available and return it.""" + + @abstractmethod + def pending_count(self) -> int: + """Number of in-flight mutants awaiting results.""" + + @abstractmethod + def signal_work_complete(self) -> None: + """Signal that no more work will be submitted. + + Called after all mutants have been submitted but before waiting for the + final results. Runners with a coordinator process use this to close the + work pipe; others treat it as a no-op. + """ + + def stop_all_workers(self) -> None: + """Terminate all in-flight workers (best effort). Default is a no-op.""" + + @abstractmethod + def shutdown(self) -> None: + """Called after all mutants are tested. Clean up resources.""" + + @abstractmethod + def collect_stats(self, tests: Iterable[str] | None) -> int: + """Run stats collection. Returns an exit code.""" + + @abstractmethod + def run_clean_tests(self, tests: Iterable[str]) -> int: + """Run the clean (unmutated) tests. Returns an exit code.""" + + @abstractmethod + def run_forced_fail(self) -> int: + """Run the forced-fail test. Returns an exit code.""" + + @abstractmethod + def list_all_tests(self) -> ListAllTestsResult: + """List all tests in the test suite.""" + + +class ForkRunner(MutantRunner): + """Runner that uses os.fork() for process isolation. + + This is the traditional mutmut approach - fast, but it can misbehave with + libraries like gevent, grpc, and torch when forking from a parent process + that has already imported test code. For those, use HotForkRunner instead. + """ + + def __init__(self, max_workers: int, test_runner: TestRunner, debug: bool = False) -> None: + self.max_workers = max_workers + self.test_runner = test_runner + self.debug = debug + self._running: dict[int, RunningWorker] = {} # pid -> RunningWorker + self._no_tests_results: list[MutantResult] = [] + + def startup(self) -> None: + # Freeze the GC so the forked children inherit a stable heap and do not + # thrash collecting objects the parent already owns. + gc.freeze() + + def submit(self, mutant_name: str, tests: list[str], cpu_time_limit: int, estimated_time: float) -> None: + if not tests: + self._no_tests_results.append(MutantResult(mutant_name=mutant_name, exit_code=33, duration=0.0)) + return + + pid = os.fork() + if pid == 0: + # In the child. + set_mutant_under_test(mutant_name) + setproctitle(f"mutmut: {mutant_name}") + + # Run fast tests first. + tests_sorted = sorted(tests, key=lambda test_name: state().duration_by_test[test_name]) + + # Signal SIGXCPU after the CPU limit, and SIGKILL one second later if + # it is still running. + limit = cpu_time_limit + int(process_time()) + resource.setrlimit(resource.RLIMIT_CPU, (limit, limit + 1)) + + if not self.debug: + sys.stdout = sys.stderr = open(os.devnull, "w") + + result = self.test_runner.run_tests(mutant_name=mutant_name, tests=tests_sorted) + os._exit(result) + else: + # In the parent. + cfg = config() + wall_time_limit_s = (estimated_time + cfg.timeout_constant) * cfg.timeout_multiplier + register_timeout(pid=pid, timeout_s=wall_time_limit_s) + self._running[pid] = RunningWorker(mutant_name, datetime.now(), estimated_time) + + def has_capacity(self) -> bool: + return len(self._running) < self.max_workers + + def wait_for_result(self, timeout: float | None = None) -> MutantResult: + if self._no_tests_results: + return self._no_tests_results.pop(0) + + pid, wait_status = os.wait() + exit_code = os.waitstatus_to_exitcode(wait_status) + + worker = self._running.pop(pid) + duration = (datetime.now() - worker.start_time).total_seconds() + return MutantResult(mutant_name=worker.mutant_name, exit_code=exit_code, duration=duration) + + def pending_count(self) -> int: + # Queued no-test results count as pending: they are submitted work that + # nobody has collected yet, even though they never occupied a worker slot. + return len(self._running) + len(self._no_tests_results) + + def signal_work_complete(self) -> None: + """No-op for ForkRunner (there is no orchestrator to signal).""" + + def stop_all_workers(self) -> None: + for pid in list(self._running): + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + + def shutdown(self) -> None: + while self._running: + try: + self.wait_for_result() + except ChildProcessError: + break + gc.unfreeze() + + def collect_stats(self, tests: Iterable[str] | None) -> int: + # Already in a clean process, so run stats directly without forking. + return self.test_runner.run_stats(tests=tests or []) + + def run_clean_tests(self, tests: Iterable[str]) -> int: + return self.test_runner.run_tests(mutant_name=None, tests=tests) + + def run_forced_fail(self) -> int: + return self.test_runner.run_forced_fail() + + def list_all_tests(self) -> ListAllTestsResult: + return self.test_runner.list_all_tests() + + +def get_mutant_runner(max_workers: int = 1) -> MutantRunner: + """Create a MutantRunner based on the configured ``process_isolation``. + + Args: + max_workers: Maximum number of concurrent workers. + + Returns: + A MutantRunner instance. + """ + if max_workers < 1: + raise ValueError("max_workers must be at least 1") + + if config().process_isolation == ProcessIsolation.HOT_FORK: + raise NotImplementedError("process_isolation = 'hot-fork' is not available yet in this build; use 'fork'.") + + pytest_runner = PytestRunner() + pytest_runner.prepare_main_test_run() + return ForkRunner(max_workers=max_workers, test_runner=pytest_runner, debug=config().debug) diff --git a/tests/mutation/test_mutation.py b/tests/mutation/test_mutation.py index 5db569d4..8a6986e6 100644 --- a/tests/mutation/test_mutation.py +++ b/tests/mutation/test_mutation.py @@ -14,21 +14,22 @@ import mutmut from mutmut.__main__ import CatchOutput -from mutmut.__main__ import MutmutProgrammaticFailException from mutmut.__main__ import _apply_config_change_invalidation from mutmut.__main__ import _changed_dependency_files from mutmut.__main__ import _cleanup_stale_stats from mutmut.__main__ import _invalidate_stale_dependency_edges from mutmut.__main__ import _refresh_change_detection_baseline +from mutmut.__main__ import _register_mutant_result from mutmut.__main__ import _report_watched_file_changes from mutmut.__main__ import _reset_mutant_results from mutmut.__main__ import compute_watched_file_hashes from mutmut.__main__ import git_changed_non_py_files from mutmut.__main__ import git_head from mutmut.__main__ import git_tracked_non_py_files -from mutmut.__main__ import record_trampoline_hit from mutmut.__main__ import run_forced_fail_test from mutmut.configuration import Config +from mutmut.configuration import ProcessIsolation +from mutmut.core import MutmutProgrammaticFailException from mutmut.mutation.data import MutantLineSpans from mutmut.mutation.data import SourceFileMutationData from mutmut.mutation.diff_apply import apply_mutant @@ -40,9 +41,11 @@ from mutmut.mutation.trampoline_templates import mangle_function_name from mutmut.state import reset_state from mutmut.state import state +from mutmut.stats import record_trampoline_hit from mutmut.utils.format_utils import get_mutant_name from mutmut.utils.format_utils import mangled_name_from_mutant_name from mutmut.utils.format_utils import orig_function_and_class_names_from_key +from mutmut.workers.isolation import MutantResult def mutants_for_source( @@ -1516,7 +1519,7 @@ def test_record_trampoline_hit_records_caller(monkeypatch): cfg.source_paths = [] cfg.resolved_mutated_source_paths = [] cfg.track_dependencies = True - monkeypatch.setattr(mutmut.__main__, "config", lambda: cfg) + monkeypatch.setattr(mutmut.stats, "config", lambda: cfg) record_trampoline_hit("my_module.x_foo", caller="my_module.x_bar") @@ -1535,7 +1538,7 @@ def test_record_trampoline_hit_skips_caller_when_disabled(monkeypatch): cfg.source_paths = [] cfg.resolved_mutated_source_paths = [] cfg.track_dependencies = False - monkeypatch.setattr(mutmut.__main__, "config", lambda: cfg) + monkeypatch.setattr(mutmut.stats, "config", lambda: cfg) record_trampoline_hit("my_module.x_foo", caller="my_module.x_bar") @@ -1622,6 +1625,7 @@ def _config_for_invalidation(**overrides): cache_invalidation_exclude=[], on_dependency_change="warn", use_git_change_detection=True, + process_isolation=ProcessIsolation.FORK, ) base.update(overrides) return Config(**base) @@ -1995,3 +1999,53 @@ def test_baseline_records_git_files_for_gitless_fallback(tmp_path, monkeypatch): assert "config.yaml" in _changed_dependency_files() reset_state() + + +class TestRegisterMutantResult: + """_register_mutant_result writes a finished worker's verdict onto its meta file.""" + + @staticmethod + def _mutation_data(tmp_path, mutant_name): + data = SourceFileMutationData(path="mymod.py") + data.meta_path = tmp_path / "mymod.py.meta" + data.exit_code_by_key = {mutant_name: None} + return data + + def test_records_exit_code_and_duration(self, tmp_path): + mutant_name = "mymod.x_foo__mutmut_1" + data = self._mutation_data(tmp_path, mutant_name) + + _register_mutant_result( + MutantResult(mutant_name=mutant_name, exit_code=1, duration=1.5), + {mutant_name: data}, + ) + + assert data.exit_code_by_key[mutant_name] == 1 + assert data.durations_by_key[mutant_name] == 1.5 + + def test_persists_the_result(self, tmp_path): + mutant_name = "mymod.x_foo__mutmut_1" + data = self._mutation_data(tmp_path, mutant_name) + + _register_mutant_result( + MutantResult(mutant_name=mutant_name, exit_code=33, duration=0.0), + {mutant_name: data}, + ) + + reloaded = SourceFileMutationData(path="mymod.py") + reloaded.meta_path = data.meta_path + reloaded.load() + assert reloaded.exit_code_by_key[mutant_name] == 33 + + def test_rejects_a_mutant_the_meta_file_does_not_know(self, tmp_path): + """A name mismatch must fail loudly instead of inventing a phantom mutant.""" + data = self._mutation_data(tmp_path, "mymod.x_foo__mutmut_1") + unknown = "mymod.x_foo__mutmut_99" + + with pytest.raises(AssertionError, match=unknown): + _register_mutant_result( + MutantResult(mutant_name=unknown, exit_code=1, duration=0.1), + {unknown: data}, + ) + + assert unknown not in data.exit_code_by_key diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 21a3f74b..5a4794d2 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -3,6 +3,7 @@ import pytest from mutmut.configuration import Config +from mutmut.configuration import ProcessIsolation from mutmut.configuration import _config_reader from mutmut.configuration import _guess_source_paths from mutmut.configuration import _load_config @@ -72,6 +73,7 @@ def _get_config(only_mutate: list[str], do_not_mutate: list[str]) -> Config: cache_invalidation_exclude=[], on_dependency_change="warn", use_git_change_detection=True, + process_isolation=ProcessIsolation.FORK, ) def test_ignores_non_python_files(self): diff --git a/tests/workers/test_isolation.py b/tests/workers/test_isolation.py index 36e20510..71dcbe10 100644 --- a/tests/workers/test_isolation.py +++ b/tests/workers/test_isolation.py @@ -4,6 +4,7 @@ import pytest +from mutmut.workers.isolation import ForkRunner from mutmut.workers.isolation import OrchestratorCrashError from mutmut.workers.isolation import run_in_fork from mutmut.workers.isolation import run_in_fork_with_result @@ -177,3 +178,62 @@ def test_can_be_raised_and_caught(self): assert exc_info.value.exit_code == 255 assert exc_info.value.lost_mutants == ["test_mutant"] assert exc_info.value.crash_log == "/tmp/crash.log" + + +class TestForkRunnerNoTestMutants: + """A mutant with no tests is resolved without forking, but it is still a result + the caller has to be able to collect. See the drain loop in MutantRunner's docstring.""" + + @staticmethod + def _runner(max_workers=4): + # test_runner is never touched on the no-tests path. + return ForkRunner(max_workers=max_workers, test_runner=object(), debug=False) + + @staticmethod + def _submit_without_tests(runner, mutant_name="mod.x_foo__mutmut_1"): + runner.submit(mutant_name, [], cpu_time_limit=1, estimated_time=0.0) + + def test_submitting_without_tests_does_not_fork(self, monkeypatch): + """No tests means no work, so no child process should be spawned.""" + + def explode(): + raise AssertionError("os.fork() must not be called for a mutant with no tests") + + monkeypatch.setattr(os, "fork", explode) + self._submit_without_tests(self._runner()) + + def test_queued_result_counts_as_pending(self): + """pending_count() drives the caller's drain loop, so it has to see queued results.""" + runner = self._runner() + self._submit_without_tests(runner) + assert runner.pending_count() == 1 + + def test_queued_result_does_not_consume_a_worker_slot(self): + """Nothing is running, so capacity is unaffected however many are queued.""" + runner = self._runner(max_workers=1) + self._submit_without_tests(runner, "mod.x_foo__mutmut_1") + self._submit_without_tests(runner, "mod.x_foo__mutmut_2") + assert runner.has_capacity() + + def test_documented_drain_loop_collects_the_result(self): + """The submit/pending_count/wait_for_result loop must not drop the mutant.""" + runner = self._runner() + self._submit_without_tests(runner) + + runner.signal_work_complete() + drained = [] + while runner.pending_count() > 0: + drained.append(runner.wait_for_result()) + + assert [(r.mutant_name, r.exit_code) for r in drained] == [("mod.x_foo__mutmut_1", 33)] + + def test_queued_results_are_drained_in_submission_order(self): + """Results are reported against mutant names, so order must stay stable.""" + runner = self._runner() + for i in (1, 2, 3): + self._submit_without_tests(runner, f"mod.x_foo__mutmut_{i}") + + drained = [runner.wait_for_result().mutant_name for _ in range(runner.pending_count())] + + assert drained == ["mod.x_foo__mutmut_1", "mod.x_foo__mutmut_2", "mod.x_foo__mutmut_3"] + assert runner.pending_count() == 0 From 76cc1e353591679a0c16449f2f3dda9f53c3199e Mon Sep 17 00:00:00 2001 From: nicklafleur <55208706+nicklafleur@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:14:11 -0400 Subject: [PATCH 3/5] feat(runners): add HotForkRunner for fork-safe mutation testing Add a single-orchestrator process-isolation strategy for projects whose test setup makes the parent fork-unsafe (gevent monkey-patching, grpc, torch): parent (clean) -> orchestrator (imports pytest once) -> N grandchildren The parent never imports pytest/conftest. The orchestrator imports pytest a single time, warms up (configurable via hot_fork_warmup: collect/import/none), then forks one grandchild per mutant, streaming results back over a pipe and reaping via a SIGCHLD self-pipe. If the orchestrator crashes, in-flight mutants are re-submitted to a fresh orchestrator up to max_orchestrator_restarts times before raising OrchestratorCrashError. Stats/clean-test/forced-fail/test-listing all run in short-lived forks (StatsResult carries the collected mapping back to the parent), so the parent stays clean throughout. Selected via process_isolation = "hot-fork"; get_mutant_runner() now builds it. Supporting pieces: HotForkWarmup config + validation, TestRunner.warm_up(), models/results.StatsResult, and utils/logging_utils for the orchestrator's file-only logging and crash logs. Validated end-to-end: on the my_lib project, hot-fork produces byte-identical verdicts to fork (112 mutants, same 37/64/10/1 distribution). Adds a subprocess e2e smoke test (hot_fork_basic) plus factory/config-validation unit tests. --- e2e_projects/hot_fork_basic/pyproject.toml | 14 + .../hot_fork_basic/src/hf_calc/__init__.py | 14 + .../hot_fork_basic/tests/test_hf_calc.py | 15 + src/mutmut/configuration.py | 35 ++ src/mutmut/models/__init__.py | 1 + src/mutmut/models/results.py | 44 ++ src/mutmut/runners/harness.py | 39 ++ src/mutmut/utils/logging_utils.py | 74 +++ src/mutmut/workers/isolation.py | 552 +++++++++++++++++- tests/e2e/test_e2e_hot_fork.py | 54 ++ tests/mutation/test_mutation.py | 6 + tests/test_configuration.py | 34 ++ tests/workers/test_isolation.py | 116 ++++ 13 files changed, 997 insertions(+), 1 deletion(-) create mode 100644 e2e_projects/hot_fork_basic/pyproject.toml create mode 100644 e2e_projects/hot_fork_basic/src/hf_calc/__init__.py create mode 100644 e2e_projects/hot_fork_basic/tests/test_hf_calc.py create mode 100644 src/mutmut/models/__init__.py create mode 100644 src/mutmut/models/results.py create mode 100644 src/mutmut/utils/logging_utils.py create mode 100644 tests/e2e/test_e2e_hot_fork.py diff --git a/e2e_projects/hot_fork_basic/pyproject.toml b/e2e_projects/hot_fork_basic/pyproject.toml new file mode 100644 index 00000000..88b8f9cd --- /dev/null +++ b/e2e_projects/hot_fork_basic/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "hf-calc" +version = "0.1.0" +requires-python = ">=3.10" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.mutmut] +process_isolation = "hot-fork" + +[tool.pytest.ini_options] +asyncio_default_fixture_loop_scope = "function" diff --git a/e2e_projects/hot_fork_basic/src/hf_calc/__init__.py b/e2e_projects/hot_fork_basic/src/hf_calc/__init__.py new file mode 100644 index 00000000..e302c89b --- /dev/null +++ b/e2e_projects/hot_fork_basic/src/hf_calc/__init__.py @@ -0,0 +1,14 @@ +def add(a, b): + return a + b + + +def sub(a, b): + return a - b + + +def mul(a, b): + return a * b + + +def untested(x): + return x + 1 diff --git a/e2e_projects/hot_fork_basic/tests/test_hf_calc.py b/e2e_projects/hot_fork_basic/tests/test_hf_calc.py new file mode 100644 index 00000000..ad535d84 --- /dev/null +++ b/e2e_projects/hot_fork_basic/tests/test_hf_calc.py @@ -0,0 +1,15 @@ +from hf_calc import add +from hf_calc import mul +from hf_calc import sub + + +def test_add(): + assert add(1, 2) == 3 + + +def test_sub(): + assert sub(5, 2) == 3 + + +def test_mul(): + assert mul(3, 4) == 12 diff --git a/src/mutmut/configuration.py b/src/mutmut/configuration.py index ef5a3cb2..0d612840 100644 --- a/src/mutmut/configuration.py +++ b/src/mutmut/configuration.py @@ -29,6 +29,24 @@ class ProcessIsolation(str, Enum): HOT_FORK = "hot-fork" # Fork-safe orchestrator for gevent/grpc/torch. +class HotForkWarmup(str, Enum): + """Warmup strategy for the hot-fork orchestrator. + + Controls what the orchestrator does after importing the test runner but + before forking any grandchildren: + + - COLLECT: run ``pytest --collect-only`` to pre-load conftest, plugins, and + test modules (default; biggest speedup for most projects). + - IMPORT: import the modules listed in ``preload_modules_file`` (useful when + test collection has side effects you do not want shared). + - NONE: import nothing extra beyond what running a test needs. + """ + + COLLECT = "collect" + IMPORT = "import" + NONE = "none" + + def _config_reader() -> Callable[[str, Any], Any]: path = Path("pyproject.toml") if path.exists(): @@ -141,6 +159,13 @@ def _load_config() -> Config: valid = [e.value for e in ProcessIsolation] raise ValueError(f"Invalid process_isolation value: {isolation_str!r}. Expected one of: {valid}") from None + warmup_str = s("hot_fork_warmup", "collect") + try: + hot_fork_warmup = HotForkWarmup(warmup_str) + except ValueError: + valid = [e.value for e in HotForkWarmup] + raise ValueError(f"Invalid hot_fork_warmup value: {warmup_str!r}. Expected one of: {valid}") from None + return Config( only_mutate=only_mutate, do_not_mutate=do_not_mutate, @@ -177,6 +202,11 @@ def _load_config() -> Config: on_dependency_change=s("on_dependency_change", "warn"), use_git_change_detection=s("use_git_change_detection", True), process_isolation=process_isolation, + hot_fork_warmup=hot_fork_warmup, + max_orchestrator_restarts=s("max_orchestrator_restarts", 3), + preload_modules_file=s("preload_modules_file", None), + log_to_file=s("log_to_file", False), + log_file_path=s("log_file_path", "mutants/mutmut-debug.log"), ) @@ -207,6 +237,11 @@ class Config: on_dependency_change: str use_git_change_detection: bool process_isolation: ProcessIsolation + hot_fork_warmup: HotForkWarmup + max_orchestrator_restarts: int + preload_modules_file: str | None + log_to_file: bool + log_file_path: str def config_fingerprint(self) -> dict[str, str]: """Hash the config fields that can change cached mutant *results*, grouped so the diff --git a/src/mutmut/models/__init__.py b/src/mutmut/models/__init__.py new file mode 100644 index 00000000..e6fabf88 --- /dev/null +++ b/src/mutmut/models/__init__.py @@ -0,0 +1 @@ +"""Data models for mutmut.""" diff --git a/src/mutmut/models/results.py b/src/mutmut/models/results.py new file mode 100644 index 00000000..44ca6640 --- /dev/null +++ b/src/mutmut/models/results.py @@ -0,0 +1,44 @@ +"""Result data models for worker/orchestrator communication.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass +class StatsResult: + """Stats collected by a forked child, serialized back to the parent. + + The hot-fork runner collects stats inside a forked child (so the parent + never imports pytest). The child cannot mutate the parent's ``state()`` + directly, so it packs the collected data into this picklable structure and + the parent merges it back in. + """ + + exit_code: int + tests_by_mangled_function_name: dict[str, set[str]] + duration_by_test: dict[str, float] + stats_time: float + function_dependencies: dict[str, set[str]] + + def to_dict(self) -> dict[str, Any]: + """Serialize to a picklable dict (sets become lists).""" + return { + "exit_code": self.exit_code, + "tests_by_mangled_function_name": {k: list(v) for k, v in self.tests_by_mangled_function_name.items()}, + "duration_by_test": self.duration_by_test, + "stats_time": self.stats_time, + "function_dependencies": {k: list(v) for k, v in self.function_dependencies.items()}, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> StatsResult: + """Deserialize from the dict produced by :meth:`to_dict`.""" + return cls( + exit_code=data["exit_code"], + tests_by_mangled_function_name={k: set(v) for k, v in data["tests_by_mangled_function_name"].items()}, + duration_by_test=data["duration_by_test"], + stats_time=data["stats_time"], + function_dependencies={k: set(v) for k, v in data["function_dependencies"].items()}, + ) diff --git a/src/mutmut/runners/harness.py b/src/mutmut/runners/harness.py index 6e550afa..c2f929f0 100644 --- a/src/mutmut/runners/harness.py +++ b/src/mutmut/runners/harness.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING from typing import Any +from mutmut.configuration import HotForkWarmup from mutmut.configuration import config from mutmut.state import state from mutmut.stats import save_stats @@ -49,6 +50,16 @@ def collect_main_test_coverage(self, cov: Coverage) -> int: def list_all_tests(self) -> ListAllTestsResult: raise NotImplementedError() + def warm_up(self) -> None: + """Pre-import expensive modules so forked children inherit them. + + Called by HotForkRunner inside the orchestrator after the test runner is + created. Importing pytest (and optionally running collection) here means + the grandchildren fork with everything already in memory. The default is + a no-op for runners that do not benefit from it. + """ + return + def collected_test_names() -> set[str]: return set(state().duration_by_test.keys()) @@ -82,6 +93,34 @@ def __init__(self) -> None: self._pytest_add_cli_args: list[str] = config().pytest_add_cli_args self._pytest_add_cli_args_test_selection: list[str] = config().pytest_add_cli_args_test_selection + def warm_up(self) -> None: + """Pre-load test infrastructure per the ``hot_fork_warmup`` config. + + - COLLECT (default): run ``pytest --collect-only`` to import conftest, + plugins, and test modules (biggest speedup for most projects). + - IMPORT: import the modules listed in ``preload_modules_file``. + - NONE: import nothing beyond what running a test already needs. + """ + warmup = config().hot_fork_warmup + + if warmup == HotForkWarmup.COLLECT: + with change_cwd("mutants"): + self.execute_pytest(["--collect-only", "-qqq"] + self._pytest_add_cli_args_test_selection) + elif warmup == HotForkWarmup.IMPORT: + preload_file = config().preload_modules_file + if preload_file: + import importlib + + with open(preload_file) as f: + for line in f: + module_name = line.strip() + if module_name and not module_name.startswith("#"): + try: + importlib.import_module(module_name) + except ImportError: + pass # Best effort. + # HotForkWarmup.NONE -> no-op. + # noinspection PyMethodMayBeStatic def execute_pytest(self, params: list[str], **kwargs: Any) -> int: import pytest diff --git a/src/mutmut/utils/logging_utils.py b/src/mutmut/utils/logging_utils.py new file mode 100644 index 00000000..05350316 --- /dev/null +++ b/src/mutmut/utils/logging_utils.py @@ -0,0 +1,74 @@ +"""File-based logging helpers, used mainly by the hot-fork orchestrator. + +Child/orchestrator processes cannot easily log to the console without +corrupting the interactive terminal output, so mutmut logs to a rotating file +instead. Logging is opt-in (``log_to_file``/``debug`` config) and off by default. +""" + +from __future__ import annotations + +import logging +from logging.handlers import RotatingFileHandler +from pathlib import Path + +from mutmut.configuration import config + +# Without a handler, logging falls back to lastResort, which writes WARNING+ to +# stderr and corrupts the interactive terminal. Keep mutmut's logs silent unless +# file logging is explicitly enabled. +logging.getLogger("mutmut").addHandler(logging.NullHandler()) + + +def get_logger(name: str) -> logging.Logger: + """Return a logger under the ``mutmut.`` namespace.""" + return logging.getLogger(name if name.startswith("mutmut.") else f"mutmut.{name}") + + +logger = get_logger(__name__) + + +_file_handler: logging.Handler | None = None + + +def setup_file_logging(log_file: str | None = None, level: int = logging.DEBUG) -> None: + """Attach a rotating file handler to the ``mutmut`` logger (idempotent). + + Useful for debugging child processes which cannot log to the console. + + Args: + log_file: Path to the log file (defaults to ``config().log_file_path``). + level: Logging level (default: DEBUG). + """ + global _file_handler + + if _file_handler is not None: + return + + log_path = Path(log_file if log_file is not None else config().log_file_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + + _file_handler = RotatingFileHandler( + log_path, + maxBytes=10 * 1024 * 1024, # 10 MB + backupCount=3, + ) + _file_handler.setLevel(level) + _file_handler.setFormatter( + logging.Formatter( + "%(asctime)s.%(msecs)03d [%(process)d] %(name)s %(levelname)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) + + # File only - do not propagate to the root logger (which would hit stdout). + root_logger = logging.getLogger("mutmut") + root_logger.addHandler(_file_handler) + root_logger.setLevel(level) + root_logger.propagate = False + + logger.debug(f"File logging initialized: {log_path}") + + +def get_log_file_path() -> Path: + """Return the configured debug-log file path.""" + return Path(config().log_file_path) diff --git a/src/mutmut/workers/isolation.py b/src/mutmut/workers/isolation.py index a451eea7..1810c7b1 100644 --- a/src/mutmut/workers/isolation.py +++ b/src/mutmut/workers/isolation.py @@ -15,28 +15,38 @@ from __future__ import annotations import gc +import logging import os import pickle import resource +import select import signal +import struct import sys +import time +import traceback from abc import ABC from abc import abstractmethod from collections.abc import Callable from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime +from queue import Empty from time import process_time from typing import Any from typing import NamedTuple from mutmut.configuration import ProcessIsolation from mutmut.configuration import config +from mutmut.models.results import StatsResult from mutmut.mutation.trampoline import set_mutant_under_test from mutmut.runners.harness import ListAllTestsResult from mutmut.runners.harness import PytestRunner from mutmut.runners.harness import TestRunner from mutmut.state import state +from mutmut.utils.logging_utils import get_log_file_path +from mutmut.utils.logging_utils import get_logger +from mutmut.utils.logging_utils import setup_file_logging from mutmut.utils.safe_setproctitle import safe_setproctitle as setproctitle from mutmut.workers.timeout import register_timeout @@ -131,6 +141,40 @@ def run_in_fork(fn: Callable[..., int], *args: Any, **kwargs: Any) -> int: return os.waitstatus_to_exitcode(status) +# Pipe messaging. Length-prefixed frames read with raw os.read() rather than a +# buffered reader: the receivers select() on the raw fd, and a buffered reader +# can pull several frames into userspace at once, after which select() reports +# "not readable" and the already-received frames are never processed. +def _write_all(fd: int, data: bytes) -> None: + """Write every byte, looping over short writes.""" + view = memoryview(data) + while view: + view = view[os.write(fd, view) :] + + +def _read_exactly(fd: int, size: int) -> bytes: + """Read exactly ``size`` bytes. Raises EOFError if the pipe closes first.""" + chunks = bytearray() + while len(chunks) < size: + chunk = os.read(fd, size - len(chunks)) + if not chunk: + raise EOFError("pipe closed mid-message" if chunks else "pipe closed") + chunks += chunk + return bytes(chunks) + + +def send_message(fd: int, payload: Any) -> None: + """Send one length-prefixed pickled message.""" + body = pickle.dumps(payload) + _write_all(fd, struct.pack("!I", len(body)) + body) + + +def recv_message(fd: int) -> Any: + """Receive one length-prefixed pickled message. Raises EOFError at EOF.""" + (size,) = struct.unpack("!I", _read_exactly(fd, 4)) + return pickle.loads(_read_exactly(fd, size)) + + class OrchestratorCrashError(Exception): """Raised when the hot-fork orchestrator crashes unexpectedly. @@ -267,6 +311,506 @@ def list_all_tests(self) -> ListAllTestsResult: """List all tests in the test suite.""" +class HotForkRunner(MutantRunner): + """Fork-safe mutation runner using a single hot orchestrator. + + Architecture:: + + Parent (clean) -> Orchestrator (imports pytest) -> N concurrent children + + The parent never imports pytest/conftest, so it stays fork-safe. The + orchestrator imports pytest exactly once, then forks a grandchild per + mutant; each grandchild runs one mutant's tests and exits. This is both + faster than forking a fresh pytest per mutant (one import instead of N) and + compatible with fork-unsafe libraries like gevent, grpc, and torch. + + If the orchestrator crashes, in-flight mutants are re-submitted to a fresh + orchestrator up to ``max_restarts`` times before an OrchestratorCrashError + is raised. + """ + + class RunningChild(NamedTuple): + """Info about a grandchild currently running a mutant's tests.""" + + mutant_name: str + start_time: float + wall_timeout: float + + # Default maximum number of orchestrator restarts before giving up. + DEFAULT_MAX_RESTARTS = 3 + + def __init__( + self, + max_workers: int, + test_runner_class: type, + test_runner_args: dict[str, Any], + debug: bool = False, + max_restarts: int | None = None, + ) -> None: + self._logger = get_logger(__name__) + self.max_workers = max_workers + self.test_runner_class = test_runner_class + self.test_runner_args = test_runner_args + self.debug = debug + self.max_restarts = max_restarts if max_restarts is not None else self.DEFAULT_MAX_RESTARTS + + self.work_pipe_read: int | None = None + self.work_pipe_write: int | None = None + self.result_pipe_read: int | None = None + self.result_pipe_write: int | None = None + + self.orchestrator_pid: int | None = None + self._pending: set[str] = set() # mutant_names in flight + # mutant_name -> (tests, cpu_time_limit, estimated_time, start_time) + self._pending_work: dict[str, tuple[list[str], int, float, datetime]] = {} + self._shutting_down = False + self._restart_count = 0 + self._crash_exit_codes: list[int] = [] + self._no_tests_results: list[MutantResult] = [] + + def startup(self) -> None: + gc.freeze() + self._start_orchestrator() + + def _start_orchestrator(self) -> None: + """Fork a fresh orchestrator process with new pipes. + + Can be called more than once for crash recovery; each call creates fresh + pipes and a new orchestrator. + """ + self.work_pipe_read, self.work_pipe_write = os.pipe() + self.result_pipe_read, self.result_pipe_write = os.pipe() + + pid = os.fork() + if pid == 0: + # Child: become the orchestrator. + os.close(self.work_pipe_write) + os.close(self.result_pipe_read) + try: + self._orchestrator_main(self.work_pipe_read, self.result_pipe_write) + except Exception as e: + self._write_crash_log(e) + os._exit(1) + os._exit(0) + + # Parent: close the child's ends. + os.close(self.work_pipe_read) + os.close(self.result_pipe_write) + self.orchestrator_pid = pid + self._logger.info(f"HotForkRunner started orchestrator (pid={pid})") + + def _restart_orchestrator_with_pending_work(self, exit_code: int = -1) -> None: + """Restart the orchestrator and re-submit all pending work. + + Raises OrchestratorCrashError once ``max_restarts`` is exceeded. + """ + self._restart_count += 1 + self._crash_exit_codes.append(exit_code) + + crash_log_path = get_log_file_path().parent / ".orchestrator-crash.log" + pending_mutants = list(self._pending) + self._logger.error( + f"Orchestrator crashed with exit code {exit_code}. " + f"Check {crash_log_path} and {get_log_file_path()} for details." + ) + self._logger.error(f"Pending mutants at time of crash ({len(pending_mutants)}): {pending_mutants}") + + if self._restart_count > self.max_restarts: + raise OrchestratorCrashError( + exit_code=-1, + lost_mutants=list(self._pending), + crash_log=str(crash_log_path) if crash_log_path.exists() else None, + ) + + lost_count = len(self._pending) + self._logger.warning( + f"Orchestrator crashed, restarting (attempt {self._restart_count}/{self.max_restarts}), " + f"re-submitting {lost_count} pending mutant(s)" + ) + + pending_work_copy = dict(self._pending_work) + + self._start_orchestrator() + + if self.work_pipe_write is None: + raise RuntimeError("Failed to restart orchestrator - work pipe not created") + for mutant_name, (tests, cpu_time_limit, estimated_time, _) in pending_work_copy.items(): + send_message(self.work_pipe_write, (mutant_name, list(tests), cpu_time_limit)) + self._pending_work[mutant_name] = (tests, cpu_time_limit, estimated_time, datetime.now()) + self._logger.debug(f"Re-submitted {mutant_name} to new orchestrator") + + self._logger.info(f"Orchestrator restarted, {lost_count} mutant(s) re-submitted") + + def _write_crash_log(self, exception: Exception) -> None: + """Best-effort dump of orchestrator crash info for debugging.""" + crash_file = get_log_file_path().parent / ".orchestrator-crash.log" + try: + crash_file.parent.mkdir(parents=True, exist_ok=True) + with open(crash_file, "w") as f: + f.write(f"Orchestrator crash at {datetime.now()}\n") + f.write(f"Exception: {exception}\n") + f.write(traceback.format_exc()) + except Exception: + pass + + def _setup_sigchld_pipe(self) -> tuple[int, int]: + """Set up a self-pipe so SIGCHLD wakes the orchestrator's select().""" + sigchld_pipe_r, sigchld_pipe_w = os.pipe() + os.set_blocking(sigchld_pipe_r, False) + os.set_blocking(sigchld_pipe_w, False) + + def sigchld_handler(signum: int, frame: Any) -> None: + # Write a byte to wake up select(); ignore a full pipe. + try: + os.write(sigchld_pipe_w, b"c") + except (BlockingIOError, OSError): + pass + + signal.signal(signal.SIGCHLD, sigchld_handler) + return sigchld_pipe_r, sigchld_pipe_w + + def _wait_for_child_event(self, sigchld_pipe_r: int, timeout: float | None) -> bool: + """Block until a child exits (SIGCHLD) or the timeout elapses. + + Returns True if a child may be ready to reap, False on timeout. + """ + try: + readable, _, _ = select.select([sigchld_pipe_r], [], [], timeout) + if readable: + try: + while os.read(sigchld_pipe_r, 1024): + pass + except BlockingIOError: + pass # Expected once the pipe is drained. + return True + return False + except InterruptedError: + return True # Interrupted by a signal; check anyway. + + def _orchestrator_main(self, work_fd: int, result_fd: int) -> None: + """Orchestrator: import pytest once, then fork a grandchild per mutant.""" + # The parent owns shutdown, so ignore SIGINT here. + signal.signal(signal.SIGINT, signal.SIG_IGN) + + if config().log_to_file or config().debug: + setup_file_logging() + orchestrator_logger = get_logger("mutmut.hotfork.orchestrator") + orchestrator_logger.info(f"Hot-fork orchestrator starting (pid={os.getpid()})") + + test_runner: TestRunner = self.test_runner_class(**self.test_runner_args) + + # Warm up with stdout/stderr suppressed so collection output does not + # corrupt the interactive terminal. + old_stdout, old_stderr = sys.stdout, sys.stderr + sys.stdout = sys.stderr = open(os.devnull, "w") + try: + test_runner.warm_up() + finally: + sys.stdout, sys.stderr = old_stdout, old_stderr + orchestrator_logger.info("Test runner initialized, ready for work") + + # Set up the SIGCHLD pipe AFTER warm_up: libraries like gevent may + # monkey-patch signals during import/collection. + sigchld_pipe_r, sigchld_pipe_w = self._setup_sigchld_pipe() + orchestrator_logger.debug("SIGCHLD notification pipe set up") + + running: dict[int, HotForkRunner.RunningChild] = {} + + def terminate(signum: int, frame: Any) -> None: + # The parent SIGTERMs us on Ctrl-C. Take the grandchildren down too, + # otherwise they are reparented to init and keep burning CPU until + # their RLIMIT_CPU fires. + for child_pid in list(running): + try: + os.kill(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + os._exit(0) + + signal.signal(signal.SIGTERM, terminate) + + while True: + self._reap_children(running, result_fd, orchestrator_logger, sigchld_pipe_r, block=False) + + while len(running) >= self.max_workers: + self._reap_children(running, result_fd, orchestrator_logger, sigchld_pipe_r, block=True, timeout=1.0) + + readable, _, _ = select.select([work_fd, sigchld_pipe_r], [], [], 1.0) + + if sigchld_pipe_r in readable: + self._reap_children(running, result_fd, orchestrator_logger, sigchld_pipe_r, block=False) + + if work_fd not in readable: + continue + + try: + msg = recv_message(work_fd) + except EOFError: + break + + if msg is None: + break + + mutant_name, tests, cpu_time_limit = msg + + # Wall-clock timeout is shorter than the CPU limit: multi-threaded + # code can burn N*wall CPU seconds across N cores, so half the CPU + # limit is a reasonable wall bound. + wall_timeout = cpu_time_limit / 2 + orchestrator_logger.debug( + f"Received mutant: {mutant_name} ({len(tests)} tests, " + f"cpu_limit={cpu_time_limit}s, wall_timeout={wall_timeout}s)" + ) + + child_pid = os.fork() + if child_pid == 0: + # Grandchild: run this mutant's tests under a CPU limit. + worker_logger = get_logger(f"mutmut.hotfork.worker.{os.getpid()}") + worker_logger.debug(f"Starting {mutant_name} ({len(tests)} tests)") + + sys.stdout = sys.stderr = open(os.devnull, "w") + + limit = cpu_time_limit + int(process_time()) + resource.setrlimit(resource.RLIMIT_CPU, (limit, limit + 1)) + + set_mutant_under_test(mutant_name) + try: + exit_code = test_runner.run_tests(mutant_name=mutant_name, tests=tests) + except Exception: + exit_code = -1 + + worker_logger.debug(f"Finished {mutant_name}: exit={exit_code}") + os._exit(exit_code) + + running[child_pid] = self.RunningChild(mutant_name, time.time(), wall_timeout) + # A background thread sends SIGXCPU when the wall timeout expires. + register_timeout(child_pid, wall_timeout) + + orchestrator_logger.info("Work queue exhausted, waiting for remaining children") + while running: + self._reap_children(running, result_fd, orchestrator_logger, sigchld_pipe_r, block=True, timeout=1.0) + + try: + os.close(sigchld_pipe_r) + os.close(sigchld_pipe_w) + except OSError: + pass + orchestrator_logger.info("Orchestrator shutting down cleanly") + + def _reap_children( + self, + running: dict[int, RunningChild], + result_fd: int, + orchestrator_logger: logging.Logger, + sigchld_pipe_r: int, + block: bool, + timeout: float | None = None, + ) -> None: + """Reap completed grandchildren and stream their results to the parent.""" + if block: + if not self._wait_for_child_event(sigchld_pipe_r, timeout): + return # Timeout reached, no child ready. + + # Reap ALL ready children in a loop: the kernel coalesces SIGCHLD, so a + # single signal can cover several exits. Reaping only one per signal + # would leave children unreaped and could hang select() forever. + while True: + try: + pid, status = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + return # No children. + if pid == 0: + return # No more children ready. + + # Unknown pids (coverage helpers, pytest plugins, ...) are ignored. + if pid not in running: + continue + + child = running.pop(pid) + exit_code = os.waitstatus_to_exitcode(status) + duration = time.time() - child.start_time + orchestrator_logger.debug(f"Completed {child.mutant_name}: exit={exit_code} ({duration:.3f}s)") + send_message(result_fd, (child.mutant_name, exit_code)) + + def submit(self, mutant_name: str, tests: list[str], cpu_time_limit: int, estimated_time: float) -> None: + if not tests: + # Matches ForkRunner: nothing to run, so resolve it here as "no tests" + # rather than paying for a round trip through the orchestrator. + self._no_tests_results.append(MutantResult(mutant_name=mutant_name, exit_code=33, duration=0.0)) + return + + if self.work_pipe_write is None: + raise RuntimeError("HotForkRunner not started - call startup() first") + send_message(self.work_pipe_write, (mutant_name, list(tests), cpu_time_limit)) + self._pending.add(mutant_name) + self._pending_work[mutant_name] = (list(tests), cpu_time_limit, estimated_time, datetime.now()) + + def has_capacity(self) -> bool: + return len(self._pending) < self.max_workers + + def signal_work_complete(self) -> None: + """Close the work pipe so the orchestrator sees EOF and drains workers.""" + if self.work_pipe_write is None: + return + try: + os.close(self.work_pipe_write) + except OSError: + pass + finally: + self.work_pipe_write = None + self._logger.debug("Work pipe closed, orchestrator will drain remaining workers") + + def _check_orchestrator_alive(self) -> None: + """Detect an orchestrator crash and restart it, re-submitting pending work. + + A clean exit (code 0) is left alone. Raises OrchestratorCrashError once + the restart budget is exhausted. + """ + if self.orchestrator_pid is None: + return + try: + pid, status = os.waitpid(self.orchestrator_pid, os.WNOHANG) + if pid == self.orchestrator_pid: + exit_code = os.waitstatus_to_exitcode(status) + self.orchestrator_pid = None + + if exit_code == 0: + self._logger.debug(f"Orchestrator (pid={pid}) exited cleanly") + return + + self._logger.warning(f"Orchestrator (pid={pid}) crashed with exit code {exit_code}") + self._restart_orchestrator_with_pending_work(exit_code=exit_code) + except ChildProcessError: + self._logger.warning("Orchestrator process not found") + self.orchestrator_pid = None + self._restart_orchestrator_with_pending_work(exit_code=-1) + + def wait_for_result(self, timeout: float | None = None) -> MutantResult: + if self._no_tests_results: + return self._no_tests_results.pop(0) + if self.result_pipe_read is None: + raise RuntimeError("HotForkRunner not started - call startup() first") + while True: + self._check_orchestrator_alive() + + r, _, _ = select.select([self.result_pipe_read], [], [], timeout or 1.0) + if not r: + if timeout is not None: + raise Empty() + continue + + try: + mutant_name, exit_code = recv_message(self.result_pipe_read) + except EOFError as err: + self._check_orchestrator_alive() + raise OrchestratorCrashError(exit_code=-1, lost_mutants=list(self._pending), crash_log=None) from err + + self._pending.discard(mutant_name) + self._pending_work.pop(mutant_name, None) + + return MutantResult(mutant_name=mutant_name, exit_code=exit_code, duration=0.0) + + def pending_count(self) -> int: + # Queued no-test results are submitted work nobody has collected yet. + return len(self._pending) + len(self._no_tests_results) + + def stop_all_workers(self) -> None: + if self.orchestrator_pid: + try: + os.kill(self.orchestrator_pid, signal.SIGTERM) + except ProcessLookupError: + pass + + def shutdown(self) -> None: + """Close the work pipe, drain remaining results, and reap the orchestrator.""" + if self._shutting_down: + return + self._shutting_down = True + + self._logger.info("HotForkRunner shutting down") + + self.signal_work_complete() + + # Drain any results still in flight so completed work is captured. + while self._pending: + try: + result = self.wait_for_result(timeout=1.0) + self._pending.discard(result.mutant_name) + self._pending_work.pop(result.mutant_name, None) + except (Empty, OrchestratorCrashError): + break + + if self.orchestrator_pid: + try: + os.waitpid(self.orchestrator_pid, 0) + except ChildProcessError: + pass + + if self.result_pipe_read is not None: + try: + os.close(self.result_pipe_read) + except OSError: + pass + + gc.unfreeze() + self._logger.info("HotForkRunner shutdown complete") + + def collect_stats(self, tests: Iterable[str] | None) -> int: + """Collect stats in a forked child so the parent never imports pytest. + + The child runs stats and packs the collected mapping into a StatsResult, + which the parent merges back into ``state()``. + """ + tests_list = list(tests) if tests is not None else None + + def _run_stats() -> dict[str, Any]: + child_runner: TestRunner = self.test_runner_class(**self.test_runner_args) + exit_code = child_runner.run_stats(tests=tests_list or []) + return StatsResult( + exit_code=exit_code, + tests_by_mangled_function_name=dict(state().tests_by_mangled_function_name), + duration_by_test=dict(state().duration_by_test), + stats_time=state().stats_time or 0.0, + function_dependencies=dict(state().function_dependencies), + ).to_dict() + + result = StatsResult.from_dict(run_in_fork_with_result(_run_stats)) + + for k, v in result.tests_by_mangled_function_name.items(): + state().tests_by_mangled_function_name[k] |= v + state().duration_by_test.update(result.duration_by_test) + state().stats_time = result.stats_time + for k, v in result.function_dependencies.items(): + state().function_dependencies[k] = v + + return result.exit_code + + def run_clean_tests(self, tests: Iterable[str]) -> int: + tests_list = list(tests) + + def _run_tests() -> int: + child_runner: TestRunner = self.test_runner_class(**self.test_runner_args) + return child_runner.run_tests(mutant_name=None, tests=tests_list) + + return run_in_fork(_run_tests) + + def run_forced_fail(self) -> int: + def _run_forced_fail() -> int: + child_runner: TestRunner = self.test_runner_class(**self.test_runner_args) + return child_runner.run_forced_fail() + + return run_in_fork(_run_forced_fail) + + def list_all_tests(self) -> ListAllTestsResult: + def _list_all_tests() -> dict[str, Any]: + child_runner: TestRunner = self.test_runner_class(**self.test_runner_args) + result = child_runner.list_all_tests() + return {"ids": list(result.ids)} + + data = run_in_fork_with_result(_list_all_tests) + return ListAllTestsResult(ids=set(data["ids"])) + + class ForkRunner(MutantRunner): """Runner that uses os.fork() for process isolation. @@ -382,7 +926,13 @@ def get_mutant_runner(max_workers: int = 1) -> MutantRunner: raise ValueError("max_workers must be at least 1") if config().process_isolation == ProcessIsolation.HOT_FORK: - raise NotImplementedError("process_isolation = 'hot-fork' is not available yet in this build; use 'fork'.") + return HotForkRunner( + max_workers=max_workers, + test_runner_class=PytestRunner, + test_runner_args={}, + debug=config().debug, + max_restarts=config().max_orchestrator_restarts, + ) pytest_runner = PytestRunner() pytest_runner.prepare_main_test_run() diff --git a/tests/e2e/test_e2e_hot_fork.py b/tests/e2e/test_e2e_hot_fork.py new file mode 100644 index 00000000..2b715290 --- /dev/null +++ b/tests/e2e/test_e2e_hot_fork.py @@ -0,0 +1,54 @@ +"""End-to-end smoke test for the hot-fork process-isolation runner. + +The hot-fork orchestrator forks itself and installs signal handlers, so it is +exercised in a subprocess (with a hard timeout) rather than in-process, keeping +any hang or signal interaction out of the pytest session running this test. +""" + +import json +import shutil +import subprocess +import sys +from collections import Counter +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +E2E_PROJECTS = REPO_ROOT / "e2e_projects" + + +def _read_verdicts(project_path: Path) -> dict[str, int]: + verdicts: dict[str, int] = {} + for meta in sorted((project_path / "mutants").rglob("*.meta")): + data = json.loads(meta.read_text()) + verdicts.update(data["exit_code_by_key"]) + return verdicts + + +def test_hot_fork_runs_end_to_end(tmp_path: Path): + project = tmp_path / "hot_fork_basic" + shutil.copytree(E2E_PROJECTS / "hot_fork_basic", project) + + result = subprocess.run( + [sys.executable, "-m", "mutmut", "run"], + cwd=project, + capture_output=True, + text=True, + timeout=240, + ) + + assert result.returncode == 0, f"mutmut run failed:\n{result.stdout}\n{result.stderr}" + + verdicts = _read_verdicts(project) + assert verdicts, "no mutant results were written" + + counts = Counter(verdicts.values()) + # 0 = survived, 1 = killed, 33 = no tests. The tested functions (add/sub/mul) + # must produce some killed mutants, and `untested` must produce no-tests ones. + assert counts.get(1, 0) > 0, f"expected some killed mutants, got {dict(counts)}" + assert counts.get(33, 0) > 0, f"expected some no-tests mutants, got {dict(counts)}" + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/tests/mutation/test_mutation.py b/tests/mutation/test_mutation.py index 8a6986e6..c885b726 100644 --- a/tests/mutation/test_mutation.py +++ b/tests/mutation/test_mutation.py @@ -28,6 +28,7 @@ from mutmut.__main__ import git_tracked_non_py_files from mutmut.__main__ import run_forced_fail_test from mutmut.configuration import Config +from mutmut.configuration import HotForkWarmup from mutmut.configuration import ProcessIsolation from mutmut.core import MutmutProgrammaticFailException from mutmut.mutation.data import MutantLineSpans @@ -1626,6 +1627,11 @@ def _config_for_invalidation(**overrides): on_dependency_change="warn", use_git_change_detection=True, process_isolation=ProcessIsolation.FORK, + hot_fork_warmup=HotForkWarmup.COLLECT, + max_orchestrator_restarts=3, + preload_modules_file=None, + log_to_file=False, + log_file_path="mutants/mutmut-debug.log", ) base.update(overrides) return Config(**base) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 5a4794d2..a2df6374 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -3,6 +3,7 @@ import pytest from mutmut.configuration import Config +from mutmut.configuration import HotForkWarmup from mutmut.configuration import ProcessIsolation from mutmut.configuration import _config_reader from mutmut.configuration import _guess_source_paths @@ -47,6 +48,34 @@ def test_reset_clears_singleton(self, in_tmp_dir: Path): assert config1 is not config2 +class TestProcessIsolationConfig: + def _write_pyproject(self, in_tmp_dir: Path, body: str) -> None: + (in_tmp_dir / "src").mkdir() + (in_tmp_dir / "pyproject.toml").write_text(f"[tool.mutmut]\n{body}\n") + + def test_defaults(self, in_tmp_dir: Path): + self._write_pyproject(in_tmp_dir, "") + cfg = config() + assert cfg.process_isolation == ProcessIsolation.FORK + assert cfg.hot_fork_warmup == HotForkWarmup.COLLECT + assert cfg.max_orchestrator_restarts == 3 + assert cfg.preload_modules_file is None + + def test_hot_fork_selected(self, in_tmp_dir: Path): + self._write_pyproject(in_tmp_dir, 'process_isolation = "hot-fork"') + assert config().process_isolation == ProcessIsolation.HOT_FORK + + def test_invalid_process_isolation_raises(self, in_tmp_dir: Path): + self._write_pyproject(in_tmp_dir, 'process_isolation = "bogus"') + with pytest.raises(ValueError, match="Invalid process_isolation value"): + _load_config() + + def test_invalid_hot_fork_warmup_raises(self, in_tmp_dir: Path): + self._write_pyproject(in_tmp_dir, 'hot_fork_warmup = "bogus"') + with pytest.raises(ValueError, match="Invalid hot_fork_warmup value"): + _load_config() + + class TestShouldMutateFile: @staticmethod def _get_config(only_mutate: list[str], do_not_mutate: list[str]) -> Config: @@ -74,6 +103,11 @@ def _get_config(only_mutate: list[str], do_not_mutate: list[str]) -> Config: on_dependency_change="warn", use_git_change_detection=True, process_isolation=ProcessIsolation.FORK, + hot_fork_warmup=HotForkWarmup.COLLECT, + max_orchestrator_restarts=3, + preload_modules_file=None, + log_to_file=False, + log_file_path="mutants/mutmut-debug.log", ) def test_ignores_non_python_files(self): diff --git a/tests/workers/test_isolation.py b/tests/workers/test_isolation.py index 71dcbe10..c64c8114 100644 --- a/tests/workers/test_isolation.py +++ b/tests/workers/test_isolation.py @@ -1,13 +1,21 @@ """Tests for fork isolation utilities.""" import os +import threading import pytest +from mutmut.configuration import ProcessIsolation +from mutmut.configuration import config +from mutmut.configuration import reset_config from mutmut.workers.isolation import ForkRunner +from mutmut.workers.isolation import HotForkRunner from mutmut.workers.isolation import OrchestratorCrashError +from mutmut.workers.isolation import get_mutant_runner +from mutmut.workers.isolation import recv_message from mutmut.workers.isolation import run_in_fork from mutmut.workers.isolation import run_in_fork_with_result +from mutmut.workers.isolation import send_message @pytest.mark.skipif(os.name == "nt", reason="Forking not supported on Windows") @@ -237,3 +245,111 @@ def test_queued_results_are_drained_in_submission_order(self): assert drained == ["mod.x_foo__mutmut_1", "mod.x_foo__mutmut_2", "mod.x_foo__mutmut_3"] assert runner.pending_count() == 0 + + +class TestGetMutantRunner: + """Tests for the get_mutant_runner factory.""" + + @pytest.fixture + def in_project_dir(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "src").mkdir() + reset_config() + return tmp_path + + def test_selects_fork_by_default(self, in_project_dir): + runner = get_mutant_runner(2) + assert isinstance(runner, ForkRunner) + assert runner.max_workers == 2 + + def test_selects_hot_fork(self, in_project_dir, monkeypatch): + monkeypatch.setattr(config(), "process_isolation", ProcessIsolation.HOT_FORK) + runner = get_mutant_runner(4) + assert isinstance(runner, HotForkRunner) + assert runner.max_workers == 4 + assert runner.max_restarts == config().max_orchestrator_restarts + + def test_rejects_zero_workers(self, in_project_dir): + with pytest.raises(ValueError, match="at least 1"): + get_mutant_runner(0) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX pipes required") +class TestPipeMessaging: + """Framed pipe messages must stay in step with select() on the raw fd. + + A buffered reader can pull several frames into userspace at once, after which + select() reports "not readable" and the already-received frames are never + processed - the receiver loops forever waiting for results it already has.""" + + def test_roundtrips_a_message(self): + r, w = os.pipe() + try: + send_message(w, ("mod.x_foo__mutmut_1", 0)) + assert recv_message(r) == ("mod.x_foo__mutmut_1", 0) + finally: + os.close(r) + os.close(w) + + def test_reading_one_message_leaves_the_rest_selectable(self): + import select + + r, w = os.pipe() + try: + for i in range(5): + send_message(w, (f"mod.x_foo__mutmut_{i}", 0)) + + assert recv_message(r) == ("mod.x_foo__mutmut_0", 0) + # The remaining four must still be visible to select(), or the + # receive loop would block on them forever. + assert select.select([r], [], [], 0)[0], "remaining messages are not selectable" + assert [recv_message(r)[0] for _ in range(4)] == [f"mod.x_foo__mutmut_{i}" for i in range(1, 5)] + finally: + os.close(r) + os.close(w) + + def test_survives_a_payload_larger_than_the_pipe_buffer(self): + # 128k of test ids exceeds both PIPE_BUF and the 64k pipe capacity, so + # the write must be looped and the read must reassemble it. + payload = ("mod.x_foo__mutmut_1", [f"tests/test_{i}.py::test_case" for i in range(5000)], 10) + r, w = os.pipe() + reader = threading.Thread(target=lambda: received.append(recv_message(r))) + received: list = [] + try: + reader.start() + send_message(w, payload) + reader.join(timeout=10) + assert not reader.is_alive(), "receiver blocked on a large message" + assert received == [payload] + finally: + os.close(r) + os.close(w) + + def test_reports_eof_when_the_writer_closes(self): + r, w = os.pipe() + os.close(w) + try: + with pytest.raises(EOFError): + recv_message(r) + finally: + os.close(r) + + +class TestHotForkRunnerNoTestMutants: + """HotForkRunner resolves a mutant with no tests the same way ForkRunner does.""" + + @staticmethod + def _runner(): + return HotForkRunner(max_workers=4, test_runner_class=object, test_runner_args={}) + + def test_no_test_mutant_is_resolved_without_an_orchestrator(self): + runner = self._runner() # never started, so there is no work pipe + runner.submit("mod.x_foo__mutmut_1", [], cpu_time_limit=1, estimated_time=0.0) + assert runner.pending_count() == 1 + + def test_no_test_mutant_reports_exit_code_33(self): + runner = self._runner() + runner.submit("mod.x_foo__mutmut_1", [], cpu_time_limit=1, estimated_time=0.0) + result = runner.wait_for_result() + assert (result.mutant_name, result.exit_code) == ("mod.x_foo__mutmut_1", 33) + assert runner.pending_count() == 0 From d18d7220c0cce5fcbfbdc11ad9535983135bd80e Mon Sep 17 00:00:00 2001 From: nicklafleur <55208706+nicklafleur@users.noreply.github.com> Date: Sat, 12 Sep 2026 10:07:10 -0400 Subject: [PATCH 4/5] misc: rename and doc --- ARCHITECTURE.rst | 14 ++ HISTORY.rst | 2 + README.rst | 102 ++++++++++ e2e_projects/benchmark_1k/README.md | 4 +- e2e_projects/benchmark_1k/mutmut_preload.txt | 2 +- e2e_projects/benchmark_1k/pyproject.toml | 4 +- e2e_projects/benchmark_1k/run_benchmark.py | 4 +- .../pyproject.toml | 4 +- .../src/fs_calc}/__init__.py | 0 .../tests/test_fs_calc.py} | 6 +- src/mutmut/configuration.py | 37 ++-- src/mutmut/models/results.py | 4 +- src/mutmut/runners/harness.py | 14 +- src/mutmut/utils/logging_utils.py | 4 +- src/mutmut/workers/isolation.py | 177 +++++++++--------- tests/e2e/test_e2e_forkserver.py | 18 ++ tests/e2e/test_e2e_hot_fork.py | 54 ------ tests/mutation/test_mutation.py | 6 +- tests/test_configuration.py | 22 +-- tests/workers/test_isolation.py | 50 ++--- 20 files changed, 311 insertions(+), 217 deletions(-) rename e2e_projects/{hot_fork_basic => forkserver_basic}/pyproject.toml (81%) rename e2e_projects/{hot_fork_basic/src/hf_calc => forkserver_basic/src/fs_calc}/__init__.py (100%) rename e2e_projects/{hot_fork_basic/tests/test_hf_calc.py => forkserver_basic/tests/test_fs_calc.py} (64%) create mode 100644 tests/e2e/test_e2e_forkserver.py delete mode 100644 tests/e2e/test_e2e_hot_fork.py diff --git a/ARCHITECTURE.rst b/ARCHITECTURE.rst index 6c6cf394..ba64433c 100644 --- a/ARCHITECTURE.rst +++ b/ARCHITECTURE.rst @@ -48,3 +48,17 @@ We finally check, which mutations are caught by the test suite. For each mutant, we execute the test suite. If any of the tests fails, we successfully killed the mutant. To optimize performance, we only execute the tests that could cover the mutant and sort them by mutation time. We also skip mutants, which already have a result from a previous run. The results are stored in the ``.meta`` files. + +Every mutant is tested in its own process. Which process that one is forked from is +owned by a ``MutantRunner`` (``src/mutmut/workers/isolation.py``), selected by the +``process_isolation`` config. ``ForkRunner`` forks each worker straight from mutmut's +main process, which by this point has imported pytest and run the suite once to collect +stats. ``ForkServerRunner`` instead keeps the main process free of pytest: it forks a +dedicated fork server child, and that child imports pytest once and forks a worker per +mutant, streaming results back over a length-prefixed pipe. That extra generation exists +for test setups that are not fork-safe, where inheriting the main process would hang or +crash the workers. + +The runner also owns the surrounding test operations (stats collection, clean tests, +forced fail, test listing) so that, under ``forkserver``, those too can run in short-lived +forks and leave the main process clean. diff --git a/HISTORY.rst b/HISTORY.rst index 8ab9a46f..87d12341 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,8 @@ Unreleased * Mutate the condition of ternary expressions (`#196`, `#546`) +* Add a ``process_isolation`` config for choosing which process mutant workers are forked from. The new ``forkserver`` strategy keeps mutmut's main process free of pytest and your ``conftest.py``, routing every fork through a dedicated fork server process, for test setups that are not fork-safe (``gevent``, ``grpc``, ``torch``). Tune what it preloads with ``forkserver_warmup`` (`#578`, `#566`) + * Add a `badge` command that writes a shields.io endpoint file, for publishing the mutation score (`#549`) * Support python3.15 (`#551`) diff --git a/README.rst b/README.rst index 1e76b4fd..6b8a074f 100644 --- a/README.rst +++ b/README.rst @@ -236,6 +236,108 @@ macOS at your own risk, or to disable it on other platforms), set ``use_setproct use_setproctitle = false +Process isolation (``fork`` vs ``forkserver``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: toml + + # pyproject.toml + [tool.mutmut] + process_isolation = "fork" # or "forkserver" + +Both strategies require ``os.fork``, so they are POSIX-only. + +``fork`` (default) + Mutmut's main process imports pytest and runs your test suite to collect + stats, then forks one child per mutant. This is fast and needs no + configuration, but every mutant worker inherits whatever your test setup + left behind in that process: monkey-patched sockets, running event loops, + background threads, open database connections, CUDA contexts. If any of + that is not fork-safe, the workers can hang, segfault, or report nonsense. + +``forkserver`` + Mutmut's main process never imports pytest or your ``conftest.py``. It forks + a dedicated *fork server* child, and that child does the importing once and + forks a worker per mutant:: + + main process (clean) -> fork server (imports pytest) -> N mutant workers + + Because the fork server only imports and forks, and never runs a test + itself, every worker starts from the same predictable state. This is the + same design as the ``forkserver`` start method in Python's + ``multiprocessing``, and it is still only one pytest import for the whole + run, so it is not slower than ``fork`` in the normal case. + +**Which one should you use?** Start with ``fork``. Switch to ``forkserver`` if +your run hangs, crashes, or gives results you cannot reproduce by running the +tests by hand, and your test setup touches something known to be fork-unsafe, +such as ``gevent.monkey.patch_all()``, ``grpc``, or ``torch``. It is also the +better choice if you simply do not want your session-scoped setup living in the +process that forks mutants. + + +Tuning what the fork server preloads +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``forkserver_warmup`` controls how much the fork server imports before it starts +forking workers. It only applies when ``process_isolation = "forkserver"``. +Anything imported in the fork server is inherited by every worker, and anything +left out is imported in each worker. + +.. code-block:: toml + + # pyproject.toml + [tool.mutmut] + process_isolation = "forkserver" + forkserver_warmup = "collect" # "collect" (default), "import" or "none" + +``collect`` (default) + Run ``pytest --collect-only``, which imports your ``conftest.py``, your + plugins and your test modules. This is the right default for most projects + and usually the fastest. + +``import`` + Import only the modules listed in ``preload_modules_file`` (one dotted + module name per line; blank lines and ``#`` comments are ignored, and + modules that fail to import are skipped). Use this when test *collection* + itself has side effects you do not want every worker to share, but you still + want your slow imports warmed up. + +``none`` + Import nothing beyond what running a single test requires. This is the + slowest option, and the most isolated. It is mostly useful for working out + whether a warmup side effect is what is breaking your run. + +.. code-block:: toml + + # pyproject.toml + [tool.mutmut] + process_isolation = "forkserver" + forkserver_warmup = "import" + preload_modules_file = "mutmut_preload.txt" + +If the fork server itself crashes, mutmut starts a new one and re-submits the +mutants that were in flight, up to ``max_forkserver_restarts`` times (default +``3``) before giving up with a ``ForkServerCrashError``: + +.. code-block:: toml + + # pyproject.toml + [tool.mutmut] + max_forkserver_restarts = 3 + +The fork server and its workers cannot write to the console without mangling +mutmut's progress output, so they log to a file instead. Logging is off unless +you ask for it with ``log_to_file`` (or ``debug``): + +.. code-block:: toml + + # pyproject.toml + [tool.mutmut] + log_to_file = true + log_file_path = "mutants/mutmut-debug.log" + + Disabling mutation on specific code ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/e2e_projects/benchmark_1k/README.md b/e2e_projects/benchmark_1k/README.md index f5f4ab20..b9deac7b 100644 --- a/e2e_projects/benchmark_1k/README.md +++ b/e2e_projects/benchmark_1k/README.md @@ -1,10 +1,10 @@ # Benchmark 1K -A synthetic benchmark project with 1000 mutants for validating mutmut's process isolation and hot-fork warmup strategy performance. +A synthetic benchmark project with 1000 mutants for validating mutmut's process isolation and forkserver warmup strategy performance. **TL;DR:** - `fork` is fastest and nearly immune to import delays (requires fork-safe libraries) -- `collect` (hot-fork default) is 2-9x faster than `import`/`none` depending on import cost +- `collect` (forkserver default) is 2-9x faster than `import`/`none` depending on import cost - Higher import delays dramatically penalize `import` and `none` strategies diff --git a/e2e_projects/benchmark_1k/mutmut_preload.txt b/e2e_projects/benchmark_1k/mutmut_preload.txt index 6aeb0c2c..cc726ce4 100644 --- a/e2e_projects/benchmark_1k/mutmut_preload.txt +++ b/e2e_projects/benchmark_1k/mutmut_preload.txt @@ -1,5 +1,5 @@ # Modules to preload for the 'import' warmup strategy -# These are imported in the orchestrator and inherited by forked workers +# These are imported in the fork server and inherited by forked workers pytest #normally you would avoid importing the module itself in real projects diff --git a/e2e_projects/benchmark_1k/pyproject.toml b/e2e_projects/benchmark_1k/pyproject.toml index e3fa0d99..4b4e3b9b 100644 --- a/e2e_projects/benchmark_1k/pyproject.toml +++ b/e2e_projects/benchmark_1k/pyproject.toml @@ -14,6 +14,6 @@ packages = ["src/benchmark"] [tool.mutmut] log_to_file = true source_paths = ["src/"] -process_isolation = "hot-fork" +process_isolation = "forkserver" debug = false -hot_fork_warmup = "none" +forkserver_warmup = "none" diff --git a/e2e_projects/benchmark_1k/run_benchmark.py b/e2e_projects/benchmark_1k/run_benchmark.py index 9164ff3a..56d1202d 100644 --- a/e2e_projects/benchmark_1k/run_benchmark.py +++ b/e2e_projects/benchmark_1k/run_benchmark.py @@ -69,10 +69,10 @@ def run_mutmut( show_output: bool = False, ) -> dict: """Run mutmut with specified strategy.""" - config = get_pyproject_content(debug=verbose, process_isolation=strategy if strategy == "fork" else "hot-fork") + config = get_pyproject_content(debug=verbose, process_isolation=strategy if strategy == "fork" else "forkserver") if strategy != "fork": - config += f'hot_fork_warmup = "{strategy}"\n' + config += f'forkserver_warmup = "{strategy}"\n' if strategy == "import": config += 'preload_modules_file = "mutmut_preload.txt"\n' diff --git a/e2e_projects/hot_fork_basic/pyproject.toml b/e2e_projects/forkserver_basic/pyproject.toml similarity index 81% rename from e2e_projects/hot_fork_basic/pyproject.toml rename to e2e_projects/forkserver_basic/pyproject.toml index 88b8f9cd..a3b673ba 100644 --- a/e2e_projects/hot_fork_basic/pyproject.toml +++ b/e2e_projects/forkserver_basic/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "hf-calc" +name = "fs-calc" version = "0.1.0" requires-python = ">=3.10" @@ -8,7 +8,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.mutmut] -process_isolation = "hot-fork" +process_isolation = "forkserver" [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" diff --git a/e2e_projects/hot_fork_basic/src/hf_calc/__init__.py b/e2e_projects/forkserver_basic/src/fs_calc/__init__.py similarity index 100% rename from e2e_projects/hot_fork_basic/src/hf_calc/__init__.py rename to e2e_projects/forkserver_basic/src/fs_calc/__init__.py diff --git a/e2e_projects/hot_fork_basic/tests/test_hf_calc.py b/e2e_projects/forkserver_basic/tests/test_fs_calc.py similarity index 64% rename from e2e_projects/hot_fork_basic/tests/test_hf_calc.py rename to e2e_projects/forkserver_basic/tests/test_fs_calc.py index ad535d84..749b2dc2 100644 --- a/e2e_projects/hot_fork_basic/tests/test_hf_calc.py +++ b/e2e_projects/forkserver_basic/tests/test_fs_calc.py @@ -1,6 +1,6 @@ -from hf_calc import add -from hf_calc import mul -from hf_calc import sub +from fs_calc import add +from fs_calc import mul +from fs_calc import sub def test_add(): diff --git a/src/mutmut/configuration.py b/src/mutmut/configuration.py index 0d612840..a1b7650c 100644 --- a/src/mutmut/configuration.py +++ b/src/mutmut/configuration.py @@ -21,18 +21,29 @@ class ProcessIsolation(str, Enum): """Valid values for the ``process_isolation`` config. + Chooses *which process* each mutant worker is forked from. Named after the + ``multiprocessing`` start methods of the same name, which make the same + distinction: + + - FORK: fork straight from mutmut's main process, which has already imported + pytest and run the suite to collect stats. Fastest, but every worker + inherits whatever the test setup left in that process. + - FORKSERVER: keep the main process free of pytest and fork a dedicated + server process that does nothing but import and fork workers. For test + setups that are not fork-safe (gevent, grpc, torch). + Subclassing ``str`` allows direct string comparison while still giving us validation and IDE support. """ - FORK = "fork" # Default: fork the (test-polluted) parent per mutant. - HOT_FORK = "hot-fork" # Fork-safe orchestrator for gevent/grpc/torch. + FORK = "fork" + FORKSERVER = "forkserver" -class HotForkWarmup(str, Enum): - """Warmup strategy for the hot-fork orchestrator. +class ForkServerWarmup(str, Enum): + """Warmup strategy for the fork server. - Controls what the orchestrator does after importing the test runner but + Controls what the fork server does after importing the test runner but before forking any grandchildren: - COLLECT: run ``pytest --collect-only`` to pre-load conftest, plugins, and @@ -159,12 +170,12 @@ def _load_config() -> Config: valid = [e.value for e in ProcessIsolation] raise ValueError(f"Invalid process_isolation value: {isolation_str!r}. Expected one of: {valid}") from None - warmup_str = s("hot_fork_warmup", "collect") + warmup_str = s("forkserver_warmup", "collect") try: - hot_fork_warmup = HotForkWarmup(warmup_str) + forkserver_warmup = ForkServerWarmup(warmup_str) except ValueError: - valid = [e.value for e in HotForkWarmup] - raise ValueError(f"Invalid hot_fork_warmup value: {warmup_str!r}. Expected one of: {valid}") from None + valid = [e.value for e in ForkServerWarmup] + raise ValueError(f"Invalid forkserver_warmup value: {warmup_str!r}. Expected one of: {valid}") from None return Config( only_mutate=only_mutate, @@ -202,8 +213,8 @@ def _load_config() -> Config: on_dependency_change=s("on_dependency_change", "warn"), use_git_change_detection=s("use_git_change_detection", True), process_isolation=process_isolation, - hot_fork_warmup=hot_fork_warmup, - max_orchestrator_restarts=s("max_orchestrator_restarts", 3), + forkserver_warmup=forkserver_warmup, + max_forkserver_restarts=s("max_forkserver_restarts", 3), preload_modules_file=s("preload_modules_file", None), log_to_file=s("log_to_file", False), log_file_path=s("log_file_path", "mutants/mutmut-debug.log"), @@ -237,8 +248,8 @@ class Config: on_dependency_change: str use_git_change_detection: bool process_isolation: ProcessIsolation - hot_fork_warmup: HotForkWarmup - max_orchestrator_restarts: int + forkserver_warmup: ForkServerWarmup + max_forkserver_restarts: int preload_modules_file: str | None log_to_file: bool log_file_path: str diff --git a/src/mutmut/models/results.py b/src/mutmut/models/results.py index 44ca6640..ea72490d 100644 --- a/src/mutmut/models/results.py +++ b/src/mutmut/models/results.py @@ -1,4 +1,4 @@ -"""Result data models for worker/orchestrator communication.""" +"""Result data models for worker/fork server communication.""" from __future__ import annotations @@ -10,7 +10,7 @@ class StatsResult: """Stats collected by a forked child, serialized back to the parent. - The hot-fork runner collects stats inside a forked child (so the parent + The forkserver runner collects stats inside a forked child (so the parent never imports pytest). The child cannot mutate the parent's ``state()`` directly, so it packs the collected data into this picklable structure and the parent merges it back in. diff --git a/src/mutmut/runners/harness.py b/src/mutmut/runners/harness.py index c2f929f0..349f8741 100644 --- a/src/mutmut/runners/harness.py +++ b/src/mutmut/runners/harness.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING from typing import Any -from mutmut.configuration import HotForkWarmup +from mutmut.configuration import ForkServerWarmup from mutmut.configuration import config from mutmut.state import state from mutmut.stats import save_stats @@ -53,7 +53,7 @@ def list_all_tests(self) -> ListAllTestsResult: def warm_up(self) -> None: """Pre-import expensive modules so forked children inherit them. - Called by HotForkRunner inside the orchestrator after the test runner is + Called by ForkServerRunner inside the fork server after the test runner is created. Importing pytest (and optionally running collection) here means the grandchildren fork with everything already in memory. The default is a no-op for runners that do not benefit from it. @@ -94,19 +94,19 @@ def __init__(self) -> None: self._pytest_add_cli_args_test_selection: list[str] = config().pytest_add_cli_args_test_selection def warm_up(self) -> None: - """Pre-load test infrastructure per the ``hot_fork_warmup`` config. + """Pre-load test infrastructure per the ``forkserver_warmup`` config. - COLLECT (default): run ``pytest --collect-only`` to import conftest, plugins, and test modules (biggest speedup for most projects). - IMPORT: import the modules listed in ``preload_modules_file``. - NONE: import nothing beyond what running a test already needs. """ - warmup = config().hot_fork_warmup + warmup = config().forkserver_warmup - if warmup == HotForkWarmup.COLLECT: + if warmup == ForkServerWarmup.COLLECT: with change_cwd("mutants"): self.execute_pytest(["--collect-only", "-qqq"] + self._pytest_add_cli_args_test_selection) - elif warmup == HotForkWarmup.IMPORT: + elif warmup == ForkServerWarmup.IMPORT: preload_file = config().preload_modules_file if preload_file: import importlib @@ -119,7 +119,7 @@ def warm_up(self) -> None: importlib.import_module(module_name) except ImportError: pass # Best effort. - # HotForkWarmup.NONE -> no-op. + # ForkServerWarmup.NONE -> no-op. # noinspection PyMethodMayBeStatic def execute_pytest(self, params: list[str], **kwargs: Any) -> int: diff --git a/src/mutmut/utils/logging_utils.py b/src/mutmut/utils/logging_utils.py index 05350316..aee8135b 100644 --- a/src/mutmut/utils/logging_utils.py +++ b/src/mutmut/utils/logging_utils.py @@ -1,6 +1,6 @@ -"""File-based logging helpers, used mainly by the hot-fork orchestrator. +"""File-based logging helpers, used mainly by the fork server. -Child/orchestrator processes cannot easily log to the console without +Forked processes cannot easily log to the console without corrupting the interactive terminal output, so mutmut logs to a rotating file instead. Logging is opt-in (``log_to_file``/``debug`` config) and off by default. """ diff --git a/src/mutmut/workers/isolation.py b/src/mutmut/workers/isolation.py index 1810c7b1..cf2a3f7c 100644 --- a/src/mutmut/workers/isolation.py +++ b/src/mutmut/workers/isolation.py @@ -8,8 +8,9 @@ The low-level ``run_in_fork*`` helpers run operations in forked children so the parent stays clean. On top of them, ``MutantRunner`` abstracts the process isolation strategy used to test each mutant. ``ForkRunner`` is the traditional -os.fork()-per-mutant approach; ``HotForkRunner`` (single orchestrator) will be -layered on later for fork-unsafe libraries. +os.fork()-per-mutant approach; ``ForkServerRunner`` keeps the main process clean +by routing every fork through a dedicated server process, for fork-unsafe +libraries. """ from __future__ import annotations @@ -175,10 +176,10 @@ def recv_message(fd: int) -> Any: return pickle.loads(_read_exactly(fd, size)) -class OrchestratorCrashError(Exception): - """Raised when the hot-fork orchestrator crashes unexpectedly. +class ForkServerCrashError(Exception): + """Raised when the fork server crashes unexpectedly. - The orchestrator manages all mutant test runs. If it crashes, any + The fork server manages all mutant test runs. If it crashes, any in-flight mutants are lost. The user can resume by running `mutmut run` again - completed results are preserved. """ @@ -190,7 +191,7 @@ def __init__(self, exit_code: int, lost_mutants: list[str], crash_log: str | Non # Build detailed message details = [ - f"Hot-fork orchestrator crashed unexpectedly (exit code: {exit_code})", + f"Fork server crashed unexpectedly (exit code: {exit_code})", f"Lost {len(lost_mutants)} in-flight mutant(s):", ] for m in lost_mutants[:10]: @@ -311,21 +312,21 @@ def list_all_tests(self) -> ListAllTestsResult: """List all tests in the test suite.""" -class HotForkRunner(MutantRunner): - """Fork-safe mutation runner using a single hot orchestrator. +class ForkServerRunner(MutantRunner): + """Fork-safe mutation runner using a single fork server. Architecture:: - Parent (clean) -> Orchestrator (imports pytest) -> N concurrent children + Parent (clean) -> Fork server (imports pytest) -> N concurrent children The parent never imports pytest/conftest, so it stays fork-safe. The - orchestrator imports pytest exactly once, then forks a grandchild per + fork server imports pytest exactly once, then forks a grandchild per mutant; each grandchild runs one mutant's tests and exits. This is both faster than forking a fresh pytest per mutant (one import instead of N) and compatible with fork-unsafe libraries like gevent, grpc, and torch. - If the orchestrator crashes, in-flight mutants are re-submitted to a fresh - orchestrator up to ``max_restarts`` times before an OrchestratorCrashError + If the fork server crashes, in-flight mutants are re-submitted to a fresh + fork server up to ``max_restarts`` times before a ForkServerCrashError is raised. """ @@ -336,7 +337,7 @@ class RunningChild(NamedTuple): start_time: float wall_timeout: float - # Default maximum number of orchestrator restarts before giving up. + # Default maximum number of fork server restarts before giving up. DEFAULT_MAX_RESTARTS = 3 def __init__( @@ -359,7 +360,7 @@ def __init__( self.result_pipe_read: int | None = None self.result_pipe_write: int | None = None - self.orchestrator_pid: int | None = None + self.forkserver_pid: int | None = None self._pending: set[str] = set() # mutant_names in flight # mutant_name -> (tests, cpu_time_limit, estimated_time, start_time) self._pending_work: dict[str, tuple[list[str], int, float, datetime]] = {} @@ -370,24 +371,24 @@ def __init__( def startup(self) -> None: gc.freeze() - self._start_orchestrator() + self._start_forkserver() - def _start_orchestrator(self) -> None: - """Fork a fresh orchestrator process with new pipes. + def _start_forkserver(self) -> None: + """Start a fresh fork server process with new pipes. Can be called more than once for crash recovery; each call creates fresh - pipes and a new orchestrator. + pipes and a new fork server. """ self.work_pipe_read, self.work_pipe_write = os.pipe() self.result_pipe_read, self.result_pipe_write = os.pipe() pid = os.fork() if pid == 0: - # Child: become the orchestrator. + # Child: become the fork server. os.close(self.work_pipe_write) os.close(self.result_pipe_read) try: - self._orchestrator_main(self.work_pipe_read, self.result_pipe_write) + self._forkserver_main(self.work_pipe_read, self.result_pipe_write) except Exception as e: self._write_crash_log(e) os._exit(1) @@ -396,27 +397,27 @@ def _start_orchestrator(self) -> None: # Parent: close the child's ends. os.close(self.work_pipe_read) os.close(self.result_pipe_write) - self.orchestrator_pid = pid - self._logger.info(f"HotForkRunner started orchestrator (pid={pid})") + self.forkserver_pid = pid + self._logger.info(f"Started fork server (pid={pid})") - def _restart_orchestrator_with_pending_work(self, exit_code: int = -1) -> None: - """Restart the orchestrator and re-submit all pending work. + def _restart_forkserver_with_pending_work(self, exit_code: int = -1) -> None: + """Restart the fork server and re-submit all pending work. - Raises OrchestratorCrashError once ``max_restarts`` is exceeded. + Raises ForkServerCrashError once ``max_restarts`` is exceeded. """ self._restart_count += 1 self._crash_exit_codes.append(exit_code) - crash_log_path = get_log_file_path().parent / ".orchestrator-crash.log" + crash_log_path = get_log_file_path().parent / ".forkserver-crash.log" pending_mutants = list(self._pending) self._logger.error( - f"Orchestrator crashed with exit code {exit_code}. " + f"Fork server crashed with exit code {exit_code}. " f"Check {crash_log_path} and {get_log_file_path()} for details." ) self._logger.error(f"Pending mutants at time of crash ({len(pending_mutants)}): {pending_mutants}") if self._restart_count > self.max_restarts: - raise OrchestratorCrashError( + raise ForkServerCrashError( exit_code=-1, lost_mutants=list(self._pending), crash_log=str(crash_log_path) if crash_log_path.exists() else None, @@ -424,37 +425,37 @@ def _restart_orchestrator_with_pending_work(self, exit_code: int = -1) -> None: lost_count = len(self._pending) self._logger.warning( - f"Orchestrator crashed, restarting (attempt {self._restart_count}/{self.max_restarts}), " + f"Fork server crashed, restarting (attempt {self._restart_count}/{self.max_restarts}), " f"re-submitting {lost_count} pending mutant(s)" ) pending_work_copy = dict(self._pending_work) - self._start_orchestrator() + self._start_forkserver() if self.work_pipe_write is None: - raise RuntimeError("Failed to restart orchestrator - work pipe not created") + raise RuntimeError("Failed to restart fork server - work pipe not created") for mutant_name, (tests, cpu_time_limit, estimated_time, _) in pending_work_copy.items(): send_message(self.work_pipe_write, (mutant_name, list(tests), cpu_time_limit)) self._pending_work[mutant_name] = (tests, cpu_time_limit, estimated_time, datetime.now()) - self._logger.debug(f"Re-submitted {mutant_name} to new orchestrator") + self._logger.debug(f"Re-submitted {mutant_name} to new fork server") - self._logger.info(f"Orchestrator restarted, {lost_count} mutant(s) re-submitted") + self._logger.info(f"Fork server restarted, {lost_count} mutant(s) re-submitted") def _write_crash_log(self, exception: Exception) -> None: - """Best-effort dump of orchestrator crash info for debugging.""" - crash_file = get_log_file_path().parent / ".orchestrator-crash.log" + """Best-effort dump of fork server crash info for debugging.""" + crash_file = get_log_file_path().parent / ".forkserver-crash.log" try: crash_file.parent.mkdir(parents=True, exist_ok=True) with open(crash_file, "w") as f: - f.write(f"Orchestrator crash at {datetime.now()}\n") + f.write(f"Fork server crash at {datetime.now()}\n") f.write(f"Exception: {exception}\n") f.write(traceback.format_exc()) except Exception: pass def _setup_sigchld_pipe(self) -> tuple[int, int]: - """Set up a self-pipe so SIGCHLD wakes the orchestrator's select().""" + """Set up a self-pipe so SIGCHLD wakes the fork server's select().""" sigchld_pipe_r, sigchld_pipe_w = os.pipe() os.set_blocking(sigchld_pipe_r, False) os.set_blocking(sigchld_pipe_w, False) @@ -487,15 +488,15 @@ def _wait_for_child_event(self, sigchld_pipe_r: int, timeout: float | None) -> b except InterruptedError: return True # Interrupted by a signal; check anyway. - def _orchestrator_main(self, work_fd: int, result_fd: int) -> None: - """Orchestrator: import pytest once, then fork a grandchild per mutant.""" + def _forkserver_main(self, work_fd: int, result_fd: int) -> None: + """Fork server: import pytest once, then fork a grandchild per mutant.""" # The parent owns shutdown, so ignore SIGINT here. signal.signal(signal.SIGINT, signal.SIG_IGN) if config().log_to_file or config().debug: setup_file_logging() - orchestrator_logger = get_logger("mutmut.hotfork.orchestrator") - orchestrator_logger.info(f"Hot-fork orchestrator starting (pid={os.getpid()})") + forkserver_logger = get_logger("mutmut.forkserver") + forkserver_logger.info(f"Fork server starting (pid={os.getpid()})") test_runner: TestRunner = self.test_runner_class(**self.test_runner_args) @@ -507,14 +508,14 @@ def _orchestrator_main(self, work_fd: int, result_fd: int) -> None: test_runner.warm_up() finally: sys.stdout, sys.stderr = old_stdout, old_stderr - orchestrator_logger.info("Test runner initialized, ready for work") + forkserver_logger.info("Test runner initialized, ready for work") # Set up the SIGCHLD pipe AFTER warm_up: libraries like gevent may # monkey-patch signals during import/collection. sigchld_pipe_r, sigchld_pipe_w = self._setup_sigchld_pipe() - orchestrator_logger.debug("SIGCHLD notification pipe set up") + forkserver_logger.debug("SIGCHLD notification pipe set up") - running: dict[int, HotForkRunner.RunningChild] = {} + running: dict[int, ForkServerRunner.RunningChild] = {} def terminate(signum: int, frame: Any) -> None: # The parent SIGTERMs us on Ctrl-C. Take the grandchildren down too, @@ -530,15 +531,15 @@ def terminate(signum: int, frame: Any) -> None: signal.signal(signal.SIGTERM, terminate) while True: - self._reap_children(running, result_fd, orchestrator_logger, sigchld_pipe_r, block=False) + self._reap_children(running, result_fd, forkserver_logger, sigchld_pipe_r, block=False) while len(running) >= self.max_workers: - self._reap_children(running, result_fd, orchestrator_logger, sigchld_pipe_r, block=True, timeout=1.0) + self._reap_children(running, result_fd, forkserver_logger, sigchld_pipe_r, block=True, timeout=1.0) readable, _, _ = select.select([work_fd, sigchld_pipe_r], [], [], 1.0) if sigchld_pipe_r in readable: - self._reap_children(running, result_fd, orchestrator_logger, sigchld_pipe_r, block=False) + self._reap_children(running, result_fd, forkserver_logger, sigchld_pipe_r, block=False) if work_fd not in readable: continue @@ -557,7 +558,7 @@ def terminate(signum: int, frame: Any) -> None: # code can burn N*wall CPU seconds across N cores, so half the CPU # limit is a reasonable wall bound. wall_timeout = cpu_time_limit / 2 - orchestrator_logger.debug( + forkserver_logger.debug( f"Received mutant: {mutant_name} ({len(tests)} tests, " f"cpu_limit={cpu_time_limit}s, wall_timeout={wall_timeout}s)" ) @@ -565,7 +566,7 @@ def terminate(signum: int, frame: Any) -> None: child_pid = os.fork() if child_pid == 0: # Grandchild: run this mutant's tests under a CPU limit. - worker_logger = get_logger(f"mutmut.hotfork.worker.{os.getpid()}") + worker_logger = get_logger(f"mutmut.forkserver.worker.{os.getpid()}") worker_logger.debug(f"Starting {mutant_name} ({len(tests)} tests)") sys.stdout = sys.stderr = open(os.devnull, "w") @@ -586,22 +587,22 @@ def terminate(signum: int, frame: Any) -> None: # A background thread sends SIGXCPU when the wall timeout expires. register_timeout(child_pid, wall_timeout) - orchestrator_logger.info("Work queue exhausted, waiting for remaining children") + forkserver_logger.info("Work queue exhausted, waiting for remaining children") while running: - self._reap_children(running, result_fd, orchestrator_logger, sigchld_pipe_r, block=True, timeout=1.0) + self._reap_children(running, result_fd, forkserver_logger, sigchld_pipe_r, block=True, timeout=1.0) try: os.close(sigchld_pipe_r) os.close(sigchld_pipe_w) except OSError: pass - orchestrator_logger.info("Orchestrator shutting down cleanly") + forkserver_logger.info("Fork server shutting down cleanly") def _reap_children( self, running: dict[int, RunningChild], result_fd: int, - orchestrator_logger: logging.Logger, + forkserver_logger: logging.Logger, sigchld_pipe_r: int, block: bool, timeout: float | None = None, @@ -629,18 +630,18 @@ def _reap_children( child = running.pop(pid) exit_code = os.waitstatus_to_exitcode(status) duration = time.time() - child.start_time - orchestrator_logger.debug(f"Completed {child.mutant_name}: exit={exit_code} ({duration:.3f}s)") + forkserver_logger.debug(f"Completed {child.mutant_name}: exit={exit_code} ({duration:.3f}s)") send_message(result_fd, (child.mutant_name, exit_code)) def submit(self, mutant_name: str, tests: list[str], cpu_time_limit: int, estimated_time: float) -> None: if not tests: # Matches ForkRunner: nothing to run, so resolve it here as "no tests" - # rather than paying for a round trip through the orchestrator. + # rather than paying for a round trip through the fork server. self._no_tests_results.append(MutantResult(mutant_name=mutant_name, exit_code=33, duration=0.0)) return if self.work_pipe_write is None: - raise RuntimeError("HotForkRunner not started - call startup() first") + raise RuntimeError("ForkServerRunner not started - call startup() first") send_message(self.work_pipe_write, (mutant_name, list(tests), cpu_time_limit)) self._pending.add(mutant_name) self._pending_work[mutant_name] = (list(tests), cpu_time_limit, estimated_time, datetime.now()) @@ -649,7 +650,7 @@ def has_capacity(self) -> bool: return len(self._pending) < self.max_workers def signal_work_complete(self) -> None: - """Close the work pipe so the orchestrator sees EOF and drains workers.""" + """Close the work pipe so the fork server sees EOF and drains workers.""" if self.work_pipe_write is None: return try: @@ -658,40 +659,40 @@ def signal_work_complete(self) -> None: pass finally: self.work_pipe_write = None - self._logger.debug("Work pipe closed, orchestrator will drain remaining workers") + self._logger.debug("Work pipe closed, fork server will drain remaining workers") - def _check_orchestrator_alive(self) -> None: - """Detect an orchestrator crash and restart it, re-submitting pending work. + def _check_forkserver_alive(self) -> None: + """Detect a fork server crash and restart it, re-submitting pending work. - A clean exit (code 0) is left alone. Raises OrchestratorCrashError once + A clean exit (code 0) is left alone. Raises ForkServerCrashError once the restart budget is exhausted. """ - if self.orchestrator_pid is None: + if self.forkserver_pid is None: return try: - pid, status = os.waitpid(self.orchestrator_pid, os.WNOHANG) - if pid == self.orchestrator_pid: + pid, status = os.waitpid(self.forkserver_pid, os.WNOHANG) + if pid == self.forkserver_pid: exit_code = os.waitstatus_to_exitcode(status) - self.orchestrator_pid = None + self.forkserver_pid = None if exit_code == 0: - self._logger.debug(f"Orchestrator (pid={pid}) exited cleanly") + self._logger.debug(f"Fork server (pid={pid}) exited cleanly") return - self._logger.warning(f"Orchestrator (pid={pid}) crashed with exit code {exit_code}") - self._restart_orchestrator_with_pending_work(exit_code=exit_code) + self._logger.warning(f"Fork server (pid={pid}) crashed with exit code {exit_code}") + self._restart_forkserver_with_pending_work(exit_code=exit_code) except ChildProcessError: - self._logger.warning("Orchestrator process not found") - self.orchestrator_pid = None - self._restart_orchestrator_with_pending_work(exit_code=-1) + self._logger.warning("Fork server process not found") + self.forkserver_pid = None + self._restart_forkserver_with_pending_work(exit_code=-1) def wait_for_result(self, timeout: float | None = None) -> MutantResult: if self._no_tests_results: return self._no_tests_results.pop(0) if self.result_pipe_read is None: - raise RuntimeError("HotForkRunner not started - call startup() first") + raise RuntimeError("ForkServerRunner not started - call startup() first") while True: - self._check_orchestrator_alive() + self._check_forkserver_alive() r, _, _ = select.select([self.result_pipe_read], [], [], timeout or 1.0) if not r: @@ -702,8 +703,8 @@ def wait_for_result(self, timeout: float | None = None) -> MutantResult: try: mutant_name, exit_code = recv_message(self.result_pipe_read) except EOFError as err: - self._check_orchestrator_alive() - raise OrchestratorCrashError(exit_code=-1, lost_mutants=list(self._pending), crash_log=None) from err + self._check_forkserver_alive() + raise ForkServerCrashError(exit_code=-1, lost_mutants=list(self._pending), crash_log=None) from err self._pending.discard(mutant_name) self._pending_work.pop(mutant_name, None) @@ -715,19 +716,19 @@ def pending_count(self) -> int: return len(self._pending) + len(self._no_tests_results) def stop_all_workers(self) -> None: - if self.orchestrator_pid: + if self.forkserver_pid: try: - os.kill(self.orchestrator_pid, signal.SIGTERM) + os.kill(self.forkserver_pid, signal.SIGTERM) except ProcessLookupError: pass def shutdown(self) -> None: - """Close the work pipe, drain remaining results, and reap the orchestrator.""" + """Close the work pipe, drain remaining results, and reap the fork server.""" if self._shutting_down: return self._shutting_down = True - self._logger.info("HotForkRunner shutting down") + self._logger.info("ForkServerRunner shutting down") self.signal_work_complete() @@ -737,12 +738,12 @@ def shutdown(self) -> None: result = self.wait_for_result(timeout=1.0) self._pending.discard(result.mutant_name) self._pending_work.pop(result.mutant_name, None) - except (Empty, OrchestratorCrashError): + except (Empty, ForkServerCrashError): break - if self.orchestrator_pid: + if self.forkserver_pid: try: - os.waitpid(self.orchestrator_pid, 0) + os.waitpid(self.forkserver_pid, 0) except ChildProcessError: pass @@ -753,7 +754,7 @@ def shutdown(self) -> None: pass gc.unfreeze() - self._logger.info("HotForkRunner shutdown complete") + self._logger.info("ForkServerRunner shutdown complete") def collect_stats(self, tests: Iterable[str] | None) -> int: """Collect stats in a forked child so the parent never imports pytest. @@ -816,7 +817,7 @@ class ForkRunner(MutantRunner): This is the traditional mutmut approach - fast, but it can misbehave with libraries like gevent, grpc, and torch when forking from a parent process - that has already imported test code. For those, use HotForkRunner instead. + that has already imported test code. For those, use ForkServerRunner instead. """ def __init__(self, max_workers: int, test_runner: TestRunner, debug: bool = False) -> None: @@ -882,7 +883,7 @@ def pending_count(self) -> int: return len(self._running) + len(self._no_tests_results) def signal_work_complete(self) -> None: - """No-op for ForkRunner (there is no orchestrator to signal).""" + """No-op for ForkRunner (there is no fork server to signal).""" def stop_all_workers(self) -> None: for pid in list(self._running): @@ -925,13 +926,13 @@ def get_mutant_runner(max_workers: int = 1) -> MutantRunner: if max_workers < 1: raise ValueError("max_workers must be at least 1") - if config().process_isolation == ProcessIsolation.HOT_FORK: - return HotForkRunner( + if config().process_isolation == ProcessIsolation.FORKSERVER: + return ForkServerRunner( max_workers=max_workers, test_runner_class=PytestRunner, test_runner_args={}, debug=config().debug, - max_restarts=config().max_orchestrator_restarts, + max_restarts=config().max_forkserver_restarts, ) pytest_runner = PytestRunner() diff --git a/tests/e2e/test_e2e_forkserver.py b/tests/e2e/test_e2e_forkserver.py new file mode 100644 index 00000000..b38bdb4b --- /dev/null +++ b/tests/e2e/test_e2e_forkserver.py @@ -0,0 +1,18 @@ +from inline_snapshot import snapshot + +from tests.e2e.e2e_utils import run_mutmut_on_project + + +def test_forkserver_result_snapshot(): + """The forkserver_basic project sets ``process_isolation = "forkserver"``.""" + assert run_mutmut_on_project("forkserver_basic") == snapshot( + { + "mutants/src/fs_calc/__init__.py.meta": { + "fs_calc.x_add__mutmut_1": 1, + "fs_calc.x_sub__mutmut_1": 1, + "fs_calc.x_mul__mutmut_1": 1, + "fs_calc.x_untested__mutmut_1": 33, + "fs_calc.x_untested__mutmut_2": 33, + } + } + ) diff --git a/tests/e2e/test_e2e_hot_fork.py b/tests/e2e/test_e2e_hot_fork.py deleted file mode 100644 index 2b715290..00000000 --- a/tests/e2e/test_e2e_hot_fork.py +++ /dev/null @@ -1,54 +0,0 @@ -"""End-to-end smoke test for the hot-fork process-isolation runner. - -The hot-fork orchestrator forks itself and installs signal handlers, so it is -exercised in a subprocess (with a hard timeout) rather than in-process, keeping -any hang or signal interaction out of the pytest session running this test. -""" - -import json -import shutil -import subprocess -import sys -from collections import Counter -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).resolve().parent.parent.parent -E2E_PROJECTS = REPO_ROOT / "e2e_projects" - - -def _read_verdicts(project_path: Path) -> dict[str, int]: - verdicts: dict[str, int] = {} - for meta in sorted((project_path / "mutants").rglob("*.meta")): - data = json.loads(meta.read_text()) - verdicts.update(data["exit_code_by_key"]) - return verdicts - - -def test_hot_fork_runs_end_to_end(tmp_path: Path): - project = tmp_path / "hot_fork_basic" - shutil.copytree(E2E_PROJECTS / "hot_fork_basic", project) - - result = subprocess.run( - [sys.executable, "-m", "mutmut", "run"], - cwd=project, - capture_output=True, - text=True, - timeout=240, - ) - - assert result.returncode == 0, f"mutmut run failed:\n{result.stdout}\n{result.stderr}" - - verdicts = _read_verdicts(project) - assert verdicts, "no mutant results were written" - - counts = Counter(verdicts.values()) - # 0 = survived, 1 = killed, 33 = no tests. The tested functions (add/sub/mul) - # must produce some killed mutants, and `untested` must produce no-tests ones. - assert counts.get(1, 0) > 0, f"expected some killed mutants, got {dict(counts)}" - assert counts.get(33, 0) > 0, f"expected some no-tests mutants, got {dict(counts)}" - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/tests/mutation/test_mutation.py b/tests/mutation/test_mutation.py index c885b726..0ff0d198 100644 --- a/tests/mutation/test_mutation.py +++ b/tests/mutation/test_mutation.py @@ -28,7 +28,7 @@ from mutmut.__main__ import git_tracked_non_py_files from mutmut.__main__ import run_forced_fail_test from mutmut.configuration import Config -from mutmut.configuration import HotForkWarmup +from mutmut.configuration import ForkServerWarmup from mutmut.configuration import ProcessIsolation from mutmut.core import MutmutProgrammaticFailException from mutmut.mutation.data import MutantLineSpans @@ -1627,8 +1627,8 @@ def _config_for_invalidation(**overrides): on_dependency_change="warn", use_git_change_detection=True, process_isolation=ProcessIsolation.FORK, - hot_fork_warmup=HotForkWarmup.COLLECT, - max_orchestrator_restarts=3, + forkserver_warmup=ForkServerWarmup.COLLECT, + max_forkserver_restarts=3, preload_modules_file=None, log_to_file=False, log_file_path="mutants/mutmut-debug.log", diff --git a/tests/test_configuration.py b/tests/test_configuration.py index a2df6374..33a7438b 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -3,7 +3,7 @@ import pytest from mutmut.configuration import Config -from mutmut.configuration import HotForkWarmup +from mutmut.configuration import ForkServerWarmup from mutmut.configuration import ProcessIsolation from mutmut.configuration import _config_reader from mutmut.configuration import _guess_source_paths @@ -57,22 +57,22 @@ def test_defaults(self, in_tmp_dir: Path): self._write_pyproject(in_tmp_dir, "") cfg = config() assert cfg.process_isolation == ProcessIsolation.FORK - assert cfg.hot_fork_warmup == HotForkWarmup.COLLECT - assert cfg.max_orchestrator_restarts == 3 + assert cfg.forkserver_warmup == ForkServerWarmup.COLLECT + assert cfg.max_forkserver_restarts == 3 assert cfg.preload_modules_file is None - def test_hot_fork_selected(self, in_tmp_dir: Path): - self._write_pyproject(in_tmp_dir, 'process_isolation = "hot-fork"') - assert config().process_isolation == ProcessIsolation.HOT_FORK + def test_forkserver_selected(self, in_tmp_dir: Path): + self._write_pyproject(in_tmp_dir, 'process_isolation = "forkserver"') + assert config().process_isolation == ProcessIsolation.FORKSERVER def test_invalid_process_isolation_raises(self, in_tmp_dir: Path): self._write_pyproject(in_tmp_dir, 'process_isolation = "bogus"') with pytest.raises(ValueError, match="Invalid process_isolation value"): _load_config() - def test_invalid_hot_fork_warmup_raises(self, in_tmp_dir: Path): - self._write_pyproject(in_tmp_dir, 'hot_fork_warmup = "bogus"') - with pytest.raises(ValueError, match="Invalid hot_fork_warmup value"): + def test_invalid_forkserver_warmup_raises(self, in_tmp_dir: Path): + self._write_pyproject(in_tmp_dir, 'forkserver_warmup = "bogus"') + with pytest.raises(ValueError, match="Invalid forkserver_warmup value"): _load_config() @@ -103,8 +103,8 @@ def _get_config(only_mutate: list[str], do_not_mutate: list[str]) -> Config: on_dependency_change="warn", use_git_change_detection=True, process_isolation=ProcessIsolation.FORK, - hot_fork_warmup=HotForkWarmup.COLLECT, - max_orchestrator_restarts=3, + forkserver_warmup=ForkServerWarmup.COLLECT, + max_forkserver_restarts=3, preload_modules_file=None, log_to_file=False, log_file_path="mutants/mutmut-debug.log", diff --git a/tests/workers/test_isolation.py b/tests/workers/test_isolation.py index c64c8114..ba8becc0 100644 --- a/tests/workers/test_isolation.py +++ b/tests/workers/test_isolation.py @@ -9,8 +9,8 @@ from mutmut.configuration import config from mutmut.configuration import reset_config from mutmut.workers.isolation import ForkRunner -from mutmut.workers.isolation import HotForkRunner -from mutmut.workers.isolation import OrchestratorCrashError +from mutmut.workers.isolation import ForkServerCrashError +from mutmut.workers.isolation import ForkServerRunner from mutmut.workers.isolation import get_mutant_runner from mutmut.workers.isolation import recv_message from mutmut.workers.isolation import run_in_fork @@ -119,17 +119,17 @@ def create_marker(): assert marker.read_text() == "created" -class TestOrchestratorCrashError: - """Tests for OrchestratorCrashError exception.""" +class TestForkServerCrashError: + """Tests for ForkServerCrashError exception.""" def test_error_message_includes_exit_code(self): """Exit code is included in message.""" - err = OrchestratorCrashError(exit_code=1, lost_mutants=[]) + err = ForkServerCrashError(exit_code=1, lost_mutants=[]) assert "exit code: 1" in str(err) def test_error_message_lists_lost_mutants(self): """Lost mutants are listed in message.""" - err = OrchestratorCrashError(exit_code=1, lost_mutants=["mutant_1", "mutant_2"]) + err = ForkServerCrashError(exit_code=1, lost_mutants=["mutant_1", "mutant_2"]) assert "mutant_1" in str(err) assert "mutant_2" in str(err) assert "2 in-flight mutant(s)" in str(err) @@ -137,7 +137,7 @@ def test_error_message_lists_lost_mutants(self): def test_truncates_long_mutant_list(self): """Only first 10 mutants shown, rest summarized.""" mutants = [f"mutant_{i}" for i in range(15)] - err = OrchestratorCrashError(exit_code=1, lost_mutants=mutants) + err = ForkServerCrashError(exit_code=1, lost_mutants=mutants) assert "mutant_0" in str(err) assert "mutant_9" in str(err) @@ -146,42 +146,42 @@ def test_truncates_long_mutant_list(self): def test_includes_resume_instructions(self): """Message includes how to resume.""" - err = OrchestratorCrashError(exit_code=1, lost_mutants=[]) + err = ForkServerCrashError(exit_code=1, lost_mutants=[]) assert "mutmut run" in str(err) def test_includes_crash_log_path(self): """Crash log path is shown if provided.""" - err = OrchestratorCrashError(exit_code=1, lost_mutants=[], crash_log="mutants/.orchestrator-crash.log") - assert ".orchestrator-crash.log" in str(err) + err = ForkServerCrashError(exit_code=1, lost_mutants=[], crash_log="mutants/.forkserver-crash.log") + assert ".forkserver-crash.log" in str(err) def test_attributes_accessible(self): """Exception attributes are accessible.""" - err = OrchestratorCrashError(exit_code=42, lost_mutants=["a", "b"], crash_log="/path/to/log") + err = ForkServerCrashError(exit_code=42, lost_mutants=["a", "b"], crash_log="/path/to/log") assert err.exit_code == 42 assert err.lost_mutants == ["a", "b"] assert err.crash_log == "/path/to/log" def test_empty_lost_mutants(self): """Works correctly with empty lost mutants list.""" - err = OrchestratorCrashError(exit_code=0, lost_mutants=[]) + err = ForkServerCrashError(exit_code=0, lost_mutants=[]) assert "0 in-flight mutant(s)" in str(err) def test_no_crash_log(self): """Works correctly without crash log.""" - err = OrchestratorCrashError(exit_code=1, lost_mutants=["m1"]) + err = ForkServerCrashError(exit_code=1, lost_mutants=["m1"]) # Should not raise and should not include "Crash log:" msg = str(err) assert "Crash log:" not in msg def test_is_exception(self): - """OrchestratorCrashError is an Exception subclass.""" - err = OrchestratorCrashError(exit_code=1, lost_mutants=[]) + """ForkServerCrashError is an Exception subclass.""" + err = ForkServerCrashError(exit_code=1, lost_mutants=[]) assert isinstance(err, Exception) def test_can_be_raised_and_caught(self): """Exception can be raised and caught properly.""" - with pytest.raises(OrchestratorCrashError) as exc_info: - raise OrchestratorCrashError(exit_code=255, lost_mutants=["test_mutant"], crash_log="/tmp/crash.log") + with pytest.raises(ForkServerCrashError) as exc_info: + raise ForkServerCrashError(exit_code=255, lost_mutants=["test_mutant"], crash_log="/tmp/crash.log") assert exc_info.value.exit_code == 255 assert exc_info.value.lost_mutants == ["test_mutant"] @@ -262,12 +262,12 @@ def test_selects_fork_by_default(self, in_project_dir): assert isinstance(runner, ForkRunner) assert runner.max_workers == 2 - def test_selects_hot_fork(self, in_project_dir, monkeypatch): - monkeypatch.setattr(config(), "process_isolation", ProcessIsolation.HOT_FORK) + def test_selects_forkserver(self, in_project_dir, monkeypatch): + monkeypatch.setattr(config(), "process_isolation", ProcessIsolation.FORKSERVER) runner = get_mutant_runner(4) - assert isinstance(runner, HotForkRunner) + assert isinstance(runner, ForkServerRunner) assert runner.max_workers == 4 - assert runner.max_restarts == config().max_orchestrator_restarts + assert runner.max_restarts == config().max_forkserver_restarts def test_rejects_zero_workers(self, in_project_dir): with pytest.raises(ValueError, match="at least 1"): @@ -335,14 +335,14 @@ def test_reports_eof_when_the_writer_closes(self): os.close(r) -class TestHotForkRunnerNoTestMutants: - """HotForkRunner resolves a mutant with no tests the same way ForkRunner does.""" +class TestForkServerRunnerNoTestMutants: + """ForkServerRunner resolves a mutant with no tests the same way ForkRunner does.""" @staticmethod def _runner(): - return HotForkRunner(max_workers=4, test_runner_class=object, test_runner_args={}) + return ForkServerRunner(max_workers=4, test_runner_class=object, test_runner_args={}) - def test_no_test_mutant_is_resolved_without_an_orchestrator(self): + def test_no_test_mutant_is_resolved_without_a_forkserver(self): runner = self._runner() # never started, so there is no work pipe runner.submit("mod.x_foo__mutmut_1", [], cpu_time_limit=1, estimated_time=0.0) assert runner.pending_count() == 1 From 6766645afafc374849aab44202a3f8ca32c9112a Mon Sep 17 00:00:00 2001 From: nicklafleur <55208706+nicklafleur@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:31:10 -0400 Subject: [PATCH 5/5] fix(coverage): gather coverage in a forked child (#528) The coverage pre-run used to import the test suite into the main process and then evict every module it had loaded, so the stats run imported them a second time. gather_coverage now runs in a throwaway fork and sends back only the measured line numbers. Also: - run_in_fork_with_result carries the child's traceback across the pipe - e2e regression guard: a dependency that refuses a second init per process --- ARCHITECTURE.rst | 7 ++ HISTORY.rst | 2 + .../ignore_single_init_dep.py | 21 ++++ .../main/test_mutate_only_covered_lines.py | 5 + src/mutmut/code_coverage.py | 36 +++--- src/mutmut/workers/isolation.py | 71 +++++++----- tests/test_code_coverage.py | 98 ++++++++++++++++ tests/workers/test_isolation.py | 109 ++++++++++++++++++ 8 files changed, 298 insertions(+), 51 deletions(-) create mode 100644 e2e_projects/mutate_only_covered_lines/src/mutate_only_covered_lines/ignore_single_init_dep.py diff --git a/ARCHITECTURE.rst b/ARCHITECTURE.rst index ba64433c..73f11574 100644 --- a/ARCHITECTURE.rst +++ b/ARCHITECTURE.rst @@ -15,6 +15,13 @@ We start by copying ``source_paths`` to ``mutants/`` and then mutate the ``*.py` The mutated files contains the original code and the mutants. The active mutant is stored in process-local state (and mirrored to the ``MUTANT_UNDER_TEST`` environment variable). If a mutant is not enabled, it will run the original code. +When ``mutate_only_covered_lines`` is set, we also run the test suite under coverage.py +first, to find out which lines are worth mutating. That run happens in a forked child +(``store_lines_covered_by_tests``) which sends back only the line numbers it measured. +Running it in the main process would import the test suite and all of its dependencies +there, and the later phases import the same modules again -- which some native extensions +do not survive. + Collecting tests and stats ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/HISTORY.rst b/HISTORY.rst index 87d12341..1d75f1e5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,8 @@ Unreleased * Fix mutants stopped by the CPU-time limit being reported as killed instead of timed out (`#565`) +* Fix ``mutate_only_covered_lines`` breaking when the test suite imports a dependency that cannot be initialized twice in one process, which raised for numpy and libyaml, produced misleading type errors for cryptography, and segfaulted for asyncpg. Coverage is now gathered in a separate process, so the modules the coverage run imports are no longer unloaded and re-imported in the main process (`#528`, `#566`) + * Mutate the condition of ternary expressions (`#196`, `#546`) * Add a ``process_isolation`` config for choosing which process mutant workers are forked from. The new ``forkserver`` strategy keeps mutmut's main process free of pytest and your ``conftest.py``, routing every fork through a dedicated fork server process, for test setups that are not fork-safe (``gevent``, ``grpc``, ``torch``). Tune what it preloads with ``forkserver_warmup`` (`#578`, `#566`) diff --git a/e2e_projects/mutate_only_covered_lines/src/mutate_only_covered_lines/ignore_single_init_dep.py b/e2e_projects/mutate_only_covered_lines/src/mutate_only_covered_lines/ignore_single_init_dep.py new file mode 100644 index 00000000..8853d72b --- /dev/null +++ b/e2e_projects/mutate_only_covered_lines/src/mutate_only_covered_lines/ignore_single_init_dep.py @@ -0,0 +1,21 @@ +"""Stands in for a dependency that cannot be initialized twice in one process. + +Regression guard for #528, where numpy and libyaml raise, cryptography fails a type +check and asyncpg segfaults. Named to match ``do_not_mutate = ["*ignore*"]``, so it is +copied and imported like any dependency but stays out of the result snapshot. +""" + +import os + +# Outside the module, or it would be evicted with it. Keyed by pid so forks, which +# inherit a copy of the environment, each get their own initialization. +_INITIALIZED_IN_PID = "MUTMUT_E2E_SINGLE_INIT_DEP_PID" + +if os.environ.get(_INITIALIZED_IN_PID) == str(os.getpid()): + raise ImportError("cannot load module more than once per process") + +os.environ[_INITIALIZED_IN_PID] = str(os.getpid()) + + +def single_init_value(): + return 42 diff --git a/e2e_projects/mutate_only_covered_lines/tests/main/test_mutate_only_covered_lines.py b/e2e_projects/mutate_only_covered_lines/tests/main/test_mutate_only_covered_lines.py index 3e68e49c..44af91e7 100644 --- a/e2e_projects/mutate_only_covered_lines/tests/main/test_mutate_only_covered_lines.py +++ b/e2e_projects/mutate_only_covered_lines/tests/main/test_mutate_only_covered_lines.py @@ -1,4 +1,6 @@ from mutate_only_covered_lines.ignore_me import this_function_shall_NOT_be_mutated +# Raises on a second import in the same process; see the module docstring (#528). +from mutate_only_covered_lines.ignore_single_init_dep import single_init_value from mutate_only_covered_lines import hello_mutate_only_covered_lines, mutate_only_covered_lines_multiline, function_with_pragma, do_not_mutate_external_ommited_function from mutate_only_covered_lines.exclude_me import excluded_function, excluded_branch, excluded_by_coverage_config, excluded_case @@ -33,3 +35,6 @@ def test_excluded_by_coverage_config(): def test_excluded_case(): assert excluded_case("a") == 1 assert excluded_case("b") == 2 + +def test_single_init_dep(): + assert single_init_value() == 42 diff --git a/src/mutmut/code_coverage.py b/src/mutmut/code_coverage.py index 9992b950..235e5536 100644 --- a/src/mutmut/code_coverage.py +++ b/src/mutmut/code_coverage.py @@ -1,12 +1,9 @@ from __future__ import annotations -import importlib -import sys from collections.abc import Iterable from dataclasses import dataclass from dataclasses import field from pathlib import Path -from types import ModuleType from typing import TYPE_CHECKING import coverage @@ -59,15 +56,22 @@ def get_excluded_lines_for_file(filename: str, excluded_lines: dict[str, set[int return set(excluded_lines.get(abs_filename, ())) -# Gathers coverage for the given source files and -# Returns the covered and excluded lines of each of them -# Since this is run on the source files before we create mutations, -# we need to unload any modules that get loaded during the test run def gather_coverage(runner: TestRunner, source_files: Iterable[Path]) -> CoverageInfo: - # We want to unload any python modules that get loaded - # because we plan to mutate them and want them to be reloaded - modules = dict(sys.modules) + """Gathers coverage for the given source files in a throwaway child process. + Returns the covered and excluded lines of each of them. Forking keeps the test + suite's imports out of the main process, which imports the same modules again + later and cannot always survive it (#528). + """ + # TODO: (#397) mutmut.workers.isolation imports POSIX-only modules + from mutmut.workers.isolation import run_in_fork_with_result + + result: CoverageInfo = run_in_fork_with_result(_gather_coverage_in_this_process, runner, source_files) + return result + + +def _gather_coverage_in_this_process(runner: TestRunner, source_files: Iterable[Path]) -> CoverageInfo: + """The actual measurement. Pollutes the calling process; see gather_coverage.""" mutants_path = Path("mutants") # Run the tests and gather coverage @@ -85,8 +89,6 @@ def gather_coverage(runner: TestRunner, source_files: Iterable[Path]) -> Coverag info.covered_lines[abs_filename] = set(coverage_data.lines(abs_filename) or []) info.excluded_lines[abs_filename] = _excluded_lines(cov, abs_filename) - _unload_modules_not_in(modules) - return info @@ -101,13 +103,3 @@ def _excluded_lines(cov: coverage.Coverage, abs_filename: str) -> set[int]: return set() return set(excluded) - - -# Unloads modules that are not in the 'modules' list -def _unload_modules_not_in(modules: dict[str, ModuleType]) -> None: - for name in list(sys.modules): - if name == "mutmut.code_coverage": - continue - if name not in modules: - sys.modules.pop(name, None) - importlib.invalidate_caches() diff --git a/src/mutmut/workers/isolation.py b/src/mutmut/workers/isolation.py index cf2a3f7c..1a308ccb 100644 --- a/src/mutmut/workers/isolation.py +++ b/src/mutmut/workers/isolation.py @@ -36,6 +36,7 @@ from time import process_time from typing import Any from typing import NamedTuple +from typing import TypeVar from mutmut.configuration import ProcessIsolation from mutmut.configuration import config @@ -51,6 +52,16 @@ from mutmut.utils.safe_setproctitle import safe_setproctitle as setproctitle from mutmut.workers.timeout import register_timeout +T = TypeVar("T") + + +def _with_child_traceback(message: str, data: dict[str, Any] | None) -> str: + """Append the child's traceback to *message* when the child sent one""" + child_traceback = (data or {}).get("traceback") + if not child_traceback: + return message + return f"{message}\n\nTraceback from the child process:\n{child_traceback}" + def run_in_fork_with_result(fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: """Fork a child to run a function, return its result via pipe. @@ -83,7 +94,7 @@ def run_in_fork_with_result(fn: Callable[..., Any], *args: Any, **kwargs: Any) - except Exception as e: try: with os.fdopen(write_fd, "wb") as f: - pickle.dump({"ok": False, "error": str(e)}, f) + pickle.dump({"ok": False, "error": str(e), "traceback": traceback.format_exc()}, f) except Exception: pass os._exit(1) @@ -107,10 +118,10 @@ def run_in_fork_with_result(fn: Callable[..., Any], *args: Any, **kwargs: Any) - error_msg = f"Child exited with code {exit_code}" if data and not data.get("ok") and "error" in data: error_msg += f": {data['error']}" - raise ChildProcessError(error_msg) + raise ChildProcessError(_with_child_traceback(error_msg, data)) if not data.get("ok"): - raise ChildProcessError(f"Child failed: {data.get('error', 'unknown')}") + raise ChildProcessError(_with_child_traceback(f"Child failed: {data.get('error', 'unknown')}", data)) return data["value"] @@ -343,7 +354,7 @@ class RunningChild(NamedTuple): def __init__( self, max_workers: int, - test_runner_class: type, + test_runner_class: type[TestRunner], test_runner_args: dict[str, Any], debug: bool = False, max_restarts: int | None = None, @@ -756,16 +767,30 @@ def shutdown(self) -> None: gc.unfreeze() self._logger.info("ForkServerRunner shutdown complete") - def collect_stats(self, tests: Iterable[str] | None) -> int: - """Collect stats in a forked child so the parent never imports pytest. + # All test-running operations go through these two, so the parent only ever holds + # test_runner_class and never imports pytest or a conftest itself. + def _run_in_child(self, operation: Callable[[TestRunner], T]) -> T: + """Run `operation` against a fresh TestRunner in a child, return its result.""" - The child runs stats and packs the collected mapping into a StatsResult, - which the parent merges back into ``state()``. - """ + def in_child() -> T: + return operation(self.test_runner_class(**self.test_runner_args)) + + result: T = run_in_fork_with_result(in_child) + return result + + def _exit_code_from_child(self, operation: Callable[[TestRunner], int]) -> int: + """Run `operation` against a fresh TestRunner in a child, return its code.""" + + def in_child() -> int: + return operation(self.test_runner_class(**self.test_runner_args)) + + return run_in_fork(in_child) + + def collect_stats(self, tests: Iterable[str] | None) -> int: + """Collect stats in a child, then merge what it measured back into state().""" tests_list = list(tests) if tests is not None else None - def _run_stats() -> dict[str, Any]: - child_runner: TestRunner = self.test_runner_class(**self.test_runner_args) + def run_stats(child_runner: TestRunner) -> dict[str, Any]: exit_code = child_runner.run_stats(tests=tests_list or []) return StatsResult( exit_code=exit_code, @@ -775,7 +800,7 @@ def _run_stats() -> dict[str, Any]: function_dependencies=dict(state().function_dependencies), ).to_dict() - result = StatsResult.from_dict(run_in_fork_with_result(_run_stats)) + result = StatsResult.from_dict(self._run_in_child(run_stats)) for k, v in result.tests_by_mangled_function_name.items(): state().tests_by_mangled_function_name[k] |= v @@ -788,27 +813,15 @@ def _run_stats() -> dict[str, Any]: def run_clean_tests(self, tests: Iterable[str]) -> int: tests_list = list(tests) - - def _run_tests() -> int: - child_runner: TestRunner = self.test_runner_class(**self.test_runner_args) - return child_runner.run_tests(mutant_name=None, tests=tests_list) - - return run_in_fork(_run_tests) + return self._exit_code_from_child( + lambda child_runner: child_runner.run_tests(mutant_name=None, tests=tests_list) + ) def run_forced_fail(self) -> int: - def _run_forced_fail() -> int: - child_runner: TestRunner = self.test_runner_class(**self.test_runner_args) - return child_runner.run_forced_fail() - - return run_in_fork(_run_forced_fail) + return self._exit_code_from_child(lambda child_runner: child_runner.run_forced_fail()) def list_all_tests(self) -> ListAllTestsResult: - def _list_all_tests() -> dict[str, Any]: - child_runner: TestRunner = self.test_runner_class(**self.test_runner_args) - result = child_runner.list_all_tests() - return {"ids": list(result.ids)} - - data = run_in_fork_with_result(_list_all_tests) + data = self._run_in_child(lambda child_runner: {"ids": list(child_runner.list_all_tests().ids)}) return ListAllTestsResult(ids=set(data["ids"])) diff --git a/tests/test_code_coverage.py b/tests/test_code_coverage.py index b3a357d5..92807fe3 100644 --- a/tests/test_code_coverage.py +++ b/tests/test_code_coverage.py @@ -1,6 +1,15 @@ +import importlib +import os +import sys +from pathlib import Path + import coverage +import pytest +from mutmut.code_coverage import CoverageInfo from mutmut.code_coverage import _excluded_lines +from mutmut.code_coverage import _gather_coverage_in_this_process +from mutmut.code_coverage import gather_coverage from mutmut.code_coverage import get_excluded_lines_for_file @@ -49,3 +58,92 @@ def test_excluded_lines_honours_the_projects_coverage_config(tmp_path, monkeypat excluded = _excluded_lines(coverage.Coverage(data_file=None), str(source_file)) assert excluded == {2, 3} + + +class _RunnerThatImports: + """Stands in for a TestRunner whose test run imports a dependency. + + The measurement only ever calls ``collect_main_test_coverage``, so this is + enough to observe what the run leaves behind in ``sys.modules``. + """ + + def __init__(self, module_name: str) -> None: + self.module_name = module_name + + def collect_main_test_coverage(self, cov: coverage.Coverage) -> int: + importlib.import_module(self.module_name) + return 0 + + +def _minimal_project(tmp_path, monkeypatch) -> None: + """chdir somewhere config() can load; importing isolation reads it at import time.""" + (tmp_path / "src").mkdir(exist_ok=True) + monkeypatch.chdir(tmp_path) + + +def _importable_module(tmp_path, monkeypatch, name: str) -> str: + (tmp_path / f"{name}.py").write_text("value = 1\n") + monkeypatch.syspath_prepend(str(tmp_path)) + _minimal_project(tmp_path, monkeypatch) + return name + + +def test_the_measurement_leaves_the_modules_it_imported_loaded(tmp_path, monkeypatch): + """Regression test for #528: the eviction that broke numpy and friends is gone.""" + name = _importable_module(tmp_path, monkeypatch, "imported_by_the_coverage_run") + try: + _gather_coverage_in_this_process(_RunnerThatImports(name), []) + assert name in sys.modules + finally: + sys.modules.pop(name, None) + + +@pytest.mark.skipif(os.name == "nt", reason="Forking not supported on Windows") +class TestGatherCoverageIsolatesTheCaller: + """The other half of #528: dropping the eviction is only safe if the imports never land here.""" + + def test_results_come_back_but_imports_do_not(self, tmp_path, monkeypatch): + name = _importable_module(tmp_path, monkeypatch, "imported_by_the_test_run") + + info = gather_coverage(_RunnerThatImports(name), []) + + assert isinstance(info, CoverageInfo) + # The import the run did never reached this process. + assert name not in sys.modules + + def test_measurements_cross_the_fork_boundary(self, tmp_path, monkeypatch): + """A real file's covered and excluded lines survive the round trip.""" + _minimal_project(tmp_path, monkeypatch) + (tmp_path / "mutants").mkdir() + (tmp_path / "mutants" / "measured.py").write_text( + "def covered():\n return 1\n\ndef never_called(): # pragma: no cover\n return 2\n" + ) + monkeypatch.syspath_prepend(str(tmp_path / "mutants")) + + class _RunnerThatRunsTheCode: + def collect_main_test_coverage(self, cov: coverage.Coverage) -> int: + with cov.collect(): + import measured + + measured.covered() + return 0 + + info = gather_coverage(_RunnerThatRunsTheCode(), [Path("measured.py")]) + + key = str((tmp_path / "mutants" / "measured.py").absolute()) + # Both `def` lines run at import; only never_called's body (5) does not. + assert info.covered_lines[key] == {1, 2, 4} + # Not an exact match: which lines of an excluded statement get reported is a + # coverage.py version detail. All this test needs is that they came back. + assert 4 in info.excluded_lines[key] + + def test_child_failure_is_reported_to_the_caller(self, tmp_path, monkeypatch): + """A measurement that blows up must not look like empty coverage.""" + _minimal_project(tmp_path, monkeypatch) + + class _RunnerThatFails: + def collect_main_test_coverage(self, cov: coverage.Coverage) -> int: + raise RuntimeError("conftest exploded") + + with pytest.raises(ChildProcessError, match="conftest exploded"): + gather_coverage(_RunnerThatFails(), []) diff --git a/tests/workers/test_isolation.py b/tests/workers/test_isolation.py index ba8becc0..26eb66d4 100644 --- a/tests/workers/test_isolation.py +++ b/tests/workers/test_isolation.py @@ -1,13 +1,19 @@ """Tests for fork isolation utilities.""" import os +import sys import threading +from pathlib import Path +from types import ModuleType import pytest from mutmut.configuration import ProcessIsolation from mutmut.configuration import config from mutmut.configuration import reset_config +from mutmut.runners.harness import ListAllTestsResult +from mutmut.state import reset_state +from mutmut.state import state from mutmut.workers.isolation import ForkRunner from mutmut.workers.isolation import ForkServerCrashError from mutmut.workers.isolation import ForkServerRunner @@ -72,6 +78,22 @@ def crash(): assert "specific error message" in str(exc_info.value) + def test_child_crash_includes_child_traceback(self): + """Without this the parent only reports where it forked, not what failed.""" + + def crash(): + def inner_frame_with_a_distinctive_name(): + raise RuntimeError("boom") + + inner_frame_with_a_distinctive_name() + + with pytest.raises(ChildProcessError) as exc_info: + run_in_fork_with_result(crash) + + message = str(exc_info.value) + assert "Traceback from the child process:" in message + assert "inner_frame_with_a_distinctive_name" in message + def test_no_temp_files_created(self, tmp_path, monkeypatch): """Pipe-based transport doesn't create temp files.""" monkeypatch.chdir(tmp_path) @@ -353,3 +375,90 @@ def test_no_test_mutant_reports_exit_code_33(self): result = runner.wait_for_result() assert (result.mutant_name, result.exit_code) == ("mod.x_foo__mutmut_1", 33) assert runner.pending_count() == 0 + + +class _RecordingTestRunner: + """A TestRunner that records its pid to disk and dirties its own sys.modules. + + The pid file is visible to the parent either way; the marker module only if the + operation ran in the parent. + """ + + MARKER_MODULE = "imported_by_the_test_runner" + pid_file = "" # Set by the test before the operation runs; inherited by the fork. + + def _record(self) -> None: + sys.modules[self.MARKER_MODULE] = ModuleType(self.MARKER_MODULE) + Path(self.pid_file).write_text(str(os.getpid())) + + def run_stats(self, tests): + self._record() + state().duration_by_test["test_from_the_child"] = 1.5 + state().tests_by_mangled_function_name["mod.x_foo"].add("test_from_the_child") + return 0 + + def run_tests(self, *, mutant_name, tests): + self._record() + return 0 + + def run_forced_fail(self): + self._record() + return 1 + + def list_all_tests(self): + self._record() + return ListAllTestsResult(ids={"test_from_the_child"}) + + +@pytest.mark.skipif(os.name == "nt", reason="Forking not supported on Windows") +class TestForkServerRunnerNeverRunsTestsInTheMainProcess: + """Under forkserver, no test execution may happen in the main process. + + A conftest that calls gevent.monkey.patch_all() must not run where we fork from. + """ + + @pytest.fixture(autouse=True) + def runner(self, tmp_path, monkeypatch): + monkeypatch.setattr(_RecordingTestRunner, "pid_file", str(tmp_path / "ran_in_pid")) + self.pid_file = tmp_path / "ran_in_pid" + reset_state() + yield + sys.modules.pop(_RecordingTestRunner.MARKER_MODULE, None) + + def _runner(self): + return ForkServerRunner(max_workers=1, test_runner_class=_RecordingTestRunner, test_runner_args={}) + + def _assert_ran_in_a_child(self): + assert self.pid_file.exists(), "the operation never ran" + assert self.pid_file.read_text() != str(os.getpid()), "the operation ran in the main process" + assert _RecordingTestRunner.MARKER_MODULE not in sys.modules, ( + "the test runner's imports leaked into the main process" + ) + + def test_collect_stats_runs_in_a_child(self): + assert self._runner().collect_stats(tests=None) == 0 + self._assert_ran_in_a_child() + + def test_collect_stats_still_merges_what_the_child_measured(self): + self._runner().collect_stats(tests=None) + # The child cannot touch the parent's state(), so these came back over the pipe. + assert state().duration_by_test["test_from_the_child"] == 1.5 + assert state().tests_by_mangled_function_name["mod.x_foo"] == {"test_from_the_child"} + + def test_run_clean_tests_runs_in_a_child(self): + assert self._runner().run_clean_tests(tests=["test_a"]) == 0 + self._assert_ran_in_a_child() + + def test_run_forced_fail_runs_in_a_child(self): + assert self._runner().run_forced_fail() == 1 + self._assert_ran_in_a_child() + + def test_list_all_tests_runs_in_a_child(self): + assert self._runner().list_all_tests().ids == {"test_from_the_child"} + self._assert_ran_in_a_child() + + def test_the_parent_never_holds_a_test_runner_instance(self): + """The parent keeps the class, so it cannot run tests even by mistake.""" + runner = self._runner() + assert runner.test_runner_class is _RecordingTestRunner + assert not any(isinstance(value, _RecordingTestRunner) for value in vars(runner).values())