diff --git a/HISTORY.rst b/HISTORY.rst
index 8ab9a46f..f591d70e 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -4,6 +4,8 @@ Changelog
 Unreleased
 ~~~~~~~~~~
 
+* Fix obsolete test names never being removed from ``mutants/mutmut-stats.json`` (`#564`)
+
 * Fix ``# pragma: no mutate block`` being silently ignored when placed on an ``else``, ``except``, ``except*`` or ``finally`` header, and on the ``try`` line of a ``try``/``except*`` (`#559`)
 
 * Fix ``# pragma: no mutate`` being silently ignored on ``match`` and ``case`` headers (`#554`)
diff --git a/src/mutmut/runners/harness.py b/src/mutmut/runners/harness.py
index 6e550afa..5282263f 100644
--- a/src/mutmut/runners/harness.py
+++ b/src/mutmut/runners/harness.py
@@ -60,18 +60,18 @@ def __init__(self, *, ids: set[str]) -> None:
         self.ids = ids
 
     def clear_out_obsolete_test_names(self) -> None:
-        count_before = sum(len(x) for x in state().tests_by_mangled_function_name)
+        known = collected_test_names().union(*state().tests_by_mangled_function_name.values())
+        obsolete = known - self.ids
+        if not obsolete:
+            return
         state().tests_by_mangled_function_name = defaultdict(
             set,
-            **{
-                k: {test_name for test_name in test_names if test_name in self.ids}
-                for k, test_names in state().tests_by_mangled_function_name.items()
-            },
+            **{k: test_names - obsolete for k, test_names in state().tests_by_mangled_function_name.items()},
         )
-        count_after = sum(len(x) for x in state().tests_by_mangled_function_name)
-        if count_before != count_after:
-            print(f"Removed {count_before - count_after} obsolete test names")
-            save_stats()
+        for test_name in obsolete:
+            state().duration_by_test.pop(test_name, None)
+        print(f"Removed {len(obsolete)} obsolete test names")
+        save_stats()
 
     def new_tests(self) -> set[str]:
         return self.ids - collected_test_names()
diff --git a/tests/runners/__init__.py b/tests/runners/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/runners/test_harness.py b/tests/runners/test_harness.py
new file mode 100644
index 00000000..5fa3f7bf
--- /dev/null
+++ b/tests/runners/test_harness.py
@@ -0,0 +1,71 @@
+"""Tests for the incremental-stats bookkeeping in ``ListAllTestsResult``."""
+
+import json
+from collections import defaultdict
+from pathlib import Path
+
+import pytest
+
+from mutmut.runners.harness import ListAllTestsResult
+from mutmut.state import state
+
+STATS_FILE = Path("mutants/mutmut-stats.json")
+
+
+@pytest.fixture(autouse=True)
+def isolated_stats(tmp_path, monkeypatch):
+    # save_stats() writes mutants/mutmut-stats.json relative to the cwd, and it
+    # loads the config on the way, which needs a guessable source dir to exist.
+    monkeypatch.chdir(tmp_path)
+    (tmp_path / "mutants").mkdir()
+    (tmp_path / "src").mkdir()
+    saved_tests = state().tests_by_mangled_function_name
+    saved_durations = state().duration_by_test
+    state().tests_by_mangled_function_name = defaultdict(set)
+    state().duration_by_test = defaultdict(float)
+    yield
+    state().tests_by_mangled_function_name = saved_tests
+    state().duration_by_test = saved_durations
+
+
+class TestClearOutObsoleteTestNames:
+    def test_obsolete_test_names_are_dropped_and_persisted(self, capsys):
+        state().tests_by_mangled_function_name["pkg.foo.x_add"] |= {
+            "tests/test_foo.py::test_add",
+            "tests/test_gone.py::test_add",
+        }
+        state().tests_by_mangled_function_name["pkg.foo.x_sub"].add("tests/test_gone.py::test_sub")
+
+        ListAllTestsResult(ids={"tests/test_foo.py::test_add"}).clear_out_obsolete_test_names()
+
+        assert dict(state().tests_by_mangled_function_name) == {
+            "pkg.foo.x_add": {"tests/test_foo.py::test_add"},
+            "pkg.foo.x_sub": set(),
+        }
+        assert "Removed 2 obsolete test names" in capsys.readouterr().out
+        on_disk = json.loads(STATS_FILE.read_text())
+        assert on_disk["tests_by_mangled_function_name"] == {
+            "pkg.foo.x_add": ["tests/test_foo.py::test_add"],
+            "pkg.foo.x_sub": [],
+        }
+
+    def test_obsolete_test_durations_are_dropped_and_persisted(self, capsys):
+        state().tests_by_mangled_function_name["pkg.foo.x_add"].add("tests/test_foo.py::test_add")
+        state().duration_by_test["tests/test_foo.py::test_add"] = 0.1
+        state().duration_by_test["tests/test_gone.py::test_add"] = 0.2
+
+        ListAllTestsResult(ids={"tests/test_foo.py::test_add"}).clear_out_obsolete_test_names()
+
+        assert dict(state().duration_by_test) == {"tests/test_foo.py::test_add": 0.1}
+        assert "Removed 1 obsolete test names" in capsys.readouterr().out
+        on_disk = json.loads(STATS_FILE.read_text())
+        assert on_disk["duration_by_test"] == {"tests/test_foo.py::test_add": 0.1}
+
+    def test_nothing_obsolete_leaves_stats_file_untouched(self, capsys):
+        state().tests_by_mangled_function_name["pkg.foo.x_add"].add("tests/test_foo.py::test_add")
+
+        ListAllTestsResult(ids={"tests/test_foo.py::test_add"}).clear_out_obsolete_test_names()
+
+        assert dict(state().tests_by_mangled_function_name) == {"pkg.foo.x_add": {"tests/test_foo.py::test_add"}}
+        assert capsys.readouterr().out == ""
+        assert not STATS_FILE.exists()