diff --git a/HISTORY.rst b/HISTORY.rst
index 95a74b65..7b2b9b7d 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -4,6 +4,8 @@ Changelog
 Unreleased
 ~~~~~~~~~~
 
+* Fix obsolete test names never being removed from ``mutants/mutmut-stats.json``: the counters guarding the save iterated dict keys instead of the test sets, so the cleanup was applied in memory but never persisted (`#564`)
+
 * Fix ``# pragma: no mutate block`` being silently ignored when placed on an ``else``, ``except``, ``except*`` or ``finally`` header, and on the ``try`` line of a ``try``/``except*``
 
 * Fix mutants being reported as survived when a test uses ``patch.dict(os.environ, ..., clear=True)`` (`#511`)
diff --git a/src/mutmut/runners/harness.py b/src/mutmut/runners/harness.py
index 6e550afa..f68863f4 100644
--- a/src/mutmut/runners/harness.py
+++ b/src/mutmut/runners/harness.py
@@ -60,7 +60,7 @@ def __init__(self, *, ids: set[str]) -> None:
         self.ids = ids
 
     def clear_out_obsolete_test_names(self) -> None:
-        count_before = sum(len(x) for x in state().tests_by_mangled_function_name)
+        count_before = sum(len(x) for x in state().tests_by_mangled_function_name.values())
         state().tests_by_mangled_function_name = defaultdict(
             set,
             **{
@@ -68,7 +68,7 @@ def clear_out_obsolete_test_names(self) -> None:
                 for k, test_names in state().tests_by_mangled_function_name.items()
             },
         )
-        count_after = sum(len(x) for x in state().tests_by_mangled_function_name)
+        count_after = sum(len(x) for x in state().tests_by_mangled_function_name.values())
         if count_before != count_after:
             print(f"Removed {count_before - count_after} obsolete test names")
             save_stats()
diff --git a/tests/runners/__init__.py b/tests/runners/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/runners/test_harness.py b/tests/runners/test_harness.py
new file mode 100644
index 00000000..d6c36f0e
--- /dev/null
+++ b/tests/runners/test_harness.py
@@ -0,0 +1,56 @@
+"""Tests for the incremental-stats bookkeeping in ``ListAllTestsResult``."""
+
+import json
+from collections import defaultdict
+from pathlib import Path
+
+import pytest
+
+from mutmut.runners.harness import ListAllTestsResult
+from mutmut.state import state
+
+STATS_FILE = Path("mutants/mutmut-stats.json")
+
+
+@pytest.fixture(autouse=True)
+def isolated_stats(tmp_path, monkeypatch):
+    # save_stats() writes mutants/mutmut-stats.json relative to the cwd, and it
+    # loads the config on the way, which needs a guessable source dir to exist.
+    monkeypatch.chdir(tmp_path)
+    (tmp_path / "mutants").mkdir()
+    (tmp_path / "src").mkdir()
+    saved = state().tests_by_mangled_function_name
+    state().tests_by_mangled_function_name = defaultdict(set)
+    yield
+    state().tests_by_mangled_function_name = saved
+
+
+class TestClearOutObsoleteTestNames:
+    def test_obsolete_test_names_are_dropped_and_persisted(self, capsys):
+        state().tests_by_mangled_function_name["pkg.foo.x_add"] |= {
+            "tests/test_foo.py::test_add",
+            "tests/test_gone.py::test_add",
+        }
+        state().tests_by_mangled_function_name["pkg.foo.x_sub"].add("tests/test_gone.py::test_sub")
+
+        ListAllTestsResult(ids={"tests/test_foo.py::test_add"}).clear_out_obsolete_test_names()
+
+        assert dict(state().tests_by_mangled_function_name) == {
+            "pkg.foo.x_add": {"tests/test_foo.py::test_add"},
+            "pkg.foo.x_sub": set(),
+        }
+        assert "Removed 2 obsolete test names" in capsys.readouterr().out
+        on_disk = json.loads(STATS_FILE.read_text())
+        assert on_disk["tests_by_mangled_function_name"] == {
+            "pkg.foo.x_add": ["tests/test_foo.py::test_add"],
+            "pkg.foo.x_sub": [],
+        }
+
+    def test_nothing_obsolete_leaves_stats_file_untouched(self, capsys):
+        state().tests_by_mangled_function_name["pkg.foo.x_add"].add("tests/test_foo.py::test_add")
+
+        ListAllTestsResult(ids={"tests/test_foo.py::test_add"}).clear_out_obsolete_test_names()
+
+        assert dict(state().tests_by_mangled_function_name) == {"pkg.foo.x_add": {"tests/test_foo.py::test_add"}}
+        assert capsys.readouterr().out == ""
+        assert not STATS_FILE.exists()