ci(mutation): mutmut pilot over the stats, gates and cache key

.github/.licenserc.yaml

@@ -45,6 +45,7 @@ header:
     - 'dokimi/requirements.lock.txt'
     - 'dokimi/requirements-docs.lock.txt'
     - 'dokimi/requirements-ci.lock.txt'
+    - 'dokimi/requirements-mutation.lock.txt'
     # Read by agents verbatim: a header would become part of the instructions,
     # persona prompt or skill being measured.
     - 'AGENTS.md'

.github/renovate.json5

@@ -51,7 +51,7 @@
   // and updates the SHA and the tag together.
   "pre-commit": { enabled: true },
   "pip-compile": {
-    managerFilePatterns: ["/(^|/)requirements(-ci)?\\.lock\\.txt$/"],
+    managerFilePatterns: ["/(^|/)requirements(-ci|-mutation)?\\.lock\\.txt$/"],
   },
   // Inside a range: bump the lock only. Outside it: move the range too.
   rangeStrategy: "update-lockfile",

.github/workflows/mutation.yml

@@ -0,0 +1,100 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: mutation
+
+# Mutation pilot: mutmut over the measurement core (the statistics, the gate
+# decisions and the response-cache key; [tool.mutmut] in dokimi/pyproject.toml
+# lists the modules and the tests run against them). A surviving mutant is a
+# change to that code no test notices, so a wrong number would ship silently.
+# Informational: the job reports the score per module and the survivors in the
+# step summary and never fails on them. Weekly and on demand, not per PR: a full
+# run takes tens of minutes. Diff-only runs on PRs are a follow-up.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '17 5 * * 1'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+defaults:
+  run:
+    working-directory: dokimi
+
+jobs:
+  mutmut:
+    name: mutmut (pilot modules)
+    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: '3.13'
+      # The runtime lock for the tests, and mutmut's own lock (requirements-mutation.txt),
+      # which no other workflow installs.
+      - name: Install deps
+        run: pip install --require-hashes -r requirements.lock.txt -r requirements-mutation.lock.txt
+      - name: Run mutmut
+        # A crash of the run is reported below, not failed on. The step's own timeout
+        # sits inside the job's 90 minutes: a run that outlives the step is cancelled
+        # as a failed step, which continue-on-error absorbs, and the summary step
+        # still gets its time. A job-level timeout would fail the whole job instead.
+        timeout-minutes: 75
+        continue-on-error: true
+        run: mutmut run --max-children "$(nproc)"
+      - name: Score and survivors
+        if: always()
+        run: |
+          mutmut results --all true > mutmut-results.txt || true
+          python - <<'EOF' >> "$GITHUB_STEP_SUMMARY"
+          import collections
+
+          counts = collections.defaultdict(collections.Counter)
+          survivors = collections.defaultdict(list)
+          with open("mutmut-results.txt", encoding="utf-8") as fh:
+              for line in fh:
+                  name, sep, status = line.strip().rpartition(": ")
+                  if not sep:
+                      continue
+                  # <module>.x_<function>__mutmut_<n>, or x<sep><class><sep><method>
+                  module = name.rsplit(".", 1)[0]
+                  counts[module][status] += 1
+                  if status == "survived":
+                      survivors[module].append(name)
+
+          print("## Mutation pilot\n")
+          if not counts:
+              print("No results: see the `Run mutmut` step log.")
+          else:
+              print("| Module | Killed | Survived | Other | Score |")
+              print("|---|---:|---:|---:|---:|")
+              for module in sorted(counts):
+                  c = counts[module]
+                  killed, survived = c["killed"], c["survived"]
+                  other = sum(c.values()) - killed - survived
+                  scored = killed + survived
+                  score = f"{100 * killed / scored:.1f}%" if scored else "n/a"
+                  print(f"| `{module}` | {killed} | {survived} | {other} | {score} |")
+              unchecked = sum(c["not checked"] for c in counts.values())
+              unscored = sorted(m for m, c in counts.items() if not c["killed"] + c["survived"])
+              if unchecked or unscored:
+                  print(f"\n> **Incomplete run: do not read the scores as final.** {unchecked} mutants were "
+                        "not checked (a step timeout or a crash cut the run short)"
+                        + (f"; nothing was scored in {', '.join(f'`{m}`' for m in unscored)}" if unscored else "")
+                        + ". See the `Run mutmut` step log.")
+              print("\nOther: no tests, timeout, suspicious or not checked.")
+              print("`mutmut show <name>` prints a mutant's diff.\n")
+              for module in sorted(survivors):
+                  print(f"<details><summary>{module}: {len(survivors[module])} survivors</summary>\n")
+                  print("```")
+                  print("\n".join(survivors[module]))
+                  print("```\n</details>\n")
+          EOF

.github/workflows/pip-audit.yml

@@ -49,7 +49,8 @@ jobs:
       - name: Install tools
         run: pip install --require-hashes -r requirements-ci.lock.txt
       # The locks are the full closure (requirements-docs.lock.txt for the
-      # documentation site, requirements-ci.lock.txt for the tools), so --no-deps --disable-pip audits them as written
+      # documentation site, requirements-ci.lock.txt for the tools,
+      # requirements-mutation.lock.txt for the weekly mutation run), so --no-deps --disable-pip audits them as written
       # without resolving or installing anything. Each --ignore-vuln is
       # an advisory SECURITY.md documents and dokimi/osv-scanner.toml ignores;
       # add or drop them together, in all three.
@@ -60,7 +61,8 @@ jobs:
       - name: pip-audit
         run: |
           sed -e 's/ *\\$//' -e 's/ *;.*//' \
-              requirements.lock.txt requirements-docs.lock.txt requirements-ci.lock.txt | sort -u \
+              requirements.lock.txt requirements-docs.lock.txt requirements-ci.lock.txt \
+              requirements-mutation.lock.txt | sort -u \
             | awk -F'==' '/^[A-Za-z0-9]/ { print > (ENVIRON["RUNNER_TEMP"] "/pins" seen[$1]++ ".txt") }'
           for f in "$RUNNER_TEMP"/pins*.txt; do
             pip-audit -r "$f" --no-deps --disable-pip --progress-spinner off \

.gitignore

@@ -13,6 +13,9 @@ coverage.json
 /dokimi/outcomes-result.json
 /dokimi/journeys-result.json
 /dokimi/journeys-result.md
+# mutmut's working copy and results (`mutmut run`, the mutation workflow)
+/dokimi/mutants/
+/dokimi/mutmut-results.txt
 
 # Local environments and credentials
 .venv/

dokimi/CONTRIBUTING.md

@@ -149,19 +149,36 @@ pre-commit run --all-files                     # the whole tree, as CI's pre-com
 mypy and pytest are not hooks: they need the full environment and pytest takes
 minutes. `make ci-local` runs both; `make hooks` runs mypy before a push.
 
+**Mutation pilot.** mutmut (pinned in `requirements-mutation.txt`, locked apart from the tools lock) mutates the
+measurement core: the statistics in `dokimi/eval/stats.py` and
+`binary_stats.py`, the score, regression, critical-band and trust gates in
+`dokimi/gate/` (not `novelty_gate.py`) and the response-cache key in
+`dokimi/caching.py` (the list is `only_mutate` under `[tool.mutmut]` in
+`pyproject.toml`). A mutant that survives is a change to that code, such as
+a flipped comparison, an off-by-one or a dropped term, that no test notices.
+The `mutation` workflow runs it weekly and on demand and writes the score per
+module and the survivors to the job summary; it never fails a build. From
+`dokimi/`: `mutmut run --max-children 8`, then `mutmut results` and
+`mutmut show <name>` for a survivor's diff. A test that kills a survivor
+asserts the value the mutant changes. Some survivors are equivalent (the mutant
+computes the same result, e.g. a `<` that differs only at a value no input
+reaches); leave those alone. `mutants/` is scratch and git-ignored.
+
 Dependencies install from `requirements.lock.txt`; the docs build's from
 `requirements-docs.lock.txt` on top of it, and the tools CI, `make dev`, the
 release job and the eval image use (`requirements-ci.txt`) from
 `requirements-ci.lock.txt`. That lock also pins the build backend, setuptools:
 package-smoke and the release job build with `python -m build --no-isolation`,
 so the published sdist and wheel are not built by a backend resolved fresh from
-PyPI. When you change a range in
+PyPI. The mutation workflow adds mutmut's closure
+(`requirements-mutation.txt`) from `requirements-mutation.lock.txt`, which
+nothing else installs, and pip-audit audits with the others. When you change a range in
 `requirements*.txt`, regenerate the lock with the command in its header: run it
 from `dokimi/`, and add `--upgrade` to pick up new versions (without it, uv
 keeps the current pins). The command's `--exclude-newer=P7D` skips releases
 less than 7 days old; if a security fix is newer than that, its vulnerability PR
 fails to resolve, so regenerate the lock by hand without the flag. The docs lock
-and the tools lock (`requirements-ci.lock.txt`) are constrained to the main one,
+and the tools locks (`requirements-ci.lock.txt`, `requirements-mutation.lock.txt`) are constrained to the main one,
 so regenerate them (command in each header) after the main lock changes too;
 `tests/test_dependency_locks.py` fails while a pin they share differs.
 

dokimi/SECURITY.md

@@ -45,7 +45,8 @@ The project is pre-1.0; only the latest `main` is supported. Fixes land on
 ## Known advisories in dependencies
 
 `pip-audit` against the locks (`requirements.lock.txt`,
-`requirements-docs.lock.txt` and `requirements-ci.lock.txt`) reports two advisories with no
+`requirements-docs.lock.txt`, `requirements-ci.lock.txt` and
+`requirements-mutation.lock.txt`) reports two advisories with no
 upstream fix as of 2026-10-05:
 
 - **diskcache 5.6.3 — PYSEC-2026-2447** (CVE-2025-69872, GHSA-w8v5-vhqr-4h9v;

dokimi/changelog.d/20261021_160000_napetrov_mutation-pilot.md

@@ -0,0 +1,11 @@
+### Added
+
+- **Mutation pilot over the measurement core** — `.github/workflows/mutation.yml`
+  runs mutmut 3.8.0 (hash-pinned in its own `requirements-mutation.lock.txt`, installed by that workflow alone) weekly and on demand over
+  `eval/stats.py`, `eval/binary_stats.py`, the score, regression, critical-band and
+  trust gates and `caching.py`, and writes the score per module and the surviving
+  mutants to the job summary. It never fails a build. New tests kill the survivors
+  that were real gaps (inclusive thresholds, the Wilcoxon veto, per-condition metric
+  keys, the bootstrap tail index, the cache key's role and sampling parameters), which
+  takes the pilot from 75.3% to 92.0% of mutants killed. Configuration:
+  `[tool.mutmut]` in `pyproject.toml`; usage: CONTRIBUTING.md.

dokimi/pyproject.toml

@@ -266,3 +266,42 @@ filterwarnings = [
 # Test order is shuffled and `random` reseeded per test (pytest-randomly), so a
 # test that leans on another's leftovers fails. The header prints the seed;
 # `--randomly-seed=<seed>` replays that order, `-p no:randomly` keeps file order.
+
+# Mutation pilot (mutmut 3.x, pinned in requirements-mutation.txt; run by
+# .github/workflows/mutation.yml, not by CI). The measurement core: the
+# statistics, the gate decisions and the cache key, where a surviving mutant is
+# a wrong number that ships silently. `source_paths` is what mutmut copies into
+# mutants/ and imports from there, so it is the whole package; `only_mutate` is
+# the pilot scope, and the test selection is the tests of those modules only.
+# gate/novelty_gate.py is left out: evaluate_novelty is one 440-line function,
+# and mutmut 3 writes a copy of the whole function per mutant (the generated
+# file passed 26 MB before the first test ran).
+# Usage, from dokimi/: `mutmut run`, then `mutmut results` / `mutmut show <id>`.
+[tool.mutmut]
+source_paths = ["dokimi"]
+only_mutate = [
+  "dokimi/eval/stats.py",
+  "dokimi/eval/binary_stats.py",
+  "dokimi/gate/critical_bands.py",
+  "dokimi/gate/regression.py",
+  "dokimi/gate/score_gate.py",
+  "dokimi/gate/trust_gate.py",
+  "dokimi/caching.py",
+]
+pytest_add_cli_args = ["-p", "no:randomly", "-p", "no:cacheprovider"]
+pytest_add_cli_args_test_selection = [
+  "tests/test_stats.py",
+  "tests/test_stats_properties.py",
+  "tests/test_binary_stats.py",
+  "tests/test_novelty_stats_point_biserial.py",
+  "tests/test_pr26_properties.py",
+  "tests/test_skill_lift_summary.py",
+  "tests/test_model_compare.py",
+  "tests/test_repo_compare.py",
+  "tests/test_critical_bands.py",
+  "tests/test_regression.py",
+  "tests/test_score_gate.py",
+  "tests/test_gate_call_sites.py",
+  "tests/test_trust_gate.py",
+  "tests/test_caching.py",
+]