diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml index 14220caa..c59d4106 100644 --- a/.github/.licenserc.yaml +++ b/.github/.licenserc.yaml @@ -45,6 +45,7 @@ header: - 'dokimi/requirements.lock.txt' - 'dokimi/requirements-docs.lock.txt' - 'dokimi/requirements-ci.lock.txt' + - 'dokimi/requirements-mutation.lock.txt' # Read by agents verbatim: a header would become part of the instructions, # persona prompt or skill being measured. - 'AGENTS.md' diff --git a/.github/renovate.json5 b/.github/renovate.json5 index aa27fa19..44bc34df 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -51,7 +51,7 @@ // and updates the SHA and the tag together. "pre-commit": { enabled: true }, "pip-compile": { - managerFilePatterns: ["/(^|/)requirements(-ci)?\\.lock\\.txt$/"], + managerFilePatterns: ["/(^|/)requirements(-ci|-mutation)?\\.lock\\.txt$/"], }, // Inside a range: bump the lock only. Outside it: move the range too. rangeStrategy: "update-lockfile", diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml new file mode 100644 index 00000000..3498547e --- /dev/null +++ b/.github/workflows/mutation.yml @@ -0,0 +1,100 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: mutation + +# Mutation pilot: mutmut over the measurement core (the statistics, the gate +# decisions and the response-cache key; [tool.mutmut] in dokimi/pyproject.toml +# lists the modules and the tests run against them). A surviving mutant is a +# change to that code no test notices, so a wrong number would ship silently. +# Informational: the job reports the score per module and the survivors in the +# step summary and never fails on them. Weekly and on demand, not per PR: a full +# run takes tens of minutes. Diff-only runs on PRs are a follow-up. +on: + workflow_dispatch: + schedule: + - cron: '17 5 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +defaults: + run: + working-directory: dokimi + +jobs: + mutmut: + name: mutmut (pilot modules) + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} + timeout-minutes: 90 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + # The runtime lock for the tests, and mutmut's own lock (requirements-mutation.txt), + # which no other workflow installs. + - name: Install deps + run: pip install --require-hashes -r requirements.lock.txt -r requirements-mutation.lock.txt + - name: Run mutmut + # A crash of the run is reported below, not failed on. The step's own timeout + # sits inside the job's 90 minutes: a run that outlives the step is cancelled + # as a failed step, which continue-on-error absorbs, and the summary step + # still gets its time. A job-level timeout would fail the whole job instead. + timeout-minutes: 75 + continue-on-error: true + run: mutmut run --max-children "$(nproc)" + - name: Score and survivors + if: always() + run: | + mutmut results --all true > mutmut-results.txt || true + python - <<'EOF' >> "$GITHUB_STEP_SUMMARY" + import collections + + counts = collections.defaultdict(collections.Counter) + survivors = collections.defaultdict(list) + with open("mutmut-results.txt", encoding="utf-8") as fh: + for line in fh: + name, sep, status = line.strip().rpartition(": ") + if not sep: + continue + # .x___mutmut_, or x + module = name.rsplit(".", 1)[0] + counts[module][status] += 1 + if status == "survived": + survivors[module].append(name) + + print("## Mutation pilot\n") + if not counts: + print("No results: see the `Run mutmut` step log.") + else: + print("| Module | Killed | Survived | Other | Score |") + print("|---|---:|---:|---:|---:|") + for module in sorted(counts): + c = counts[module] + killed, survived = c["killed"], c["survived"] + other = sum(c.values()) - killed - survived + scored = killed + survived + score = f"{100 * killed / scored:.1f}%" if scored else "n/a" + print(f"| `{module}` | {killed} | {survived} | {other} | {score} |") + unchecked = sum(c["not checked"] for c in counts.values()) + unscored = sorted(m for m, c in counts.items() if not c["killed"] + c["survived"]) + if unchecked or unscored: + print(f"\n> **Incomplete run: do not read the scores as final.** {unchecked} mutants were " + "not checked (a step timeout or a crash cut the run short)" + + (f"; nothing was scored in {', '.join(f'`{m}`' for m in unscored)}" if unscored else "") + + ". See the `Run mutmut` step log.") + print("\nOther: no tests, timeout, suspicious or not checked.") + print("`mutmut show ` prints a mutant's diff.\n") + for module in sorted(survivors): + print(f"
{module}: {len(survivors[module])} survivors\n") + print("```") + print("\n".join(survivors[module])) + print("```\n
\n") + EOF diff --git a/.github/workflows/pip-audit.yml b/.github/workflows/pip-audit.yml index ed816b4c..cd5f1a34 100644 --- a/.github/workflows/pip-audit.yml +++ b/.github/workflows/pip-audit.yml @@ -49,7 +49,8 @@ jobs: - name: Install tools run: pip install --require-hashes -r requirements-ci.lock.txt # The locks are the full closure (requirements-docs.lock.txt for the - # documentation site, requirements-ci.lock.txt for the tools), so --no-deps --disable-pip audits them as written + # documentation site, requirements-ci.lock.txt for the tools, + # requirements-mutation.lock.txt for the weekly mutation run), so --no-deps --disable-pip audits them as written # without resolving or installing anything. Each --ignore-vuln is # an advisory SECURITY.md documents and dokimi/osv-scanner.toml ignores; # add or drop them together, in all three. @@ -60,7 +61,8 @@ jobs: - name: pip-audit run: | sed -e 's/ *\\$//' -e 's/ *;.*//' \ - requirements.lock.txt requirements-docs.lock.txt requirements-ci.lock.txt | sort -u \ + requirements.lock.txt requirements-docs.lock.txt requirements-ci.lock.txt \ + requirements-mutation.lock.txt | sort -u \ | awk -F'==' '/^[A-Za-z0-9]/ { print > (ENVIRON["RUNNER_TEMP"] "/pins" seen[$1]++ ".txt") }' for f in "$RUNNER_TEMP"/pins*.txt; do pip-audit -r "$f" --no-deps --disable-pip --progress-spinner off \ diff --git a/.gitignore b/.gitignore index 8fd0d7de..edfa580a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ coverage.json /dokimi/outcomes-result.json /dokimi/journeys-result.json /dokimi/journeys-result.md +# mutmut's working copy and results (`mutmut run`, the mutation workflow) +/dokimi/mutants/ +/dokimi/mutmut-results.txt # Local environments and credentials .venv/ diff --git a/dokimi/CONTRIBUTING.md b/dokimi/CONTRIBUTING.md index f1f9d16a..e7dab426 100644 --- a/dokimi/CONTRIBUTING.md +++ b/dokimi/CONTRIBUTING.md @@ -149,19 +149,36 @@ pre-commit run --all-files # the whole tree, as CI's pre-com mypy and pytest are not hooks: they need the full environment and pytest takes minutes. `make ci-local` runs both; `make hooks` runs mypy before a push. +**Mutation pilot.** mutmut (pinned in `requirements-mutation.txt`, locked apart from the tools lock) mutates the +measurement core: the statistics in `dokimi/eval/stats.py` and +`binary_stats.py`, the score, regression, critical-band and trust gates in +`dokimi/gate/` (not `novelty_gate.py`) and the response-cache key in +`dokimi/caching.py` (the list is `only_mutate` under `[tool.mutmut]` in +`pyproject.toml`). A mutant that survives is a change to that code, such as +a flipped comparison, an off-by-one or a dropped term, that no test notices. +The `mutation` workflow runs it weekly and on demand and writes the score per +module and the survivors to the job summary; it never fails a build. From +`dokimi/`: `mutmut run --max-children 8`, then `mutmut results` and +`mutmut show ` for a survivor's diff. A test that kills a survivor +asserts the value the mutant changes. Some survivors are equivalent (the mutant +computes the same result, e.g. a `<` that differs only at a value no input +reaches); leave those alone. `mutants/` is scratch and git-ignored. + Dependencies install from `requirements.lock.txt`; the docs build's from `requirements-docs.lock.txt` on top of it, and the tools CI, `make dev`, the release job and the eval image use (`requirements-ci.txt`) from `requirements-ci.lock.txt`. That lock also pins the build backend, setuptools: package-smoke and the release job build with `python -m build --no-isolation`, so the published sdist and wheel are not built by a backend resolved fresh from -PyPI. When you change a range in +PyPI. The mutation workflow adds mutmut's closure +(`requirements-mutation.txt`) from `requirements-mutation.lock.txt`, which +nothing else installs, and pip-audit audits with the others. When you change a range in `requirements*.txt`, regenerate the lock with the command in its header: run it from `dokimi/`, and add `--upgrade` to pick up new versions (without it, uv keeps the current pins). The command's `--exclude-newer=P7D` skips releases less than 7 days old; if a security fix is newer than that, its vulnerability PR fails to resolve, so regenerate the lock by hand without the flag. The docs lock -and the tools lock (`requirements-ci.lock.txt`) are constrained to the main one, +and the tools locks (`requirements-ci.lock.txt`, `requirements-mutation.lock.txt`) are constrained to the main one, so regenerate them (command in each header) after the main lock changes too; `tests/test_dependency_locks.py` fails while a pin they share differs. diff --git a/dokimi/SECURITY.md b/dokimi/SECURITY.md index f6aa045e..e50b1456 100644 --- a/dokimi/SECURITY.md +++ b/dokimi/SECURITY.md @@ -45,7 +45,8 @@ The project is pre-1.0; only the latest `main` is supported. Fixes land on ## Known advisories in dependencies `pip-audit` against the locks (`requirements.lock.txt`, -`requirements-docs.lock.txt` and `requirements-ci.lock.txt`) reports two advisories with no +`requirements-docs.lock.txt`, `requirements-ci.lock.txt` and +`requirements-mutation.lock.txt`) reports two advisories with no upstream fix as of 2026-10-05: - **diskcache 5.6.3 — PYSEC-2026-2447** (CVE-2025-69872, GHSA-w8v5-vhqr-4h9v; diff --git a/dokimi/changelog.d/20261021_160000_napetrov_mutation-pilot.md b/dokimi/changelog.d/20261021_160000_napetrov_mutation-pilot.md new file mode 100644 index 00000000..23b8228f --- /dev/null +++ b/dokimi/changelog.d/20261021_160000_napetrov_mutation-pilot.md @@ -0,0 +1,11 @@ +### Added + +- **Mutation pilot over the measurement core** — `.github/workflows/mutation.yml` + runs mutmut 3.8.0 (hash-pinned in its own `requirements-mutation.lock.txt`, installed by that workflow alone) weekly and on demand over + `eval/stats.py`, `eval/binary_stats.py`, the score, regression, critical-band and + trust gates and `caching.py`, and writes the score per module and the surviving + mutants to the job summary. It never fails a build. New tests kill the survivors + that were real gaps (inclusive thresholds, the Wilcoxon veto, per-condition metric + keys, the bootstrap tail index, the cache key's role and sampling parameters), which + takes the pilot from 75.3% to 92.0% of mutants killed. Configuration: + `[tool.mutmut]` in `pyproject.toml`; usage: CONTRIBUTING.md. diff --git a/dokimi/pyproject.toml b/dokimi/pyproject.toml index 19dd7f01..6060a9bb 100644 --- a/dokimi/pyproject.toml +++ b/dokimi/pyproject.toml @@ -266,3 +266,42 @@ filterwarnings = [ # Test order is shuffled and `random` reseeded per test (pytest-randomly), so a # test that leans on another's leftovers fails. The header prints the seed; # `--randomly-seed=` replays that order, `-p no:randomly` keeps file order. + +# Mutation pilot (mutmut 3.x, pinned in requirements-mutation.txt; run by +# .github/workflows/mutation.yml, not by CI). The measurement core: the +# statistics, the gate decisions and the cache key, where a surviving mutant is +# a wrong number that ships silently. `source_paths` is what mutmut copies into +# mutants/ and imports from there, so it is the whole package; `only_mutate` is +# the pilot scope, and the test selection is the tests of those modules only. +# gate/novelty_gate.py is left out: evaluate_novelty is one 440-line function, +# and mutmut 3 writes a copy of the whole function per mutant (the generated +# file passed 26 MB before the first test ran). +# Usage, from dokimi/: `mutmut run`, then `mutmut results` / `mutmut show `. +[tool.mutmut] +source_paths = ["dokimi"] +only_mutate = [ + "dokimi/eval/stats.py", + "dokimi/eval/binary_stats.py", + "dokimi/gate/critical_bands.py", + "dokimi/gate/regression.py", + "dokimi/gate/score_gate.py", + "dokimi/gate/trust_gate.py", + "dokimi/caching.py", +] +pytest_add_cli_args = ["-p", "no:randomly", "-p", "no:cacheprovider"] +pytest_add_cli_args_test_selection = [ + "tests/test_stats.py", + "tests/test_stats_properties.py", + "tests/test_binary_stats.py", + "tests/test_novelty_stats_point_biserial.py", + "tests/test_pr26_properties.py", + "tests/test_skill_lift_summary.py", + "tests/test_model_compare.py", + "tests/test_repo_compare.py", + "tests/test_critical_bands.py", + "tests/test_regression.py", + "tests/test_score_gate.py", + "tests/test_gate_call_sites.py", + "tests/test_trust_gate.py", + "tests/test_caching.py", +] diff --git a/dokimi/requirements-mutation.lock.txt b/dokimi/requirements-mutation.lock.txt new file mode 100644 index 00000000..b2807d12 --- /dev/null +++ b/dokimi/requirements-mutation.lock.txt @@ -0,0 +1,481 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile requirements-mutation.txt --constraint=requirements.lock.txt --universal --python-version=3.11 --exclude-newer=P7D --exclude-newer-package=certifi=false --generate-hashes --output-file=requirements-mutation.lock.txt +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via + # -c requirements.lock.txt + # mutmut +colorama==0.4.6 ; sys_platform == 'win32' \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via + # -c requirements.lock.txt + # pytest +coverage==7.16.2 \ + --hash=sha256:00d3eb96e9988c45f50cccd1f1496571ac5c1f91386ac02c4d55516eeda19a24 \ + --hash=sha256:01c6908bc613b420c26c818fe948e1b97dfd041a53c98b01c63bd8321f5c9aae \ + --hash=sha256:066429634299e14dd2d511e1e85f8f9cecc500781f6b41907c0dd6f1baea7e63 \ + --hash=sha256:0993d0e90858c03943d3cb152e068a20dd4707924deec84dd2230261baae3b1b \ + --hash=sha256:0dcbcfcc059117284c603ff8cb61a65872512882f84a8cf0339241f7f7c2f148 \ + --hash=sha256:0fd7a86fdda7cb6d616d178654bd0ad6bc0f3f33c2e478aa598500a1a9e34eda \ + --hash=sha256:11d28e9123a9156cb405d8d27b44256c9a58fb5decc2073a8f17862057e3aa0f \ + --hash=sha256:11e597173af1dc33d5f8a7332ada544199269a223af1ee1770ddd5e245ad0fe8 \ + --hash=sha256:126d1af8804d7224421fe991ff65d3ce649081560df7a98b1a5ffff07f9923bd \ + --hash=sha256:14253fc7bb15749b849795a06f5d3b6d8bc3fb8a4b5ddc341faf7a89dce205fc \ + --hash=sha256:152877cdc8a07264882cfcd503ba56a3ef6cba56a70e8c70f6eb8ffd7384789a \ + --hash=sha256:17228fbca0f22976f797be94e975dcd237799c657d49551c7de1e0654d1202e9 \ + --hash=sha256:191803c4996b499fcd78c2ad5e5f767dcc53cb4dc6de6d6a741b443a1821ef02 \ + --hash=sha256:1a37c6e478cf687e1aa30a593d19c92c02fad9d122b51ab73f51b8dc7a0c0fc9 \ + --hash=sha256:1c569a9fd25505f1cd6bea90588818f90373ce90e2632e2cacf19ddbd6e14fdb \ + --hash=sha256:1d56e4d21c56d2046447733f8b118409597db48c01efe898ee9ac24e858ec2d6 \ + --hash=sha256:1d5d0e3b660506fb84f995814e3118a21efdc0c8eb80127da1be627d90093c17 \ + --hash=sha256:1f15254427c9b33eedac4f198eaf9e356eb4f6214551afb43da6194a2c088ad7 \ + --hash=sha256:218d742afca2b5ad5ca759e93eddedfbcc6eadf8322f080dcefc40b7bd4e2d48 \ + --hash=sha256:22957cef43ce038641de78ba995de7568d2d6a37c6ddbf7fa0fd7d1ae2344d91 \ + --hash=sha256:23219888477edd736b6fcaec1272d47d93b926e999641ffea7e53a1738e70b2b \ + --hash=sha256:251aed777c47c77aba047096d4542889db089227655711dfc2b9c54ef0e15e35 \ + --hash=sha256:28ff850182a67d117990fa2ce5ea1032836d8c9630dae867e8bdd3bff4533b79 \ + --hash=sha256:29309ccc86b7f33df7db12813c299f215bbbc470ed6292d0bedd63ffae1ebf64 \ + --hash=sha256:2aca0bdfa9e91621d5b09d815357bf63def4fc0e9cb66da67bf2cf93f3b1a6f5 \ + --hash=sha256:30c1b65d529e46569899fadca59e4a87c1faf2886923f1307ba61e654d4f3c20 \ + --hash=sha256:35f37886699cb9abd29958247d718628d5bc6f39e623dff66a09e546c42a7e03 \ + --hash=sha256:382d3346d56b0eec1b793d53a4c88799c8053f516aa3a8d7c44315696954bacf \ + --hash=sha256:396bb16e04ce04efbb3df91456ae4e3da918e69ecdf67fb711b0a0fdf35ccce0 \ + --hash=sha256:3e7f99698ba3a7d13988bdd984b7ebf13af4dbe2166dc8502eef90d77603b0a4 \ + --hash=sha256:3e861f1071dcc2fec1e88bef0920f6b1eaa66a143555b4f8ab79ba2b0f30ef55 \ + --hash=sha256:3f43bac1856ba269b905302778d4df433d6006489a192174ad77ac528e395032 \ + --hash=sha256:40c0f00899fe6181ae7f434ceb200e51f5ee4b8ed10e3b5f0b605f0cae15da87 \ + --hash=sha256:414c26dfdb96aac2d570a54e03008f001e32eb2d413705365503648c6bd361d8 \ + --hash=sha256:4358b9c8c0125b460407f3017c6cce8156e904b32772c5630d27112f52bdbfe5 \ + --hash=sha256:444889f7f66b74e4455c0a97e0e166dd41177f1dca8c0239a47cff25e05ba7e1 \ + --hash=sha256:44f21e407b278efdfc1ee5e481e00518bd1d500310a30a5fbf2bcbedfef4aaf0 \ + --hash=sha256:4cc4f73aa3fabc36e32046d6cd2971405948d8a903636508a3d3b2f9128b3a95 \ + --hash=sha256:4dbbd1155ca46e6e0b6b89d204428c56ef6a459af21333f365d135a2820e5a09 \ + --hash=sha256:4ee546b9e4872ffa194bf07ac87bfa1202ebb824d0795dc1ef22f175545ca90a \ + --hash=sha256:5139009b5efd2194fc168ee9362f0e191ba612ef5d29242f9269c22f9b8f80c7 \ + --hash=sha256:5375ebd99038021b35e99dc88255022912c06565d316212f4a576e4b08d30f5d \ + --hash=sha256:5397e21a90dde0e9c6896b77ded8f0be26b66f8b22b33aed41f6043ed95d55e6 \ + --hash=sha256:57ff3783f99d75a1e81dd56a9737eb5665e6736a5d93258ba596b6dcad8fd05b \ + --hash=sha256:58d4a54c6ea672afef66d49be922a2c69826c5ae1a42a9cd94f0c9c2bacdf800 \ + --hash=sha256:59c3926585e1cd1f2190f4b2ac9014de1bbeaf0d5d0587b0dc6b0aa90d17896a \ + --hash=sha256:5a27b731c171e43dc8b5f32b76a5051dde2ec9b9366c87028f08a7088ebc2c7b \ + --hash=sha256:5b3146d2317c75f70df2509066d979dadd941f7021cdf9b5db4bcd8568258e25 \ + --hash=sha256:5dca0bb66b4c3d624ba047887bf70270030c150692d543cb501293dc38a9f4b5 \ + --hash=sha256:611a44e5229a59d7483ce830160e1a0e85f700562c7a5651c7c63fb8f4eb528c \ + --hash=sha256:648352b94507179d82637292e7ae8802508d95f78e2f00a705a50b6c48011681 \ + --hash=sha256:6a75180829efb8ae62b4aded25be6ddca1c888d138d2d82e21d93bfbd88f41cb \ + --hash=sha256:705e5af11d34647efdc170c7840b6857c81cf74be96419a553f237e68e62cb72 \ + --hash=sha256:723dcdab91357159b722935b500ee8abc0a66c8c432e1e9fabf4cc7598952de8 \ + --hash=sha256:724bd0f1e81856b35e59fc98cf7b4e544a3cb662e4e0864dca73d4326ee9d808 \ + --hash=sha256:732d950e51f3ba4fb6209c73250f3e8924fefca42953ee04a9e65d8c02414d7d \ + --hash=sha256:736fde09ea39646d11f8e3b76bd3425c075aa4dd45f24891970bb77c14ff20f5 \ + --hash=sha256:7a076277ca9f5750cc230f0f578ebd2620cec60255b25707361699fef6fb465c \ + --hash=sha256:7b3bce4a0d05401d70b7d0d5ca783e686bc9d30e81dbd7d980d532609bf809e4 \ + --hash=sha256:7b451c68218c150f616bc9649783ec8de76a59792c759b43aa0c9c0466a465e4 \ + --hash=sha256:7d0732c83746bc24123c581a85d9dd96b70ddb538c9076020aa1a041790361e9 \ + --hash=sha256:7ed238d227e23cc300c3d464babdaf9f6ddc740aa1b15a77ae96136e6a7c4516 \ + --hash=sha256:80d3f7b48d43ee8fc5e8707a8adb43d743a5a1a85256c25a24f9d6d0e2238fa6 \ + --hash=sha256:80e9fdb4c3d926b6ba721d4bf7435bdb869c3527ae7803290361d0ab73db13b6 \ + --hash=sha256:848893e1d361448c113dc2f0913503522a6f7be231d0e38333d2a22d9698a011 \ + --hash=sha256:893ea9cf86cb8d2546812ac93d973aaf2ee1fb45110a873b014214fd23e3725e \ + --hash=sha256:8afd9bf35cc6a1f22eb3634808fa8e0b91902459c5721ef2e4461dfe771d7f08 \ + --hash=sha256:8be099e979fc42559328a21828281b4578304191ae46ed4e80a407048a82eee6 \ + --hash=sha256:8e209591f7c41ae4a9171335cf6156afda0b21de73b02f73f5aa95b2d5fbb08d \ + --hash=sha256:8fc15cc8d0d06e873c00ef18e1372d605f9aaf3de27d8c24e50782e75bc8b843 \ + --hash=sha256:9174f0af24e5eff248b9dbfe76ec5275a3d19d37edbc2810543f12cf97347a34 \ + --hash=sha256:921415102a90637fcc2e3f169f61dad7699ecf690e8639fc21b813acbedc0967 \ + --hash=sha256:967d72c835d7a8cf0af99ec813a2d06e3db6df706402f1fe85b31b437645f495 \ + --hash=sha256:98d9c97f51b334b0adce7b964442a9af33c1a00c6ac856984cc5dc8d18f81c75 \ + --hash=sha256:99704f73721e23859112072d522076e11c31744fc96b5652e5dd2018aa4359f7 \ + --hash=sha256:9a75a4704ff640e46170042eec1f984385a121227c505d5a16ad8e495f452541 \ + --hash=sha256:9acc7f7ec4a1b5f89bd929fde5b8a714f6fafdc6cc18725413d510aa082b47ad \ + --hash=sha256:9c6afdd69218202bc1758c9a14b86b8cf1084f37ed2ca143e567a103772b16d1 \ + --hash=sha256:9cdf19874e0d247f32f03609200370343c3c7aa260b191d8c2bb251d36198283 \ + --hash=sha256:9e1d0ced76318bab499693ff25f64faa343415187cb2e4d7befdfdd391a1cf6a \ + --hash=sha256:9fd670ac43b709c575aefc25bf52d8a598a3bc5017bddfd0a179152ab06a2deb \ + --hash=sha256:a0f2285329dac10ab08f79cb11f5692c497018e6c7c511f95e6fd63a70b8f831 \ + --hash=sha256:a2fac6895eb299a2e52d7bbb8fb3903502b9da8d3f5309ceb16ec40c646b58ee \ + --hash=sha256:a336eec40e3520d369b8a6cdabb4f596e69a8b42927ca074aa1452fed943238a \ + --hash=sha256:a4624f80732f6b427ac58f1f59c577a0994a12e8174b5af6a027b4b58795d4c3 \ + --hash=sha256:a56ac4fa5a75c7e182e8f62600cfb4aff43c5ed7356a034f3557659c3bec1d90 \ + --hash=sha256:a678c0b6b22086ec2427359d22e37445d4a792f5fdbbc744112c7dade65cad02 \ + --hash=sha256:a740ea6f083c6db7b926534d159508f80ba275ab35e722522de0d18d0f56e55f \ + --hash=sha256:a90700f743e29aa3d75a6ff5f01953176a889c00e526194bc4d281731b88d99d \ + --hash=sha256:a9a638be322a8d76a41cdb17781c7f82aaee6a66493d8ffb7e2c09ee22423d99 \ + --hash=sha256:a9cd3de0a5bfe7b0e21ee10e1a14e3d61bf52efc88217ab1d95d6ace6970bd46 \ + --hash=sha256:aa62c85046473959c13ba9edca9dc90a77d5c1095b1ba313556314d77fe5b036 \ + --hash=sha256:aba5c63b7afdc749cc9eae943d5b868cba2b261a176378fa1c5a30bc8bc89982 \ + --hash=sha256:ac0f3b379c94acc2f7dce5f5f0b24d44fa1cc6a509717ef83dfee07450c2117c \ + --hash=sha256:af2a2a8c7c74de0559e0c368d94c8def9e16c58faaee33a0bf081057c4227e3b \ + --hash=sha256:af98ad5ed9d6daaca956201e00bb429a7eb2b080426686f70a20353e0f9839f5 \ + --hash=sha256:afdf43b72ef3876c1fe66423b91466e37877c9e81e8cec70542b7e8525b9d1b7 \ + --hash=sha256:b88841e654f09732804809e435b3e005a929ffd9998b872b7b213957b8759cb8 \ + --hash=sha256:bb2fc905bbf4e6b7f40806ea79e31515abf6349594cdf0adf27c4215f0463204 \ + --hash=sha256:bb4ffe96aa663cee727659db5a2afeb38c95f8677b747d447b90d6d4874ea2c5 \ + --hash=sha256:bc0b0ac781d489304b741269857f1f8338b7a26b1b89c06c0344658001ec0035 \ + --hash=sha256:bf1bd822ec4e387ed245bed0d71151582cf7be9e5309bc4145eefe36083d5878 \ + --hash=sha256:c19cd6d025c1673f22afcd22c7df8a662d779e05d8e3fa6820c22afb895b0206 \ + --hash=sha256:c3305c38a2fa21a4254f2ace7dd9ef5fc569c9a558b66e7017650b3d637fb95e \ + --hash=sha256:c85d54e7e8a2ca932fe8399301af9b8d5907ea2a455ffaff6e7d1208db83b943 \ + --hash=sha256:ca64d9f1f384f151b9511bec01126072acd2f313439f8ed015a22d8790aab6fa \ + --hash=sha256:cce2bc991293f15cc4084ca116827b5900c5f34e1a54dfe83f10ab5c43162eb7 \ + --hash=sha256:d6276d78f6fca7d0ac066d5da4165c5acd07829e8305c2cb900b738fb3a75a72 \ + --hash=sha256:d93db87adb6b1c1b408dce4763314b55d76a9f589e96783a84ac9e7689e48bdf \ + --hash=sha256:db5f8394e17f877a625b257f2ba0ce8e728a499c2c1579ad66220272cd3df510 \ + --hash=sha256:db76506aa5416081f3e8974ae0f7965c58ada0bb0ef7339ac86099588dbb20d3 \ + --hash=sha256:dba2edfb054f6d4a08df9d1637c39a5aa3865bca6617c13c86be21e45658a59c \ + --hash=sha256:dcf4bc2aab4e16b1c4c0c2005918f23a7dd5d7821ddae82caed9e3342dc2fcce \ + --hash=sha256:e1fa594c887365b69745f25a416806e61085dd07b94c9eae68a6e20730629b23 \ + --hash=sha256:e6c52d3307824ff93b39efd99e4185d557db40bd841452abfb32e5d9151ca162 \ + --hash=sha256:eb57acff4a74246ae513c142d4b36e18c389c3aed8661914a53f7cd0071031b2 \ + --hash=sha256:f80bd9f9633eafc73d0a913ba2645c96ba58bba1befc30590f7c0fbfde59d865 \ + --hash=sha256:f8475460aa33ee28ac896ab1156d0bb3b6c639f7f8383c2677d3359eb35f8205 \ + --hash=sha256:fb2bde05838fffae1a1bf75e5d411a6cac3e4e9bb97e6640fed8cd47888b33f0 \ + --hash=sha256:fb9d92ecfe2d5b494367c67f7446f8b75b68d8d0c8cf3bc3e6997478be25d9e2 \ + --hash=sha256:fd3d72233eb8b48acc94fa57d44e2d32ce8e7abed02882ccb6d855ccc4ed33ec + # via + # -c requirements.lock.txt + # mutmut +iniconfig==2.3.1 \ + --hash=sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960 \ + --hash=sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7 + # via + # -c requirements.lock.txt + # pytest +libcst==1.9.0 \ + --hash=sha256:02ee2dbdb5c218116f16a021350fc267a14be205b50d81c33f5d73857ab6fbf7 \ + --hash=sha256:03165a264653bb77f6a11b412ae09c08bdb0c25864f3b8d42b816ac64b9d4b9e \ + --hash=sha256:087b58a9afe076bb08e2d726478e1f16cb928d67ffa9092817e033c335de522a \ + --hash=sha256:0c7b548512db25af9c2997a95fa731bd6b6928ecbad6c0915d7482d8bb42d34f \ + --hash=sha256:0ef45740663d0e6d8159b9eb6db436c4eb8dd0808c30812501542d65cd44c030 \ + --hash=sha256:107593af46945593e7825821793393262bc4fa1d3ea24c3ed487b61269bbbdf8 \ + --hash=sha256:1260b5d070a3324447a53a00387225a55472a62077ce01922d30a2a78cc4b138 \ + --hash=sha256:1879e5bd97f124acc787ae73cf52975cea46ac0282db5e8d9296173fbe87fa21 \ + --hash=sha256:2b150c4f298fe54eb0fe73abf71f57db74d070796140a8c206c9615b6861f01e \ + --hash=sha256:32395244edfe6538e0ea2bf82051d60103d3f54861274805c4fb3745efb70a85 \ + --hash=sha256:329ffda6b0ef708ff22876f982ec886a509c8f73ac1ccb0190693d35cb09aa0f \ + --hash=sha256:3cdbafbc992b38ea005ef37e20243c4177879020c82d414789fbb7fa22d6d3a9 \ + --hash=sha256:3e5b684191a0462b2d40a73261ea7f4da6a49e7a030b7e0fceca46bebb51dfe5 \ + --hash=sha256:3f735e3acb3afc75d50803fde1598b467bf05d12d6854025df4398aebd75a14c \ + --hash=sha256:4090105fa4ad114766d0f8b34418d9a3b359bc8645da860a447e19770b5705d1 \ + --hash=sha256:444e84c76cd035cd2fe136838c1a524d34b08216521f6f5093df8a6f6cfa5799 \ + --hash=sha256:45808c03528b3ad40b14095348a918e08d98c47b4a125d631cb78e817c0a5b16 \ + --hash=sha256:496c24e0d3240bc7da45dae543aff3f5b6509978c39262d6b84ed2fb999dded2 \ + --hash=sha256:497d5329345f1f5df84e41b0bbd00204b64a2fd30dfe3cfaeaebca633a31e877 \ + --hash=sha256:4adf97bb1aff8039b0bd4991e2f838be6e4fad552821b26b71a5d1a653c2582f \ + --hash=sha256:4d382fba04077eb556a1ea4295a4482e192aa93639c5a07ba0885841965ea0c0 \ + --hash=sha256:4fb016ad6ec8485991770dcff3fe072b0296eff6fefe7b3b4e28fb919dea3def \ + --hash=sha256:50ab94bb2524b419056d4003032b8c67102ac800f8d85b3c4260a01746d94dbb \ + --hash=sha256:50b913e37187f00a6fb88962009364e1cf1b1284aa1762304f34b52b972f2218 \ + --hash=sha256:5302fc698ef863e1a2fb540e95793b3d5a96abc89aa6423ee9c389d7d7ec5eab \ + --hash=sha256:568288cdbfe3b4ca3ae4852cb0a439ff053dd54c841bc4995bf2b71238b5de40 \ + --hash=sha256:5705f664266742b8d9323db714f58cf06a6cde739eb09fd9ca31cd233f9806e5 \ + --hash=sha256:5891ce9cff815077614f509e3a23888c5ddb8081d6188e8ba1172c0ccc369022 \ + --hash=sha256:611cebd3bbc2014576f4dcc7b845b3c594c96ddc287a3db9b78f22eff156a7d3 \ + --hash=sha256:6e50576bad7d56459d9792cd0b0dfe5469dad13646e9a9c0a8b1ba20b269f332 \ + --hash=sha256:77d9303572cf0fbd0a2eb71aff167a4246541f43bf58b781b35b9d0bbab80ddc \ + --hash=sha256:7acfd18adcdd32dcf41ce676cf121002afcbe9ad2c72dc0c18d78465beedc228 \ + --hash=sha256:7dcddfb4905c5ef90a6fdc58dccbc049faf0056b68161042fc1e0526b20cba05 \ + --hash=sha256:8504b422c95676a8c27b517e1ac01413ece91bf356865c587ca9bdcd5708a2f7 \ + --hash=sha256:86361e2b426bd1b403e52375703c8b764e226e571adcb30442510710681edbf0 \ + --hash=sha256:8930971d2299b7006bb5d10c10038f44efb6da89d5bca2823595e31a9cdb96af \ + --hash=sha256:8bc5351d92ca6ac1cc32e097700e1161ad1ceaa4d9b2cca5abadb1e94576b325 \ + --hash=sha256:8c14abe844bec8b021111b3985474e49f5af799c020e8e40dedcac87c97a40c5 \ + --hash=sha256:8d731abe1307720ea1a52d447555e8443a6d130e0e520243c0634a58f6edbc9d \ + --hash=sha256:8f0dd08a5773d7051e105250b2a2c73d8065ea9b2d68e7e1bdc5244660e75f93 \ + --hash=sha256:951f13c193e590fc88c9582a5341a130fe53ecece51043cff4bd54e25a5e022d \ + --hash=sha256:9f92c75283030fd58fb7d6e560168a00381e0e3368ec8a026a3ec8a828a05f93 \ + --hash=sha256:a2faaf92500d0226358125630f5aab4758e8aad3f2d70a10892ec3c700781a54 \ + --hash=sha256:a5068bf6114f6f4d79af7a6c80a28d1deb50441150ea1db13b5458ab34bec159 \ + --hash=sha256:a621e261990148c1cfbe26c1798bd2375c131cf358a44a7a4659fc41c1a336e1 \ + --hash=sha256:a6615b036694f6906e30dc018b5b34898cff7ff760ab549f9dea1ada4f53495b \ + --hash=sha256:ae8b2abe4d5f52bfcad749350e4483976b7b5142f765ebab69959f2b4f99641c \ + --hash=sha256:b18c9d7316f96ab255a9a127289af0b9b7d1ebf8929e81eefdeccd951f4ba58e \ + --hash=sha256:b4fa00cb71ffbda877dc4010ae8f3330ca605af859ddc6444bea96c4f294d648 \ + --hash=sha256:b755ed4a4bc2faee849b54820023137d60d4c199e0a0822f0ff0c1bc49e49b48 \ + --hash=sha256:b8b9df30f524317b097dc53065b25dda33d6a4cc3c7c8bf4fc83ca7559c58cc0 \ + --hash=sha256:bb5d0946f2b4c6711b5d69fe4f833b364f9e7a1a2b08f88b98619dc18975099a \ + --hash=sha256:bc6aa3e7f21315dbe9868c0d13ffaac309514e8c816320c5f91230fa5b21df92 \ + --hash=sha256:bcb9f9d4fcfe2ec7a40d2c26e03a538d5d9dc189c38551eb3f94ab661afee7c0 \ + --hash=sha256:cc84aee3c293a012db722aa2d6ea87be524646852684a5a4e54c1fb98ea9072f \ + --hash=sha256:cd1a3500c41784075c4946a995d5ad89f68fa0d226b63ff3c4d78f6ea6dd23e5 \ + --hash=sha256:dacc583bcd8db56e457ae559b910673c6b5df66f694d1f1154ce0b3982158d17 \ + --hash=sha256:e465a7bc9c2b9533eb9e06d2391f8819f811b5112919d4064c9fb8565aaafa08 \ + --hash=sha256:e8d671c39a431c309476099b8ec811e412503ec0f4465f6fc907cb51c70e8e6b \ + --hash=sha256:ea490fa8540503db5f321f0268becab46eb50f710e8cec8041e241fd66f6874f \ + --hash=sha256:eccf4c57d273cdd3fe1c67b72cf9bb1bbd4547aa011824e96ccf5b7136057aa4 \ + --hash=sha256:f101e0bf8638eba80694ecc3c3abb5309eb488f5271427536cb379c5c1e0ba9e \ + --hash=sha256:f6248cb07444ab9a6733a855737a9febed8b9adca51347019348b09a3ac7dfe9 \ + --hash=sha256:fceeaf1a1fd186557973904cfd2bb3c05ad835b01d7d04e9ec1df463c0add966 + # via mutmut +linkify-it-py==2.2.0 \ + --hash=sha256:3adc40eb5af300b2605fcfdb968c24e1d780a90f1f2221af7c15e5111e94d443 \ + --hash=sha256:907acd2d17ac1fbb9ddb62c8957ccbd6158cac602231a15c3b0cd1e215f03cee + # via markdown-it-py +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via + # -c requirements.lock.txt + # mdit-py-plugins + # rich + # textual +mdit-py-plugins==0.6.1 \ + --hash=sha256:214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d \ + --hash=sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0 + # via textual +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via + # -c requirements.lock.txt + # markdown-it-py +mutmut==3.8.0 \ + --hash=sha256:4c0abcef6b8d1116f2adfaad19df12e1c621e4a402d4367ef15bf8cba497b91d \ + --hash=sha256:bc670276518f893ab356f1d28826c49b7e606e7cf89c300e3888c0b52fc1d2ed + # via -r requirements-mutation.txt +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # -c requirements.lock.txt + # pytest +platformdirs==4.13.0 \ + --hash=sha256:1aa0b0d3f224c1f07c295121e312a5a24a180d6ae5a8425ea1784b3e3863e9c0 \ + --hash=sha256:3dbcf4cd708f21cf876c4eaa90e58412bc4f033d87143f41b1493ff77c25b7e1 + # via textual +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via + # -c requirements.lock.txt + # pytest +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via + # -c requirements.lock.txt + # pytest + # rich + # textual +pytest==9.1.1 \ + --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ + --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + # via + # -c requirements.lock.txt + # mutmut +pyyaml==6.0.3 ; python_full_version != '3.13.*' \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # -c requirements.lock.txt + # libcst +pyyaml-ft==8.0.0 ; python_full_version == '3.13.*' \ + --hash=sha256:052561b89d5b2a8e1289f326d060e794c21fa068aa11255fe71d65baf18a632e \ + --hash=sha256:06237267dbcab70d4c0e9436d8f719f04a51123f0ca2694c00dd4b68c338e40b \ + --hash=sha256:0c947dce03954c7b5d38869ed4878b2e6ff1d44b08a0d84dc83fdad205ae39ab \ + --hash=sha256:30c5f1751625786c19de751e3130fc345ebcba6a86f6bddd6e1285342f4bbb69 \ + --hash=sha256:3bb4b927929b0cb162fb1605392a321e3333e48ce616cdcfa04a839271373255 \ + --hash=sha256:3fa992481155ddda2e303fcc74c79c05eddcdbc907b888d3d9ce3ff3e2adcfb0 \ + --hash=sha256:58e1015098cf8d8aec82f360789c16283b88ca670fe4275ef6c48c5e30b22a96 \ + --hash=sha256:7d10175a746be65f6feb86224df5d6bc5c049ebf52b89a88cf1cd78af5a367a8 \ + --hash=sha256:8a7f332bc565817644cdb38ffe4739e44c3e18c55793f75dddb87630f03fc254 \ + --hash=sha256:8c1306282bc958bfda31237f900eb52c9bedf9b93a11f82e1aab004c9a5657a6 \ + --hash=sha256:8c56bb46b4fda34cbb92a9446a841da3982cdde6ea13de3fbd80db7eeeab8b49 \ + --hash=sha256:8d445bf6ea16bb93c37b42fdacfb2f94c8e92a79ba9e12768c96ecde867046d1 \ + --hash=sha256:bd48d639cab5ca50ad957b6dd632c7dd3ac02a1abe0e8196a3c24a52f5db3f7a \ + --hash=sha256:cec6c92b4207004b62dfad1f0be321c9f04725e0f271c16247d8b39c3bf3ea42 \ + --hash=sha256:dab0abb46eb1780da486f022dce034b952c8ae40753627b27a626d803926483b \ + --hash=sha256:de04cfe9439565e32f178106c51dd6ca61afaa2907d143835d501d84703d3793 \ + --hash=sha256:e64fa5f3e2ceb790d50602b2fd4ec37abbd760a8c778e46354df647e7c5a4ebb + # via libcst +rich==14.3.4 \ + --hash=sha256:07e7adb4690f68864777b1450859253bed81a99a31ac321ac1817b2313558952 \ + --hash=sha256:817e02727f2b25b40ef56f5aa2217f400c8489f79ca8f46ea2b70dd5e14558a9 + # via + # -c requirements.lock.txt + # textual +setproctitle==1.3.8 \ + --hash=sha256:09cb645c6ae440286391e256203fc03522b3b0f55fda8219f698abb5c447cce7 \ + --hash=sha256:0a582308ba43afa7233d28243d9d1212e11c81d3c187a43929ec0cb70c4781c5 \ + --hash=sha256:0b15200dc328020f498b196eb59a3b3e35cb08f17bfe05b2e63e5ed47110d254 \ + --hash=sha256:0d2f756ede847e05e7759af4f2d87dedf3ed34004092216be85acd8301b1d385 \ + --hash=sha256:0e85b55eac9abfa3166ab00dac21e718ac2624a1df7d0b75559a2d3e3781cb33 \ + --hash=sha256:15aaec002066f6dd7d920c46d3bffc3e529cb4a9fac961f2dd878aad64b0e29e \ + --hash=sha256:191888bdf7c223c8b00eaa935947de0904165dee5e84d2055aecbb473d9c3f51 \ + --hash=sha256:199c2c674af749003596d3a5ccf35112d28a7cfe95a363eae5fd042c33e0253e \ + --hash=sha256:1a6f6a3b2920f4362e8d924f9e5b084f0b2692a2c140e348abd2e3a07a1def73 \ + --hash=sha256:23a193a32a9cca10c4062e2efa3377e804d0209903c734940cc6626d6dfd018a \ + --hash=sha256:26997a85aa70bb088e86f0b7899fb0c909be824d22fc025e87b0005afbe59261 \ + --hash=sha256:26da5ad9883d825b9d732fd1dcb99268a4ee545b1edd0aaa5c56f16009cccaf8 \ + --hash=sha256:322b4643f98c824b17dbf2ddb07ea7a37def8cc9e18adee320befce4bdffa271 \ + --hash=sha256:349499c0b21a940c3cb93d938013d40313711911e2fbaecd4700e2b7c236ca57 \ + --hash=sha256:35fc5e10fcba6df25d676ef908b44a36c17e7f167738ed016561185e24f4d274 \ + --hash=sha256:37a91697c3b96bac58b6936b3bf0366af96a2a6cef963fdd9aaf0593a4bd1ff2 \ + --hash=sha256:38310d9cf94959e39d22fdc0154c66c8bdbadfa790c524b04cb36da6333c7507 \ + --hash=sha256:394aca04e9f11c469f4f29d25a40a79eb4b8c9a72f438ba6390b776590f89757 \ + --hash=sha256:3a6337e7c0f50a702106e6652764117c442a0468292afb8a7406ff7c483bfc8a \ + --hash=sha256:3b256ac72a3855b434506a85cf5ca6e25441657d5dc56eb73df6e279193cb35b \ + --hash=sha256:3d2ced815b940c88097e07182c57cbfe1a0156839d1edc9be2bc0e4c74ab4eed \ + --hash=sha256:4151e5b53f7fb6a39fac2e3db35d70761bac27c0f6b924aeb4a6f67e92ad56e3 \ + --hash=sha256:4756a96435917494ec2f27f0da46bc05eecaa1a7e072d57220ecb2d46ea8a3d3 \ + --hash=sha256:4a765e0c80a2453480be27c4fc22ba1c0771867040c5f91e4221940d78a4abfb \ + --hash=sha256:4d6a0245679b74e0fda974edb672cd1f2351b0d762279b6776ca3a065c9322a4 \ + --hash=sha256:50beee618377ce6fd35767a9327f2da594b36778eb10b503ee2dc9b7c801f433 \ + --hash=sha256:50f9bce172b6f9ead1c8fe813d85b1dde57ab503a445f982026ac3a16c719e1e \ + --hash=sha256:53c92b933a47fdc9bcf5033834ebd640bdba8ddc8437e1ab2310682806c31c0e \ + --hash=sha256:59239b6e5007b1bec79ef444a390b3bcd59cbcb2686c9910da5a42f1e99f738c \ + --hash=sha256:5b6372be8ff8afc56e0d50036ff618a29bfef72f52946f316e3a65fe25266800 \ + --hash=sha256:5c05aff52e6eb9c4d2ea86cf0d0a60b1487ed570263a1b13bb30d9f96aa8ba97 \ + --hash=sha256:5cff3bdbe6be556cc19bd072ff1ed9b5a8f4cbf356cd19b1ca94506b3e2b29f6 \ + --hash=sha256:61aac898128dade36438da9387fea570f00cd519ad5231d3b268913afc2fc329 \ + --hash=sha256:63348d86ae60b071a898e4241b41da1346d8c36f1a60bf3b698ae5eb35b79865 \ + --hash=sha256:65279b2df2951b64878da5382363e730d242486179c440c8653fe39aa3cc9e62 \ + --hash=sha256:656ffe282ec0ca18308eebb100898c79b4f5698a93228d50d2f2a4a09474d7e0 \ + --hash=sha256:6579e2a6fd0bf3128c06141a87a49423e145a3ad1631fa9cd43886668ce4ea8c \ + --hash=sha256:66656b71c3eb1f7f57feb4579822bebc750c2a71e6090bc0aa7ec161cdecd30b \ + --hash=sha256:66aad6ca59e4197d8e49de384474d8b9ecc6e10989523ca7339cecf7ef2a34bf \ + --hash=sha256:6a740a6de2675da29594612eca3c962ceefc063fb62f43fc5a8f84d547ce0d7a \ + --hash=sha256:741b1ae7e1cd68745b8b677daba73685dab2d1b8bb12500e6537f3482a91150f \ + --hash=sha256:7855290194637a3c5a2e37c423efbf5813fde97a6b63ab99feade5e05d73dc6c \ + --hash=sha256:78cbc62de67dad0a1dbadb1b2bed1bb4b283d73b460ba5fdce16727d27396dd6 \ + --hash=sha256:7b77c8671974288cf192e481ed63e928f1086762e97e8845d09d65ba121f8aaa \ + --hash=sha256:7c8a2beab229c818ae36ca3f7d769db95dd748c76b857851e0a883d947419ba5 \ + --hash=sha256:7cdd3d6e5e8abcee233157615d88dbb80527a5a4eccb95a93fbf92e6d2c8f9dd \ + --hash=sha256:8086259943fc75eb07e11c1d08600709b9ce4fb2ec9bbd2490556c47d3ead135 \ + --hash=sha256:823d7d8790a4f8484c7fa65468fcd9d009f13ffe5f0bc89ec0e0d2c165ac0335 \ + --hash=sha256:84003eeaf2d7460390bbff1dbd7bee5b1213e7ae47a489f0fb7aad6acd2b045c \ + --hash=sha256:84320e61f3906cc550b900ab1ba02ec53edd2d2e1e917f0873cba16592801c89 \ + --hash=sha256:861380b2244983d12b6fe00e4a49f16f9946932245a252dd16c1f5ef7d1be632 \ + --hash=sha256:8879d303d4e3559efdba033ff198c6869010f7958412ec2047674881859fdd7f \ + --hash=sha256:8bb543b4b58ad839f7f0d335a7f4b40cef6989952e7239de3532de0b7771607e \ + --hash=sha256:8c0a7753f5ddaf0ecd1fc8df3e2106611d4290f0508e643e723cb5000b7d6c2f \ + --hash=sha256:8e286e4c4e90063072a6e088a8d6526a7ffceb47429db88ba50ba58eeebcc758 \ + --hash=sha256:97b35b15f81363a314baca76674c19027dce19eed1590abd51c2d285830efd50 \ + --hash=sha256:9c7b9b6ba3d0f6e504be5978854966168fb95b17ce126eab9a6132fe15f3e540 \ + --hash=sha256:9e36ee7300584a53fd7ac5f3853323411bde1cf6c081921bd3754a73b4071245 \ + --hash=sha256:a279b136ed97407209d2dc9c30d1e65405d1a4a06ab162cedd9c4159b51f8f20 \ + --hash=sha256:a4a782f9ab26e662a890c1a7d865c452c83ed554de44aa83da35fa5bdcc79921 \ + --hash=sha256:ac4e037d276ce27069cad0083c4611b45232278bd2fbac33f06ef9438c171a1e \ + --hash=sha256:ae87e7b3e6414b3dd887adb72d2c4ea9f210c5f48efafd295ade4a58c760d29d \ + --hash=sha256:b0e70a5d294e4c8cf7dc94e015fca6d4913779a424937e66b429fba82d8eb5e2 \ + --hash=sha256:b705f01a531dba8fbbab90e30d432af4cc70a79c0a33b238e2c061816140cad0 \ + --hash=sha256:b8954fc72fdd49588c8b8d615eabc27dfd8eb6d0cffe987fc5df02934816fb19 \ + --hash=sha256:ba59516fcab4b621b80966a32f1195770696c113a948d00845f35b762e335db7 \ + --hash=sha256:bbe67d7ac416087996e39494325d94ef9f4ac1ed33823f90de83c73925665cd2 \ + --hash=sha256:bcdf74fe8a6e3b31bed1bc7d3e2a38cfbda3263d2750492ea0725dbc2db5aacf \ + --hash=sha256:bef8c5f8820c70e8761171409052056ffd247023b001a6254287a0694c3d8b1c \ + --hash=sha256:c09f15930910dd895ac113b16c0b3ac2051dcbd44b0f017f94cb51349ba11583 \ + --hash=sha256:c284642b229ba7b7b927fb044aba03694b880ab572db3f07a10adc786db1ff7b \ + --hash=sha256:c3cb92b3239e8094c2454a2160e45417490d4fce69cd09ac846a0c3e43360fb4 \ + --hash=sha256:c667cc9c4cd949814b4ba8be70632a6454637647f3feea2724e02d3d3e35c66d \ + --hash=sha256:c85ff5b000eef8a280333ca158483a76f86bd374816add5c016a9f4294471289 \ + --hash=sha256:cafe209d064a6efb88cb45a03e97981ff8832802b2b5d009dde0197a3b7b41c8 \ + --hash=sha256:cc711e639a3e978978ab3bbb01f8f106715f5c23bbd99ced5af2404528bbaa6c \ + --hash=sha256:ce48dbf74e8ec618d74cbaffe1159a4c32a4428de598f0ccdb0782bae618a72a \ + --hash=sha256:d4289e209e805d7d276a534f062092974fbe6814889d4c132c247205ebb49a65 \ + --hash=sha256:d63adfa4be8f9dbac4be6b359e2a8eb15a733545ef613d02a12ae27a4d40e975 \ + --hash=sha256:dd3a8fcee7d8a23e44781144830dee97b53ca4e0380bdd2a0e05b13913ce5725 \ + --hash=sha256:df807e7f036c19235320b6b5137291891d660f10ccb0b50dc46bc8385be18379 \ + --hash=sha256:e03a3bdd844e20a611518f7532e951c63d25bb27ccae7d7eb02100542540c14f \ + --hash=sha256:e5e6208cd0ce0562887925850d22bffd85aa6d5709f6890d3cdac8668e5aefb9 \ + --hash=sha256:e9f643bd97374aeb7e4274e5c361277bf1428e8e55246cee6823fa3525dc2c04 \ + --hash=sha256:eadfb2ec0948a791846f7cc411c18ff5bd8359e8102cf78d39b8a28bd98b7c6a \ + --hash=sha256:ecb5101325e0254694e55d46ff936926296626bf81c902b60328ead152170bfd \ + --hash=sha256:ecf44731e351c6100885120cecf53a0f43a1bb975a92480de1f9e022ab542bc1 \ + --hash=sha256:f215ec9237d5f8e96d0205be6e2141292c20b133343c2153b6d4934f523ef422 \ + --hash=sha256:f523d34ad72b811be61e5027a046ad4670b54fc3c6384056f06b59f77d671af0 \ + --hash=sha256:f5916010de8b6045d7c63d98bb3f7a1e13f8c9c1046e859f2fd8de79c4e2d865 \ + --hash=sha256:f599401675bdcac21176ca385ea3139fc04c1f8e26724dc59f4fc9df1d1528ef \ + --hash=sha256:f5df0503093ec8c3c7e1da53920e79c62ab653b51b136c4736caa252552a8732 \ + --hash=sha256:f5f1d7e0d57cd75febd70c6fc26c2ea06024ab2006011abb3094b9fea54801ee \ + --hash=sha256:f61345393548bc72844d837d62fcf63444b0f5dac418544beb6d4f628a1c5df3 \ + --hash=sha256:f7bd9f7a369a2914102c375d98998c97d669306372732c850dd8319d5a92f515 \ + --hash=sha256:f7de4da27fa4481e8a7a4ef58c908ffd2b996847c3d7fbf27b5c53ce0984c408 \ + --hash=sha256:f85b4d87273e0ec63568433bbb8ccf160b548c49b53576fe010bad9f698b8586 \ + --hash=sha256:f92f6ded50c0ae7b0fed612be77c5345ed4a4578f99b98c7f8258d0e599eff16 \ + --hash=sha256:f9a49d8103df362272c625641a4f7b104ae584db21d0f1eec3b7a5d0d7750ed8 \ + --hash=sha256:fbd9ceb8c091c125e5892757e4d16af06674dc96edd7b5bf1e627b1ae7650c66 + # via mutmut +textual==8.2.8 \ + --hash=sha256:267375fd402dc8d981457212efa71f0e3365fd17bba144ba9bb3ed7563cb374a \ + --hash=sha256:3f106a9fbc73e39dd266c9712432087de78a6d644084c7c241d6a25c3169115b + # via mutmut +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # -c requirements.lock.txt + # textual diff --git a/dokimi/requirements-mutation.txt b/dokimi/requirements-mutation.txt new file mode 100644 index 00000000..a9d30de6 --- /dev/null +++ b/dokimi/requirements-mutation.txt @@ -0,0 +1,9 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# The mutation pilot's tool (.github/workflows/mutation.yml), compiled with hashes +# into requirements-mutation.lock.txt. Kept apart from requirements-ci.txt: only the +# weekly mutation workflow installs it, so CI, the release job, pip-audit's tools +# step and the eval image do not carry its closure (libcst, textual, ...). +# An exact pin: mutmut's config keys changed incompatibly between 2.x and 3.x. +mutmut==3.8.0 diff --git a/dokimi/tests/test_binary_stats.py b/dokimi/tests/test_binary_stats.py index f60ccbda..75bdf300 100644 --- a/dokimi/tests/test_binary_stats.py +++ b/dokimi/tests/test_binary_stats.py @@ -173,3 +173,54 @@ def test_small_pvalues_keep_significant_digits(): assert res["p_exact"] == pytest.approx(2 / 2 ** 20, rel=1e-2) assert res["p_exact"] > 0.0 assert math.isfinite(res["p_exact"]) + + +# Mutation pilot: the disclosure fields and the CI parameters reach the result. + + +def test_repeats_of_one_task_are_flagged_as_pseudo_replicated(): + res = mcnemar_test( + [True, True, True, False], [False, False, True, False], units=["a", "a", "b", "b"] + ) + assert (res["n"], res["n_units"]) == (4, 2) + assert res["pseudo_replicated"] is True + + +def test_one_unit_per_pair_is_not_pseudo_replicated(): + res = mcnemar_test([True, True, False], [False, True, False], units=["a", "b", "c"]) + assert (res["n"], res["n_units"]) == (3, 3) + assert res["pseudo_replicated"] is False + + +def test_a_units_list_of_the_wrong_length_is_rejected(): + with pytest.raises(ValueError, match="units must be one per pair: got 1 for 2 pairs"): + mcnemar_test([True, False], [False, False], units=["a"]) + + +def test_the_interval_follows_the_confidence_and_the_seed(): + treat = [True] * 6 + [False] * 4 + control = [False] * 3 + [True] * 3 + [False] * 4 + wide = mcnemar_test(treat, control) + narrow = mcnemar_test(treat, control, confidence=0.5) + assert narrow["ci_confidence"] == 0.5 + assert wide["ci_lo"] < narrow["ci_lo"] <= narrow["ci_hi"] < wide["ci_hi"] + deltas = [1.0] * 3 + [0.0] * 7 + boot = binary_stats.bootstrap_ci(deltas, seed=7) + seeded = mcnemar_test(treat, control, seed=7) + assert (seeded["ci_lo"], seeded["ci_hi"]) == (boot["lo"], boot["hi"]) + + +def test_the_seed_reaches_an_interval_that_depends_on_it(): + # 6 gained, 6 lost, 3 both-pass: the bootstrap bounds differ between seeds (the case above gives + # (0.0, 0.6) for every seed, so it cannot tell a forwarded seed from a dropped one). + treat = [True] * 6 + [False] * 6 + [True] * 3 + control = [False] * 6 + [True] * 6 + [True] * 3 + default = mcnemar_test(treat, control) + seeded = mcnemar_test(treat, control, seed=7) + assert (default["ci_lo"], default["ci_hi"]) == (-0.4, 0.4667) + assert (seeded["ci_lo"], seeded["ci_hi"]) == (-0.4667, 0.4667) + + +def test_the_paired_delta_keeps_four_decimals(): + res = mcnemar_test([True, False, False], [False, False, False]) + assert res["paired_pass_rate_delta"] == 0.3333 diff --git a/dokimi/tests/test_caching.py b/dokimi/tests/test_caching.py index e2e5e3e9..06b9a937 100644 --- a/dokimi/tests/test_caching.py +++ b/dokimi/tests/test_caching.py @@ -7,6 +7,8 @@ from dataclasses import replace +import hashlib + import json import pytest @@ -105,6 +107,11 @@ def test_openai_needs_no_breakpoint(): ("amazon-bedrock", "meta.llama3-70b-instruct-v1:0", False), ("google-vertex", "claude-3-5-haiku@20241022", True), ("google-vertex", "gemini-2.0-flash", False), + ("bedrock", "anthropic.claude-3-5-sonnet-20241022-v2:0", True), + ("vertex_ai", "claude-3-5-haiku@20241022", True), + ("openrouter", "anthropic/claude-3.5-sonnet", True), + ("openrouter", "openai/gpt-4o", False), + ("Anthropic", "", True), ("openai", "gpt-6.1-sol", False), ("", "", False), ], @@ -388,3 +395,92 @@ def test_no_temp_files_are_left_behind(monkeypatch, tmp_path): monkeypatch.setenv(caching.RESPONSE_CACHE_ENV, str(tmp_path)) caching.store_response("k", "answer", _usage()) assert [p.name for p in tmp_path.iterdir()] == ["k.json"] + + +# --------------------------------------------------------------------------- +# Mutation pilot: what the breakpoint looks like, and what the key separates +# --------------------------------------------------------------------------- + +BEDROCK_CLAUDE = {"model": "anthropic.claude-3-5-sonnet-20241022-v2:0", "provider": "bedrock"} +_EPHEMERAL = {"type": "ephemeral"} + + +def test_the_model_name_decides_the_prefix_breakpoint_on_a_shared_provider(): + messages, cached = caching.build_messages("q", cache_prefix=LONG, **BEDROCK_CLAUDE) + assert cached is True + assert messages[-1]["content"] == [ + {"type": "text", "text": LONG, "cache_control": _EPHEMERAL}, + {"type": "text", "text": "q"}, + ] + + +def test_the_model_name_decides_the_system_breakpoint_on_a_shared_provider(): + convo = [{"role": "system", "content": LONG}, {"role": "user", "content": "q"}] + out, cached = caching.annotate_system_breakpoint(convo, **BEDROCK_CLAUDE) + assert cached is True + assert out[0] == { + "role": "system", + "content": [{"type": "text", "text": LONG, "cache_control": _EPHEMERAL}], + } + llama = {"model": "meta.llama3-70b-instruct-v1:0", "provider": "bedrock"} + assert caching.annotate_system_breakpoint(convo, **llama) == (convo, False) + + +def _key(messages, extra=None): + return caching.response_cache_key(litellm_model="m", messages=messages, extra=extra) + + +def test_sampling_parameters_change_the_key_and_their_order_does_not(): + user = [{"role": "user", "content": "q"}] + assert _key(user, {"temperature": 0}) != _key(user, {"temperature": 1}) + assert _key(user, {"temperature": 0, "top_p": 1}) == _key(user, {"top_p": 1, "temperature": 0}) + assert _key(user, {}) == _key(user, None) + + +def test_the_role_is_part_of_the_key(): + assert _key([{"role": "system", "content": "q"}]) != _key([{"role": "user", "content": "q"}]) + + +def test_a_text_block_list_keys_like_its_concatenation(): + blocks = [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}] + assert _key([{"role": "user", "content": blocks}]) == _key([{"role": "user", "content": "ab"}]) + + +def test_image_blocks_are_keyed_by_content_not_by_cache_control(): + def msg(url, **extra): + image = {"type": "image_url", "image_url": {"url": url}, **extra} + return [{"role": "user", "content": [image, {"type": "text", "text": "what is it?"}]}] + + assert _key(msg("a.png")) != _key(msg("b.png")) + assert _key(msg("a.png")) == _key(msg("a.png", cache_control=_EPHEMERAL)) + + +def test_the_key_of_a_known_request_does_not_change(): + """Stored entries are found by this hash: a new key format orphans every one of them. + + Change the expected value only on purpose, with a changelog entry saying the + response cache starts cold. + """ + key = caching.response_cache_key( + litellm_model="anthropic/claude-x", + messages=[ + {"role": "system", "content": "Réponds"}, + {"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}, + ], + extra={"temperature": 0}, + ) + # The hashed payload, byte for byte: sorted keys, text blocks joined, UTF-8 kept. + stored_form = ( + '{"extra": {"temperature": 0}, "messages": [{"content": "Réponds", "role": "system"}, ' + '{"content": "ab", "role": "user"}], "model": "anthropic/claude-x"}' + ) + assert key == hashlib.sha256(stored_form.encode("utf-8")).hexdigest() + + +def test_a_non_json_extra_value_keys_like_its_string(): + from pathlib import PurePosixPath + + as_path = caching.response_cache_key( + litellm_model="m", messages=[], extra={"p": PurePosixPath("/x")} + ) + assert as_path == caching.response_cache_key(litellm_model="m", messages=[], extra={"p": "/x"}) diff --git a/dokimi/tests/test_critical_bands.py b/dokimi/tests/test_critical_bands.py index b0504e75..77419565 100644 --- a/dokimi/tests/test_critical_bands.py +++ b/dokimi/tests/test_critical_bands.py @@ -4,7 +4,7 @@ """Tests for critical bands enforcement.""" import pytest -from dokimi.gate.critical_bands import check_critical_bands +from dokimi.gate.critical_bands import BandViolation, check_critical_bands def test_critical_bands_pass(): @@ -127,3 +127,55 @@ def test_critical_bands_all_conditions(): assert len(result.violations) == 4 assert all(v.violated for v in result.violations) + + +# condition -> the summary key it reads +_CONDITION_KEYS = [ + ("score_below", "score"), + ("coverage_below", "coverage"), + ("freshness_below", "freshness_lite"), + ("readability_below", "readability"), +] + + +@pytest.mark.parametrize(("condition", "key"), _CONDITION_KEYS) +def test_each_condition_reads_its_own_summary_key(condition, key): + """Only the named key is in the summary: a condition that read another key + would see the 0.0 default and report a violation.""" + spec = {"critical_bands": {"fail_on": [{"condition": condition, "value": 0.5}]}} + + result = check_critical_bands({key: 0.9}, spec) + + assert result.violations == [BandViolation(condition, 0.5, 0.9, False)] + assert result.passed + + +@pytest.mark.parametrize(("condition", "key"), _CONDITION_KEYS) +def test_each_condition_is_strictly_below(condition, key): + """"below" is strict for every condition, not just score_below.""" + spec = {"critical_bands": {"fail_on": [{"condition": condition, "value": 0.7}]}} + + at = check_critical_bands({key: 0.7}, spec) + under = check_critical_bands({key: 0.69}, spec) + + assert at.violations == [BandViolation(condition, 0.7, 0.7, False)] + assert under.violations == [BandViolation(condition, 0.7, 0.69, True)] + + +@pytest.mark.parametrize(("condition", "key"), _CONDITION_KEYS) +def test_a_missing_metric_reads_as_zero_and_violates(condition, key): + spec = {"critical_bands": {"fail_on": [{"condition": condition, "value": 0.1}]}} + + result = check_critical_bands({}, spec) + + assert result.violations == [BandViolation(condition, 0.1, 0.0, True)] + assert not result.passed + + +def test_a_band_without_a_value_has_a_zero_floor(): + """No ``value`` means a 0.0 floor, which no score in [0, 1] is below.""" + spec = {"critical_bands": {"fail_on": [{"condition": "score_below"}]}} + + result = check_critical_bands({"score": 0.0}, spec) + + assert result.violations == [BandViolation("score_below", 0.0, 0.0, False)] diff --git a/dokimi/tests/test_dependency_locks.py b/dokimi/tests/test_dependency_locks.py index 27cc0389..c6cd093a 100644 --- a/dokimi/tests/test_dependency_locks.py +++ b/dokimi/tests/test_dependency_locks.py @@ -11,23 +11,30 @@ from pathlib import Path import pytest +import yaml from packaging.markers import Marker from packaging.requirements import Requirement ROOT = Path(__file__).resolve().parents[1] # The locks .github/renovate.json5 hands to the pip-compile manager. -RENOVATE_LOCKS = ("requirements.lock.txt", "requirements-ci.lock.txt") +RENOVATE_LOCKS = ("requirements.lock.txt", "requirements-ci.lock.txt", "requirements-mutation.lock.txt") # Installed together with --require-hashes (ci.yml, docs.yml, .agent-eval/Dockerfile, # make dev), so a pin both carry has to be the same version. STACKED_LOCKS = ( ("requirements.lock.txt", "requirements-ci.lock.txt"), ("requirements.lock.txt", "requirements-docs.lock.txt"), + ("requirements.lock.txt", "requirements-mutation.lock.txt"), ("requirements-ci.lock.txt", "requirements-docs.lock.txt"), ) PIN = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)==([^\s;\\]+)") HASH = re.compile(r"--hash=sha256:[0-9a-f]{64}(?!\w)") BARE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") -HASHED_LOCKS = ("requirements.lock.txt", "requirements-ci.lock.txt", "requirements-docs.lock.txt") +HASHED_LOCKS = ( + "requirements.lock.txt", + "requirements-ci.lock.txt", + "requirements-docs.lock.txt", + "requirements-mutation.lock.txt", +) # Where the stacked locks are installed: CI's Pythons, and make dev on any OS. ENVIRONMENTS = [ { @@ -139,6 +146,29 @@ def test_every_pin_has_hashes(name: str) -> None: assert not unhashed, f"{name}: pins without a valid --hash=sha256 line: {unhashed}" +@pytest.mark.parametrize("name", ("requirements-ci.lock.txt", "requirements-mutation.lock.txt")) +def test_tools_locks_are_constrained_to_main_lock_by_header(name: str) -> None: + assert "--constraint=requirements.lock.txt" in header_command(name) + + +def test_mutation_tool_stays_out_of_the_shared_tools_lock() -> None: + """CI, the release job, pip-audit's tools step and the eval image install the tools lock; + only mutation.yml installs mutmut, from its own lock.""" + assert "mutmut" not in pins("requirements-ci.lock.txt") + assert pins("requirements-mutation.lock.txt")["mutmut"] == {"3.8.0"} + requested = (ROOT / "requirements-mutation.txt").read_text(encoding="utf-8") + assert "mutmut==3.8.0" in requested + + +@pytest.mark.parametrize("name", HASHED_LOCKS) +def test_pip_audit_audits_every_lock(name: str) -> None: + """A lock the audit job does not list ships its pins unchecked against advisories.""" + workflow = yaml.safe_load((ROOT.parent / ".github/workflows/pip-audit.yml").read_text(encoding="utf-8")) + # The audit command itself, not the file text: comments and the Install tools step name locks too. + (run,) = [s["run"] for s in workflow["jobs"]["pip-audit"]["steps"] if s.get("name") == "pip-audit"] + assert name in run.split() + + def test_tools_lock_has_every_requested_pin() -> None: """An exact pin in requirements-ci.txt (ruff, zizmor, ...) must be the version the tools lock resolves.""" locked = pins("requirements-ci.lock.txt") diff --git a/dokimi/tests/test_regression.py b/dokimi/tests/test_regression.py index 6d20f82c..3a4f12a9 100644 --- a/dokimi/tests/test_regression.py +++ b/dokimi/tests/test_regression.py @@ -3,7 +3,9 @@ """Tests for regression detection.""" -from dokimi.gate.regression import detect_regressions +import pytest + +from dokimi.gate.regression import RegressionResult, detect_regressions def test_regression_ok(): @@ -118,3 +120,126 @@ def test_regression_missing_thresholds(): result = detect_regressions(diff, spec) assert result.score_regression.severity == "OK" # -0.01 < 0.03 warn threshold + + +# The cases above all pass thresholds equal to the defaults, so they cannot tell +# a spec that is read from one that is ignored. These pin each half separately. + +_CUSTOM = { + "thresholds": { + "regressions": { + "score_drop_warn": 0.10, + "score_drop_critical": 0.20, + "metric_drop_warn": 0.01, + "metric_drop_critical": 0.02, + } + } +} + + +def test_spec_thresholds_replace_the_defaults(): + """A -0.05 score drop is WARN by default but OK under a 0.10 warn threshold, + and a -0.015 coverage drop is OK by default but WARN under a 0.01 one.""" + result = detect_regressions({"score": -0.05, "coverage": -0.015}, _CUSTOM) + + assert result.score_regression == RegressionResult( + metric="score", delta=-0.05, severity="OK", + threshold_warn=0.10, threshold_critical=0.20, + ) + assert result.metric_regressions == [ + RegressionResult( + metric="coverage", delta=-0.015, severity="WARN", + threshold_warn=0.01, threshold_critical=0.02, + ) + ] + + +def test_spec_critical_thresholds_replace_the_defaults(): + """-0.05 is only WARN under the 0.08 default; a 0.04 critical makes it CRITICAL.""" + spec = {"thresholds": {"regressions": {"score_drop_critical": 0.04, + "metric_drop_critical": 0.04}}} + result = detect_regressions({"score": -0.05, "coverage": -0.05}, spec) + + assert result.score_regression.severity == "CRITICAL" + assert result.metric_regressions[0].severity == "CRITICAL" + + +@pytest.mark.parametrize( + ("key", "delta", "severity"), + [ + # The documented defaults, each hit exactly: a drop *at* a threshold counts. + ("score", -0.03, "WARN"), + ("score", -0.08, "CRITICAL"), + ("coverage", -0.05, "WARN"), + ("coverage", -0.12, "CRITICAL"), + # Just under each default: a default lowered by 0.01 would classify these higher. + ("score", -0.029, "OK"), + ("score", -0.079, "WARN"), + ("coverage", -0.049, "OK"), + ("coverage", -0.119, "WARN"), + ], +) +def test_default_thresholds_apply_at_their_exact_values(key, delta, severity): + result = detect_regressions({key: delta}, {}) + + found = result.score_regression if key == "score" else result.metric_regressions[0] + assert (found.metric, found.severity) == (key, severity) + + +def test_a_score_drop_alone_sets_the_summary_flags(): + critical = detect_regressions({"score": -0.5}, {}) + assert critical.has_critical + assert not critical.has_warnings # a CRITICAL is not a WARN + + warn = detect_regressions({"score": -0.04}, {}) + assert warn.has_warnings + assert not warn.has_critical + + +def test_a_metric_regression_alone_sets_the_summary_flags(): + warn = detect_regressions({"score": 0.0, "coverage": -0.06}, {}) + assert warn.has_warnings + assert not warn.has_critical + + +def test_a_critical_on_a_later_tracked_metric_sets_has_critical(): + # Not the first metric in the list: has_critical must look at all of metric_regressions. + result = detect_regressions( + {"score": 0.0, "example_pass_rate": -0.5, "coverage": 0.0, "freshness_lite": 0.0, "readability": 0.0}, {} + ) + assert result.has_critical + + +@pytest.mark.parametrize("name", ["coverage", "freshness_lite", "readability", "example_pass_rate"]) +def test_every_tracked_metric_is_classified(name): + result = detect_regressions({"score": 0.0, name: -0.5}, {}) + + assert [(r.metric, r.delta, r.severity) for r in result.metric_regressions] == [ + (name, -0.5, "CRITICAL") + ] + assert result.has_critical + + +def test_an_untracked_metric_is_ignored(): + result = detect_regressions({"score": 0.0, "latency": -0.5}, {}) + + assert result.metric_regressions == [] + assert not result.has_warnings + + +def test_a_diff_without_a_score_reads_as_no_change(): + result = detect_regressions({"coverage": 0.01}, {}) + + assert (result.score_regression.delta, result.score_regression.severity) == (0.0, "OK") + + +def test_no_change_is_ok_even_with_zero_tolerance(): + """With every threshold at 0, any drop is CRITICAL, but no change is not a drop.""" + zero = {"thresholds": {"regressions": dict.fromkeys( + ("score_drop_warn", "score_drop_critical", "metric_drop_warn", "metric_drop_critical"), 0.0)}} + + unchanged = detect_regressions({"score": 0.0, "coverage": 0.0}, zero) + dropped = detect_regressions({"score": -0.001, "coverage": 0.0}, zero) + + assert not unchanged.has_warnings and not unchanged.has_critical + assert dropped.score_regression.severity == "CRITICAL" diff --git a/dokimi/tests/test_score_gate.py b/dokimi/tests/test_score_gate.py index 25d29985..945304cd 100644 --- a/dokimi/tests/test_score_gate.py +++ b/dokimi/tests/test_score_gate.py @@ -63,3 +63,21 @@ def test_each_gate_reads_only_its_own_key(): assert check_gate({"score": 0.7}, spec, "soft").passed is False assert check_gate({"score": 0.7}, spec, "hard").passed is True + + +@pytest.mark.parametrize("kind", KINDS) +def test_enabled_gate_without_min_score_lets_any_score_pass(kind): + """min_score defaults to 0.0: an enabled gate with no floor blocks nothing.""" + spec = {"future": {f"{kind}_gate": {"enabled": True}}} + + result = check_gate({"score": 0.25}, spec, kind) + + assert (result.min_score, result.passed) == (0.0, True) + + +@pytest.mark.parametrize("kind", KINDS) +def test_a_summary_without_a_score_fails_an_enabled_gate(kind): + """A missing score reads as 0.0, never as a pass: a broken run must not clear the gate.""" + result = check_gate({}, _spec(kind, min_score=0.5), kind) + + assert (result.actual_score, result.passed, result.status) == (0.0, False, "FAIL") diff --git a/dokimi/tests/test_stats.py b/dokimi/tests/test_stats.py index c2618d45..1b3bbb9b 100644 --- a/dokimi/tests/test_stats.py +++ b/dokimi/tests/test_stats.py @@ -380,6 +380,7 @@ def test_bootstrap_ci_is_pinned_to_its_seed(): assert ci["method"] == "percentile_bootstrap" assert ci["mean"] == pytest.approx(13.3333, abs=0.0001) pinned = (ci["lo"], ci["hi"]) + assert pinned == (9.1667, 20.0) # A different seed must move the interval; identical bounds would mean the # seed never reaches the resampler. other = stats.bootstrap_ci(vals, seed=7, n_resamples=1000) @@ -535,3 +536,185 @@ def test_arm_significance_leaves_the_adjusted_verdict_unknown_when_no_test_ran() # The untestable arm must not inflate the tested arm's multiplier. assert res["skill:x"]["family_size"] == 2 assert res["skill:x"]["family_tested"] == 1 + + +# --------------------------------------------------------------------------- +# Pinned values at the boundaries (mutation pilot: each test below kills a +# surviving mutant of a comparison, an off-by-one or a dropped term) +# --------------------------------------------------------------------------- + + +def test_a_wilcoxon_above_alpha_vetoes_a_significant_t_test(): + """One outlier keeps the signed-rank test above 0.05 while the t-test is below.""" + pytest.importorskip("scipy") + res = stats.paired_test([10, 11, 12, 10, 11, -1], [0] * 6) + assert res["p_ttest"] == 0.0068 + assert res["p_wilcoxon"] == 0.0625 + assert res["n_wilcoxon"] == 6 + assert res["significant"] is False + + +def test_min_pairs_for_test_is_the_smallest_n_that_can_be_significant(): + """An exact two-sided Wilcoxon cannot reach 0.05 at five pairs (smallest p 0.0625).""" + pytest.importorskip("scipy") + assert stats.MIN_PAIRS_FOR_TEST == 6 + five = stats.paired_test([1.0, 2.0, 3.0, 4.0, 5.0], [0.0] * 5) + assert (five["n"], five["p_ttest"], five["p_wilcoxon"], five["significant"]) == (5, None, None, None) + six = stats.paired_test([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], [0.0] * 6) + assert six["p_wilcoxon"] == 0.0312 + + +def test_below_min_pairs_no_test_runs(): + pytest.importorskip("scipy") + n = stats.MIN_PAIRS_FOR_TEST - 1 + res = stats.paired_test([float(i) for i in range(n)], [0.5 * i for i in range(n)]) + assert res["n"] == n + assert (res["p_ttest"], res["p_wilcoxon"], res["n_wilcoxon"]) == (None, None, 0) + assert res["significant"] is None + + +def test_two_pairs_are_enough_for_a_description(): + res = stats.paired_test([1.0, 3.0], [0.0, 0.0]) + assert res["n"] == 2 + assert res["delta_mean"] == 2.0 + assert res["cohens_d"] == 1.414 + + +def test_bootstrap_interval_uses_the_two_sided_alpha(): + # 10,000 resamples of [0, 1]: the 30th and 70th percentiles of the + # resampled means are both 0.5 at confidence 0.4. + res = stats.bootstrap_ci([0, 1], confidence=0.4) + assert (res["lo"], res["hi"]) == (0.5, 0.5) + + +def test_bootstrap_interval_pins_the_tail_indices_and_the_seed(): + # A sample whose resampled means spread out, so a one-index shift in either tail, a + # one-sided alpha, or another seed each moves the bounds (for [0, 1] they would not). + # 100 resamples at 0.8: alpha * 99 = 9.9, so the low index rounds to 10, not 9. + ci = stats.bootstrap_ci([0, 1, 2, 3, 10, 4.5, 7.5], confidence=0.8, n_resamples=100, seed=7) + assert (ci["lo"], ci["hi"]) == (2.0714, 5.7857) + wide = stats.bootstrap_ci([0, 1, 2, 3, 10], confidence=0.8, n_resamples=200, seed=7) + assert (wide["lo"], wide["hi"]) == (1.2, 5.0) + + +@pytest.mark.parametrize( + ("confidence", "n_resamples", "expected"), + [(0.95, 100, (1.2143, 6.5714)), (0.5, 50, (3.0, 4.9286))], +) +def test_bootstrap_interval_tails_are_symmetric_order_statistics(confidence, n_resamples, expected): + # Literals from the original code. Dropping round() from either index, or using n_resamples + # instead of n_resamples - 1, moves a bound by one order statistic on these (and not on the case above). + ci = stats.bootstrap_ci([0, 1, 2, 3, 10, 4.5, 7.5], confidence=confidence, n_resamples=n_resamples, seed=7) + assert (ci["lo"], ci["hi"]) == expected + + +def test_paired_test_rejects_a_length_mismatch_in_both_directions(): + assert stats.paired_test([1, 2, 3], [1, 2, 3, 4]) is None # control longer + assert stats.paired_test([1, 2, 3, 4], [1, 2, 3]) is None # treatment longer + + +def test_mean_ci_forwards_the_seed_to_the_bootstrap_fallback(monkeypatch): + seen = {} + + def fake(vals, confidence=0.95, **kw): + seen.update(kw) + return {"mean": 0.0, "lo": 0.0, "hi": 0.0, "n": len(vals)} + + monkeypatch.setattr(stats, "HAS_SCIPY", False) + monkeypatch.setattr(stats, "bootstrap_ci", fake) + stats.mean_ci([1.0, 2.0, 3.0], seed=7) + assert seen == {"seed": 7} + + +def test_bootstrap_ci_defaults_are_pinned(): + """grounding.py calls bootstrap_ci with every default, so the default seed and + n_resamples set the stored grounding_score interval.""" + ci = stats.bootstrap_ci([0.12, 0.47, 0.83, 0.31, 0.95, 0.58, 0.06, 0.74, 0.29, 0.66, 0.41, 0.88]) + assert (ci["lo"], ci["hi"]) == (0.3617, 0.6842) + + +def test_bootstrap_with_one_resample_collapses_to_it(): + res = stats.bootstrap_ci([0, 1], n_resamples=1) + assert res["lo"] == res["hi"] + assert res["lo"] in (0.0, 0.5, 1.0) + assert res["n"] == 2 + + +@pytest.mark.parametrize("confidence", [0.0, 1.0, 1.5]) +def test_bootstrap_rejects_a_confidence_outside_the_open_interval(confidence): + with pytest.raises(ValueError, match="confidence"): + stats.bootstrap_ci([0, 1], confidence=confidence) + + +def test_empty_and_single_samples_have_exact_shapes(): + assert stats.bootstrap_ci([]) == {"mean": None, "lo": None, "hi": None, "n": 0} + assert stats.bootstrap_ci([2.5]) == {"mean": 2.5, "lo": 2.5, "hi": 2.5, "n": 1} + assert stats.mean_ci([]) == { + "mean": None, "lo": None, "hi": None, "n": 0, "method": "none" + } + assert stats.mean_ci([5.0]) == {"mean": 5.0, "lo": 5.0, "hi": 5.0, "n": 1} + + +def test_zero_variance_tolerance_scales_with_the_mean(): + """At a mean of 1e6, noise of 1e-10 is below the relative tolerance.""" + pytest.importorskip("scipy") + vals = [1e6 + (1e-10 if i % 2 else -1e-10) for i in range(8)] + res = stats.mean_ci(vals) + assert res["method"] == "degenerate" + assert res["n"] == 8 + + +def test_adjusted_pvalues_are_rounded_to_four_places_by_default(): + assert stats.adjust_pvalues([0.0123456]) == [0.0123] + + +def test_paired_test_rounds_the_wilcoxon_p(): + pytest.importorskip("scipy") + res = stats.paired_test([3, 5, 4, 6, 2, 7, 5], [1, 2, 2, 1, 2, 3, 1]) + assert res["p_wilcoxon"] == round(res["p_wilcoxon"], 4) + assert 0 < res["p_wilcoxon"] < 1 + + +@pytest.mark.parametrize( + "flags,totals,n_pass,reason", + [ + ([True, False], [1, 2], 1, "fewer than 3 respondents"), + ([True] * 3, [1, 2, 3], 3, + "no variance in the item: every respondent scored it the same"), + ([True, False, True], [2, 2, 2], 2, + "no variance in the totals: nothing for the item to discriminate"), + ], +) +def test_point_biserial_says_why_it_is_undefined(flags, totals, n_pass, reason): + res = stats.point_biserial(flags, totals) + assert res["r"] is None + assert res["n"] == len(flags) + assert res["n_pass"] == n_pass + assert res["reason"] == reason + + +def test_arm_significance_passes_the_confidence_through(): + pytest.importorskip("scipy") + wide = stats.arm_significance(_factorial_evaluations(), "baseline") + narrow = stats.arm_significance(_factorial_evaluations(), "baseline", confidence=0.8) + for arm in wide: + assert wide[arm]["ci_confidence"] == 0.95 + assert narrow[arm]["ci_confidence"] == 0.8 + assert wide[arm]["ci_lo"] < narrow[arm]["ci_lo"] < narrow[arm]["ci_hi"] < wide[arm]["ci_hi"] + + +def test_arm_significance_includes_an_arm_with_two_pairs(): + res = stats.arm_significance(_factorial_evaluations()[:2], "baseline") + assert set(res) == {"docs", "skill:x", "docs+skill:x"} + assert all(sig["n"] == 2 for sig in res.values()) + + +def test_arm_significance_correction_changes_the_adjusted_p(): + pytest.importorskip("scipy") + holm = stats.arm_significance(_factorial_evaluations(), "baseline") + bh = stats.arm_significance(_factorial_evaluations(), "baseline", correction="bh") + # Holm multiplies the second-smallest p by 2, BH by 3/2. + assert holm["docs+skill:x"]["p_adjusted"] == 0.0004 + assert bh["docs+skill:x"]["p_adjusted"] == 0.0003 + assert holm["skill:x"]["significant_adjusted"] is True + assert holm["docs"]["significant_adjusted"] is False diff --git a/dokimi/tests/test_trust_gate.py b/dokimi/tests/test_trust_gate.py index 8bc9a3f1..8c520492 100644 --- a/dokimi/tests/test_trust_gate.py +++ b/dokimi/tests/test_trust_gate.py @@ -240,3 +240,225 @@ def test_report_json_lists_not_measured_checks(self): )) assert report["trust"]["verdict"] == "✅ PASS" assert report["trust"]["not_measured"] == ["retrieval_hit_rate"] + + +def _row(with_score, without_score, *, agree=None, hit=None): + ev = {"question_id": "q", "with_docs": {"aggregate": with_score}, + "without_docs": {"aggregate": without_score}} + if agree is not None: + ev["with_docs"]["agreement_score"] = agree + if hit is not None: + ev["retrieval_hit"] = hit + return ev + + +def _check(verdict, name): + return next(c for c in verdict.checks if c.name == name) + + +class TestEveryCheckReportsItsNumbers: + """One small run, every check's value worked out by hand. + + Context arm 0, 40, 60 (mean 33.33, sample sd 30.55); baseline 10, 20, 30; + panel agreement 0.1, 0.2, 0.4; two of three questions retrieved a doc; two + runs averaging 40 and 45 (sd 3.536). + """ + + def test_values_thresholds_and_severities(self): + evals = [ + _row(0, 10, agree=0.1, hit=True), + _row(40, 20, agree=0.2, hit=True), + _row(60, 30, agree=0.4, hit=False), + ] + + v = evaluate_trust(evals, multirun_with_averages=[40.0, 45.0]) + + assert [(c.name, c.passed, c.value, c.threshold, c.severity, c.measured) + for c in v.checks] == [ + ("min_questions", False, 3, 10, "fail", True), + ("zero_score_fraction", False, 0.333, 0.20, "fail", True), + ("min_with_docs_avg", True, 33.3, 30.0, "warn", True), + ("min_delta", True, 13.3, -5.0, "warn", True), + ("score_cv", False, 0.917, 0.60, "warn", True), + ("inter_rater_agreement", False, 0.233, 0.50, "warn", True), + ("retrieval_hit_rate", True, 0.667, 0.60, "warn", True), + ("multirun_variance", True, 3.54, 5.0, "fail", True), + ] + assert round(v.multirun_std, 4) == 3.5355 + assert [c.name for c in v.failures] == ["min_questions", "zero_score_fraction"] + assert [c.name for c in v.warnings] == ["score_cv", "inter_rater_agreement"] + + +class TestEveryThresholdIsInclusive: + """A value exactly at its threshold passes, for every check.""" + + def test_min_questions_at_floor(self): + assert _check(evaluate_trust(_evals(10)), "min_questions").passed + + def test_zero_fraction_at_ceiling(self): + evals = [_row(0, 0)] * 2 + [_row(70, 50)] * 8 + assert _check(evaluate_trust(evals), "zero_score_fraction").passed + + def test_context_average_at_floor(self): + check = _check(evaluate_trust(_evals(10, with_score=30, without_score=20)), "min_with_docs_avg") + assert (check.value, check.passed) == (30.0, True) + + def test_delta_at_floor(self): + check = _check(evaluate_trust(_evals(10, with_score=50, without_score=55)), "min_delta") + assert (check.value, check.passed) == (-5.0, True) + + def test_cv_at_ceiling(self): + v = evaluate_trust(_evals(10), thresholds={"max_cv": 0.0}) + check = _check(v, "score_cv") + assert (check.value, check.passed) == (0.0, True) + + def test_agreement_at_floor(self): + evals = [_row(70, 50, agree=0.5)] * 10 + check = _check(evaluate_trust(evals), "inter_rater_agreement") + assert (check.value, check.passed) == (0.5, True) + + def test_one_agreement_score_among_many_rows_is_still_judged(self): + evals = [_row(70, 50)] * 10 + [_row(70, 50, agree=0.1)] + check = _check(evaluate_trust(evals), "inter_rater_agreement") + assert (check.passed, check.value) == (False, 0.1) + + def test_hit_rate_at_floor(self): + evals = [_row(70, 50, hit=True)] * 6 + [_row(70, 50, hit=False)] * 4 + check = _check(evaluate_trust(evals), "retrieval_hit_rate") + assert (check.value, check.passed) == (0.6, True) + + def test_multirun_std_at_ceiling(self): + v = evaluate_trust(_evals(10), thresholds={"max_multirun_std": 0.0}, + multirun_with_averages=[50.0, 50.0]) + check = _check(v, "multirun_variance") + assert (check.value, check.passed) == (0.0, True) + + +class TestWhatTheChecksCount: + def test_a_question_scored_in_one_arm_only_is_not_counted(self): + evals = _evals(10) + [{"question_id": "q", "with_docs": {"aggregate": 90}}] + assert _check(evaluate_trust(evals), "min_questions").value == 10 + + def test_a_question_scored_in_the_baseline_only_is_not_counted(self): + # The other half of the both-arms filter: a baseline score with no context score + # would otherwise count, and its missing context score read as 0. + evals = _evals(10) + [{"question_id": "q", "without_docs": {"aggregate": 50}}] + v = evaluate_trust(evals) + assert _check(v, "min_questions").value == 10 + assert _check(v, "zero_score_fraction").value == 0.0 + + def test_coverage_below_the_default_floor_is_not_measured(self): + # 15 of 20 record a retrieval outcome: 75%, under the 80% default floor. + v = evaluate_trust([_row(70, 50, hit=True)] * 15 + [_row(70, 50)] * 5) + assert not _check(v, "retrieval_hit_rate").measured + assert v.not_measured == ["retrieval_hit_rate"] + + def test_a_zero_baseline_counts_as_zero(self): + check = _check(evaluate_trust(_evals(10, with_score=40, without_score=0)), "min_delta") + assert check.value == 40.0 + + def test_two_runs_are_enough_for_the_variance_check(self): + v = evaluate_trust(_evals(10), multirun_with_averages=[50.0, 70.0]) + check = _check(v, "multirun_variance") + assert (check.value, check.passed) == (14.14, False) + + +class TestScoreCv: + def test_one_question_has_no_spread(self): + check = _check(evaluate_trust([_row(50, 40)], thresholds={"min_questions": 1}), "score_cv") + assert (check.value, check.passed) == (0.0, True) + + def test_two_questions_use_the_sample_sd(self): + # sd(40, 60) = 14.142, mean 50 + check = _check(evaluate_trust([_row(40, 0), _row(60, 0)]), "score_cv") + assert check.value == 0.283 + + def test_a_mean_below_one_is_still_divided(self): + # sd(0.5, 1.5) = 0.707 over a mean of exactly 1.0 + check = _check(evaluate_trust([_row(0.5, 0), _row(1.5, 0)]), "score_cv") + assert check.value == 0.707 + + def test_an_all_zero_context_arm_reads_as_cv_one(self): + check = _check(evaluate_trust(_evals(10, with_score=0)), "score_cv") + assert (check.value, check.passed) == (1.0, False) + + +def test_a_not_measured_retrieval_check_never_warns(): + """Under the coverage floor the rate is not judged: it passes, is not a + warning, and serializes ``measured`` as false (the report JSON reads it).""" + evals = [_row(70, 50, hit=False)] * 2 + [_row(70, 50)] * 8 + + v = evaluate_trust(evals) + check = _check(v, "retrieval_hit_rate") + + assert (check.passed, check.measured, check.value) == (True, False, None) + assert (check.threshold, check.severity) == (0.60, "warn") + assert check not in v.warnings + assert v.status == "✅ PASS" + + +def test_a_legacy_artifact_serializes_measured_false(): + check = _check(evaluate_trust(_evals(10)), "retrieval_hit_rate") + assert (check.threshold, check.severity) == (0.60, "warn") + assert (check.passed, check.measured) == (True, False) + + +class TestTheRenderedBlock: + """The block a report shows, whole: each status gets its icon, each list its checks.""" + + def test_failures_and_warnings_render_apart(self): + evals = [ + _row(0, 10, agree=0.1, hit=True), + _row(40, 20, agree=0.2, hit=True), + _row(60, 30, agree=0.4, hit=False), + ] + md = format_trust_block(evaluate_trust(evals, multirun_with_averages=[40.0, 45.0])) + assert md == "\n".join([ + "## Can We Trust This Run?", + "", + "**Verdict: ❌ FAIL**", + "", + "| Check | Status | Value | Threshold | Notes |", + "|-------|--------|-------|-----------|-------|", + "| min_questions | ❌ | 3 | 10 | 3 questions evaluated (min 10) |", + "| zero_score_fraction | ❌ | 0.333 | 0.2 | 1/3 questions scored 0 in the context arm (33.3%) |", + "| min_with_docs_avg | ✅ | 33.3 | 30.0 | Context-arm avg = 33.3 (min 30.0) |", + "| min_delta | ✅ | 13.3 | -5.0 | Avg delta (context arm minus baseline) = +13.3 (min -5.0) |", + "| score_cv | ⚠️ | 0.917 | 0.6 | Score CV = 0.92 std=30.6 (max CV 0.6) |", + "| inter_rater_agreement | ⚠️ | 0.233 | 0.5 | Mean inter-rater agreement = 0.23 (min 0.5) |", + "| retrieval_hit_rate | ✅ | 0.667 | 0.6 | Retrieval hit rate = 66.7% (2/3) (min 60%) |", + "| multirun_variance | ✅ | 3.54 | 5.0 | Multi-run context-arm std = 3.54 across 2 runs (max 5.0) |", + "", + "**Blocking issues:**", + "- ❌ **min_questions**: 3 questions evaluated (min 10)", + "- ❌ **zero_score_fraction**: 1/3 questions scored 0 in the context arm (33.3%)", + "", + "**Warnings:**", + "- ⚠️ **score_cv**: Score CV = 0.92 std=30.6 (max CV 0.6)", + "- ⚠️ **inter_rater_agreement**: Mean inter-rater agreement = 0.23 (min 0.5)", + "", + ]) + + def test_a_not_measured_check_renders_as_neither_pass_nor_fail(self): + md = format_trust_block(evaluate_trust([_row(70, 50)] * 10)) + lines = md.splitlines() + assert lines[2] == "**Verdict: ✅ PASS** (1 of 6 not measured)" + assert lines[-1] == ( + "| retrieval_hit_rate | ➖ | None | 0.6 | Not measured: the evaluations " + "record no retrieval outcome (retrieval_hit) |" + ) + assert "Warnings" not in md + + def test_the_hit_rate_names_the_unrecorded_evaluations(self): + v = evaluate_trust([_row(70, 50, hit=True)] * 9 + [_row(70, 50)]) + assert _check(v, "retrieval_hit_rate").message == ( + "Retrieval hit rate = 100.0% (9/9; 1 unrecorded) (min 60%)" + ) + + def test_a_warn_only_verdict_renders_warnings_and_no_blocking_section(self): + evals = [_row(5, 5)] * 7 + [_row(100, 5)] * 3 + verdict = evaluate_trust(evals) + assert verdict.warnings and not verdict.failures + md = format_trust_block(verdict) + assert "**Warnings:**" in md + assert "**Blocking issues:**" not in md