From 5c2fe61a2859073477b1e0288ce2e622021bac78 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:34:17 -0700 Subject: [PATCH 1/7] test(report): golden output for every branch of generate_section_report Eleven cases render a whole model-compare section and compare it byte for byte with a recorded golden (one JSON), ahead of splitting the 53-complexity builder into per-section helpers. Together they reach every line and branch of the function (coverage.py --branch): treatment and baseline-only runs, a mixed pair, one and two scored arms, a pinned arm, cost rows with and without a summary, SciPy on and off, Holm and BH, difficulty buckets a run is missing from, replicate pooling with a label clash and a left-out run, and an empty section. Signed-off-by: Nikolay Petrov --- .../fixtures/model_compare_sections.json | 13 ++ dokimi/tests/test_model_compare_sections.py | 206 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 dokimi/tests/fixtures/model_compare_sections.json create mode 100644 dokimi/tests/test_model_compare_sections.py diff --git a/dokimi/tests/fixtures/model_compare_sections.json b/dokimi/tests/fixtures/model_compare_sections.json new file mode 100644 index 00000000..608e8a30 --- /dev/null +++ b/dokimi/tests/fixtures/model_compare_sections.json @@ -0,0 +1,13 @@ +{ + "baseline_only": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | — (baseline only) |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline | b baseline |\n|---|---:|| :---: | :---: |\n| easy | 3 | 60.0 | — |\n| beginner | 1 | 55.0 | — |\n| intermediate | 1 | 30.0 | — |\n| medium | 1 | 60.0 | — |\n| advanced | 1 | 65.0 | — |\n| hard | 1 | 40.0 | 56.2 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n| 2 | **b** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "baseline_only_no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 3 | 60.0 |\n| beginner | 1 | 55.0 |\n| intermediate | 1 | 30.0 |\n| medium | 1 | 60.0 |\n| advanced | 1 | 65.0 |\n| hard | 1 | 40.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "empty": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n\nCommon questions evaluated: **0**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "mixed": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `with-context` |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | — | — |\n| beginner | 1 | 75.0 | +20.0 | — | — |\n| intermediate | 1 | 45.0 | +15.0 | — | — |\n| medium | 1 | 70.0 | +10.0 | — | — |\n| advanced | 1 | 90.0 | +25.0 | — | — |\n| hard | 1 | 80.0 | +40.0 | — | 56.2 (baseline) |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| **b** | 0 | 0% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | — | — | — |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | — |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | N/A |\n", + "no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "pinned_arm": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | docs | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ |\n|---|---:| :---: | :---: |\n| easy | 9 | 68.8 | +17.1 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `docs`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n\n## Model Ranking\n\n### By absolute quality (docs)\n| Rank | Run | docs avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n", + "pooled": "## Replicate Pooling\n\nRuns sharing a configuration were averaged per question, so each row below is one configuration rather than one sample of it. Each pooled score is a mean of k runs, so its spread and confidence intervals are narrower than a single run's: that reflects the k runs spent, not a sharper measurement.\n\n- **m[n=2]'** pools 2 replicates: m_run1, m_run2; 2 question(s) dropped for not being scored in every replicate\n\n## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **m[n=2]** | solo (p) | same (p) | `with-context` |\n| **m[n=2]'** | rep (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **7**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **m[n=2]** | — | — | — | — | — | — |\n| **m[n=2]'** | $0.2469 (6/16) | 24,690 | 1,356 | 26,046 | 1.23s | 25.0% |\n\n_A pooled `[n=k]` row shows the total cost, calls and tokens of its k runs and their mean latency._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **m[n=2]** | 7 | 7 | 62.9 | 52.9 | **+10.0** | 7 | 0 |\n| **m[n=2]'** | 7 | 7 | 66.4 | 52.9 | **+13.6** | 6 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **m[n=2]** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n| **m[n=2]'** | +13.6 | 0.0033 | 0.0033 | 0.0312 | +1.8 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | m[n=2] context | m[n=2] Δ | m[n=2]' context | m[n=2]' Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 2 | 70.0 | +10.0 | 67.5 | +7.5 |\n| beginner | 1 | 65.0 | +10.0 | 70.0 | +15.0 |\n| intermediate | 1 | 40.0 | +10.0 | 42.5 | +12.5 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 75.0 | +10.0 | 82.5 | +17.5 |\n| hard | 1 | 50.0 | +10.0 | 65.0 | +25.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **m[n=2]** | 1 | 14% |\n| **m[n=2]'** | 5 | 71% |\n| tie | 1 | 14% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **m[n=2]** | **+10.0** | 7 (100%) | 0 (0%) |\n| **m[n=2]'** | **+13.6** | 6 (86%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **m[n=2]'** | 66.4 |\n| 2 | **m[n=2]** | 62.9 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **m[n=2]'** | +13.6 |\n| 2 | **m[n=2]** | +10.0 |\n", + "pooled_singletons": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **x** | x (p) | same (p) | `with-context` |\n| **y** | y (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **x** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **y** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **x** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **y** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | x context | x Δ | y context | y Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **x** | 6 | 67% |\n| **y** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **x** | **+17.1** | 6 (86%) | 1 (14%) |\n| **y** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **x** | 68.8 |\n| 2 | **y** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **x** | +17.1 |\n| 2 | **y** | +10.0 |\n", + "treatment": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "treatment_bh": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Benjamini-Hochberg); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "two_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `docs` |\n| **b** | beta (p) | same (p) | `skill:x` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n" +} diff --git a/dokimi/tests/test_model_compare_sections.py b/dokimi/tests/test_model_compare_sections.py new file mode 100644 index 00000000..0c6d03f6 --- /dev/null +++ b/dokimi/tests/test_model_compare_sections.py @@ -0,0 +1,206 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Golden output of :func:`generate_section_report`, one case per section branch. + +Each case renders a whole section and compares it byte for byte with +its entry in ``tests/fixtures/model_compare_sections.json``, so a refactor of the section +builder that moves a single character fails here. Between them the cases reach +every section and branch: treatment and baseline-only runs, a run with no +treatment beside one with, one scored arm and two, resource rows with and +without a cost summary, significance on and off (no SciPy), both corrections, +difficulty buckets a run is missing from, replicate pooling with a label clash +and a run left out of ``run_ids``, and an empty section. + +To re-record after an intended output change:: + + DOKIMI_UPDATE_GOLDEN=1 pytest tests/test_model_compare_sections.py +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from dokimi.report import model_compare +from dokimi.report.model_compare import generate_section_report + +GOLDEN = Path(__file__).parent / "fixtures" / "model_compare_sections.json" + +_DIFFICULTIES = ["easy", "medium", "hard", 1, "beginner", "advanced", "intermediate", "expert"] + + +def _run(scores: list[tuple], arm: str = "with-context", difficulties=None, **meta) -> dict: + """An ``arms.v1``-shaped artifact; *scores* holds (baseline, treatment) per question. + + A treatment of ``...`` leaves the arm out of the record (a baseline-only question); + ``None`` keeps the arm with no usable score. + """ + difficulties = difficulties or _DIFFICULTIES + evaluations = [] + for i, (base, treat) in enumerate(scores): + cond: dict = {"baseline": {"aggregate": base}} + if treat is not ...: + cond[arm] = {"aggregate": treat} + evaluations.append( + { + "question_id": f"q{i}", + "question_text": f"question {i}", + "difficulty": difficulties[i % len(difficulties)], + "scores": cond, + } + ) + return { + "baseline_arm": "baseline", + "model": "m", + "provider": "p", + "evaluations": evaluations, + **meta, + } + + +def _cost(**fields) -> dict: + return {"baseline": fields} + + +_FULL_COST = _cost( + total_cost_usd=0.12345, + cost_known_n=3, + total_llm_calls=8, + prompt_tokens=12345, + completion_tokens=678, + total_tokens=13023, + mean_latency_sec=1.234, + cache_hit_ratio=0.25, +) + +# Varying deltas so the t-test runs; one unpaired question and one with no baseline. +_A = [(50, 70), (60, 70), (40, 80), (70, 60), (55, 75), (65, 90), (30, 45), (80, None), (None, 60)] +# Constant +10 deltas: the zero-variance guard leaves the p-values N/A. +_B = [(50, 60), (60, 70), (40, 50), (70, 80), (55, 65), (65, 75), (30, 40), (80, 90), (20, 30)] +_BASE_ONLY = [(b, ...) for b, _ in _A] + + +def _cases() -> dict: + a = _run(_A, model="alpha", judge_model="judge-j", judge_provider="jp", cost_summary=_FULL_COST) + b = _run(_B, model="beta", cost_summary=_cost(total_cost_usd=1.5, n=4)) + # Every question "hard" but the one with no baseline score: the run is missing + # from the other buckets, and its "easy" bucket has no usable score at all. + lopsided = ["hard"] * 8 + ["easy"] + return { + "treatment": ([("a", a), ("b", b)], ["a", "b"], {}), + "treatment_bh": ([("a", a), ("b", b)], ["a", "b"], {"correction": "bh"}), + "no_scipy": ([("a", a), ("b", b)], ["a", "b"], {"no_scipy": True}), + "baseline_only": ( + [ + ("a", _run(_BASE_ONLY, model="alpha")), + ("b", _run(_BASE_ONLY, model="beta", difficulties=lopsided)), + ], + ["a", "b"], + {}, + ), + "baseline_only_no_scipy": ([("a", _run(_BASE_ONLY))], ["a"], {"no_scipy": True}), + "mixed": ( + [ + ("a", _run(_A, model="alpha")), + ("b", _run(_BASE_ONLY, model="beta", difficulties=lopsided)), + ], + ["a", "b"], + {}, + ), + "two_arms": ( + [ + ("a", _run(_A, arm="docs", model="alpha")), + ("b", _run(_B, arm="skill:x", model="beta")), + ], + ["a", "b"], + {}, + ), + "pinned_arm": ( + [("a", _run(_A, arm="docs", difficulties=["easy"]))], + ["a"], + {"treatment_arm": "docs"}, + ), + "pooled": ( + [ + ("m[n=2]", _run(_B, model="solo")), + ("m_run1", _run(_A, model="rep", cost_summary=_FULL_COST)), + ("m_run2", _run(_B, model="rep", cost_summary=_FULL_COST)), + ("left_out", _run(_A, model="other")), + ], + ["m[n=2]", "m_run1", "m_run2"], + {"pool_replicates": True}, + ), + "pooled_singletons": ( + [("x", _run(_A, model="x")), ("y", _run(_B, model="y"))], + ["x", "y"], + {"pool_replicates": True}, + ), + "empty": ([], [], {}), + } + + +CASES = _cases() + + +def _render(name: str, monkeypatch) -> str: + runs, run_ids, opts = CASES[name] + opts = dict(opts) + if opts.pop("no_scipy", False): + monkeypatch.setattr(model_compare, "HAS_SCIPY", False) + return "\n".join(generate_section_report(runs, run_ids, "Regular", **opts)) + + +@pytest.mark.parametrize("name", sorted(CASES)) +def test_section_report_matches_golden(name, monkeypatch): + if not model_compare.HAS_SCIPY: + pytest.skip("the goldens were recorded with SciPy available") + text = _render(name, monkeypatch) + if os.environ.get("DOKIMI_UPDATE_GOLDEN"): + goldens = json.loads(GOLDEN.read_text(encoding="utf-8")) if GOLDEN.exists() else {} + goldens[name] = text + GOLDEN.write_text( + json.dumps(goldens, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + assert text == json.loads(GOLDEN.read_text(encoding="utf-8"))[name] + + +def test_goldens_reach_every_section_branch(): + """The goldens are only a characterization if they hold each branch's marker.""" + texts = json.loads(GOLDEN.read_text(encoding="utf-8")) + assert set(texts) == set(CASES) + markers = { + "treatment": [ + "(3/8)", + "| **b** | $1.5000 | — |", + "N/A", + "Holm-Bonferroni", + "2 unscored", + "`with-context`", + "| tie | 1 |", + "| beginner | 1 |", + ], + "treatment_bh": ["Benjamini-Hochberg"], + "no_scipy": ["## Overall Summary"], + "baseline_only": [ + "Report Limitation", + "a baseline |", + "### By absolute quality (baseline)", + "Statistical tests require treatment arm", + "requires treatment arm.", + ], + "mixed": ["— (baseline only)", "(baseline) |", "| — | — |"], + "two_arms": ["Context arm", "non-baseline treatment arm"], + "pooled": ["## Replicate Pooling", "m[n=2]'", "pools 2 replicates", "A pooled `[n=k]` row"], + "empty": ["Common questions evaluated: **0**"], + } + for name, wanted in markers.items(): + for marker in wanted: + assert marker in texts[name], (name, marker) + assert "Statistical Significance" not in texts["no_scipy"] + assert "Statistical Significance" not in texts["baseline_only_no_scipy"] + assert "Resource Usage" not in texts["baseline_only"] From d139a2a4b79eeac5a48a5c92bfd2a73ef0c58815 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:40:28 -0700 Subject: [PATCH 2/7] refactor(report): split generate_section_report into per-section helpers The model-compare section builder was one 556-line function with cyclomatic complexity 53, the highest in the repo. Each report section is now its own helper (pooling, models compared, resource usage, overall summary, significance, by difficulty, head-to-head, context-arm benefit, ranking), and the builder computes the shared inputs once and concatenates them. No output changes: the goldens added in the previous commit pass byte for byte. generate_section_report goes from 53 to 4 and no new function is over 6 (ruff C901 and the ratchet's own counter agree); the file goes from 1296 to 1209 lines. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/report/model_compare.py | 847 ++++++++++++-------------- 1 file changed, 380 insertions(+), 467 deletions(-) diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py index f6fd0b38..0f69d7b5 100644 --- a/dokimi/dokimi/report/model_compare.py +++ b/dokimi/dokimi/report/model_compare.py @@ -755,208 +755,208 @@ def generate_section_report( question into a single labelled row, so the report ranks configurations rather than individual samples of one configuration. """ - - # Extract and intersect to common questions only - all_run_scores_raw: dict[str, list[dict]] = {} - for run_id, run_data in runs: - all_run_scores_raw[run_id] = extract_scores( - run_data, force_treatment_arm=treatment_arm - ) - + raw_scores = {run_id: extract_scores(data, force_treatment_arm=treatment_arm) for run_id, data in runs} replicate_note: list[str] = [] if pool_replicates: - groups = group_replicates(runs) - run_data_by_id = dict(runs) - pooled_runs: list[tuple[str, dict]] = [] - pooled_ids: list[str] = [] - pooled_scores: dict[str, list[dict]] = {} - for _key, ids in groups.items(): - present = [rid for rid in run_ids if rid in ids] - if not present: - continue - label = replicate_label(present) - while label in pooled_scores: # keep labels unique - label += "'" - pooled_scores[label] = pool_replicate_scores(all_run_scores_raw, present) - pooled_run = run_data_by_id[present[0]] - if len(present) > 1: - pooled_run = {**pooled_run, "cost_summary": pool_cost_summaries( - [run_data_by_id[rid].get("cost_summary") for rid in present])} - pooled_runs.append((label, pooled_run)) - pooled_ids.append(label) - if len(present) > 1: - seen = {s["question_id"] for rid in present for s in all_run_scores_raw.get(rid, [])} - dropped = len(seen) - len(pooled_scores[label]) - replicate_note.append( - f"- **{label}** pools {len(present)} replicates: {', '.join(present)}; " - f"{dropped} question(s) dropped for not being scored in every replicate" - ) - runs = pooled_runs - run_ids = pooled_ids - all_run_scores_raw = pooled_scores - - question_sets = [ - set(s["question_id"] for s in scores) - for scores in all_run_scores_raw.values() - ] - common_questions = set.intersection(*question_sets) if question_sets else set() - - # All metrics below use the common-question subset exclusively - all_run_scores: dict[str, list[dict]] = { - run_id: [ - s for s in all_run_scores_raw.get(run_id, []) - if s["question_id"] in common_questions - ] - for run_id in run_ids - } + runs, run_ids, raw_scores, replicate_note = _pool_section(runs, run_ids, raw_scores) + common_questions, all_run_scores = _common_scores(raw_scores, run_ids) # The arm that was actually scored, taken from the extracted records rather than the # artifact header, so the label can never disagree with the numbers beside it. Named # explicitly because a `profile:` run and a `docs` run otherwise render identically: # a reader of the Markdown alone could not tell which treatment it measured. - scored_arms = { - s["treatment_arm"] + scored = {s["treatment_arm"] for scores in all_run_scores.values() for s in scores if s.get("treatment_arm")} + arm_label = next(iter(scored)) if len(scored) == 1 else "Context arm" + # Single-arm vs multi-arm: without a scored pair there is no delta to analyse. + has_treatment = any( + s.get("with_docs") is not None and s.get("delta") is not None for scores in all_run_scores.values() for s in scores - if s.get("treatment_arm") - } - arm_label = next(iter(scored_arms)) if len(scored_arms) == 1 else "Context arm" + ) + summary_stats = {run_id: _run_summary(all_run_scores[run_id]) for run_id in run_ids} - lines: list[str] = [] + lines = _pooling_lines(replicate_note) + _models_compared_lines(runs, all_run_scores, common_questions) + if not has_treatment: + lines += _BASELINE_ONLY_LIMITATION + lines += _resource_usage_lines(runs, bool(replicate_note)) + lines += _overall_summary_lines(run_ids, summary_stats, arm_label) + if HAS_SCIPY: + lines += _significance_lines(run_ids, all_run_scores, has_treatment, correction) + lines += _difficulty_lines(run_ids, all_run_scores, has_treatment) + lines += _head_to_head_lines(run_ids, all_run_scores, common_questions, has_treatment) + lines += _benefit_lines(run_ids, summary_stats, has_treatment, arm_label if len(scored) == 1 else None) + return lines + _ranking_lines(summary_stats, has_treatment, arm_label) + + +def _pool_section( + runs: list[tuple[str, dict]], run_ids: list[str], raw_scores: dict[str, list[dict]] +) -> tuple[list[tuple[str, dict]], list[str], dict[str, list[dict]], list[str]]: + """Runs, run ids and scores with each configuration's replicates pooled into one labelled run, + plus the note naming what each pooled run holds.""" + run_data_by_id = dict(runs) + pooled_runs: list[tuple[str, dict]] = [] + pooled_scores: dict[str, list[dict]] = {} + note: list[str] = [] + for ids in group_replicates(runs).values(): + present = [rid for rid in run_ids if rid in ids] + if not present: + continue + label = replicate_label(present) + while label in pooled_scores: # keep labels unique + label += "'" + pooled_scores[label] = pool_replicate_scores(raw_scores, present) + pooled_run = run_data_by_id[present[0]] + if len(present) > 1: + pooled_run = {**pooled_run, "cost_summary": pool_cost_summaries( + [run_data_by_id[rid].get("cost_summary") for rid in present])} + seen = {s["question_id"] for rid in present for s in raw_scores.get(rid, [])} + dropped = len(seen) - len(pooled_scores[label]) + note.append( + f"- **{label}** pools {len(present)} replicates: {', '.join(present)}; " + f"{dropped} question(s) dropped for not being scored in every replicate" + ) + pooled_runs.append((label, pooled_run)) + return pooled_runs, [label for label, _ in pooled_runs], pooled_scores, note - # --- Models Compared --- - if replicate_note: - lines += [ - "## Replicate Pooling", - "", - "Runs sharing a configuration were averaged per question, so each row " - "below is one configuration rather than one sample of it. Each pooled " - "score is a mean of k runs, so its spread and confidence intervals are " - "narrower than a single run's: that reflects the k runs spent, not a " - "sharper measurement.", - "", - *replicate_note, - "", - ] - lines += [ - "## Models Compared", - "| Run ID | Answer model | Judge model | Treatment arm |", - "|---|---|---|---|", +def _common_scores( + raw_scores: dict[str, list[dict]], run_ids: list[str] +) -> tuple[set[str], dict[str, list[dict]]]: + """The question ids every run scored, and each run's scores cut to them. + + All metrics of a section use this common-question subset exclusively.""" + question_sets = [set(s["question_id"] for s in scores) for scores in raw_scores.values()] + common = set.intersection(*question_sets) if question_sets else set() + return common, { + run_id: [s for s in raw_scores.get(run_id, []) if s["question_id"] in common] for run_id in run_ids + } + + +def _pooling_lines(replicate_note: list[str]) -> list[str]: + if not replicate_note: + return [] + return [ + "## Replicate Pooling", + "", + "Runs sharing a configuration were averaged per question, so each row " + "below is one configuration rather than one sample of it. Each pooled " + "score is a mean of k runs, so its spread and confidence intervals are " + "narrower than a single run's: that reflects the k runs spent, not a " + "sharper measurement.", + "", + *replicate_note, + "", ] + + +def _models_compared_lines( + runs: list[tuple[str, dict]], all_run_scores: dict[str, list[dict]], common_questions: set[str] +) -> list[str]: + lines = ["## Models Compared", "| Run ID | Answer model | Judge model | Treatment arm |", "|---|---|---|---|"] for run_id, run_data in runs: - model = run_data.get("model", "unknown") provider = run_data.get("provider", "unknown") - judge_model = run_data.get("judge_model", "same") - judge_provider = run_data.get("judge_provider", provider) - run_arms = sorted( - { - s["treatment_arm"] - for s in all_run_scores.get(run_id, []) - if s.get("treatment_arm") - } - ) + judge = f"{run_data.get('judge_model', 'same')} ({run_data.get('judge_provider', provider)})" + run_arms = sorted({s["treatment_arm"] for s in all_run_scores.get(run_id, []) if s.get("treatment_arm")}) arm_cell = ", ".join(f"`{a}`" for a in run_arms) if run_arms else "— (baseline only)" - lines.append( - f"| **{run_id}** | {model} ({provider}) | {judge_model} ({judge_provider}) " - f"| {arm_cell} |" - ) - - lines += [ + lines.append(f"| **{run_id}** | {run_data.get('model', 'unknown')} ({provider}) | {judge} | {arm_cell} |") + return lines + [ "", f"Common questions evaluated: **{len(common_questions)}**", "_All summary, significance, ranking, and head-to-head metrics use only this common question set._", "", ] - # Detect report limitations (single-arm vs multi-arm) - has_treatment = any( - s.get("with_docs") is not None and s.get("delta") is not None - for scores in all_run_scores.values() - for s in scores - ) - - if not has_treatment: - lines += [ - "> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).", - "> ", - "> **Missing from this report:**", - "> - Context-arm vs Baseline delta analysis", - "> - Statistical significance tests", - "> - Head-to-head winner comparisons", - "> ", - "> **Available in this report:**", - "> - Absolute baseline quality scores per model", - "> - Resource usage metrics (cost, tokens, latency)", - "> - Model ranking by absolute baseline quality", - "> ", - "> **To generate full comparison with delta analysis, add a treatment arm:**", - "> ```bash", - "> python cli.py arms run \\", - "> --arms \"baseline,docs\" \\", - "> --judge \\", - "> --judge-model claude-sonnet-5-5 \\", - "> [other parameters]", - "> ```", - "> ", - "> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).", - "> ", - "> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`", - "", - ] - # --- Resource Usage --- - # Extract cost_summary from each run (baseline arm only for now) - cost_data_available = any( - run_data.get("cost_summary") for _, run_data in runs - ) - if cost_data_available: - lines += [ - "## Resource Usage (baseline arm)", - "| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |", - "|---|---:|---:|---:|---:|---:|---:|", - ] - for run_id, run_data in runs: - cost_summary = run_data.get("cost_summary", {}) - baseline_arm = run_data.get("baseline_arm", "baseline") - cs = cost_summary.get(baseline_arm, {}) - - # Extract metrics - cost_usd = cs.get("total_cost_usd") - cost_str = f"${cost_usd:.4f}" if cost_usd is not None else "—" - if cost_usd is not None: - cost_known = cs.get("cost_known_n") - total_calls = cs.get("total_llm_calls", cs.get("n")) - if cost_known is not None and total_calls is not None and cost_known < total_calls: - cost_str += f" ({cost_known}/{total_calls})" - - prompt_tok = cs.get("prompt_tokens") - prompt_str = f"{prompt_tok:,}" if prompt_tok is not None else "—" - - completion_tok = cs.get("completion_tokens") - completion_str = f"{completion_tok:,}" if completion_tok is not None else "—" - - total_tok = cs.get("total_tokens") - total_str = f"{total_tok:,}" if total_tok is not None else "—" - - latency = cs.get("mean_latency_sec") - latency_str = f"{latency:.2f}s" if latency is not None else "—" - - cache_ratio = cs.get("cache_hit_ratio") - cache_str = f"{cache_ratio:.1%}" if cache_ratio is not None else "—" - - lines.append( - f"| **{run_id}** | {cost_str} | {prompt_str} | {completion_str} | " - f"{total_str} | {latency_str} | {cache_str} |" - ) - if replicate_note: - lines += ["", "_A pooled `[n=k]` row shows the total cost, calls and tokens of its k runs " - "and their mean latency._"] +_BASELINE_ONLY_LIMITATION = [ + "> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).", + "> ", + "> **Missing from this report:**", + "> - Context-arm vs Baseline delta analysis", + "> - Statistical significance tests", + "> - Head-to-head winner comparisons", + "> ", + "> **Available in this report:**", + "> - Absolute baseline quality scores per model", + "> - Resource usage metrics (cost, tokens, latency)", + "> - Model ranking by absolute baseline quality", + "> ", + "> **To generate full comparison with delta analysis, add a treatment arm:**", + "> ```bash", + "> python cli.py arms run \\", + "> --arms \"baseline,docs\" \\", + "> --judge \\", + "> --judge-model claude-sonnet-5-5 \\", + "> [other parameters]", + "> ```", + "> ", + "> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).", + "> ", + "> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`", + "", +] + +#: Resource-usage columns after the cost: the cost-summary field and its cell format. +_RESOURCE_CELLS = (("prompt_tokens", "{:,}"), ("completion_tokens", "{:,}"), ("total_tokens", "{:,}"), + ("mean_latency_sec", "{:.2f}s"), ("cache_hit_ratio", "{:.1%}")) + + +def _cost_cell(cs: dict) -> str: + """Total cost, with the share of calls it covers when some had no known price.""" + cost_usd = cs.get("total_cost_usd") + if cost_usd is None: + return "—" + cost_known, total_calls = cs.get("cost_known_n"), cs.get("total_llm_calls", cs.get("n")) + if cost_known is not None and total_calls is not None and cost_known < total_calls: + return f"${cost_usd:.4f} ({cost_known}/{total_calls})" + return f"${cost_usd:.4f}" + + +def _resource_usage_lines(runs: list[tuple[str, dict]], pooled: bool) -> list[str]: + """Baseline-arm cost, tokens, latency and cache hits; nothing when no run has a cost summary.""" + if not any(run_data.get("cost_summary") for _, run_data in runs): + return [] + lines = [ + "## Resource Usage (baseline arm)", + "| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |", + "|---|---:|---:|---:|---:|---:|---:|", + ] + for run_id, run_data in runs: + cs = run_data.get("cost_summary", {}).get(run_data.get("baseline_arm", "baseline"), {}) + cells = [_cost_cell(cs), *("—" if cs.get(f) is None else t.format(cs.get(f)) for f, t in _RESOURCE_CELLS)] + lines.append(f"| **{run_id}** | " + " | ".join(cells) + " |") + if pooled: + lines += ["", "_A pooled `[n=k]` row shows the total cost, calls and tokens of its k runs " + "and their mean latency._"] + return lines + [""] + + +def _run_summary(scores: list[dict]) -> dict: + """One run's arm means and paired-delta counts over the common questions.""" + with_docs_scores = [s["with_docs"] for s in scores if s["with_docs"] is not None] + without_docs_scores = [s["without_docs"] for s in scores if s["without_docs"] is not None] + # Average the per-question deltas rather than differencing the two arm + # means. The arm means are filtered independently, so when any question + # is missing a score they cover different question subsets and their + # difference is not a paired delta. + paired_deltas = [s["delta"] for s in scores if s.get("delta") is not None] + # Per-run, distinct from the section-level has_treatment: a single run may + # lack a treatment arm while others in the comparison have one. + paired = bool(with_docs_scores) and bool(paired_deltas) + return { + "n": len(scores), + "n_paired": len(paired_deltas), + # Questions that reached the intersection but lack a usable pair. + "unpaired": len(scores) - len(paired_deltas), + "with_avg": _avg(with_docs_scores) if with_docs_scores else None, + "without_avg": _avg(without_docs_scores) if without_docs_scores else None, + "delta_avg": round(sum(paired_deltas) / len(paired_deltas), 1) if paired else None, + "improved": sum(1 for d in paired_deltas if d > 0) if paired else None, + "degraded": sum(1 for d in paired_deltas if d < 0) if paired else None, + } - lines += [""] - # --- Overall Summary --- - lines += [ +def _overall_summary_lines(run_ids: list[str], summary_stats: dict[str, dict], arm_label: str) -> list[str]: + lines = [ "## Overall Summary", "_Delta is the mean of per-question paired differences. Scored pairs is the" " number of questions with a usable score in both arms; Delta, Improved and" @@ -964,333 +964,246 @@ def generate_section_report( f"| Run | N | Scored pairs | {arm_label} | Baseline | Delta | Improved | Degraded |", "|---|---:|---:|---:|---:|---:|---:|---:|", ] - - summary_stats: dict[str, dict] = {} for run_id in run_ids: - scores = all_run_scores[run_id] - - # Handle both single-arm (baseline-only) and multi-arm runs - with_docs_scores = [s["with_docs"] for s in scores if s["with_docs"] is not None] - without_docs_scores = [s["without_docs"] for s in scores if s["without_docs"] is not None] - - # Per-run, distinct from the section-level has_treatment above: a single - # run may lack a treatment arm while others in the comparison have one. - run_has_treatment = bool(with_docs_scores) - with_avg = _avg(with_docs_scores) if with_docs_scores else None - without_avg = _avg(without_docs_scores) if without_docs_scores else None - - # Average the per-question deltas rather than differencing the two arm - # means. The arm means are filtered independently, so when any question - # is missing a score they cover different question subsets and their - # difference is not a paired delta. - paired_deltas = [s["delta"] for s in scores if s.get("delta") is not None] - - if run_has_treatment and paired_deltas: - delta_avg = round(sum(paired_deltas) / len(paired_deltas), 1) - improved = sum(1 for d in paired_deltas if d > 0) - degraded = sum(1 for d in paired_deltas if d < 0) - else: - delta_avg = None - improved = None - degraded = None - - # Questions that reached the intersection but lack a usable pair. - unpaired = len(scores) - len(paired_deltas) - - summary_stats[run_id] = { - "n": len(scores), - "n_paired": len(paired_deltas), - "unpaired": unpaired, - "with_avg": with_avg, - "without_avg": without_avg, - "delta_avg": delta_avg, - "improved": improved, - "degraded": degraded, - } - + st = summary_stats[run_id] # An arm with no usable score prints as "—" rather than a numeric zero, # which would read as a legitimate score of 0/100. - with_avg_str = "—" if with_avg is None else str(with_avg) - without_avg_str = "—" if without_avg is None else str(without_avg) - delta_str = "—" if delta_avg is None else f"**{_fmt_delta(delta_avg)}**" - improved_str = "—" if improved is None else str(improved) - degraded_str = "—" if degraded is None else str(degraded) + delta_str = "—" if st["delta_avg"] is None else f"**{_fmt_delta(st['delta_avg'])}**" # Make partial coverage visible: a delta over fewer pairs than questions # is weaker evidence than the question count alone suggests. - scored_str = str(len(paired_deltas)) if not unpaired else f"{len(paired_deltas)} ({unpaired} unscored)" - - lines.append( - f"| **{run_id}** | {len(scores)} | {scored_str} | {with_avg_str} | {without_avg_str} | " - f"{delta_str} | {improved_str} | {degraded_str} |" + scored_str = f"{st['n_paired']} ({st['unpaired']} unscored)" if st["unpaired"] else str(st["n_paired"]) + cells = [str(st["n"]), scored_str, _num_or_dash(st["with_avg"]), _num_or_dash(st["without_avg"]), + delta_str, _num_or_dash(st["improved"]), _num_or_dash(st["degraded"])] + lines.append(f"| **{run_id}** | " + " | ".join(cells) + " |") + return lines + [""] + + +def _adjusted_significance( + run_ids: list[str], all_run_scores: dict[str, list[dict]], correction: str +) -> tuple[dict[str, dict | None], int]: + """Each run's paired test with its p-value adjusted across the runs, and how many were tested. + + Every run in the section is one test of the same "did the treatment help?" + question, so they form a family: with k runs, the chance that at least one + clears 0.05 by luck alone is far above 0.05. Correct across the family before + calling anything significant, exactly as the arms report does across arms. + """ + sigs: dict[str, dict | None] = {} + for run_id in run_ids: + # Filter for paired scores: both baseline and treatment must be non-None + paired = [(s["with_docs"], s["without_docs"]) for s in all_run_scores[run_id] + if s["with_docs"] is not None and s["without_docs"] is not None] + if paired: + with_paired, without_paired = zip(*paired, strict=True) + sigs[run_id] = significance_test(list(with_paired), list(without_paired)) + else: + sigs[run_id] = None + + tested = [sig for rid in run_ids if (sig := sigs[rid]) and sig["p_ttest"] is not None] + adjusted = arm_stats.adjust_pvalues([sig["p_ttest"] for sig in tested], method=correction) + for sig, p_adj in zip(tested, adjusted, strict=True): + sig["p_adjusted"] = p_adj + # Same conjunction as the raw verdict, with the adjusted p in place of the raw one. + sig["significant_adjusted"] = bool( + p_adj is not None and p_adj < arm_stats.ALPHA + and (sig["p_wilcoxon"] is None or sig["p_wilcoxon"] < arm_stats.ALPHA) ) + return sigs, len(tested) - lines += [""] - # --- Statistical Significance --- - if HAS_SCIPY: - lines += [ - "## Statistical Significance of Delta (α = 0.05)", - ] +def _significance_row(run_id: str, sig: dict) -> str: + # p-values are None when no test could run (zero-variance deltas); "N/A" is + # honest where "None" would just look broken. + verdict = sig.get("significant_adjusted") + if sig["significant"] is None or verdict is None: + sig_mark = "N/A" + else: + sig_mark = "OK Yes" if verdict else "No" + p_ttest = sig["p_ttest"] if sig["p_ttest"] is not None else "N/A" + p_adj = sig.get("p_adjusted") + p_adj_str = p_adj if p_adj is not None else "N/A" + p_wilcox = sig["p_wilcoxon"] if sig["p_wilcoxon"] is not None else "N/A" + return ( + f"| **{run_id}** | {_fmt_delta(sig['delta_mean'])} | {p_ttest} | " + f"{p_adj_str} | {p_wilcox} | {_fmt_delta(sig['cohens_d'])} " + f"| {sig['effect']} | {sig_mark} |" + ) - if not has_treatment: - lines += [ - "", - "> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).", - "> Add `--arms \"baseline,docs\"` to enable delta analysis.", - "", - ] - else: - # Every run in the section is one test of the same "did the - # treatment help?" question, so they form a family: with k runs, - # the chance that at least one clears 0.05 by luck alone is far - # above 0.05. Correct across the family before calling anything - # significant, exactly as the arms report does across arms. - sigs: dict[str, dict | None] = {} - for run_id in run_ids: - scores = all_run_scores[run_id] - # Filter for paired scores: both baseline and treatment must be non-None - paired = [(s["with_docs"], s["without_docs"]) for s in scores - if s["with_docs"] is not None and s["without_docs"] is not None] - if paired: - with_paired, without_paired = zip(*paired, strict=True) - sigs[run_id] = significance_test(list(with_paired), list(without_paired)) - else: - sigs[run_id] = None - - tested = [sig for rid in run_ids if (sig := sigs[rid]) and sig["p_ttest"] is not None] - adjusted = arm_stats.adjust_pvalues( - [sig["p_ttest"] for sig in tested], method=correction - ) - for sig, p_adj in zip(tested, adjusted, strict=True): - sig["p_adjusted"] = p_adj - # Same conjunction as the raw verdict, with the adjusted p in - # place of the raw one. - sig["significant_adjusted"] = bool( - p_adj is not None and p_adj < arm_stats.ALPHA - and (sig["p_wilcoxon"] is None or sig["p_wilcoxon"] < arm_stats.ALPHA) - ) - label = "Holm-Bonferroni" if correction == "holm" else "Benjamini-Hochberg" - lines += [ - f"_p-values are adjusted across the {len(tested)} run(s) compared here " - f"({label}); the raw column is shown for reference. " - "Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._", - "", - "| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z " - "| Effect | Significant? |", - "|---|---:|---:|---:|---:|---:|---|---|", - ] - for run_id in run_ids: - sig = sigs[run_id] - if sig: - # p-values are None when no test could run (zero-variance - # deltas); "N/A" is honest where "None" would just look broken. - verdict = sig.get("significant_adjusted") - if sig["significant"] is None or verdict is None: - sig_mark = "N/A" - else: - sig_mark = "OK Yes" if verdict else "No" - p_ttest = sig["p_ttest"] if sig["p_ttest"] is not None else "N/A" - p_adj = sig.get("p_adjusted") - p_adj_str = p_adj if p_adj is not None else "N/A" - p_wilcox = sig["p_wilcoxon"] if sig["p_wilcoxon"] is not None else "N/A" - lines.append( - f"| **{run_id}** | {_fmt_delta(sig['delta_mean'])} | {p_ttest} | " - f"{p_adj_str} | {p_wilcox} | {_fmt_delta(sig['cohens_d'])} " - f"| {sig['effect']} | {sig_mark} |" - ) - lines += [""] +def _significance_lines( + run_ids: list[str], all_run_scores: dict[str, list[dict]], has_treatment: bool, correction: str +) -> list[str]: + lines = ["## Statistical Significance of Delta (α = 0.05)"] + if not has_treatment: + return lines + [ + "", + "> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).", + "> Add `--arms \"baseline,docs\"` to enable delta analysis.", + "", + "", + ] + sigs, n_tested = _adjusted_significance(run_ids, all_run_scores, correction) + label = "Holm-Bonferroni" if correction == "holm" else "Benjamini-Hochberg" + lines += [ + f"_p-values are adjusted across the {n_tested} run(s) compared here " + f"({label}); the raw column is shown for reference. " + "Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._", + "", + "| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z " + "| Effect | Significant? |", + "|---|---:|---:|---:|---:|---:|---|---|", + ] + lines += [_significance_row(run_id, sig) for run_id in run_ids if (sig := sigs[run_id])] + return lines + [""] + + +_DIFFICULTY_ORDER = ("easy", "beginner", "intermediate", "medium", "advanced", "hard") + + +def _difficulty_cells(sc: list[dict], has_treatment: bool) -> str: + """One run's cells of a difficulty row: context mean and delta, or the baseline.""" + if not sc: + return " — | — |" if has_treatment else " — |" + with_scores = [s["with_docs"] for s in sc if s["with_docs"] is not None] + if with_scores: + # Same pairing rule as the overall summary: average the per-question + # deltas, never the difference of two means filtered over different + # question subsets. + diff_deltas = [s["delta"] for s in sc if s.get("delta") is not None] + delta_cell = _fmt_delta(round(sum(diff_deltas) / len(diff_deltas), 1)) if diff_deltas else "—" + return f" {_avg(with_scores)} | {delta_cell} |" + # No treatment score in this bucket: report the baseline. + without_scores = [s["without_docs"] for s in sc if s["without_docs"] is not None] + wo = _avg(without_scores) if without_scores else None + if has_treatment: + wo_cell = "—" if wo is None else f"{wo} (baseline)" + return f" — | {wo_cell} |" + return f" {'—' if wo is None else wo} |" + - # --- By Difficulty --- - difficulty_groups: dict[str, dict[str, list]] = defaultdict(lambda: defaultdict(list)) +def _difficulty_lines(run_ids: list[str], all_run_scores: dict[str, list[dict]], has_treatment: bool) -> list[str]: + groups: dict[str, dict[str, list]] = defaultdict(lambda: defaultdict(list)) for run_id in run_ids: for s in all_run_scores[run_id]: # already common questions only - difficulty_groups[s["difficulty"]][run_id].append(s) - - if difficulty_groups: - # Adapt header to single-arm vs multi-arm scenario - if has_treatment: - header = "| Difficulty | N | " + " | ".join( - f"{rid} context | {rid} Δ" for rid in run_ids - ) + " |" - separator = "|---|---:|" + "|".join([" :---: | :---: "] * len(run_ids)) + "|" - else: - header = "| Difficulty | N | " + " | ".join( - f"{rid} baseline" for rid in run_ids - ) + " |" - separator = "|---|---:|" + "| :---: " * len(run_ids) + "|" - - lines += [ - "## By Difficulty (common questions only)", - header, - separator, - ] - for diff in ["easy", "beginner", "intermediate", "medium", "advanced", "hard"]: - if diff not in difficulty_groups: - continue - run_data = difficulty_groups[diff] - n = len(next(iter(run_data.values()))) - row = f"| {diff} | {n} |" - for run_id in run_ids: - sc = run_data.get(run_id, []) - if sc: - with_scores = [s["with_docs"] for s in sc if s["with_docs"] is not None] - without_scores = [s["without_docs"] for s in sc if s["without_docs"] is not None] - # Same pairing rule as the overall summary: average the - # per-question deltas, never the difference of two means - # filtered over different question subsets. - diff_deltas = [s["delta"] for s in sc if s.get("delta") is not None] - if with_scores: - wa = _avg(with_scores) - delta_cell = ( - _fmt_delta(round(sum(diff_deltas) / len(diff_deltas), 1)) - if diff_deltas else "—" - ) - row += f" {wa} | {delta_cell} |" - else: - # No treatment score in this bucket: report the baseline. - wo = _avg(without_scores) if without_scores else None - wo_cell = "—" if wo is None else f"{wo} (baseline)" - if has_treatment: - row += f" — | {wo_cell} |" - else: - row += f" {'—' if wo is None else wo} |" - else: - if has_treatment: - row += " — | — |" - else: - row += " — |" - lines.append(row) - lines += [""] - - # --- Head-to-Head --- - lines += [ + groups[s["difficulty"]][run_id].append(s) + if not groups: + return [] + # Adapt header to single-arm vs multi-arm scenario + if has_treatment: + header = "| Difficulty | N | " + " | ".join(f"{rid} context | {rid} Δ" for rid in run_ids) + " |" + separator = "|---|---:|" + "|".join([" :---: | :---: "] * len(run_ids)) + "|" + else: + header = "| Difficulty | N | " + " | ".join(f"{rid} baseline" for rid in run_ids) + " |" + separator = "|---|---:|" + "| :---: " * len(run_ids) + "|" + lines = ["## By Difficulty (common questions only)", header, separator] + for diff in [d for d in _DIFFICULTY_ORDER if d in groups]: + by_run = groups[diff] + cells = "".join(_difficulty_cells(by_run.get(run_id, []), has_treatment) for run_id in run_ids) + lines.append(f"| {diff} | {len(next(iter(by_run.values())))} |{cells}") + return lines + [""] + + +def _winner_counts(run_ids: list[str], all_run_scores: dict[str, list[dict]], common_questions: set[str]) -> dict: + """Per run, the questions it alone scored highest on in the context arm; shared tops under "tie".""" + winner_counts: dict[str, int] = defaultdict(int) + for qid in common_questions: + question_scores = {} + for run_id in run_ids: + score = next((s["with_docs"] for s in all_run_scores[run_id] + if s["question_id"] == qid and s["with_docs"] is not None), None) + if score is not None: + question_scores[run_id] = score + if question_scores: + max_score = max(question_scores.values()) + winners = [rid for rid, score in question_scores.items() if score == max_score] + winner_counts[winners[0] if len(winners) == 1 else "tie"] += 1 + return winner_counts + + +def _head_to_head_lines( + run_ids: list[str], all_run_scores: dict[str, list[dict]], common_questions: set[str], has_treatment: bool +) -> list[str]: + lines = [ "## Head-to-Head (context arm, common questions)", "_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._", "", ] - if not has_treatment: - lines += [ + return lines + [ "", "> **⚠️ Skipped:** Head-to-head requires treatment arm scores.", "> Single-arm runs only have baseline scores.", "", ] - else: - lines += [ - "| Winner | Count | % |", - "|---|---:|---:|", - ] - winner_counts: dict[str, int] = defaultdict(int) - for qid in common_questions: - question_scores = {} - for run_id in run_ids: - for s in all_run_scores[run_id]: - if s["question_id"] == qid and s["with_docs"] is not None: - question_scores[run_id] = s["with_docs"] - break - if question_scores: - max_score = max(question_scores.values()) - winners = [rid for rid, score in question_scores.items() if score == max_score] - if len(winners) == 1: - winner_counts[winners[0]] += 1 - else: - winner_counts["tie"] += 1 - - total = len(common_questions) - for run_id in run_ids: - count = winner_counts.get(run_id, 0) - pct = round(100 * count / total) if total > 0 else 0 - lines.append(f"| **{run_id}** | {count} | {pct}% |") - tie_count = winner_counts.get("tie", 0) - lines.append(f"| tie | {tie_count} | {round(100 * tie_count / total) if total > 0 else 0}% |") - lines += [""] + winner_counts = _winner_counts(run_ids, all_run_scores, common_questions) + total = len(common_questions) + lines += ["| Winner | Count | % |", "|---|---:|---:|"] + for label, key in [*((f"**{rid}**", rid) for rid in run_ids), ("tie", "tie")]: + count = winner_counts.get(key, 0) + lines.append(f"| {label} | {count} | {round(100 * count / total) if total > 0 else 0}% |") + return lines + [""] - # --- Context-Arm Benefit --- - lines += [ - "## Context-Arm Benefit — Delta Comparison", - ] +def _benefit_lines( + run_ids: list[str], summary_stats: dict[str, dict], has_treatment: bool, single_arm: str | None +) -> list[str]: + """Per run, the mean delta and the share of scored pairs the treatment helped and hurt.""" + lines = ["## Context-Arm Benefit — Delta Comparison"] if not has_treatment: - lines += [ + return lines + [ "", "> **⚠️ Skipped:** Delta comparison requires treatment arm.", "> Single-arm runs cannot compute context benefit.", "", ] - else: - subtitle = ( - f"_Which model benefits more from the treatment arm `{arm_label}`?_" - if len(scored_arms) == 1 - else "_Which model benefits more from the non-baseline treatment arm?_" - ) - lines += [ - subtitle, - "", - "| Run | Avg Delta | Questions context helped | Questions context hurt |", - "|---|---:|---:|---:|", - ] - for run_id in run_ids: - st = summary_stats[run_id] - # Percentages are over scored pairs, the same population the - # improved/degraded counts come from. - n_paired = st["n_paired"] - if st["improved"] is None or not n_paired: - lines.append(f"| **{run_id}** | — | — | — |") - continue - helped_pct = round(100 * st["improved"] / n_paired) - hurt_pct = round(100 * st["degraded"] / n_paired) - lines.append( - f"| **{run_id}** | **{_fmt_delta(st['delta_avg'])}** | " - f"{st['improved']} ({helped_pct}%) | {st['degraded']} ({hurt_pct}%) |" - ) - lines += [""] - - # --- Model Ranking --- lines += [ - "## Model Ranking", + f"_Which model benefits more from the treatment arm `{single_arm}`?_" + if single_arm is not None + else "_Which model benefits more from the non-baseline treatment arm?_", "", + "| Run | Avg Delta | Questions context helped | Questions context hurt |", + "|---|---:|---:|---:|", ] + for run_id in run_ids: + st = summary_stats[run_id] + # Percentages are over scored pairs, the same population the + # improved/degraded counts come from. + n_paired = st["n_paired"] + if st["improved"] is None or not n_paired: + lines.append(f"| **{run_id}** | — | — | — |") + continue + helped_pct = round(100 * st["improved"] / n_paired) + hurt_pct = round(100 * st["degraded"] / n_paired) + lines.append( + f"| **{run_id}** | **{_fmt_delta(st['delta_avg'])}** | " + f"{st['improved']} ({helped_pct}%) | {st['degraded']} ({hurt_pct}%) |" + ) + return lines + [""] - if has_treatment: - lines += [ - f"### By absolute quality ({arm_label})", - f"| Rank | Run | {arm_label} avg |", - "|---:|---|---:|", - ] - for rank, (run_id, st) in enumerate( - sorted(summary_stats.items(), key=_rank_key("with_avg"), reverse=True), 1 - ): - lines.append(f"| {rank} | **{run_id}** | {_num_or_dash(st['with_avg'])} |") - else: - lines += [ - "### By absolute quality (baseline)", - "| Rank | Run | Baseline avg |", - "|---:|---|---:|", - ] - for rank, (run_id, st) in enumerate( - sorted(summary_stats.items(), key=_rank_key("without_avg"), reverse=True), 1 - ): - lines.append(f"| {rank} | **{run_id}** | {_num_or_dash(st['without_avg'])} |") - if has_treatment: - lines += [ - "", - "### By treatment utilisation (Delta)", - "| Rank | Run | Delta |", - "|---:|---|---:|", - ] - for rank, (run_id, st) in enumerate( - sorted(summary_stats.items(), key=_rank_key("delta_avg"), reverse=True), 1 - ): - lines.append(f"| {rank} | **{run_id}** | {_fmt_delta(st['delta_avg'])} |") - else: - lines += [ +def _rank_table(summary_stats: dict[str, dict], field: str, column: str, fmt) -> list[str]: + """Runs ranked by *field*, those without a value last.""" + ranked = sorted(summary_stats.items(), key=_rank_key(field), reverse=True) + return [f"| Rank | Run | {column} |", "|---:|---|---:|", + *(f"| {rank} | **{run_id}** | {fmt(st[field])} |" for rank, (run_id, st) in enumerate(ranked, 1))] + + +def _ranking_lines(summary_stats: dict[str, dict], has_treatment: bool, arm_label: str) -> list[str]: + lines = ["## Model Ranking", ""] + if not has_treatment: + return lines + [ + "### By absolute quality (baseline)", + *_rank_table(summary_stats, "without_avg", "Baseline avg", _num_or_dash), "", "> **Note:** Treatment utilization ranking requires treatment arm.", "", + "", ] - - lines += [""] - return lines + return lines + [ + f"### By absolute quality ({arm_label})", + *_rank_table(summary_stats, "with_avg", f"{arm_label} avg", _num_or_dash), + "", + "### By treatment utilisation (Delta)", + *_rank_table(summary_stats, "delta_avg", "Delta", _fmt_delta), + "", + ] From 10b4bbe18f1112e8586b9d96388262c970d732fc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 08:14:23 -0700 Subject: [PATCH 3/7] test(report): pin the significance verdicts, zero scores, cost cell and arm cells of the section report Signed-off-by: Nikolay Petrov --- .../fixtures/model_compare_sections.json | 8 +- dokimi/tests/test_model_compare_sections.py | 80 ++++++++++++++++++- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/dokimi/tests/fixtures/model_compare_sections.json b/dokimi/tests/fixtures/model_compare_sections.json index 608e8a30..b15645e4 100644 --- a/dokimi/tests/fixtures/model_compare_sections.json +++ b/dokimi/tests/fixtures/model_compare_sections.json @@ -1,13 +1,19 @@ { "baseline_only": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | — (baseline only) |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline | b baseline |\n|---|---:|| :---: | :---: |\n| easy | 3 | 60.0 | — |\n| beginner | 1 | 55.0 | — |\n| intermediate | 1 | 30.0 | — |\n| medium | 1 | 60.0 | — |\n| advanced | 1 | 65.0 | — |\n| hard | 1 | 40.0 | 56.2 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n| 2 | **b** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", "baseline_only_no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 3 | 60.0 |\n| beginner | 1 | 55.0 |\n| intermediate | 1 | 30.0 |\n| medium | 1 | 60.0 |\n| advanced | 1 | 65.0 |\n| hard | 1 | 40.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "baseline_zero": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **3**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 3 | 0 (3 unscored) | — | 30.0 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 1 | 0.0 |\n| medium | 1 | 40.0 |\n| hard | 1 | 50.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 30.0 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", "empty": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n\nCommon questions evaluated: **0**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", "mixed": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `with-context` |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | — | — |\n| beginner | 1 | 75.0 | +20.0 | — | — |\n| intermediate | 1 | 45.0 | +15.0 | — | — |\n| medium | 1 | 70.0 | +10.0 | — | — |\n| advanced | 1 | 90.0 | +25.0 | — | — |\n| hard | 1 | 80.0 | +40.0 | — | 56.2 (baseline) |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| **b** | 0 | 0% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | — | — | — |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | — |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | N/A |\n", "no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", "pinned_arm": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | docs | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ |\n|---|---:| :---: | :---: |\n| easy | 9 | 68.8 | +17.1 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `docs`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n\n## Model Ranking\n\n### By absolute quality (docs)\n| Rank | Run | docs avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n", "pooled": "## Replicate Pooling\n\nRuns sharing a configuration were averaged per question, so each row below is one configuration rather than one sample of it. Each pooled score is a mean of k runs, so its spread and confidence intervals are narrower than a single run's: that reflects the k runs spent, not a sharper measurement.\n\n- **m[n=2]'** pools 2 replicates: m_run1, m_run2; 2 question(s) dropped for not being scored in every replicate\n\n## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **m[n=2]** | solo (p) | same (p) | `with-context` |\n| **m[n=2]'** | rep (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **7**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **m[n=2]** | — | — | — | — | — | — |\n| **m[n=2]'** | $0.2469 (6/16) | 24,690 | 1,356 | 26,046 | 1.23s | 25.0% |\n\n_A pooled `[n=k]` row shows the total cost, calls and tokens of its k runs and their mean latency._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **m[n=2]** | 7 | 7 | 62.9 | 52.9 | **+10.0** | 7 | 0 |\n| **m[n=2]'** | 7 | 7 | 66.4 | 52.9 | **+13.6** | 6 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **m[n=2]** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n| **m[n=2]'** | +13.6 | 0.0033 | 0.0033 | 0.0312 | +1.8 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | m[n=2] context | m[n=2] Δ | m[n=2]' context | m[n=2]' Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 2 | 70.0 | +10.0 | 67.5 | +7.5 |\n| beginner | 1 | 65.0 | +10.0 | 70.0 | +15.0 |\n| intermediate | 1 | 40.0 | +10.0 | 42.5 | +12.5 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 75.0 | +10.0 | 82.5 | +17.5 |\n| hard | 1 | 50.0 | +10.0 | 65.0 | +25.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **m[n=2]** | 1 | 14% |\n| **m[n=2]'** | 5 | 71% |\n| tie | 1 | 14% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **m[n=2]** | **+10.0** | 7 (100%) | 0 (0%) |\n| **m[n=2]'** | **+13.6** | 6 (86%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **m[n=2]'** | 66.4 |\n| 2 | **m[n=2]** | 62.9 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **m[n=2]'** | +13.6 |\n| 2 | **m[n=2]** | +10.0 |\n", "pooled_singletons": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **x** | x (p) | same (p) | `with-context` |\n| **y** | y (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **x** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **y** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **x** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **y** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | x context | x Δ | y context | y Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **x** | 6 | 67% |\n| **y** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **x** | **+17.1** | 6 (86%) | 1 (14%) |\n| **y** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **x** | 68.8 |\n| 2 | **y** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **x** | +17.1 |\n| 2 | **y** | +10.0 |\n", + "pooled_singletons_cost": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **x** | x (p) | same (p) | `with-context` |\n| **y** | y (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **x** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **y** | $0.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **x** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **y** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **x** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **y** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | x context | x Δ | y context | y Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **x** | 6 | 67% |\n| **y** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **x** | **+17.1** | 6 (86%) | 1 (14%) |\n| **y** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **x** | 68.8 |\n| 2 | **y** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **x** | +17.1 |\n| 2 | **y** | +10.0 |\n", "treatment": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", "treatment_bh": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Benjamini-Hochberg); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", - "two_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `docs` |\n| **b** | beta (p) | same (p) | `skill:x` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n" + "two_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `docs` |\n| **b** | beta (p) | same (p) | `skill:x` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "unscored_and_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **u** | m (p) | same (p) | `docs`, `skill:x` |\n| **v** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **u** | $2.0000 (4/5) | — | — | — | — | — |\n| **v** | $3.0000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **u** | 8 | 7 (1 unscored) | 64.3 | 56.2 | **+11.4** | 7 | 0 |\n| **v** | 8 | 7 (1 unscored) | 64.3 | 56.2 | **+11.4** | 7 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 2 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **u** | +11.4 | 0.0016 | 0.0032 | 0.0156 | +2.1 | large | OK Yes |\n| **v** | +11.4 | 0.0016 | 0.0032 | 0.0156 | +2.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | u context | u Δ | v context | v Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 2 | 70.0 | +10.0 | 70.0 | +10.0 |\n| beginner | 1 | 70.0 | +15.0 | 70.0 | +15.0 |\n| intermediate | 1 | 50.0 | +20.0 | 50.0 | +20.0 |\n| medium | 1 | 75.0 | +15.0 | 75.0 | +15.0 |\n| advanced | 1 | 70.0 | +5.0 | 70.0 | +5.0 |\n| hard | 1 | 45.0 | +5.0 | 45.0 | +5.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **u** | 0 | 0% |\n| **v** | 0 | 0% |\n| tie | 7 | 88% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **u** | **+11.4** | 7 (100%) | 0 (0%) |\n| **v** | **+11.4** | 7 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **u** | 64.3 |\n| 2 | **v** | 64.3 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **u** | +11.4 |\n| 2 | **v** | +11.4 |\n", + "verdict_holm_no": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **h1** | h1 (p) | same (p) | `with-context` |\n| **h2** | h2 (p) | same (p) | `with-context` |\n| **h3** | h3 (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **h1** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n| **h2** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n| **h3** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 3 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **h1** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n| **h2** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n| **h3** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | h1 context | h1 Δ | h2 context | h2 Δ | h3 context | h3 Δ |\n|---|---:| :---: | :---: | :---: | :---: | :---: | :---: |\n| easy | 2 | 52.5 | +2.5 | 52.5 | +2.5 | 52.5 | +2.5 |\n| beginner | 1 | 59.0 | +9.0 | 59.0 | +9.0 | 59.0 | +9.0 |\n| intermediate | 1 | 61.0 | +11.0 | 61.0 | +11.0 | 61.0 | +11.0 |\n| medium | 1 | 50.0 | 0.0 | 50.0 | 0.0 | 50.0 | 0.0 |\n| advanced | 1 | 52.0 | +2.0 | 52.0 | +2.0 | 52.0 | +2.0 |\n| hard | 1 | 64.0 | +14.0 | 64.0 | +14.0 | 64.0 | +14.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **h1** | 0 | 0% |\n| **h2** | 0 | 0% |\n| **h3** | 0 | 0% |\n| tie | 8 | 100% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **h1** | **+5.9** | 6 (75%) | 1 (12%) |\n| **h2** | **+5.9** | 6 (75%) | 1 (12%) |\n| **h3** | **+5.9** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **h1** | 55.9 |\n| 2 | **h2** | 55.9 |\n| 3 | **h3** | 55.9 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **h1** | +5.9 |\n| 2 | **h2** | +5.9 |\n| 3 | **h3** | +5.9 |\n", + "verdict_wilcoxon_no": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **w** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **w** | 6 | 6 | 55.2 | 50.0 | **+5.2** | 5 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **w** | +5.2 | 0.0384 | 0.0384 | 0.0625 | +1.1 | large | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | w context | w Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 52.5 | +2.5 |\n| beginner | 1 | 55.0 | +5.0 |\n| medium | 1 | 57.0 | +7.0 |\n| advanced | 1 | 54.0 | +4.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **w** | 6 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **w** | **+5.2** | 5 (83%) | 1 (17%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **w** | 55.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **w** | +5.2 |\n", + "zero_scores": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 8 | 8 | 33.1 | 24.4 | **+8.8** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **z** | +8.8 | 0.247 | 0.247 | 0.1719 | +0.4 | small | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 47.5 | +27.5 |\n| beginner | 1 | 0.0 | 0.0 |\n| intermediate | 1 | 50.0 | +5.0 |\n| medium | 1 | 0.0 | -30.0 |\n| advanced | 1 | 35.0 | +15.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 8 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **+8.8** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 33.1 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | +8.8 |\n" } diff --git a/dokimi/tests/test_model_compare_sections.py b/dokimi/tests/test_model_compare_sections.py index 0c6d03f6..5201492e 100644 --- a/dokimi/tests/test_model_compare_sections.py +++ b/dokimi/tests/test_model_compare_sections.py @@ -33,18 +33,26 @@ _DIFFICULTIES = ["easy", "medium", "hard", 1, "beginner", "advanced", "intermediate", "expert"] -def _run(scores: list[tuple], arm: str = "with-context", difficulties=None, **meta) -> dict: +def _run( + scores: list[tuple], + arm: str | list[str] = "with-context", + difficulties=None, + baseline_arm: str = "baseline", + **meta, +) -> dict: """An ``arms.v1``-shaped artifact; *scores* holds (baseline, treatment) per question. + *arm* may be a list: question i is scored in ``arm[i % len(arm)]``. + A treatment of ``...`` leaves the arm out of the record (a baseline-only question); ``None`` keeps the arm with no usable score. """ difficulties = difficulties or _DIFFICULTIES evaluations = [] for i, (base, treat) in enumerate(scores): - cond: dict = {"baseline": {"aggregate": base}} + cond: dict = {baseline_arm: {"aggregate": base}} if treat is not ...: - cond[arm] = {"aggregate": treat} + cond[arm[i % len(arm)] if isinstance(arm, list) else arm] = {"aggregate": treat} evaluations.append( { "question_id": f"q{i}", @@ -54,7 +62,7 @@ def _run(scores: list[tuple], arm: str = "with-context", difficulties=None, **me } ) return { - "baseline_arm": "baseline", + "baseline_arm": baseline_arm, "model": "m", "provider": "p", "evaluations": evaluations, @@ -82,6 +90,14 @@ def _cost(**fields) -> dict: # Constant +10 deltas: the zero-variance guard leaves the p-values N/A. _B = [(50, 60), (60, 70), (40, 50), (70, 80), (55, 65), (65, 75), (30, 40), (80, 90), (20, 30)] _BASE_ONLY = [(b, ...) for b, _ in _A] +# A raw t-test p of 0.024 and Wilcoxon 0.047: significant alone, not once Holm-adjusted over 3 runs. +_H = [(50, 58), (50, 50), (50, 64), (50, 47), (50, 59), (50, 52), (50, 61), (50, 56)] +# t-test p 0.038 but Wilcoxon 0.0625: adjusted p stays below 0.05, the Wilcoxon clause says No. +_W = [(50, 47), (50, 57), (50, 60), (50, 58), (50, 55), (50, 54)] +# Zeros are scores: a 0 in either arm, and a bucket whose only treatment score is 0. +_Z = [(0, 40), (30, 0), (50, 60), (40, 55), (0, 0), (20, 35), (45, 50), (10, 25)] +# Exactly one question without a usable pair. +_U = [(50, 60), (60, 75), (40, 45), (70, 80), (55, 70), (65, 70), (30, 50), (80, None)] def _cases() -> dict: @@ -140,6 +156,39 @@ def _cases() -> dict: {"pool_replicates": True}, ), "empty": ([], [], {}), + # The raw verdict is Yes for each of the three, the adjusted one No. + "verdict_holm_no": ( + [(r, _run(_H, model=r)) for r in ("h1", "h2", "h3")], + ["h1", "h2", "h3"], + {}, + ), + "verdict_wilcoxon_no": ([("w", _run(_W))], ["w"], {}), + "zero_scores": ([("z", _run(_Z))], ["z"], {}), + # A baseline-only bucket whose only score is 0 prints 0.0, not a dash. + "baseline_zero": ( + [("a", _run([(0, ...), (40, ...), (50, ...)], difficulties=["easy", "medium", "hard"]))], + ["a"], + {}, + ), + # One unscored question; a run whose questions span two arms; a cost covering 4 of + # 5 calls with the count read from ``n``, and one covering all 5. + "unscored_and_arms": ( + [ + ("u", _run(_U, arm=["docs", "skill:x"], cost_summary=_cost(total_cost_usd=2.0, cost_known_n=4, n=5))), + ("v", _run(_U, arm="docs", cost_summary=_cost(total_cost_usd=3.0, cost_known_n=5, total_llm_calls=5))), + ], + ["u", "v"], + {}, + ), + # Replicates that do not pool leave no footnote; a baseline arm that is not "baseline". + "pooled_singletons_cost": ( + [ + ("x", _run(_A, model="x", cost_summary=_FULL_COST)), + ("y", _run(_B, model="y", baseline_arm="control", cost_summary={"control": {"total_cost_usd": 0.5}, "baseline": {"total_cost_usd": 9.0}})), + ], + ["x", "y"], + {"pool_replicates": True}, + ), } @@ -197,6 +246,12 @@ def test_goldens_reach_every_section_branch(): "two_arms": ["Context arm", "non-baseline treatment arm"], "pooled": ["## Replicate Pooling", "m[n=2]'", "pools 2 replicates", "A pooled `[n=k]` row"], "empty": ["Common questions evaluated: **0**"], + "verdict_holm_no": ["| No |", "(Holm-Bonferroni)"], + "verdict_wilcoxon_no": ["| No |"], + "zero_scores": ["| beginner | 1 | 0.0 | 0.0 |", "| medium | 1 | 0.0 | -30.0 |"], + "baseline_zero": ["| easy | 1 | 0.0 |"], + "unscored_and_arms": ["7 (1 unscored)", "`docs`, `skill:x`", "(4/5)"], + "pooled_singletons_cost": ["## Resource Usage", "| **y** | $0.5000 |"], } for name, wanted in markers.items(): for marker in wanted: @@ -204,3 +259,20 @@ def test_goldens_reach_every_section_branch(): assert "Statistical Significance" not in texts["no_scipy"] assert "Statistical Significance" not in texts["baseline_only_no_scipy"] assert "Resource Usage" not in texts["baseline_only"] + assert "OK Yes" not in texts["verdict_holm_no"] and "OK Yes" not in texts["verdict_wilcoxon_no"] + assert "pooled `[n=k]` row" not in texts["pooled_singletons_cost"] + assert "(5/5)" not in texts["unscored_and_arms"] + + +@pytest.mark.parametrize( + "p, p_wilcoxon, significant", + [(0.05, None, False), (0.049, None, True), (0.049, 0.05, False), (0.049, 0.049, True)], +) +def test_adjusted_verdict_is_strictly_below_alpha(monkeypatch, p, p_wilcoxon, significant): + # A p exactly at alpha is not significant, for the t-test and for the Wilcoxon clause. + fake = {"p_ttest": p, "p_wilcoxon": p_wilcoxon, "significant": True} + monkeypatch.setattr(model_compare, "significance_test", lambda *_: dict(fake)) + scores = {"r": [{"with_docs": 60, "without_docs": 50}] * 6} + sigs, tested = model_compare._adjusted_significance(["r"], scores, "holm") + assert tested == 1 + assert sigs["r"]["significant_adjusted"] is significant From 61759a20d4e2c108d6ee1473fb8ad1f3326d7454 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 11:39:18 -0700 Subject: [PATCH 4/7] style(test): ruff format the section report goldens Signed-off-by: Nikolay Petrov --- dokimi/tests/test_model_compare_sections.py | 38 ++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/dokimi/tests/test_model_compare_sections.py b/dokimi/tests/test_model_compare_sections.py index 5201492e..18f971be 100644 --- a/dokimi/tests/test_model_compare_sections.py +++ b/dokimi/tests/test_model_compare_sections.py @@ -166,7 +166,12 @@ def _cases() -> dict: "zero_scores": ([("z", _run(_Z))], ["z"], {}), # A baseline-only bucket whose only score is 0 prints 0.0, not a dash. "baseline_zero": ( - [("a", _run([(0, ...), (40, ...), (50, ...)], difficulties=["easy", "medium", "hard"]))], + [ + ( + "a", + _run([(0, ...), (40, ...), (50, ...)], difficulties=["easy", "medium", "hard"]), + ) + ], ["a"], {}, ), @@ -174,8 +179,22 @@ def _cases() -> dict: # 5 calls with the count read from ``n``, and one covering all 5. "unscored_and_arms": ( [ - ("u", _run(_U, arm=["docs", "skill:x"], cost_summary=_cost(total_cost_usd=2.0, cost_known_n=4, n=5))), - ("v", _run(_U, arm="docs", cost_summary=_cost(total_cost_usd=3.0, cost_known_n=5, total_llm_calls=5))), + ( + "u", + _run( + _U, + arm=["docs", "skill:x"], + cost_summary=_cost(total_cost_usd=2.0, cost_known_n=4, n=5), + ), + ), + ( + "v", + _run( + _U, + arm="docs", + cost_summary=_cost(total_cost_usd=3.0, cost_known_n=5, total_llm_calls=5), + ), + ), ], ["u", "v"], {}, @@ -184,7 +203,18 @@ def _cases() -> dict: "pooled_singletons_cost": ( [ ("x", _run(_A, model="x", cost_summary=_FULL_COST)), - ("y", _run(_B, model="y", baseline_arm="control", cost_summary={"control": {"total_cost_usd": 0.5}, "baseline": {"total_cost_usd": 9.0}})), + ( + "y", + _run( + _B, + model="y", + baseline_arm="control", + cost_summary={ + "control": {"total_cost_usd": 0.5}, + "baseline": {"total_cost_usd": 9.0}, + }, + ), + ), ], ["x", "y"], {"pool_replicates": True}, From 26eafa7b7c40505f4904b1738df5d38b1558c594 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:05:29 -0700 Subject: [PATCH 5/7] chore(growth): lower model_compare.py and drop generate_section_report from the baseline Signed-off-by: Nikolay Petrov --- dokimi/scripts/growth_baseline.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 2b0f5fe8..0a25f19e 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -5,7 +5,7 @@ "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1112}, "dokimi/novelty/bundle.py": {"lines": 1016, "rationale": "grew on main in #101 before the ratchet landed; split it in its own PR"}, - "dokimi/report/model_compare.py": {"lines": 1297, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, + "dokimi/report/model_compare.py": {"lines": 1210, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, "tests/test_container_agent.py": {"lines": 1251, "rationale": "pre-existing when the gate was introduced"}, @@ -77,7 +77,6 @@ "dokimi/report/generator.py::ReportGenerator._format_markdown": {"complexity": 18}, "dokimi/report/model_compare.py::check_run_consistency": {"complexity": 13}, "dokimi/report/model_compare.py::extract_scores": {"complexity": 16}, - "dokimi/report/model_compare.py::generate_section_report": {"complexity": 53}, "dokimi/report/subject_report.py::render_subject_scorecard": {"complexity": 12}, "dokimi/report/task_runs_report.py::render_task_runs_report": {"complexity": 22}, "dokimi/repos/budget.py::budget": {"complexity": 21, "rationale": "pre-existing when the gate was introduced"}, From 08a8f5d8385a2cdac8d906aaedeb73627f424628 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 08:12:44 -0700 Subject: [PATCH 6/7] test(report): pin zero cost, latency, cache hit, improved count and baseline mean in the section goldens Signed-off-by: Nikolay Petrov --- dokimi/tests/fixtures/model_compare_sections.json | 3 ++- dokimi/tests/test_model_compare_sections.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/fixtures/model_compare_sections.json b/dokimi/tests/fixtures/model_compare_sections.json index b15645e4..78d76aad 100644 --- a/dokimi/tests/fixtures/model_compare_sections.json +++ b/dokimi/tests/fixtures/model_compare_sections.json @@ -15,5 +15,6 @@ "unscored_and_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **u** | m (p) | same (p) | `docs`, `skill:x` |\n| **v** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **u** | $2.0000 (4/5) | — | — | — | — | — |\n| **v** | $3.0000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **u** | 8 | 7 (1 unscored) | 64.3 | 56.2 | **+11.4** | 7 | 0 |\n| **v** | 8 | 7 (1 unscored) | 64.3 | 56.2 | **+11.4** | 7 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 2 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **u** | +11.4 | 0.0016 | 0.0032 | 0.0156 | +2.1 | large | OK Yes |\n| **v** | +11.4 | 0.0016 | 0.0032 | 0.0156 | +2.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | u context | u Δ | v context | v Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 2 | 70.0 | +10.0 | 70.0 | +10.0 |\n| beginner | 1 | 70.0 | +15.0 | 70.0 | +15.0 |\n| intermediate | 1 | 50.0 | +20.0 | 50.0 | +20.0 |\n| medium | 1 | 75.0 | +15.0 | 75.0 | +15.0 |\n| advanced | 1 | 70.0 | +5.0 | 70.0 | +5.0 |\n| hard | 1 | 45.0 | +5.0 | 45.0 | +5.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **u** | 0 | 0% |\n| **v** | 0 | 0% |\n| tie | 7 | 88% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **u** | **+11.4** | 7 (100%) | 0 (0%) |\n| **v** | **+11.4** | 7 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **u** | 64.3 |\n| 2 | **v** | 64.3 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **u** | +11.4 |\n| 2 | **v** | +11.4 |\n", "verdict_holm_no": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **h1** | h1 (p) | same (p) | `with-context` |\n| **h2** | h2 (p) | same (p) | `with-context` |\n| **h3** | h3 (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **h1** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n| **h2** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n| **h3** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 3 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **h1** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n| **h2** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n| **h3** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | h1 context | h1 Δ | h2 context | h2 Δ | h3 context | h3 Δ |\n|---|---:| :---: | :---: | :---: | :---: | :---: | :---: |\n| easy | 2 | 52.5 | +2.5 | 52.5 | +2.5 | 52.5 | +2.5 |\n| beginner | 1 | 59.0 | +9.0 | 59.0 | +9.0 | 59.0 | +9.0 |\n| intermediate | 1 | 61.0 | +11.0 | 61.0 | +11.0 | 61.0 | +11.0 |\n| medium | 1 | 50.0 | 0.0 | 50.0 | 0.0 | 50.0 | 0.0 |\n| advanced | 1 | 52.0 | +2.0 | 52.0 | +2.0 | 52.0 | +2.0 |\n| hard | 1 | 64.0 | +14.0 | 64.0 | +14.0 | 64.0 | +14.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **h1** | 0 | 0% |\n| **h2** | 0 | 0% |\n| **h3** | 0 | 0% |\n| tie | 8 | 100% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **h1** | **+5.9** | 6 (75%) | 1 (12%) |\n| **h2** | **+5.9** | 6 (75%) | 1 (12%) |\n| **h3** | **+5.9** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **h1** | 55.9 |\n| 2 | **h2** | 55.9 |\n| 3 | **h3** | 55.9 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **h1** | +5.9 |\n| 2 | **h2** | +5.9 |\n| 3 | **h3** | +5.9 |\n", "verdict_wilcoxon_no": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **w** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **w** | 6 | 6 | 55.2 | 50.0 | **+5.2** | 5 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **w** | +5.2 | 0.0384 | 0.0384 | 0.0625 | +1.1 | large | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | w context | w Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 52.5 | +2.5 |\n| beginner | 1 | 55.0 | +5.0 |\n| medium | 1 | 57.0 | +7.0 |\n| advanced | 1 | 54.0 | +4.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **w** | 6 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **w** | **+5.2** | 5 (83%) | 1 (17%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **w** | 55.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **w** | +5.2 |\n", - "zero_scores": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 8 | 8 | 33.1 | 24.4 | **+8.8** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **z** | +8.8 | 0.247 | 0.247 | 0.1719 | +0.4 | small | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 47.5 | +27.5 |\n| beginner | 1 | 0.0 | 0.0 |\n| intermediate | 1 | 50.0 | +5.0 |\n| medium | 1 | 0.0 | -30.0 |\n| advanced | 1 | 35.0 | +15.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 8 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **+8.8** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 33.1 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | +8.8 |\n" + "zero_scores": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 8 | 8 | 33.1 | 24.4 | **+8.8** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **z** | +8.8 | 0.247 | 0.247 | 0.1719 | +0.4 | small | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 47.5 | +27.5 |\n| beginner | 1 | 0.0 | 0.0 |\n| intermediate | 1 | 50.0 | +5.0 |\n| medium | 1 | 0.0 | -30.0 |\n| advanced | 1 | 35.0 | +15.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 8 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **+8.8** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 33.1 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | +8.8 |\n", + "zero_values": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | zero (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **z** | $0.0000 (0/5) | — | — | — | 0.00s | 0.0% |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 6 | 5 (1 unscored) | 45.0 | 45.8 | **-10.0** | 0 | 5 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 0 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 50.0 | -10.0 |\n| beginner | 1 | 45.0 | -10.0 |\n| medium | 1 | 50.0 | -10.0 |\n| advanced | 1 | — | 0.0 (baseline) |\n| hard | 1 | 30.0 | -10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 5 | 83% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **-10.0** | 0 (0%) | 5 (100%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 45.0 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | -10.0 |\n" } diff --git a/dokimi/tests/test_model_compare_sections.py b/dokimi/tests/test_model_compare_sections.py index 18f971be..8293e033 100644 --- a/dokimi/tests/test_model_compare_sections.py +++ b/dokimi/tests/test_model_compare_sections.py @@ -98,6 +98,11 @@ def _cost(**fields) -> dict: _Z = [(0, 40), (30, 0), (50, 60), (40, 55), (0, 0), (20, 35), (45, 50), (10, 25)] # Exactly one question without a usable pair. _U = [(50, 60), (60, 75), (40, 45), (70, 80), (55, 70), (65, 70), (30, 50), (80, None)] +# Nothing improved (0 of 5 pairs), and a bucket whose only score is a baseline 0. +_N = [(50, 40), (60, 50), (40, 30), (70, 60), (55, 45), (0, ...)] +_FREE_COST = _cost( + total_cost_usd=0, cost_known_n=0, total_llm_calls=5, mean_latency_sec=0, cache_hit_ratio=0 +) def _cases() -> dict: @@ -107,6 +112,7 @@ def _cases() -> dict: # from the other buckets, and its "easy" bucket has no usable score at all. lopsided = ["hard"] * 8 + ["easy"] return { + "zero_values": ([("z", _run(_N, model="zero", cost_summary=_FREE_COST))], ["z"], {}), "treatment": ([("a", a), ("b", b)], ["a", "b"], {}), "treatment_bh": ([("a", a), ("b", b)], ["a", "b"], {"correction": "bh"}), "no_scipy": ([("a", a), ("b", b)], ["a", "b"], {"no_scipy": True}), @@ -280,6 +286,11 @@ def test_goldens_reach_every_section_branch(): "verdict_wilcoxon_no": ["| No |"], "zero_scores": ["| beginner | 1 | 0.0 | 0.0 |", "| medium | 1 | 0.0 | -30.0 |"], "baseline_zero": ["| easy | 1 | 0.0 |"], + "zero_values": [ + "| **z** | $0.0000 (0/5) | — | — | — | 0.00s | 0.0% |", + "| 0 (0%) | 5 (100%) |", + "| — | 0.0 (baseline) |", + ], "unscored_and_arms": ["7 (1 unscored)", "`docs`, `skill:x`", "(4/5)"], "pooled_singletons_cost": ["## Resource Usage", "| **y** | $0.5000 |"], } From 56b041c7597fa6441f87a6470ef2b93cec59c431 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 23 Oct 2026 16:00:32 +0000 Subject: [PATCH 7/7] test(report): pin zero token counts in the zero_values golden Signed-off-by: Anton Volkov --- dokimi/tests/fixtures/model_compare_sections.json | 2 +- dokimi/tests/test_model_compare_sections.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dokimi/tests/fixtures/model_compare_sections.json b/dokimi/tests/fixtures/model_compare_sections.json index 78d76aad..115fe5f2 100644 --- a/dokimi/tests/fixtures/model_compare_sections.json +++ b/dokimi/tests/fixtures/model_compare_sections.json @@ -16,5 +16,5 @@ "verdict_holm_no": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **h1** | h1 (p) | same (p) | `with-context` |\n| **h2** | h2 (p) | same (p) | `with-context` |\n| **h3** | h3 (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **h1** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n| **h2** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n| **h3** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 3 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **h1** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n| **h2** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n| **h3** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | h1 context | h1 Δ | h2 context | h2 Δ | h3 context | h3 Δ |\n|---|---:| :---: | :---: | :---: | :---: | :---: | :---: |\n| easy | 2 | 52.5 | +2.5 | 52.5 | +2.5 | 52.5 | +2.5 |\n| beginner | 1 | 59.0 | +9.0 | 59.0 | +9.0 | 59.0 | +9.0 |\n| intermediate | 1 | 61.0 | +11.0 | 61.0 | +11.0 | 61.0 | +11.0 |\n| medium | 1 | 50.0 | 0.0 | 50.0 | 0.0 | 50.0 | 0.0 |\n| advanced | 1 | 52.0 | +2.0 | 52.0 | +2.0 | 52.0 | +2.0 |\n| hard | 1 | 64.0 | +14.0 | 64.0 | +14.0 | 64.0 | +14.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **h1** | 0 | 0% |\n| **h2** | 0 | 0% |\n| **h3** | 0 | 0% |\n| tie | 8 | 100% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **h1** | **+5.9** | 6 (75%) | 1 (12%) |\n| **h2** | **+5.9** | 6 (75%) | 1 (12%) |\n| **h3** | **+5.9** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **h1** | 55.9 |\n| 2 | **h2** | 55.9 |\n| 3 | **h3** | 55.9 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **h1** | +5.9 |\n| 2 | **h2** | +5.9 |\n| 3 | **h3** | +5.9 |\n", "verdict_wilcoxon_no": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **w** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **w** | 6 | 6 | 55.2 | 50.0 | **+5.2** | 5 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **w** | +5.2 | 0.0384 | 0.0384 | 0.0625 | +1.1 | large | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | w context | w Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 52.5 | +2.5 |\n| beginner | 1 | 55.0 | +5.0 |\n| medium | 1 | 57.0 | +7.0 |\n| advanced | 1 | 54.0 | +4.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **w** | 6 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **w** | **+5.2** | 5 (83%) | 1 (17%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **w** | 55.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **w** | +5.2 |\n", "zero_scores": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 8 | 8 | 33.1 | 24.4 | **+8.8** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **z** | +8.8 | 0.247 | 0.247 | 0.1719 | +0.4 | small | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 47.5 | +27.5 |\n| beginner | 1 | 0.0 | 0.0 |\n| intermediate | 1 | 50.0 | +5.0 |\n| medium | 1 | 0.0 | -30.0 |\n| advanced | 1 | 35.0 | +15.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 8 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **+8.8** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 33.1 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | +8.8 |\n", - "zero_values": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | zero (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **z** | $0.0000 (0/5) | — | — | — | 0.00s | 0.0% |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 6 | 5 (1 unscored) | 45.0 | 45.8 | **-10.0** | 0 | 5 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 0 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 50.0 | -10.0 |\n| beginner | 1 | 45.0 | -10.0 |\n| medium | 1 | 50.0 | -10.0 |\n| advanced | 1 | — | 0.0 (baseline) |\n| hard | 1 | 30.0 | -10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 5 | 83% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **-10.0** | 0 (0%) | 5 (100%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 45.0 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | -10.0 |\n" + "zero_values": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | zero (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **z** | $0.0000 (0/5) | 0 | 0 | 0 | 0.00s | 0.0% |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 6 | 5 (1 unscored) | 45.0 | 45.8 | **-10.0** | 0 | 5 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 0 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 50.0 | -10.0 |\n| beginner | 1 | 45.0 | -10.0 |\n| medium | 1 | 50.0 | -10.0 |\n| advanced | 1 | — | 0.0 (baseline) |\n| hard | 1 | 30.0 | -10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 5 | 83% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **-10.0** | 0 (0%) | 5 (100%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 45.0 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | -10.0 |\n" } diff --git a/dokimi/tests/test_model_compare_sections.py b/dokimi/tests/test_model_compare_sections.py index 8293e033..43bcbce7 100644 --- a/dokimi/tests/test_model_compare_sections.py +++ b/dokimi/tests/test_model_compare_sections.py @@ -101,7 +101,14 @@ def _cost(**fields) -> dict: # Nothing improved (0 of 5 pairs), and a bucket whose only score is a baseline 0. _N = [(50, 40), (60, 50), (40, 30), (70, 60), (55, 45), (0, ...)] _FREE_COST = _cost( - total_cost_usd=0, cost_known_n=0, total_llm_calls=5, mean_latency_sec=0, cache_hit_ratio=0 + total_cost_usd=0, + cost_known_n=0, + total_llm_calls=5, + prompt_tokens=0, + completion_tokens=0, + total_tokens=0, + mean_latency_sec=0, + cache_hit_ratio=0, ) @@ -287,7 +294,7 @@ def test_goldens_reach_every_section_branch(): "zero_scores": ["| beginner | 1 | 0.0 | 0.0 |", "| medium | 1 | 0.0 | -30.0 |"], "baseline_zero": ["| easy | 1 | 0.0 |"], "zero_values": [ - "| **z** | $0.0000 (0/5) | — | — | — | 0.00s | 0.0% |", + "| **z** | $0.0000 (0/5) | 0 | 0 | 0 | 0.00s | 0.0% |", "| 0 (0%) | 5 (100%) |", "| — | 0.0 (baseline) |", ],