diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py index b88cf773..cec86bf7 100644 --- a/dokimi/dokimi/report/model_compare.py +++ b/dokimi/dokimi/report/model_compare.py @@ -756,208 +756,208 @@ def generate_section_report( question into a single labelled row, so the report ranks configurations rather than individual samples of one configuration. """ - - # Extract and intersect to common questions only - all_run_scores_raw: dict[str, list[dict]] = {} - for run_id, run_data in runs: - all_run_scores_raw[run_id] = extract_scores( - run_data, force_treatment_arm=treatment_arm - ) - + raw_scores = {run_id: extract_scores(data, force_treatment_arm=treatment_arm) for run_id, data in runs} replicate_note: list[str] = [] if pool_replicates: - groups = group_replicates(runs) - run_data_by_id = dict(runs) - pooled_runs: list[tuple[str, dict]] = [] - pooled_ids: list[str] = [] - pooled_scores: dict[str, list[dict]] = {} - for _key, ids in groups.items(): - present = [rid for rid in run_ids if rid in ids] - if not present: - continue - label = replicate_label(present) - while label in pooled_scores: # keep labels unique - label += "'" - pooled_scores[label] = pool_replicate_scores(all_run_scores_raw, present) - pooled_run = run_data_by_id[present[0]] - if len(present) > 1: - pooled_run = {**pooled_run, "cost_summary": pool_cost_summaries( - [run_data_by_id[rid].get("cost_summary") for rid in present])} - pooled_runs.append((label, pooled_run)) - pooled_ids.append(label) - if len(present) > 1: - seen = {s["question_id"] for rid in present for s in all_run_scores_raw.get(rid, [])} - dropped = len(seen) - len(pooled_scores[label]) - replicate_note.append( - f"- **{label}** pools {len(present)} replicates: {', '.join(present)}; " - f"{dropped} question(s) dropped for not being scored in every replicate" - ) - runs = pooled_runs - run_ids = pooled_ids - all_run_scores_raw = pooled_scores - - question_sets = [ - set(s["question_id"] for s in scores) - for scores in all_run_scores_raw.values() - ] - common_questions = set.intersection(*question_sets) if question_sets else set() - - # All metrics below use the common-question subset exclusively - all_run_scores: dict[str, list[dict]] = { - run_id: [ - s for s in all_run_scores_raw.get(run_id, []) - if s["question_id"] in common_questions - ] - for run_id in run_ids - } + runs, run_ids, raw_scores, replicate_note = _pool_section(runs, run_ids, raw_scores) + common_questions, all_run_scores = _common_scores(raw_scores, run_ids) # The arm that was actually scored, taken from the extracted records rather than the # artifact header, so the label can never disagree with the numbers beside it. Named # explicitly because a `profile:` run and a `docs` run otherwise render identically: # a reader of the Markdown alone could not tell which treatment it measured. - scored_arms = { - s["treatment_arm"] + scored = {s["treatment_arm"] for scores in all_run_scores.values() for s in scores if s.get("treatment_arm")} + arm_label = next(iter(scored)) if len(scored) == 1 else "Context arm" + # Single-arm vs multi-arm: without a scored pair there is no delta to analyse. + has_treatment = any( + s.get("with_docs") is not None and s.get("delta") is not None for scores in all_run_scores.values() for s in scores - if s.get("treatment_arm") - } - arm_label = next(iter(scored_arms)) if len(scored_arms) == 1 else "Context arm" + ) + summary_stats = {run_id: _run_summary(all_run_scores[run_id]) for run_id in run_ids} - lines: list[str] = [] + lines = _pooling_lines(replicate_note) + _models_compared_lines(runs, all_run_scores, common_questions) + if not has_treatment: + lines += _BASELINE_ONLY_LIMITATION + lines += _resource_usage_lines(runs, bool(replicate_note)) + lines += _overall_summary_lines(run_ids, summary_stats, arm_label) + if HAS_SCIPY: + lines += _significance_lines(run_ids, all_run_scores, has_treatment, correction) + lines += _difficulty_lines(run_ids, all_run_scores, has_treatment) + lines += _head_to_head_lines(run_ids, all_run_scores, common_questions, has_treatment) + lines += _benefit_lines(run_ids, summary_stats, has_treatment, arm_label if len(scored) == 1 else None) + return lines + _ranking_lines(summary_stats, has_treatment, arm_label) + + +def _pool_section( + runs: list[tuple[str, dict]], run_ids: list[str], raw_scores: dict[str, list[dict]] +) -> tuple[list[tuple[str, dict]], list[str], dict[str, list[dict]], list[str]]: + """Runs, run ids and scores with each configuration's replicates pooled into one labelled run, + plus the note naming what each pooled run holds.""" + run_data_by_id = dict(runs) + pooled_runs: list[tuple[str, dict]] = [] + pooled_scores: dict[str, list[dict]] = {} + note: list[str] = [] + for ids in group_replicates(runs).values(): + present = [rid for rid in run_ids if rid in ids] + if not present: + continue + label = replicate_label(present) + while label in pooled_scores: # keep labels unique + label += "'" + pooled_scores[label] = pool_replicate_scores(raw_scores, present) + pooled_run = run_data_by_id[present[0]] + if len(present) > 1: + pooled_run = {**pooled_run, "cost_summary": pool_cost_summaries( + [run_data_by_id[rid].get("cost_summary") for rid in present])} + seen = {s["question_id"] for rid in present for s in raw_scores.get(rid, [])} + dropped = len(seen) - len(pooled_scores[label]) + note.append( + f"- **{label}** pools {len(present)} replicates: {', '.join(present)}; " + f"{dropped} question(s) dropped for not being scored in every replicate" + ) + pooled_runs.append((label, pooled_run)) + return pooled_runs, [label for label, _ in pooled_runs], pooled_scores, note - # --- Models Compared --- - if replicate_note: - lines += [ - "## Replicate Pooling", - "", - "Runs sharing a configuration were averaged per question, so each row " - "below is one configuration rather than one sample of it. Each pooled " - "score is a mean of k runs, so its spread and confidence intervals are " - "narrower than a single run's: that reflects the k runs spent, not a " - "sharper measurement.", - "", - *replicate_note, - "", - ] - lines += [ - "## Models Compared", - "| Run ID | Answer model | Judge model | Treatment arm |", - "|---|---|---|---|", +def _common_scores( + raw_scores: dict[str, list[dict]], run_ids: list[str] +) -> tuple[set[str], dict[str, list[dict]]]: + """The question ids every run scored, and each run's scores cut to them. + + All metrics of a section use this common-question subset exclusively.""" + question_sets = [set(s["question_id"] for s in scores) for scores in raw_scores.values()] + common = set.intersection(*question_sets) if question_sets else set() + return common, { + run_id: [s for s in raw_scores.get(run_id, []) if s["question_id"] in common] for run_id in run_ids + } + + +def _pooling_lines(replicate_note: list[str]) -> list[str]: + if not replicate_note: + return [] + return [ + "## Replicate Pooling", + "", + "Runs sharing a configuration were averaged per question, so each row " + "below is one configuration rather than one sample of it. Each pooled " + "score is a mean of k runs, so its spread and confidence intervals are " + "narrower than a single run's: that reflects the k runs spent, not a " + "sharper measurement.", + "", + *replicate_note, + "", ] + + +def _models_compared_lines( + runs: list[tuple[str, dict]], all_run_scores: dict[str, list[dict]], common_questions: set[str] +) -> list[str]: + lines = ["## Models Compared", "| Run ID | Answer model | Judge model | Treatment arm |", "|---|---|---|---|"] for run_id, run_data in runs: - model = run_data.get("model", "unknown") provider = run_data.get("provider", "unknown") - judge_model = run_data.get("judge_model", "same") - judge_provider = run_data.get("judge_provider", provider) - run_arms = sorted( - { - s["treatment_arm"] - for s in all_run_scores.get(run_id, []) - if s.get("treatment_arm") - } - ) + judge = f"{run_data.get('judge_model', 'same')} ({run_data.get('judge_provider', provider)})" + run_arms = sorted({s["treatment_arm"] for s in all_run_scores.get(run_id, []) if s.get("treatment_arm")}) arm_cell = ", ".join(f"`{a}`" for a in run_arms) if run_arms else "— (baseline only)" - lines.append( - f"| **{run_id}** | {model} ({provider}) | {judge_model} ({judge_provider}) " - f"| {arm_cell} |" - ) - - lines += [ + lines.append(f"| **{run_id}** | {run_data.get('model', 'unknown')} ({provider}) | {judge} | {arm_cell} |") + return lines + [ "", f"Common questions evaluated: **{len(common_questions)}**", "_All summary, significance, ranking, and head-to-head metrics use only this common question set._", "", ] - # Detect report limitations (single-arm vs multi-arm) - has_treatment = any( - s.get("with_docs") is not None and s.get("delta") is not None - for scores in all_run_scores.values() - for s in scores - ) - - if not has_treatment: - lines += [ - "> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).", - "> ", - "> **Missing from this report:**", - "> - Context-arm vs Baseline delta analysis", - "> - Statistical significance tests", - "> - Head-to-head winner comparisons", - "> ", - "> **Available in this report:**", - "> - Absolute baseline quality scores per model", - "> - Resource usage metrics (cost, tokens, latency)", - "> - Model ranking by absolute baseline quality", - "> ", - "> **To generate full comparison with delta analysis, add a treatment arm:**", - "> ```bash", - "> python cli.py arms run \\", - "> --arms \"baseline,docs\" \\", - "> --judge \\", - "> --judge-model claude-sonnet-5-5 \\", - "> [other parameters]", - "> ```", - "> ", - "> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).", - "> ", - "> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`", - "", - ] - # --- Resource Usage --- - # Extract cost_summary from each run (baseline arm only for now) - cost_data_available = any( - run_data.get("cost_summary") for _, run_data in runs - ) - if cost_data_available: - lines += [ - "## Resource Usage (baseline arm)", - "| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |", - "|---|---:|---:|---:|---:|---:|---:|", - ] - for run_id, run_data in runs: - cost_summary = run_data.get("cost_summary", {}) - baseline_arm = run_data.get("baseline_arm", "baseline") - cs = cost_summary.get(baseline_arm, {}) - - # Extract metrics - cost_usd = cs.get("total_cost_usd") - cost_str = f"${cost_usd:.4f}" if cost_usd is not None else "—" - if cost_usd is not None: - cost_known = cs.get("cost_known_n") - total_calls = cs.get("total_llm_calls", cs.get("n")) - if cost_known is not None and total_calls is not None and cost_known < total_calls: - cost_str += f" ({cost_known}/{total_calls})" - - prompt_tok = cs.get("prompt_tokens") - prompt_str = f"{prompt_tok:,}" if prompt_tok is not None else "—" - - completion_tok = cs.get("completion_tokens") - completion_str = f"{completion_tok:,}" if completion_tok is not None else "—" - - total_tok = cs.get("total_tokens") - total_str = f"{total_tok:,}" if total_tok is not None else "—" - - latency = cs.get("mean_latency_sec") - latency_str = f"{latency:.2f}s" if latency is not None else "—" - - cache_ratio = cs.get("cache_hit_ratio") - cache_str = f"{cache_ratio:.1%}" if cache_ratio is not None else "—" - - lines.append( - f"| **{run_id}** | {cost_str} | {prompt_str} | {completion_str} | " - f"{total_str} | {latency_str} | {cache_str} |" - ) - if replicate_note: - lines += ["", "_A pooled `[n=k]` row shows the total cost, calls and tokens of its k runs " - "and their mean latency._"] +_BASELINE_ONLY_LIMITATION = [ + "> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).", + "> ", + "> **Missing from this report:**", + "> - Context-arm vs Baseline delta analysis", + "> - Statistical significance tests", + "> - Head-to-head winner comparisons", + "> ", + "> **Available in this report:**", + "> - Absolute baseline quality scores per model", + "> - Resource usage metrics (cost, tokens, latency)", + "> - Model ranking by absolute baseline quality", + "> ", + "> **To generate full comparison with delta analysis, add a treatment arm:**", + "> ```bash", + "> python cli.py arms run \\", + "> --arms \"baseline,docs\" \\", + "> --judge \\", + "> --judge-model claude-sonnet-5-5 \\", + "> [other parameters]", + "> ```", + "> ", + "> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).", + "> ", + "> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`", + "", +] + +#: Resource-usage columns after the cost: the cost-summary field and its cell format. +_RESOURCE_CELLS = (("prompt_tokens", "{:,}"), ("completion_tokens", "{:,}"), ("total_tokens", "{:,}"), + ("mean_latency_sec", "{:.2f}s"), ("cache_hit_ratio", "{:.1%}")) + + +def _cost_cell(cs: dict) -> str: + """Total cost, with the share of calls it covers when some had no known price.""" + cost_usd = cs.get("total_cost_usd") + if cost_usd is None: + return "—" + cost_known, total_calls = cs.get("cost_known_n"), cs.get("total_llm_calls", cs.get("n")) + if cost_known is not None and total_calls is not None and cost_known < total_calls: + return f"${cost_usd:.4f} ({cost_known}/{total_calls})" + return f"${cost_usd:.4f}" + + +def _resource_usage_lines(runs: list[tuple[str, dict]], pooled: bool) -> list[str]: + """Baseline-arm cost, tokens, latency and cache hits; nothing when no run has a cost summary.""" + if not any(run_data.get("cost_summary") for _, run_data in runs): + return [] + lines = [ + "## Resource Usage (baseline arm)", + "| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |", + "|---|---:|---:|---:|---:|---:|---:|", + ] + for run_id, run_data in runs: + cs = run_data.get("cost_summary", {}).get(run_data.get("baseline_arm", "baseline"), {}) + cells = [_cost_cell(cs), *("—" if cs.get(f) is None else t.format(cs.get(f)) for f, t in _RESOURCE_CELLS)] + lines.append(f"| **{run_id}** | " + " | ".join(cells) + " |") + if pooled: + lines += ["", "_A pooled `[n=k]` row shows the total cost, calls and tokens of its k runs " + "and their mean latency._"] + return lines + [""] + + +def _run_summary(scores: list[dict]) -> dict: + """One run's arm means and paired-delta counts over the common questions.""" + with_docs_scores = [s["with_docs"] for s in scores if s["with_docs"] is not None] + without_docs_scores = [s["without_docs"] for s in scores if s["without_docs"] is not None] + # Average the per-question deltas rather than differencing the two arm + # means. The arm means are filtered independently, so when any question + # is missing a score they cover different question subsets and their + # difference is not a paired delta. + paired_deltas = [s["delta"] for s in scores if s.get("delta") is not None] + # Per-run, distinct from the section-level has_treatment: a single run may + # lack a treatment arm while others in the comparison have one. + paired = bool(with_docs_scores) and bool(paired_deltas) + return { + "n": len(scores), + "n_paired": len(paired_deltas), + # Questions that reached the intersection but lack a usable pair. + "unpaired": len(scores) - len(paired_deltas), + "with_avg": _avg(with_docs_scores) if with_docs_scores else None, + "without_avg": _avg(without_docs_scores) if without_docs_scores else None, + "delta_avg": round(sum(paired_deltas) / len(paired_deltas), 1) if paired else None, + "improved": sum(1 for d in paired_deltas if d > 0) if paired else None, + "degraded": sum(1 for d in paired_deltas if d < 0) if paired else None, + } - lines += [""] - # --- Overall Summary --- - lines += [ +def _overall_summary_lines(run_ids: list[str], summary_stats: dict[str, dict], arm_label: str) -> list[str]: + lines = [ "## Overall Summary", "_Delta is the mean of per-question paired differences. Scored pairs is the" " number of questions with a usable score in both arms; Delta, Improved and" @@ -965,333 +965,246 @@ def generate_section_report( f"| Run | N | Scored pairs | {arm_label} | Baseline | Delta | Improved | Degraded |", "|---|---:|---:|---:|---:|---:|---:|---:|", ] - - summary_stats: dict[str, dict] = {} for run_id in run_ids: - scores = all_run_scores[run_id] - - # Handle both single-arm (baseline-only) and multi-arm runs - with_docs_scores = [s["with_docs"] for s in scores if s["with_docs"] is not None] - without_docs_scores = [s["without_docs"] for s in scores if s["without_docs"] is not None] - - # Per-run, distinct from the section-level has_treatment above: a single - # run may lack a treatment arm while others in the comparison have one. - run_has_treatment = bool(with_docs_scores) - with_avg = _avg(with_docs_scores) if with_docs_scores else None - without_avg = _avg(without_docs_scores) if without_docs_scores else None - - # Average the per-question deltas rather than differencing the two arm - # means. The arm means are filtered independently, so when any question - # is missing a score they cover different question subsets and their - # difference is not a paired delta. - paired_deltas = [s["delta"] for s in scores if s.get("delta") is not None] - - if run_has_treatment and paired_deltas: - delta_avg = round(sum(paired_deltas) / len(paired_deltas), 1) - improved = sum(1 for d in paired_deltas if d > 0) - degraded = sum(1 for d in paired_deltas if d < 0) - else: - delta_avg = None - improved = None - degraded = None - - # Questions that reached the intersection but lack a usable pair. - unpaired = len(scores) - len(paired_deltas) - - summary_stats[run_id] = { - "n": len(scores), - "n_paired": len(paired_deltas), - "unpaired": unpaired, - "with_avg": with_avg, - "without_avg": without_avg, - "delta_avg": delta_avg, - "improved": improved, - "degraded": degraded, - } - + st = summary_stats[run_id] # An arm with no usable score prints as "—" rather than a numeric zero, # which would read as a legitimate score of 0/100. - with_avg_str = "—" if with_avg is None else str(with_avg) - without_avg_str = "—" if without_avg is None else str(without_avg) - delta_str = "—" if delta_avg is None else f"**{_fmt_delta(delta_avg)}**" - improved_str = "—" if improved is None else str(improved) - degraded_str = "—" if degraded is None else str(degraded) + delta_str = "—" if st["delta_avg"] is None else f"**{_fmt_delta(st['delta_avg'])}**" # Make partial coverage visible: a delta over fewer pairs than questions # is weaker evidence than the question count alone suggests. - scored_str = str(len(paired_deltas)) if not unpaired else f"{len(paired_deltas)} ({unpaired} unscored)" - - lines.append( - f"| **{run_id}** | {len(scores)} | {scored_str} | {with_avg_str} | {without_avg_str} | " - f"{delta_str} | {improved_str} | {degraded_str} |" + scored_str = f"{st['n_paired']} ({st['unpaired']} unscored)" if st["unpaired"] else str(st["n_paired"]) + cells = [str(st["n"]), scored_str, _num_or_dash(st["with_avg"]), _num_or_dash(st["without_avg"]), + delta_str, _num_or_dash(st["improved"]), _num_or_dash(st["degraded"])] + lines.append(f"| **{run_id}** | " + " | ".join(cells) + " |") + return lines + [""] + + +def _adjusted_significance( + run_ids: list[str], all_run_scores: dict[str, list[dict]], correction: str +) -> tuple[dict[str, dict | None], int]: + """Each run's paired test with its p-value adjusted across the runs, and how many were tested. + + Every run in the section is one test of the same "did the treatment help?" + question, so they form a family: with k runs, the chance that at least one + clears 0.05 by luck alone is far above 0.05. Correct across the family before + calling anything significant, exactly as the arms report does across arms. + """ + sigs: dict[str, dict | None] = {} + for run_id in run_ids: + # Filter for paired scores: both baseline and treatment must be non-None + paired = [(s["with_docs"], s["without_docs"]) for s in all_run_scores[run_id] + if s["with_docs"] is not None and s["without_docs"] is not None] + if paired: + with_paired, without_paired = zip(*paired, strict=True) + sigs[run_id] = significance_test(list(with_paired), list(without_paired)) + else: + sigs[run_id] = None + + tested = [sig for rid in run_ids if (sig := sigs[rid]) and sig["p_ttest"] is not None] + adjusted = arm_stats.adjust_pvalues([sig["p_ttest"] for sig in tested], method=correction) + for sig, p_adj in zip(tested, adjusted, strict=True): + sig["p_adjusted"] = p_adj + # Same conjunction as the raw verdict, with the adjusted p in place of the raw one. + sig["significant_adjusted"] = bool( + p_adj is not None and p_adj < arm_stats.ALPHA + and (sig["p_wilcoxon"] is None or sig["p_wilcoxon"] < arm_stats.ALPHA) ) + return sigs, len(tested) - lines += [""] - # --- Statistical Significance --- - if HAS_SCIPY: - lines += [ - "## Statistical Significance of Delta (α = 0.05)", - ] +def _significance_row(run_id: str, sig: dict) -> str: + # p-values are None when no test could run (zero-variance deltas); "N/A" is + # honest where "None" would just look broken. + verdict = sig.get("significant_adjusted") + if sig["significant"] is None or verdict is None: + sig_mark = "N/A" + else: + sig_mark = "OK Yes" if verdict else "No" + p_ttest = sig["p_ttest"] if sig["p_ttest"] is not None else "N/A" + p_adj = sig.get("p_adjusted") + p_adj_str = p_adj if p_adj is not None else "N/A" + p_wilcox = sig["p_wilcoxon"] if sig["p_wilcoxon"] is not None else "N/A" + return ( + f"| **{run_id}** | {_fmt_delta(sig['delta_mean'])} | {p_ttest} | " + f"{p_adj_str} | {p_wilcox} | {_fmt_delta(sig['cohens_d'])} " + f"| {sig['effect']} | {sig_mark} |" + ) - if not has_treatment: - lines += [ - "", - "> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).", - "> Add `--arms \"baseline,docs\"` to enable delta analysis.", - "", - ] - else: - # Every run in the section is one test of the same "did the - # treatment help?" question, so they form a family: with k runs, - # the chance that at least one clears 0.05 by luck alone is far - # above 0.05. Correct across the family before calling anything - # significant, exactly as the arms report does across arms. - sigs: dict[str, dict | None] = {} - for run_id in run_ids: - scores = all_run_scores[run_id] - # Filter for paired scores: both baseline and treatment must be non-None - paired = [(s["with_docs"], s["without_docs"]) for s in scores - if s["with_docs"] is not None and s["without_docs"] is not None] - if paired: - with_paired, without_paired = zip(*paired, strict=True) - sigs[run_id] = significance_test(list(with_paired), list(without_paired)) - else: - sigs[run_id] = None - - tested = [sig for rid in run_ids if (sig := sigs[rid]) and sig["p_ttest"] is not None] - adjusted = arm_stats.adjust_pvalues( - [sig["p_ttest"] for sig in tested], method=correction - ) - for sig, p_adj in zip(tested, adjusted, strict=True): - sig["p_adjusted"] = p_adj - # Same conjunction as the raw verdict, with the adjusted p in - # place of the raw one. - sig["significant_adjusted"] = bool( - p_adj is not None and p_adj < arm_stats.ALPHA - and (sig["p_wilcoxon"] is None or sig["p_wilcoxon"] < arm_stats.ALPHA) - ) - label = "Holm-Bonferroni" if correction == "holm" else "Benjamini-Hochberg" - lines += [ - f"_p-values are adjusted across the {len(tested)} run(s) compared here " - f"({label}); the raw column is shown for reference. " - "Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._", - "", - "| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z " - "| Effect | Significant? |", - "|---|---:|---:|---:|---:|---:|---|---|", - ] - for run_id in run_ids: - sig = sigs[run_id] - if sig: - # p-values are None when no test could run (zero-variance - # deltas); "N/A" is honest where "None" would just look broken. - verdict = sig.get("significant_adjusted") - if sig["significant"] is None or verdict is None: - sig_mark = "N/A" - else: - sig_mark = "OK Yes" if verdict else "No" - p_ttest = sig["p_ttest"] if sig["p_ttest"] is not None else "N/A" - p_adj = sig.get("p_adjusted") - p_adj_str = p_adj if p_adj is not None else "N/A" - p_wilcox = sig["p_wilcoxon"] if sig["p_wilcoxon"] is not None else "N/A" - lines.append( - f"| **{run_id}** | {_fmt_delta(sig['delta_mean'])} | {p_ttest} | " - f"{p_adj_str} | {p_wilcox} | {_fmt_delta(sig['cohens_d'])} " - f"| {sig['effect']} | {sig_mark} |" - ) - lines += [""] +def _significance_lines( + run_ids: list[str], all_run_scores: dict[str, list[dict]], has_treatment: bool, correction: str +) -> list[str]: + lines = ["## Statistical Significance of Delta (α = 0.05)"] + if not has_treatment: + return lines + [ + "", + "> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).", + "> Add `--arms \"baseline,docs\"` to enable delta analysis.", + "", + "", + ] + sigs, n_tested = _adjusted_significance(run_ids, all_run_scores, correction) + label = "Holm-Bonferroni" if correction == "holm" else "Benjamini-Hochberg" + lines += [ + f"_p-values are adjusted across the {n_tested} run(s) compared here " + f"({label}); the raw column is shown for reference. " + "Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._", + "", + "| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z " + "| Effect | Significant? |", + "|---|---:|---:|---:|---:|---:|---|---|", + ] + lines += [_significance_row(run_id, sig) for run_id in run_ids if (sig := sigs[run_id])] + return lines + [""] + + +_DIFFICULTY_ORDER = ("easy", "beginner", "intermediate", "medium", "advanced", "hard") + + +def _difficulty_cells(sc: list[dict], has_treatment: bool) -> str: + """One run's cells of a difficulty row: context mean and delta, or the baseline.""" + if not sc: + return " — | — |" if has_treatment else " — |" + with_scores = [s["with_docs"] for s in sc if s["with_docs"] is not None] + if with_scores: + # Same pairing rule as the overall summary: average the per-question + # deltas, never the difference of two means filtered over different + # question subsets. + diff_deltas = [s["delta"] for s in sc if s.get("delta") is not None] + delta_cell = _fmt_delta(round(sum(diff_deltas) / len(diff_deltas), 1)) if diff_deltas else "—" + return f" {_avg(with_scores)} | {delta_cell} |" + # No treatment score in this bucket: report the baseline. + without_scores = [s["without_docs"] for s in sc if s["without_docs"] is not None] + wo = _avg(without_scores) if without_scores else None + if has_treatment: + wo_cell = "—" if wo is None else f"{wo} (baseline)" + return f" — | {wo_cell} |" + return f" {'—' if wo is None else wo} |" + - # --- By Difficulty --- - difficulty_groups: dict[str, dict[str, list]] = defaultdict(lambda: defaultdict(list)) +def _difficulty_lines(run_ids: list[str], all_run_scores: dict[str, list[dict]], has_treatment: bool) -> list[str]: + groups: dict[str, dict[str, list]] = defaultdict(lambda: defaultdict(list)) for run_id in run_ids: for s in all_run_scores[run_id]: # already common questions only - difficulty_groups[s["difficulty"]][run_id].append(s) - - if difficulty_groups: - # Adapt header to single-arm vs multi-arm scenario - if has_treatment: - header = "| Difficulty | N | " + " | ".join( - f"{rid} context | {rid} Δ" for rid in run_ids - ) + " |" - separator = "|---|---:|" + "|".join([" :---: | :---: "] * len(run_ids)) + "|" - else: - header = "| Difficulty | N | " + " | ".join( - f"{rid} baseline" for rid in run_ids - ) + " |" - separator = "|---|---:|" + "| :---: " * len(run_ids) + "|" - - lines += [ - "## By Difficulty (common questions only)", - header, - separator, - ] - for diff in ["easy", "beginner", "intermediate", "medium", "advanced", "hard"]: - if diff not in difficulty_groups: - continue - run_data = difficulty_groups[diff] - n = len(next(iter(run_data.values()))) - row = f"| {diff} | {n} |" - for run_id in run_ids: - sc = run_data.get(run_id, []) - if sc: - with_scores = [s["with_docs"] for s in sc if s["with_docs"] is not None] - without_scores = [s["without_docs"] for s in sc if s["without_docs"] is not None] - # Same pairing rule as the overall summary: average the - # per-question deltas, never the difference of two means - # filtered over different question subsets. - diff_deltas = [s["delta"] for s in sc if s.get("delta") is not None] - if with_scores: - wa = _avg(with_scores) - delta_cell = ( - _fmt_delta(round(sum(diff_deltas) / len(diff_deltas), 1)) - if diff_deltas else "—" - ) - row += f" {wa} | {delta_cell} |" - else: - # No treatment score in this bucket: report the baseline. - wo = _avg(without_scores) if without_scores else None - wo_cell = "—" if wo is None else f"{wo} (baseline)" - if has_treatment: - row += f" — | {wo_cell} |" - else: - row += f" {'—' if wo is None else wo} |" - else: - if has_treatment: - row += " — | — |" - else: - row += " — |" - lines.append(row) - lines += [""] - - # --- Head-to-Head --- - lines += [ + groups[s["difficulty"]][run_id].append(s) + if not groups: + return [] + # Adapt header to single-arm vs multi-arm scenario + if has_treatment: + header = "| Difficulty | N | " + " | ".join(f"{rid} context | {rid} Δ" for rid in run_ids) + " |" + separator = "|---|---:|" + "|".join([" :---: | :---: "] * len(run_ids)) + "|" + else: + header = "| Difficulty | N | " + " | ".join(f"{rid} baseline" for rid in run_ids) + " |" + separator = "|---|---:|" + "| :---: " * len(run_ids) + "|" + lines = ["## By Difficulty (common questions only)", header, separator] + for diff in [d for d in _DIFFICULTY_ORDER if d in groups]: + by_run = groups[diff] + cells = "".join(_difficulty_cells(by_run.get(run_id, []), has_treatment) for run_id in run_ids) + lines.append(f"| {diff} | {len(next(iter(by_run.values())))} |{cells}") + return lines + [""] + + +def _winner_counts(run_ids: list[str], all_run_scores: dict[str, list[dict]], common_questions: set[str]) -> dict: + """Per run, the questions it alone scored highest on in the context arm; shared tops under "tie".""" + winner_counts: dict[str, int] = defaultdict(int) + for qid in common_questions: + question_scores = {} + for run_id in run_ids: + score = next((s["with_docs"] for s in all_run_scores[run_id] + if s["question_id"] == qid and s["with_docs"] is not None), None) + if score is not None: + question_scores[run_id] = score + if question_scores: + max_score = max(question_scores.values()) + winners = [rid for rid, score in question_scores.items() if score == max_score] + winner_counts[winners[0] if len(winners) == 1 else "tie"] += 1 + return winner_counts + + +def _head_to_head_lines( + run_ids: list[str], all_run_scores: dict[str, list[dict]], common_questions: set[str], has_treatment: bool +) -> list[str]: + lines = [ "## Head-to-Head (context arm, common questions)", "_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._", "", ] - if not has_treatment: - lines += [ + return lines + [ "", "> **⚠️ Skipped:** Head-to-head requires treatment arm scores.", "> Single-arm runs only have baseline scores.", "", ] - else: - lines += [ - "| Winner | Count | % |", - "|---|---:|---:|", - ] - winner_counts: dict[str, int] = defaultdict(int) - for qid in common_questions: - question_scores = {} - for run_id in run_ids: - for s in all_run_scores[run_id]: - if s["question_id"] == qid and s["with_docs"] is not None: - question_scores[run_id] = s["with_docs"] - break - if question_scores: - max_score = max(question_scores.values()) - winners = [rid for rid, score in question_scores.items() if score == max_score] - if len(winners) == 1: - winner_counts[winners[0]] += 1 - else: - winner_counts["tie"] += 1 - - total = len(common_questions) - for run_id in run_ids: - count = winner_counts.get(run_id, 0) - pct = round(100 * count / total) if total > 0 else 0 - lines.append(f"| **{run_id}** | {count} | {pct}% |") - tie_count = winner_counts.get("tie", 0) - lines.append(f"| tie | {tie_count} | {round(100 * tie_count / total) if total > 0 else 0}% |") - lines += [""] + winner_counts = _winner_counts(run_ids, all_run_scores, common_questions) + total = len(common_questions) + lines += ["| Winner | Count | % |", "|---|---:|---:|"] + for label, key in [*((f"**{rid}**", rid) for rid in run_ids), ("tie", "tie")]: + count = winner_counts.get(key, 0) + lines.append(f"| {label} | {count} | {round(100 * count / total) if total > 0 else 0}% |") + return lines + [""] - # --- Context-Arm Benefit --- - lines += [ - "## Context-Arm Benefit — Delta Comparison", - ] +def _benefit_lines( + run_ids: list[str], summary_stats: dict[str, dict], has_treatment: bool, single_arm: str | None +) -> list[str]: + """Per run, the mean delta and the share of scored pairs the treatment helped and hurt.""" + lines = ["## Context-Arm Benefit — Delta Comparison"] if not has_treatment: - lines += [ + return lines + [ "", "> **⚠️ Skipped:** Delta comparison requires treatment arm.", "> Single-arm runs cannot compute context benefit.", "", ] - else: - subtitle = ( - f"_Which model benefits more from the treatment arm `{arm_label}`?_" - if len(scored_arms) == 1 - else "_Which model benefits more from the non-baseline treatment arm?_" - ) - lines += [ - subtitle, - "", - "| Run | Avg Delta | Questions context helped | Questions context hurt |", - "|---|---:|---:|---:|", - ] - for run_id in run_ids: - st = summary_stats[run_id] - # Percentages are over scored pairs, the same population the - # improved/degraded counts come from. - n_paired = st["n_paired"] - if st["improved"] is None or not n_paired: - lines.append(f"| **{run_id}** | — | — | — |") - continue - helped_pct = round(100 * st["improved"] / n_paired) - hurt_pct = round(100 * st["degraded"] / n_paired) - lines.append( - f"| **{run_id}** | **{_fmt_delta(st['delta_avg'])}** | " - f"{st['improved']} ({helped_pct}%) | {st['degraded']} ({hurt_pct}%) |" - ) - lines += [""] - - # --- Model Ranking --- lines += [ - "## Model Ranking", + f"_Which model benefits more from the treatment arm `{single_arm}`?_" + if single_arm is not None + else "_Which model benefits more from the non-baseline treatment arm?_", "", + "| Run | Avg Delta | Questions context helped | Questions context hurt |", + "|---|---:|---:|---:|", ] + for run_id in run_ids: + st = summary_stats[run_id] + # Percentages are over scored pairs, the same population the + # improved/degraded counts come from. + n_paired = st["n_paired"] + if st["improved"] is None or not n_paired: + lines.append(f"| **{run_id}** | — | — | — |") + continue + helped_pct = round(100 * st["improved"] / n_paired) + hurt_pct = round(100 * st["degraded"] / n_paired) + lines.append( + f"| **{run_id}** | **{_fmt_delta(st['delta_avg'])}** | " + f"{st['improved']} ({helped_pct}%) | {st['degraded']} ({hurt_pct}%) |" + ) + return lines + [""] - if has_treatment: - lines += [ - f"### By absolute quality ({arm_label})", - f"| Rank | Run | {arm_label} avg |", - "|---:|---|---:|", - ] - for rank, (run_id, st) in enumerate( - sorted(summary_stats.items(), key=_rank_key("with_avg"), reverse=True), 1 - ): - lines.append(f"| {rank} | **{run_id}** | {_num_or_dash(st['with_avg'])} |") - else: - lines += [ - "### By absolute quality (baseline)", - "| Rank | Run | Baseline avg |", - "|---:|---|---:|", - ] - for rank, (run_id, st) in enumerate( - sorted(summary_stats.items(), key=_rank_key("without_avg"), reverse=True), 1 - ): - lines.append(f"| {rank} | **{run_id}** | {_num_or_dash(st['without_avg'])} |") - if has_treatment: - lines += [ - "", - "### By treatment utilisation (Delta)", - "| Rank | Run | Delta |", - "|---:|---|---:|", - ] - for rank, (run_id, st) in enumerate( - sorted(summary_stats.items(), key=_rank_key("delta_avg"), reverse=True), 1 - ): - lines.append(f"| {rank} | **{run_id}** | {_fmt_delta(st['delta_avg'])} |") - else: - lines += [ +def _rank_table(summary_stats: dict[str, dict], field: str, column: str, fmt) -> list[str]: + """Runs ranked by *field*, those without a value last.""" + ranked = sorted(summary_stats.items(), key=_rank_key(field), reverse=True) + return [f"| Rank | Run | {column} |", "|---:|---|---:|", + *(f"| {rank} | **{run_id}** | {fmt(st[field])} |" for rank, (run_id, st) in enumerate(ranked, 1))] + + +def _ranking_lines(summary_stats: dict[str, dict], has_treatment: bool, arm_label: str) -> list[str]: + lines = ["## Model Ranking", ""] + if not has_treatment: + return lines + [ + "### By absolute quality (baseline)", + *_rank_table(summary_stats, "without_avg", "Baseline avg", _num_or_dash), "", "> **Note:** Treatment utilization ranking requires treatment arm.", "", + "", ] - - lines += [""] - return lines + return lines + [ + f"### By absolute quality ({arm_label})", + *_rank_table(summary_stats, "with_avg", f"{arm_label} avg", _num_or_dash), + "", + "### By treatment utilisation (Delta)", + *_rank_table(summary_stats, "delta_avg", "Delta", _fmt_delta), + "", + ] diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 0c237b9f..8a2e441c 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -4,7 +4,7 @@ "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, - "dokimi/report/model_compare.py": {"lines": 1297, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, + "dokimi/report/model_compare.py": {"lines": 1210, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, "tests/test_container_agent.py": {"lines": 1251, "rationale": "pre-existing when the gate was introduced"}, @@ -76,7 +76,6 @@ "dokimi/report/generator.py::ReportGenerator._format_markdown": {"complexity": 18}, "dokimi/report/model_compare.py::check_run_consistency": {"complexity": 13}, "dokimi/report/model_compare.py::extract_scores": {"complexity": 16}, - "dokimi/report/model_compare.py::generate_section_report": {"complexity": 53}, "dokimi/report/subject_report.py::render_subject_scorecard": {"complexity": 12}, "dokimi/report/task_runs_report.py::render_task_runs_report": {"complexity": 22}, "dokimi/repos/budget.py::budget": {"complexity": 21, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/fixtures/model_compare_sections.json b/dokimi/tests/fixtures/model_compare_sections.json new file mode 100644 index 00000000..115fe5f2 --- /dev/null +++ b/dokimi/tests/fixtures/model_compare_sections.json @@ -0,0 +1,20 @@ +{ + "baseline_only": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | — (baseline only) |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline | b baseline |\n|---|---:|| :---: | :---: |\n| easy | 3 | 60.0 | — |\n| beginner | 1 | 55.0 | — |\n| intermediate | 1 | 30.0 | — |\n| medium | 1 | 60.0 | — |\n| advanced | 1 | 65.0 | — |\n| hard | 1 | 40.0 | 56.2 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n| 2 | **b** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "baseline_only_no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 3 | 60.0 |\n| beginner | 1 | 55.0 |\n| intermediate | 1 | 30.0 |\n| medium | 1 | 60.0 |\n| advanced | 1 | 65.0 |\n| hard | 1 | 40.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "baseline_zero": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **3**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 3 | 0 (3 unscored) | — | 30.0 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 1 | 0.0 |\n| medium | 1 | 40.0 |\n| hard | 1 | 50.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 30.0 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "empty": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n\nCommon questions evaluated: **0**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "mixed": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `with-context` |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | — | — |\n| beginner | 1 | 75.0 | +20.0 | — | — |\n| intermediate | 1 | 45.0 | +15.0 | — | — |\n| medium | 1 | 70.0 | +10.0 | — | — |\n| advanced | 1 | 90.0 | +25.0 | — | — |\n| hard | 1 | 80.0 | +40.0 | — | 56.2 (baseline) |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| **b** | 0 | 0% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | — | — | — |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | — |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | N/A |\n", + "no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "pinned_arm": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | docs | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ |\n|---|---:| :---: | :---: |\n| easy | 9 | 68.8 | +17.1 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `docs`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n\n## Model Ranking\n\n### By absolute quality (docs)\n| Rank | Run | docs avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n", + "pooled": "## Replicate Pooling\n\nRuns sharing a configuration were averaged per question, so each row below is one configuration rather than one sample of it. Each pooled score is a mean of k runs, so its spread and confidence intervals are narrower than a single run's: that reflects the k runs spent, not a sharper measurement.\n\n- **m[n=2]'** pools 2 replicates: m_run1, m_run2; 2 question(s) dropped for not being scored in every replicate\n\n## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **m[n=2]** | solo (p) | same (p) | `with-context` |\n| **m[n=2]'** | rep (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **7**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **m[n=2]** | — | — | — | — | — | — |\n| **m[n=2]'** | $0.2469 (6/16) | 24,690 | 1,356 | 26,046 | 1.23s | 25.0% |\n\n_A pooled `[n=k]` row shows the total cost, calls and tokens of its k runs and their mean latency._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **m[n=2]** | 7 | 7 | 62.9 | 52.9 | **+10.0** | 7 | 0 |\n| **m[n=2]'** | 7 | 7 | 66.4 | 52.9 | **+13.6** | 6 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **m[n=2]** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n| **m[n=2]'** | +13.6 | 0.0033 | 0.0033 | 0.0312 | +1.8 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | m[n=2] context | m[n=2] Δ | m[n=2]' context | m[n=2]' Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 2 | 70.0 | +10.0 | 67.5 | +7.5 |\n| beginner | 1 | 65.0 | +10.0 | 70.0 | +15.0 |\n| intermediate | 1 | 40.0 | +10.0 | 42.5 | +12.5 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 75.0 | +10.0 | 82.5 | +17.5 |\n| hard | 1 | 50.0 | +10.0 | 65.0 | +25.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **m[n=2]** | 1 | 14% |\n| **m[n=2]'** | 5 | 71% |\n| tie | 1 | 14% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **m[n=2]** | **+10.0** | 7 (100%) | 0 (0%) |\n| **m[n=2]'** | **+13.6** | 6 (86%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **m[n=2]'** | 66.4 |\n| 2 | **m[n=2]** | 62.9 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **m[n=2]'** | +13.6 |\n| 2 | **m[n=2]** | +10.0 |\n", + "pooled_singletons": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **x** | x (p) | same (p) | `with-context` |\n| **y** | y (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **x** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **y** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **x** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **y** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | x context | x Δ | y context | y Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **x** | 6 | 67% |\n| **y** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **x** | **+17.1** | 6 (86%) | 1 (14%) |\n| **y** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **x** | 68.8 |\n| 2 | **y** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **x** | +17.1 |\n| 2 | **y** | +10.0 |\n", + "pooled_singletons_cost": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **x** | x (p) | same (p) | `with-context` |\n| **y** | y (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **x** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **y** | $0.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **x** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **y** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **x** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **y** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | x context | x Δ | y context | y Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **x** | 6 | 67% |\n| **y** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **x** | **+17.1** | 6 (86%) | 1 (14%) |\n| **y** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **x** | 68.8 |\n| 2 | **y** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **x** | +17.1 |\n| 2 | **y** | +10.0 |\n", + "treatment": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "treatment_bh": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Benjamini-Hochberg); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "two_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `docs` |\n| **b** | beta (p) | same (p) | `skill:x` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n| **b** | +10.0 | N/A | N/A | N/A | N/A | unknown | N/A |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", + "unscored_and_arms": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **u** | m (p) | same (p) | `docs`, `skill:x` |\n| **v** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **u** | $2.0000 (4/5) | — | — | — | — | — |\n| **v** | $3.0000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **u** | 8 | 7 (1 unscored) | 64.3 | 56.2 | **+11.4** | 7 | 0 |\n| **v** | 8 | 7 (1 unscored) | 64.3 | 56.2 | **+11.4** | 7 | 0 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 2 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **u** | +11.4 | 0.0016 | 0.0032 | 0.0156 | +2.1 | large | OK Yes |\n| **v** | +11.4 | 0.0016 | 0.0032 | 0.0156 | +2.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | u context | u Δ | v context | v Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 2 | 70.0 | +10.0 | 70.0 | +10.0 |\n| beginner | 1 | 70.0 | +15.0 | 70.0 | +15.0 |\n| intermediate | 1 | 50.0 | +20.0 | 50.0 | +20.0 |\n| medium | 1 | 75.0 | +15.0 | 75.0 | +15.0 |\n| advanced | 1 | 70.0 | +5.0 | 70.0 | +5.0 |\n| hard | 1 | 45.0 | +5.0 | 45.0 | +5.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **u** | 0 | 0% |\n| **v** | 0 | 0% |\n| tie | 7 | 88% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the non-baseline treatment arm?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **u** | **+11.4** | 7 (100%) | 0 (0%) |\n| **v** | **+11.4** | 7 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (Context arm)\n| Rank | Run | Context arm avg |\n|---:|---|---:|\n| 1 | **u** | 64.3 |\n| 2 | **v** | 64.3 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **u** | +11.4 |\n| 2 | **v** | +11.4 |\n", + "verdict_holm_no": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **h1** | h1 (p) | same (p) | `with-context` |\n| **h2** | h2 (p) | same (p) | `with-context` |\n| **h3** | h3 (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **h1** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n| **h2** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n| **h3** | 8 | 8 | 55.9 | 50.0 | **+5.9** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 3 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **h1** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n| **h2** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n| **h3** | +5.9 | 0.024 | 0.072 | 0.0469 | +1.0 | large | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | h1 context | h1 Δ | h2 context | h2 Δ | h3 context | h3 Δ |\n|---|---:| :---: | :---: | :---: | :---: | :---: | :---: |\n| easy | 2 | 52.5 | +2.5 | 52.5 | +2.5 | 52.5 | +2.5 |\n| beginner | 1 | 59.0 | +9.0 | 59.0 | +9.0 | 59.0 | +9.0 |\n| intermediate | 1 | 61.0 | +11.0 | 61.0 | +11.0 | 61.0 | +11.0 |\n| medium | 1 | 50.0 | 0.0 | 50.0 | 0.0 | 50.0 | 0.0 |\n| advanced | 1 | 52.0 | +2.0 | 52.0 | +2.0 | 52.0 | +2.0 |\n| hard | 1 | 64.0 | +14.0 | 64.0 | +14.0 | 64.0 | +14.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **h1** | 0 | 0% |\n| **h2** | 0 | 0% |\n| **h3** | 0 | 0% |\n| tie | 8 | 100% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **h1** | **+5.9** | 6 (75%) | 1 (12%) |\n| **h2** | **+5.9** | 6 (75%) | 1 (12%) |\n| **h3** | **+5.9** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **h1** | 55.9 |\n| 2 | **h2** | 55.9 |\n| 3 | **h3** | 55.9 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **h1** | +5.9 |\n| 2 | **h2** | +5.9 |\n| 3 | **h3** | +5.9 |\n", + "verdict_wilcoxon_no": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **w** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **w** | 6 | 6 | 55.2 | 50.0 | **+5.2** | 5 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **w** | +5.2 | 0.0384 | 0.0384 | 0.0625 | +1.1 | large | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | w context | w Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 52.5 | +2.5 |\n| beginner | 1 | 55.0 | +5.0 |\n| medium | 1 | 57.0 | +7.0 |\n| advanced | 1 | 54.0 | +4.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **w** | 6 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **w** | **+5.2** | 5 (83%) | 1 (17%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **w** | 55.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **w** | +5.2 |\n", + "zero_scores": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | m (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **8**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 8 | 8 | 33.1 | 24.4 | **+8.8** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **z** | +8.8 | 0.247 | 0.247 | 0.1719 | +0.4 | small | No |\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 47.5 | +27.5 |\n| beginner | 1 | 0.0 | 0.0 |\n| intermediate | 1 | 50.0 | +5.0 |\n| medium | 1 | 0.0 | -30.0 |\n| advanced | 1 | 35.0 | +15.0 |\n| hard | 1 | 60.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 8 | 100% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **+8.8** | 6 (75%) | 1 (12%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 33.1 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | +8.8 |\n", + "zero_values": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **z** | zero (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **6**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **z** | $0.0000 (0/5) | 0 | 0 | 0 | 0.00s | 0.0% |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **z** | 6 | 5 (1 unscored) | 45.0 | 45.8 | **-10.0** | 0 | 5 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 0 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n\n## By Difficulty (common questions only)\n| Difficulty | N | z context | z Δ |\n|---|---:| :---: | :---: |\n| easy | 2 | 50.0 | -10.0 |\n| beginner | 1 | 45.0 | -10.0 |\n| medium | 1 | 50.0 | -10.0 |\n| advanced | 1 | — | 0.0 (baseline) |\n| hard | 1 | 30.0 | -10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **z** | 5 | 83% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **z** | **-10.0** | 0 (0%) | 5 (100%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **z** | 45.0 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **z** | -10.0 |\n" +} diff --git a/dokimi/tests/test_model_compare_sections.py b/dokimi/tests/test_model_compare_sections.py new file mode 100644 index 00000000..43bcbce7 --- /dev/null +++ b/dokimi/tests/test_model_compare_sections.py @@ -0,0 +1,326 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Golden output of :func:`generate_section_report`, one case per section branch. + +Each case renders a whole section and compares it byte for byte with +its entry in ``tests/fixtures/model_compare_sections.json``, so a refactor of the section +builder that moves a single character fails here. Between them the cases reach +every section and branch: treatment and baseline-only runs, a run with no +treatment beside one with, one scored arm and two, resource rows with and +without a cost summary, significance on and off (no SciPy), both corrections, +difficulty buckets a run is missing from, replicate pooling with a label clash +and a run left out of ``run_ids``, and an empty section. + +To re-record after an intended output change:: + + DOKIMI_UPDATE_GOLDEN=1 pytest tests/test_model_compare_sections.py +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from dokimi.report import model_compare +from dokimi.report.model_compare import generate_section_report + +GOLDEN = Path(__file__).parent / "fixtures" / "model_compare_sections.json" + +_DIFFICULTIES = ["easy", "medium", "hard", 1, "beginner", "advanced", "intermediate", "expert"] + + +def _run( + scores: list[tuple], + arm: str | list[str] = "with-context", + difficulties=None, + baseline_arm: str = "baseline", + **meta, +) -> dict: + """An ``arms.v1``-shaped artifact; *scores* holds (baseline, treatment) per question. + + *arm* may be a list: question i is scored in ``arm[i % len(arm)]``. + + A treatment of ``...`` leaves the arm out of the record (a baseline-only question); + ``None`` keeps the arm with no usable score. + """ + difficulties = difficulties or _DIFFICULTIES + evaluations = [] + for i, (base, treat) in enumerate(scores): + cond: dict = {baseline_arm: {"aggregate": base}} + if treat is not ...: + cond[arm[i % len(arm)] if isinstance(arm, list) else arm] = {"aggregate": treat} + evaluations.append( + { + "question_id": f"q{i}", + "question_text": f"question {i}", + "difficulty": difficulties[i % len(difficulties)], + "scores": cond, + } + ) + return { + "baseline_arm": baseline_arm, + "model": "m", + "provider": "p", + "evaluations": evaluations, + **meta, + } + + +def _cost(**fields) -> dict: + return {"baseline": fields} + + +_FULL_COST = _cost( + total_cost_usd=0.12345, + cost_known_n=3, + total_llm_calls=8, + prompt_tokens=12345, + completion_tokens=678, + total_tokens=13023, + mean_latency_sec=1.234, + cache_hit_ratio=0.25, +) + +# Varying deltas so the t-test runs; one unpaired question and one with no baseline. +_A = [(50, 70), (60, 70), (40, 80), (70, 60), (55, 75), (65, 90), (30, 45), (80, None), (None, 60)] +# Constant +10 deltas: the zero-variance guard leaves the p-values N/A. +_B = [(50, 60), (60, 70), (40, 50), (70, 80), (55, 65), (65, 75), (30, 40), (80, 90), (20, 30)] +_BASE_ONLY = [(b, ...) for b, _ in _A] +# A raw t-test p of 0.024 and Wilcoxon 0.047: significant alone, not once Holm-adjusted over 3 runs. +_H = [(50, 58), (50, 50), (50, 64), (50, 47), (50, 59), (50, 52), (50, 61), (50, 56)] +# t-test p 0.038 but Wilcoxon 0.0625: adjusted p stays below 0.05, the Wilcoxon clause says No. +_W = [(50, 47), (50, 57), (50, 60), (50, 58), (50, 55), (50, 54)] +# Zeros are scores: a 0 in either arm, and a bucket whose only treatment score is 0. +_Z = [(0, 40), (30, 0), (50, 60), (40, 55), (0, 0), (20, 35), (45, 50), (10, 25)] +# Exactly one question without a usable pair. +_U = [(50, 60), (60, 75), (40, 45), (70, 80), (55, 70), (65, 70), (30, 50), (80, None)] +# Nothing improved (0 of 5 pairs), and a bucket whose only score is a baseline 0. +_N = [(50, 40), (60, 50), (40, 30), (70, 60), (55, 45), (0, ...)] +_FREE_COST = _cost( + total_cost_usd=0, + cost_known_n=0, + total_llm_calls=5, + prompt_tokens=0, + completion_tokens=0, + total_tokens=0, + mean_latency_sec=0, + cache_hit_ratio=0, +) + + +def _cases() -> dict: + a = _run(_A, model="alpha", judge_model="judge-j", judge_provider="jp", cost_summary=_FULL_COST) + b = _run(_B, model="beta", cost_summary=_cost(total_cost_usd=1.5, n=4)) + # Every question "hard" but the one with no baseline score: the run is missing + # from the other buckets, and its "easy" bucket has no usable score at all. + lopsided = ["hard"] * 8 + ["easy"] + return { + "zero_values": ([("z", _run(_N, model="zero", cost_summary=_FREE_COST))], ["z"], {}), + "treatment": ([("a", a), ("b", b)], ["a", "b"], {}), + "treatment_bh": ([("a", a), ("b", b)], ["a", "b"], {"correction": "bh"}), + "no_scipy": ([("a", a), ("b", b)], ["a", "b"], {"no_scipy": True}), + "baseline_only": ( + [ + ("a", _run(_BASE_ONLY, model="alpha")), + ("b", _run(_BASE_ONLY, model="beta", difficulties=lopsided)), + ], + ["a", "b"], + {}, + ), + "baseline_only_no_scipy": ([("a", _run(_BASE_ONLY))], ["a"], {"no_scipy": True}), + "mixed": ( + [ + ("a", _run(_A, model="alpha")), + ("b", _run(_BASE_ONLY, model="beta", difficulties=lopsided)), + ], + ["a", "b"], + {}, + ), + "two_arms": ( + [ + ("a", _run(_A, arm="docs", model="alpha")), + ("b", _run(_B, arm="skill:x", model="beta")), + ], + ["a", "b"], + {}, + ), + "pinned_arm": ( + [("a", _run(_A, arm="docs", difficulties=["easy"]))], + ["a"], + {"treatment_arm": "docs"}, + ), + "pooled": ( + [ + ("m[n=2]", _run(_B, model="solo")), + ("m_run1", _run(_A, model="rep", cost_summary=_FULL_COST)), + ("m_run2", _run(_B, model="rep", cost_summary=_FULL_COST)), + ("left_out", _run(_A, model="other")), + ], + ["m[n=2]", "m_run1", "m_run2"], + {"pool_replicates": True}, + ), + "pooled_singletons": ( + [("x", _run(_A, model="x")), ("y", _run(_B, model="y"))], + ["x", "y"], + {"pool_replicates": True}, + ), + "empty": ([], [], {}), + # The raw verdict is Yes for each of the three, the adjusted one No. + "verdict_holm_no": ( + [(r, _run(_H, model=r)) for r in ("h1", "h2", "h3")], + ["h1", "h2", "h3"], + {}, + ), + "verdict_wilcoxon_no": ([("w", _run(_W))], ["w"], {}), + "zero_scores": ([("z", _run(_Z))], ["z"], {}), + # A baseline-only bucket whose only score is 0 prints 0.0, not a dash. + "baseline_zero": ( + [ + ( + "a", + _run([(0, ...), (40, ...), (50, ...)], difficulties=["easy", "medium", "hard"]), + ) + ], + ["a"], + {}, + ), + # One unscored question; a run whose questions span two arms; a cost covering 4 of + # 5 calls with the count read from ``n``, and one covering all 5. + "unscored_and_arms": ( + [ + ( + "u", + _run( + _U, + arm=["docs", "skill:x"], + cost_summary=_cost(total_cost_usd=2.0, cost_known_n=4, n=5), + ), + ), + ( + "v", + _run( + _U, + arm="docs", + cost_summary=_cost(total_cost_usd=3.0, cost_known_n=5, total_llm_calls=5), + ), + ), + ], + ["u", "v"], + {}, + ), + # Replicates that do not pool leave no footnote; a baseline arm that is not "baseline". + "pooled_singletons_cost": ( + [ + ("x", _run(_A, model="x", cost_summary=_FULL_COST)), + ( + "y", + _run( + _B, + model="y", + baseline_arm="control", + cost_summary={ + "control": {"total_cost_usd": 0.5}, + "baseline": {"total_cost_usd": 9.0}, + }, + ), + ), + ], + ["x", "y"], + {"pool_replicates": True}, + ), + } + + +CASES = _cases() + + +def _render(name: str, monkeypatch) -> str: + runs, run_ids, opts = CASES[name] + opts = dict(opts) + if opts.pop("no_scipy", False): + monkeypatch.setattr(model_compare, "HAS_SCIPY", False) + return "\n".join(generate_section_report(runs, run_ids, "Regular", **opts)) + + +@pytest.mark.parametrize("name", sorted(CASES)) +def test_section_report_matches_golden(name, monkeypatch): + if not model_compare.HAS_SCIPY: + pytest.skip("the goldens were recorded with SciPy available") + text = _render(name, monkeypatch) + if os.environ.get("DOKIMI_UPDATE_GOLDEN"): + goldens = json.loads(GOLDEN.read_text(encoding="utf-8")) if GOLDEN.exists() else {} + goldens[name] = text + GOLDEN.write_text( + json.dumps(goldens, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + assert text == json.loads(GOLDEN.read_text(encoding="utf-8"))[name] + + +def test_goldens_reach_every_section_branch(): + """The goldens are only a characterization if they hold each branch's marker.""" + texts = json.loads(GOLDEN.read_text(encoding="utf-8")) + assert set(texts) == set(CASES) + markers = { + "treatment": [ + "(3/8)", + "| **b** | $1.5000 | — |", + "N/A", + "Holm-Bonferroni", + "2 unscored", + "`with-context`", + "| tie | 1 |", + "| beginner | 1 |", + ], + "treatment_bh": ["Benjamini-Hochberg"], + "no_scipy": ["## Overall Summary"], + "baseline_only": [ + "Report Limitation", + "a baseline |", + "### By absolute quality (baseline)", + "Statistical tests require treatment arm", + "requires treatment arm.", + ], + "mixed": ["— (baseline only)", "(baseline) |", "| — | — |"], + "two_arms": ["Context arm", "non-baseline treatment arm"], + "pooled": ["## Replicate Pooling", "m[n=2]'", "pools 2 replicates", "A pooled `[n=k]` row"], + "empty": ["Common questions evaluated: **0**"], + "verdict_holm_no": ["| No |", "(Holm-Bonferroni)"], + "verdict_wilcoxon_no": ["| No |"], + "zero_scores": ["| beginner | 1 | 0.0 | 0.0 |", "| medium | 1 | 0.0 | -30.0 |"], + "baseline_zero": ["| easy | 1 | 0.0 |"], + "zero_values": [ + "| **z** | $0.0000 (0/5) | 0 | 0 | 0 | 0.00s | 0.0% |", + "| 0 (0%) | 5 (100%) |", + "| — | 0.0 (baseline) |", + ], + "unscored_and_arms": ["7 (1 unscored)", "`docs`, `skill:x`", "(4/5)"], + "pooled_singletons_cost": ["## Resource Usage", "| **y** | $0.5000 |"], + } + for name, wanted in markers.items(): + for marker in wanted: + assert marker in texts[name], (name, marker) + assert "Statistical Significance" not in texts["no_scipy"] + assert "Statistical Significance" not in texts["baseline_only_no_scipy"] + assert "Resource Usage" not in texts["baseline_only"] + assert "OK Yes" not in texts["verdict_holm_no"] and "OK Yes" not in texts["verdict_wilcoxon_no"] + assert "pooled `[n=k]` row" not in texts["pooled_singletons_cost"] + assert "(5/5)" not in texts["unscored_and_arms"] + + +@pytest.mark.parametrize( + "p, p_wilcoxon, significant", + [(0.05, None, False), (0.049, None, True), (0.049, 0.05, False), (0.049, 0.049, True)], +) +def test_adjusted_verdict_is_strictly_below_alpha(monkeypatch, p, p_wilcoxon, significant): + # A p exactly at alpha is not significant, for the t-test and for the Wilcoxon clause. + fake = {"p_ttest": p, "p_wilcoxon": p_wilcoxon, "significant": True} + monkeypatch.setattr(model_compare, "significance_test", lambda *_: dict(fake)) + scores = {"r": [{"with_docs": 60, "without_docs": 50}] * 6} + sigs, tested = model_compare._adjusted_significance(["r"], scores, "holm") + assert tested == 1 + assert sigs["r"]["significant_adjusted"] is significant