Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 18 additions & 11 deletions dokimi/dokimi/repos/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,22 +229,29 @@ def _provenance_warnings(cells: list[dict[str, Any]], baseline: str, treatments:
if selector_matches(parse_selector(sel), c)}
if len(versions) > 1:
out.append(f"{text} vs {baseline}: cells ran on different harness versions ({', '.join(sorted(versions))})")
arms = [[c for c in cells if selector_matches(parse_selector(sel), c) and (not models or c["model"] in models)]
for sel in (baseline, text)]
for model in sorted({c["model"] for c in arms[0]} & {c["model"] for c in arms[1]}):
base, treat = ({m for c in arm if c["model"] == model for m in resolved_models(c)} for arm in arms)
if base and treat and not _models_equivalent(base, treat):
out.append(f"{text} vs {baseline}: model {model} was answered by {', '.join(sorted(base))} in "
f"baseline, {', '.join(sorted(treat))} in treatment")
uptake = uptake_counts(arms[1])
if uptake and uptake["cells"] and not uptake["used"]:
out.append(f"{text}: the agent used none of what the arm added in {uptake['cells']} cell(s); "
"a delta measures the patch's text, not the tool")
out += _arm_warnings(cells, baseline, text, models)
compared = [c for c in cells if (not models or c["model"] in models)
and any(selector_matches(parse_selector(sel), c) for sel in (baseline, *treatments))]
return out + mismatch_warnings(model_mismatch(compared))


def _arm_warnings(cells: list[dict[str, Any]], baseline: str, text: str, models: list[str] | None) -> list[str]:
"""Resolved-model mismatches between ``baseline`` and ``text``, and a treatment whose agent used nothing it added."""
out = []
arms = [[c for c in cells if selector_matches(parse_selector(sel), c) and (not models or c["model"] in models)]
for sel in (baseline, text)]
for model in sorted({c["model"] for c in arms[0]} & {c["model"] for c in arms[1]}):
base, treat = ({m for c in arm if c["model"] == model for m in resolved_models(c)} for arm in arms)
if base and treat and not _models_equivalent(base, treat):
out.append(f"{text} vs {baseline}: model {model} was answered by {', '.join(sorted(base))} in "
f"baseline, {', '.join(sorted(treat))} in treatment")
uptake = uptake_counts(arms[1])
if uptake and uptake["cells"] and not uptake["used"]:
out.append(f"{text}: the agent used none of what the arm added in {uptake['cells']} cell(s); "
"a delta measures the patch's text, not the tool")
return out


def _built_from_tree(stamp: dict[str, Any]) -> bool:
"""Whether the stamped image is the one ``repo run`` tags from the tree (``dokimi/<id>:<sha12>``, or
``:worktree-<digest>`` for a working tree), not one ``--image`` named."""
Expand Down
1 change: 0 additions & 1 deletion dokimi/scripts/growth_baseline.json
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@
"dokimi/report/subject_report.py::render_subject_scorecard": {"complexity": 12},
"dokimi/report/task_runs_report.py::render_task_runs_report": {"complexity": 22},
"dokimi/repos/budget.py::budget": {"complexity": 21, "rationale": "pre-existing when the gate was introduced"},
"dokimi/repos/compare.py::_provenance_warnings": {"complexity": 11, "rationale": "went over the cap on main in #153 (patch-arm uptake warnings) after #93 seeded this baseline; split it in its own PR"},
"dokimi/repos/environment/__init__.py::inspect_repo": {"complexity": 25, "rationale": "pre-existing when the gate was introduced"},
"dokimi/repos/environment/checks.py::_python": {"complexity": 11, "rationale": "pre-existing when the gate was introduced"},
"dokimi/repos/environment/risks.py::_runs_repository_code": {"complexity": 12, "rationale": "pre-existing when the gate was introduced"},
Expand Down
Loading