diff --git a/dokimi/changelog.d/20261021_160000_napetrov_process-in-report.md b/dokimi/changelog.d/20261021_160000_napetrov_process-in-report.md new file mode 100644 index 00000000..50ef001c --- /dev/null +++ b/dokimi/changelog.d/20261021_160000_napetrov_process-in-report.md @@ -0,0 +1,12 @@ +### Added + +- **`repo report` and `repo compare` show `result.json` `process` counts** — + `repo report` prints, under each (task, arm, model) row, the mean of each + `process` count over the cells that have one (`tool_calls`, `commands`, + `failed_commands`, `denied_commands`, `guidance_read` as a number of paths, + and `counter:` for each `process_metrics` counter); the JSON row has + them under `process`. `repo compare` adds each count a compared cell + reports as a descriptive measure (`process:`, `counter:`): a + paired delta with a t interval and a raw p, outside Holm's family. Cells + whose `process` is absent or `null` are left out, so batches written before + `process` existed compare as before. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 76de3508..98f2a792 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -293,7 +293,8 @@ next release. interval, grader failures, missing cells, cost). - `dokimi/repos/compare.py` — `repo compare`: treatment arms against a baseline arm over one or more batches, one value per task and arm (gate - share, score, cost, turns, chosen `grade.json` metrics), with an interval + share, score, cost, turns, chosen `grade.json` metrics, `result.json` + `process` counts), with an interval over tasks (bootstrap for gate and score, t for the rest), a sign-flip permutation p and Holm's correction. - `dokimi/repos/selfcheck.py` — `repo tasks check`: on each task diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 00aec556..0cfcb12b 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1001,12 +1001,17 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo - The report gives, per (task, arm, model), passes over finished cells with a Wilson 95% interval (score-only tasks: the mean of `score.metric`), grader failures (counted as failures too), missing cells, and mean cost and turns. + When cells carry `result.json` `process` (§4.1), a line under the row gives + the mean of each count over the cells that have it (`guidance_read` counted + as paths; `counter:` for `process_metrics`); a `null` or absent + `process` is left out, not counted as zero. - `repo compare` reduces cells to one value per (task, arm, model) first: repetitions share a task's difficulty and are not paired across arms, so the task is the unit. Per treatment arm, model and measure (gate share, score, - cost, turns, each `--metric` from `grade.json`) it gives the mean per-task + cost, turns, each `--metric` from `grade.json`, and each `process` count a + compared cell reports as `process:` or `counter:`) it gives the mean per-task delta against the baseline with a 95% interval over tasks (a bootstrap for - gate and score, a t interval for cost, turns and metrics, `-` for a single + gate and score, a t interval for cost, turns, metrics and process counts, `-` for a single task), the tasks up, down and tied, and a two-sided sign-flip permutation p (exact up to 16 changed tasks; at one cell per task and arm it equals McNemar's exact test). Holm's correction runs over the gate and score comparisons of one call; the diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index aacc38c7..2a9af476 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -41,6 +41,10 @@ measurement failed, not that the task had no room. Any other case has no flag: ``None``. The flag is a note; p and Holm are unchanged. +Every ``result.json`` ``process`` count a cell of the compared arms reports +(:mod:`.process_counts`: ``process:``, ``counter:``) is a measure +too, the per-task mean over the cells that have it, descriptive like cost. + Each selector's batches are reported with the stamp ``repo run`` pins a batch to (``tree``, ``eval_dir`` digest, ``image``) and the harness versions of its cells. A warning is printed, not an error, when a bare ``ARM`` pools cells of @@ -70,6 +74,7 @@ from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci +from .process_counts import process_order, process_values from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, resolved_models, unmeasured_note, uptake_counts) @@ -141,6 +146,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None base_vals = _task_values([c for c in cells if selector_matches(base_sel, c)], metrics) rows: list[dict[str, Any]] = [] unpaired: list[dict[str, Any]] = [] + measures += _process_measures(cells, [baseline, *treatments]) for text in treatments: treat_vals = _task_values([c for c in cells if selector_matches(parse_selector(text), c)], metrics) unpaired += _one_arm_models(text, base_vals, treat_vals, models) @@ -253,6 +259,13 @@ def _built_from_tree(stamp: dict[str, Any]) -> bool: return repo.startswith("dokimi/") and (tag.startswith("worktree-") or bool(revision) and tag == revision[:12]) +def _process_measures(cells: list[dict[str, Any]], selectors: list[str]) -> list[str]: + """The ``process`` counters any cell of the compared arms reports; none when no cell has ``process``.""" + sels = [parse_selector(text) for text in selectors] + found = {k for c in cells if any(selector_matches(s, c) for s in sels) for k in process_values(c)} + return sorted(found, key=process_order) + + def _check_metrics(metrics: tuple[str, ...], cells: list[dict[str, Any]]) -> None: """A ``--metric`` that no finished cell reports is a typo or not a number: refuse it, not print nothing.""" unknown = [m for m in metrics if not any(m in c["metrics"] for c in cells)] @@ -289,28 +302,42 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], if b != t: # batches of different eval trees: the primary result of this task has no pair unpaired.append(_unpaired(treatment, model, task, None, f"{b} task in baseline, {t} in treatment")) for measure in measures: - pairs = [] - for task in shared: - base, treat = base_vals[(model, task)], treat_vals[(model, task)] - if measure in PRIMARY and not base["kind"] == treat["kind"] == measure: - continue # a task of the other kind, or one listed above - pair = _pair(task, base, treat, measure) - if pair is None: - lacking = [arm for arm, v in (("baseline", base), ("treatment", treat)) - if v["values"].get(measure) is None] - unpaired.append(_unpaired(treatment, model, task, measure, - f"no {measure} in {' or '.join(lacking)}")) - else: - pairs.append(pair) - if pairs: - flag = None - if measure in PRIMARY: - tops = [(base_vals[(model, p["task"])]["top"], treat_vals[(model, p["task"])]["top"]) for p in pairs] - failed = [base_vals[(model, p["task"])]["grader_failed"] - or treat_vals[(model, p["task"])]["grader_failed"] for p in pairs] - flag = headroom(pairs, tops, failed) - rows.append({"treatment": treatment, "model": model, "measure": measure, - **paired_summary(pairs, measure, seed=seed), "headroom": flag}) + if measure.startswith(("process:", "counter:")) and not all( + any(vals[(model, t)]["values"].get(measure) is not None for t in shared) for vals in (base_vals, treat_vals)): + continue # an arm that reports no such count at all (an older batch) leaves no task out of a pair + row = _compare_measure(treatment, model, measure, shared, base_vals, treat_vals, seed, unpaired) + if row is not None: + rows.append(row) + + +def _compare_measure(treatment: str, model: str, measure: str, shared: list[str], + base_vals: dict[tuple[str, str], dict[str, Any]], + treat_vals: dict[tuple[str, str], dict[str, Any]], seed: int, + unpaired: list[dict[str, Any]]) -> dict[str, Any] | None: + """The row of one measure over the shared tasks of one (treatment, model), or None with no pair.""" + pairs = [] + for task in shared: + base, treat = base_vals[(model, task)], treat_vals[(model, task)] + if measure in PRIMARY and not base["kind"] == treat["kind"] == measure: + continue # a task of the other kind, or one listed above + pair = _pair(task, base, treat, measure) + if pair is None: + lacking = [arm for arm, v in (("baseline", base), ("treatment", treat)) + if v["values"].get(measure) is None] + unpaired.append(_unpaired(treatment, model, task, measure, + f"no {measure} in {' or '.join(lacking)}")) + else: + pairs.append(pair) + if not pairs: + return None + flag = None + if measure in PRIMARY: + tops = [(base_vals[(model, p["task"])]["top"], treat_vals[(model, p["task"])]["top"]) for p in pairs] + failed = [base_vals[(model, p["task"])]["grader_failed"] + or treat_vals[(model, p["task"])]["grader_failed"] for p in pairs] + flag = headroom(pairs, tops, failed) + return {"treatment": treatment, "model": model, "measure": measure, + **paired_summary(pairs, measure, seed=seed), "headroom": flag} def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | None]], @@ -504,6 +531,7 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ found = {"cost_usd": [r["agent"].get("cost_usd") for r in recs], "turns": [r["agent"].get("num_turns") for r in recs], **{f"metric:{name}": [r["metrics"].get(name) for r in recs] for name in metrics}} + found.update(_process_lists(recs)) if score_task: found["score"] = [0.0 if r["grade"]["status"] == "grader_failure" else (r["grade"]["score"] or 0.0) for r in recs] @@ -560,6 +588,12 @@ def _finite(values: list[Any]) -> list[float]: return [float(v) for v in values if isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v)] +def _process_lists(recs: list[dict[str, Any]]) -> dict[str, list[float | None]]: + """Each process measure any of ``recs`` reports, with every cell's value (``None`` where it has none).""" + values = [process_values(r) for r in recs] + return {name: [v.get(name) for v in values] for name in {k for v in values for k in v}} + + def _mean(values: list[Any]) -> float | None: nums = _finite(values) return sum(nums) / len(nums) if nums else None diff --git a/dokimi/dokimi/repos/process_counts.py b/dokimi/dokimi/repos/process_counts.py new file mode 100644 index 00000000..9fc7be95 --- /dev/null +++ b/dokimi/dokimi/repos/process_counts.py @@ -0,0 +1,68 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``result.json`` ``process`` counts (contract §4.1) as measures of ``repo report`` and ``repo compare``. + +Each count is a number per cell: ``process:`` for the framework's counts +(``guidance_read``, a list of paths, is counted) and ``counter:`` for +each manifest ``process_metrics`` counter. A cell whose ``process`` is absent +(written before it existed) or ``null`` (its trace could not be read) has none +and is left out of a mean, not counted as zero; so is a value that is not a +finite number. +""" + +from __future__ import annotations + +import math +from collections import defaultdict +from typing import Any + + +#: ``result.json`` ``process`` keys that are counts (``guidance_read`` is a list of paths, counted). +PROCESS_KEYS = ("tool_calls", "commands", "failed_commands", "denied_commands", "guidance_read") + + +def process_values(rec: dict[str, Any]) -> dict[str, float]: + """A cell's ``process`` counters as numbers: ``process:`` and ``counter:``; ``{}`` without one.""" + found = rec.get("process") + if not isinstance(found, dict): + return {} + out = {} + for key in PROCESS_KEYS: + value = found.get(key) + count = len(value) if key == "guidance_read" and isinstance(value, list) else value + if (num := _count(count)) is not None: + out[f"process:{key}"] = num + counters = found.get("counters") + for name, value in (counters.items() if isinstance(counters, dict) else ()): + if (num := _count(value)) is not None: + out[f"counter:{name}"] = num + return out + + +def process_means(recs: list[dict[str, Any]]) -> dict[str, float]: + """Per process measure, the mean over the cells that report it, in a stable order.""" + values: dict[str, list[float]] = defaultdict(list) + for rec in recs: + for key, value in process_values(rec).items(): + values[key].append(value) + return {key: round(sum(v) / len(v), 4) for key, v in sorted(values.items(), key=lambda kv: process_order(kv[0]))} + + +def process_order(key: str) -> tuple[int, str]: + """Sort key of process measures: the framework's counts in contract order, then the counters by name.""" + name = key.removeprefix("process:") + return (PROCESS_KEYS.index(name), "") if name in PROCESS_KEYS else (len(PROCESS_KEYS), key) + + +def _count(value: Any) -> float | None: + """A finite number as a float; ``None`` for anything else (a bool, a string, NaN).""" + ok = isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + return float(value) if ok else None + + +def process_line(means: dict[str, float] | None, width: int) -> list[str]: + """The row's process means on one indented line; none for a row without them.""" + if not means: + return [] + return [" " * (width + 4) + "process " + ", ".join(f"{k.removeprefix('process:')} {v:g}" for k, v in means.items())] diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 576bb28d..3a9aca2e 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -20,6 +20,9 @@ A warning is printed when the trace says a model other than the requested one answered a cell (``model_resolved``): a harness alias can pin an older model. + +A row's ``process`` holds the mean ``result.json`` ``process`` counts of its +cells (:mod:`.process_counts`), printed on a line of its own under the row. """ from __future__ import annotations @@ -32,6 +35,7 @@ import math +from .process_counts import process_line, process_means from .runner import UNSUPPORTED_FILE, slug @@ -91,6 +95,7 @@ def summarize(batch_dir: Path) -> dict[str, Any]: "cost_usd": _mean([r["agent"]["cost_usd"] for r in recs]), "total_cost_usd": round(sum(r["agent"]["cost_usd"] or 0 for r in recs), 6), "turns": _mean([r["agent"]["num_turns"] for r in recs]), + "process": process_means(recs), "networks": sorted({str(r["agent"].get("network")) for r in recs}), "model_resolved": sorted({m for r in recs for m in resolved_models(r)})} uptake = uptake_counts(recs) @@ -136,6 +141,7 @@ def format_report(summary: dict[str, Any]) -> str: + unmeasured_note(r["uptake"])) if notes: lines.append(" " * (width + 2) + " " + ", ".join(notes)) + lines += process_line(r.get("process"), width) if summary["missing"]: shown = summary["missing"][:10] lines.append("missing (no result.json; rerun the batch): " + ", ".join(shown) diff --git a/dokimi/tests/test_repo_process_counts.py b/dokimi/tests/test_repo_process_counts.py new file mode 100644 index 00000000..4ae7b5f7 --- /dev/null +++ b/dokimi/tests/test_repo_process_counts.py @@ -0,0 +1,135 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``result.json`` ``process`` counts in `repo report` and `repo compare`, over hand-written batches.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.repos.compare import compare, format_compare +from dokimi.repos.process_counts import process_values +from dokimi.repos.report import format_report, summarize +from tests.test_repo_compare import _cell, _row + + + +def _with_process(batch: Path, process) -> None: + """Give every cell of ``batch`` a ``process`` object (``process(cell_name)``), as #157's result.json has.""" + for path in sorted(batch.glob("*/result.json")): + rec = json.loads(path.read_text()) + rec["process"] = process(path.parent.name) + path.write_text(json.dumps(rec)) + + +def test_process_counters_are_descriptive_measures(tmp_path): + for task in ("a", "b", "c"): + for rep in (1, 2): + _cell(tmp_path, task, "guidance:none", rep) + _cell(tmp_path, task, "guidance:raw", rep) + + def process(name): + raw = "__guidance-raw__" in name + return {"tool_calls": 10 if raw else 14, "commands": 4, "failed_commands": 0 if raw else 2, + "denied_commands": 0, "guidance_read": ["AGENTS.md"] if raw else [], + "counters": {"bazel": 3 if raw else 1}} + + _with_process(tmp_path, process) + summary = compare([tmp_path], "guidance:none") + measures = [r["measure"] for r in summary["comparisons"]] + assert measures[:3] == ["gate", "cost_usd", "turns"] + assert measures[3:] == ["process:tool_calls", "process:commands", "process:failed_commands", + "process:denied_commands", "process:guidance_read", "counter:bazel"] + row = _row(summary, "process:tool_calls") + assert (row["n_tasks"], row["baseline_mean"], row["treatment_mean"]) == (3, 14.0, 10.0) + assert row["delta"]["mean"] == pytest.approx(-4.0) and row["down"] == 3 and row["p_holm"] is None + assert _row(summary, "process:guidance_read")["delta"]["mean"] == pytest.approx(1.0) + assert _row(summary, "counter:bazel")["delta"]["mean"] == pytest.approx(2.0) + assert "counter:bazel" in format_compare(summary) + + +def test_process_absent_or_null_is_skipped(tmp_path): + for task in ("a", "b"): + _cell(tmp_path, task, "guidance:none", 1) + _cell(tmp_path, task, "guidance:raw", 1) + summary = compare([tmp_path], "guidance:none") + assert not any(r["measure"].startswith(("process:", "counter:")) for r in summary["comparisons"]) + # a null process (the trace could not be read) and a cell before process existed are left out, not zero + _with_process(tmp_path, lambda name: None if name.startswith("a__") else {"tool_calls": 5, "counters": None}) + summary = compare([tmp_path], "guidance:none") + assert [r["measure"] for r in summary["comparisons"]][3:] == ["process:tool_calls"] + assert _row(summary, "process:tool_calls")["n_tasks"] == 1 + assert {(u["task"], u["measure"]) for u in summary["unpaired"]} == {("a", "process:tool_calls")} + + +@pytest.mark.parametrize("arm_with_process", ["__guidance-raw__", "__guidance-none__"]) +def test_process_on_one_arm_only_does_not_flood_unpaired(tmp_path, arm_with_process): + # One arm's batch predates `process`: each shared task would otherwise be unpaired once per measure. + for task in ("a", "b", "zz"): + _cell(tmp_path, task, "guidance:raw", 1) + if task != "zz": + _cell(tmp_path, task, "guidance:none", 1) + _with_process(tmp_path, lambda name: {"tool_calls": 3, "counters": {"bazel": 1}} + if arm_with_process in name else None) + summary = compare([tmp_path], "guidance:none") + assert not any(r["measure"].startswith(("process:", "counter:")) for r in summary["comparisons"]) + assert [(u["task"], u["measure"]) for u in summary["unpaired"]] == [("zz", None)] + + +@pytest.mark.parametrize("null_rep", [1, 2, 3]) +def test_a_missing_process_value_in_a_task_is_not_zero(tmp_path, null_rep): + # Whichever rep has no process (the first one included), the task keeps the measure from the other reps. + for rep in (1, 2, 3): + _cell(tmp_path, "a", "guidance:none", rep) + _cell(tmp_path, "a", "guidance:raw", rep) + counts = {"guidance-none": 10, "guidance-raw": 4} + + def process(name): + if name.endswith(f"r{null_rep}"): + return None + return {"tool_calls": counts[name.split("__")[1]] + int(name[-1])} + + _with_process(tmp_path, process) + row = _row(compare([tmp_path], "guidance:none"), "process:tool_calls") + reps = [r for r in (1, 2, 3) if r != null_rep] + assert row["baseline_mean"] == sum(10 + r for r in reps) / 2 + assert row["treatment_mean"] == sum(4 + r for r in reps) / 2 + + +def test_process_measures_come_only_from_the_compared_arms(tmp_path): + for arm in ("guidance:none", "guidance:raw", "guidance:other"): + _cell(tmp_path, "a", arm, 1) + _with_process(tmp_path, lambda name: {"tool_calls": 1, "counters": {"only_other": 1}} + if "__guidance-other__" in name else {"tool_calls": 1}) + summary = compare([tmp_path], "guidance:none", ["guidance:raw"]) + assert [r["measure"] for r in summary["comparisons"]][3:] == ["process:tool_calls"] + + +def test_report_shows_mean_process_counters(tmp_path): + for rep in (1, 2, 3): + _cell(tmp_path, "a", "guidance:raw", rep) + _cell(tmp_path, "b", "guidance:raw", 1) + counts = {"r1": 4, "r2": 8} # r3 has a null process: left out of the mean + _with_process(tmp_path, lambda name: None if not name.startswith("a__") or name.endswith("r3") else { + "tool_calls": counts[name[-2:]], "commands": 2, "failed_commands": 1, "denied_commands": 0, + "guidance_read": ["AGENTS.md", "docs/x.md"], "counters": {"bazel": 1, "bad": "x"}}) + rows = {r["task"]: r for r in summarize(tmp_path)["rows"]} + assert rows["a"]["process"] == {"process:tool_calls": 6.0, "process:commands": 2.0, + "process:failed_commands": 1.0, "process:denied_commands": 0.0, + "process:guidance_read": 2.0, "counter:bazel": 1.0} + assert rows["b"]["process"] == {} + text = format_report(summarize(tmp_path)) + assert "process tool_calls 6, commands 2, failed_commands 1, denied_commands 0, guidance_read 2, " \ + "counter:bazel 1" in text + assert text.count("process ") == 1 + + +def test_process_values_skip_what_is_not_a_count(): + assert process_values({}) == {} and process_values({"process": None}) == {} + got = process_values({"process": {"tool_calls": True, "commands": float("nan"), "failed_commands": "3", + "denied_commands": 1, "guidance_read": "AGENTS.md", + "counters": {"ok": 2, "bad": None}}}) + assert got == {"process:denied_commands": 1.0, "counter:ok": 2.0}