From afea3050c80429cbb0d6719b7bf5013f3e38d3fb Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:52:05 -0700 Subject: [PATCH 1/3] feat(repo): show result.json process counts in repo report and repo compare repo report gives each row the mean of every process count over the cells that have one; repo compare adds them as descriptive measures (process:, counter:), outside Holm's family. An absent or null process is skipped, so older batches compare as before. Signed-off-by: Nikolay Petrov --- ...61021_160000_napetrov_process-in-report.md | 12 +++ dokimi/docs/architecture.md | 3 +- dokimi/docs/repo-evaluation-contract.md | 9 +- dokimi/dokimi/repos/compare.py | 20 ++++ dokimi/dokimi/repos/process_counts.py | 68 ++++++++++++++ dokimi/dokimi/repos/report.py | 6 ++ dokimi/tests/test_repo_process_counts.py | 92 +++++++++++++++++++ 7 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 dokimi/changelog.d/20261021_160000_napetrov_process-in-report.md create mode 100644 dokimi/dokimi/repos/process_counts.py create mode 100644 dokimi/tests/test_repo_process_counts.py diff --git a/dokimi/changelog.d/20261021_160000_napetrov_process-in-report.md b/dokimi/changelog.d/20261021_160000_napetrov_process-in-report.md new file mode 100644 index 00000000..50ef001c --- /dev/null +++ b/dokimi/changelog.d/20261021_160000_napetrov_process-in-report.md @@ -0,0 +1,12 @@ +### Added + +- **`repo report` and `repo compare` show `result.json` `process` counts** — + `repo report` prints, under each (task, arm, model) row, the mean of each + `process` count over the cells that have one (`tool_calls`, `commands`, + `failed_commands`, `denied_commands`, `guidance_read` as a number of paths, + and `counter:` for each `process_metrics` counter); the JSON row has + them under `process`. `repo compare` adds each count a compared cell + reports as a descriptive measure (`process:`, `counter:`): a + paired delta with a t interval and a raw p, outside Holm's family. Cells + whose `process` is absent or `null` are left out, so batches written before + `process` existed compare as before. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 76de3508..98f2a792 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -293,7 +293,8 @@ next release. interval, grader failures, missing cells, cost). - `dokimi/repos/compare.py` — `repo compare`: treatment arms against a baseline arm over one or more batches, one value per task and arm (gate - share, score, cost, turns, chosen `grade.json` metrics), with an interval + share, score, cost, turns, chosen `grade.json` metrics, `result.json` + `process` counts), with an interval over tasks (bootstrap for gate and score, t for the rest), a sign-flip permutation p and Holm's correction. - `dokimi/repos/selfcheck.py` — `repo tasks check`: on each task diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index f3fbc398..54d0d330 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1000,12 +1000,17 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo - The report gives, per (task, arm, model), passes over finished cells with a Wilson 95% interval (score-only tasks: the mean of `score.metric`), grader failures (counted as failures too), missing cells, and mean cost and turns. + When cells carry `result.json` `process` (§4.1), a line under the row gives + the mean of each count over the cells that have it (`guidance_read` counted + as paths; `counter:` for `process_metrics`); a `null` or absent + `process` is left out, not counted as zero. - `repo compare` reduces cells to one value per (task, arm, model) first: repetitions share a task's difficulty and are not paired across arms, so the task is the unit. Per treatment arm, model and measure (gate share, score, - cost, turns, each `--metric` from `grade.json`) it gives the mean per-task + cost, turns, each `--metric` from `grade.json`, and each `process` count a + compared cell reports as `process:` or `counter:`) it gives the mean per-task delta against the baseline with a 95% interval over tasks (a bootstrap for - gate and score, a t interval for cost, turns and metrics, `-` for a single + gate and score, a t interval for cost, turns, metrics and process counts, `-` for a single task), the tasks up, down and tied, and a two-sided sign-flip permutation p (exact up to 16 changed tasks; at one cell per task and arm it equals McNemar's exact test). Holm's correction runs over the gate and score comparisons of one call; the diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 48ed1f61..a87d160e 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -28,6 +28,10 @@ the gate and score comparisons of one invocation; cost, turns and metrics are descriptive and keep their raw p. +Every ``result.json`` ``process`` count a cell of the compared arms reports +(:mod:`.process_counts`: ``process:``, ``counter:``) is a measure +too, the per-task mean over the cells that have it, descriptive like cost. + Each selector's batches are reported with the stamp ``repo run`` pins a batch to (``tree``, ``eval_dir`` digest, ``image``) and the harness versions of its cells. A warning is printed, not an error, when a bare ``ARM`` pools cells of @@ -57,6 +61,7 @@ from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci +from .process_counts import process_order, process_values from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, resolved_models, unmeasured_note, uptake_counts) @@ -128,6 +133,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None base_vals = _task_values([c for c in cells if selector_matches(base_sel, c)], metrics) rows: list[dict[str, Any]] = [] unpaired: list[dict[str, Any]] = [] + measures += _process_measures(cells, [baseline, *treatments]) for text in treatments: treat_vals = _task_values([c for c in cells if selector_matches(parse_selector(text), c)], metrics) unpaired += _one_arm_models(text, base_vals, treat_vals, models) @@ -240,6 +246,13 @@ def _built_from_tree(stamp: dict[str, Any]) -> bool: return repo.startswith("dokimi/") and (tag.startswith("worktree-") or bool(revision) and tag == revision[:12]) +def _process_measures(cells: list[dict[str, Any]], selectors: list[str]) -> list[str]: + """The ``process`` counters any cell of the compared arms reports; none when no cell has ``process``.""" + sels = [parse_selector(text) for text in selectors] + found = {k for c in cells if any(selector_matches(s, c) for s in sels) for k in process_values(c)} + return sorted(found, key=process_order) + + def _check_metrics(metrics: tuple[str, ...], cells: list[dict[str, Any]]) -> None: """A ``--metric`` that no finished cell reports is a typo or not a number: refuse it, not print nothing.""" unknown = [m for m in metrics if not any(m in c["metrics"] for c in cells)] @@ -444,6 +457,7 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[ found = {"cost_usd": [r["agent"].get("cost_usd") for r in recs], "turns": [r["agent"].get("num_turns") for r in recs], **{f"metric:{name}": [r["metrics"].get(name) for r in recs] for name in metrics}} + found.update(_process_lists(recs)) if score_task: found["score"] = [0.0 if r["grade"]["status"] == "grader_failure" else (r["grade"]["score"] or 0.0) for r in recs] @@ -479,6 +493,12 @@ def _finite(values: list[Any]) -> list[float]: return [float(v) for v in values if isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v)] +def _process_lists(recs: list[dict[str, Any]]) -> dict[str, list[float | None]]: + """Each process measure any of ``recs`` reports, with every cell's value (``None`` where it has none).""" + values = [process_values(r) for r in recs] + return {name: [v.get(name) for v in values] for name in {k for v in values for k in v}} + + def _mean(values: list[Any]) -> float | None: nums = _finite(values) return sum(nums) / len(nums) if nums else None diff --git a/dokimi/dokimi/repos/process_counts.py b/dokimi/dokimi/repos/process_counts.py new file mode 100644 index 00000000..9fc7be95 --- /dev/null +++ b/dokimi/dokimi/repos/process_counts.py @@ -0,0 +1,68 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``result.json`` ``process`` counts (contract §4.1) as measures of ``repo report`` and ``repo compare``. + +Each count is a number per cell: ``process:`` for the framework's counts +(``guidance_read``, a list of paths, is counted) and ``counter:`` for +each manifest ``process_metrics`` counter. A cell whose ``process`` is absent +(written before it existed) or ``null`` (its trace could not be read) has none +and is left out of a mean, not counted as zero; so is a value that is not a +finite number. +""" + +from __future__ import annotations + +import math +from collections import defaultdict +from typing import Any + + +#: ``result.json`` ``process`` keys that are counts (``guidance_read`` is a list of paths, counted). +PROCESS_KEYS = ("tool_calls", "commands", "failed_commands", "denied_commands", "guidance_read") + + +def process_values(rec: dict[str, Any]) -> dict[str, float]: + """A cell's ``process`` counters as numbers: ``process:`` and ``counter:``; ``{}`` without one.""" + found = rec.get("process") + if not isinstance(found, dict): + return {} + out = {} + for key in PROCESS_KEYS: + value = found.get(key) + count = len(value) if key == "guidance_read" and isinstance(value, list) else value + if (num := _count(count)) is not None: + out[f"process:{key}"] = num + counters = found.get("counters") + for name, value in (counters.items() if isinstance(counters, dict) else ()): + if (num := _count(value)) is not None: + out[f"counter:{name}"] = num + return out + + +def process_means(recs: list[dict[str, Any]]) -> dict[str, float]: + """Per process measure, the mean over the cells that report it, in a stable order.""" + values: dict[str, list[float]] = defaultdict(list) + for rec in recs: + for key, value in process_values(rec).items(): + values[key].append(value) + return {key: round(sum(v) / len(v), 4) for key, v in sorted(values.items(), key=lambda kv: process_order(kv[0]))} + + +def process_order(key: str) -> tuple[int, str]: + """Sort key of process measures: the framework's counts in contract order, then the counters by name.""" + name = key.removeprefix("process:") + return (PROCESS_KEYS.index(name), "") if name in PROCESS_KEYS else (len(PROCESS_KEYS), key) + + +def _count(value: Any) -> float | None: + """A finite number as a float; ``None`` for anything else (a bool, a string, NaN).""" + ok = isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + return float(value) if ok else None + + +def process_line(means: dict[str, float] | None, width: int) -> list[str]: + """The row's process means on one indented line; none for a row without them.""" + if not means: + return [] + return [" " * (width + 4) + "process " + ", ".join(f"{k.removeprefix('process:')} {v:g}" for k, v in means.items())] diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 576bb28d..3a9aca2e 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -20,6 +20,9 @@ A warning is printed when the trace says a model other than the requested one answered a cell (``model_resolved``): a harness alias can pin an older model. + +A row's ``process`` holds the mean ``result.json`` ``process`` counts of its +cells (:mod:`.process_counts`), printed on a line of its own under the row. """ from __future__ import annotations @@ -32,6 +35,7 @@ import math +from .process_counts import process_line, process_means from .runner import UNSUPPORTED_FILE, slug @@ -91,6 +95,7 @@ def summarize(batch_dir: Path) -> dict[str, Any]: "cost_usd": _mean([r["agent"]["cost_usd"] for r in recs]), "total_cost_usd": round(sum(r["agent"]["cost_usd"] or 0 for r in recs), 6), "turns": _mean([r["agent"]["num_turns"] for r in recs]), + "process": process_means(recs), "networks": sorted({str(r["agent"].get("network")) for r in recs}), "model_resolved": sorted({m for r in recs for m in resolved_models(r)})} uptake = uptake_counts(recs) @@ -136,6 +141,7 @@ def format_report(summary: dict[str, Any]) -> str: + unmeasured_note(r["uptake"])) if notes: lines.append(" " * (width + 2) + " " + ", ".join(notes)) + lines += process_line(r.get("process"), width) if summary["missing"]: shown = summary["missing"][:10] lines.append("missing (no result.json; rerun the batch): " + ", ".join(shown) diff --git a/dokimi/tests/test_repo_process_counts.py b/dokimi/tests/test_repo_process_counts.py new file mode 100644 index 00000000..e1412ada --- /dev/null +++ b/dokimi/tests/test_repo_process_counts.py @@ -0,0 +1,92 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``result.json`` ``process`` counts in `repo report` and `repo compare`, over hand-written batches.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.repos.compare import compare, format_compare +from dokimi.repos.process_counts import process_values +from dokimi.repos.report import format_report, summarize +from tests.test_repo_compare import _cell, _row + + + +def _with_process(batch: Path, process) -> None: + """Give every cell of ``batch`` a ``process`` object (``process(cell_name)``), as #157's result.json has.""" + for path in sorted(batch.glob("*/result.json")): + rec = json.loads(path.read_text()) + rec["process"] = process(path.parent.name) + path.write_text(json.dumps(rec)) + + +def test_process_counters_are_descriptive_measures(tmp_path): + for task in ("a", "b", "c"): + for rep in (1, 2): + _cell(tmp_path, task, "guidance:none", rep) + _cell(tmp_path, task, "guidance:raw", rep) + + def process(name): + raw = "__guidance-raw__" in name + return {"tool_calls": 10 if raw else 14, "commands": 4, "failed_commands": 0 if raw else 2, + "denied_commands": 0, "guidance_read": ["AGENTS.md"] if raw else [], + "counters": {"bazel": 3 if raw else 1}} + + _with_process(tmp_path, process) + summary = compare([tmp_path], "guidance:none") + measures = [r["measure"] for r in summary["comparisons"]] + assert measures[:3] == ["gate", "cost_usd", "turns"] + assert measures[3:] == ["process:tool_calls", "process:commands", "process:failed_commands", + "process:denied_commands", "process:guidance_read", "counter:bazel"] + row = _row(summary, "process:tool_calls") + assert (row["n_tasks"], row["baseline_mean"], row["treatment_mean"]) == (3, 14.0, 10.0) + assert row["delta"]["mean"] == pytest.approx(-4.0) and row["down"] == 3 and row["p_holm"] is None + assert _row(summary, "process:guidance_read")["delta"]["mean"] == pytest.approx(1.0) + assert _row(summary, "counter:bazel")["delta"]["mean"] == pytest.approx(2.0) + assert "counter:bazel" in format_compare(summary) + + +def test_process_absent_or_null_is_skipped(tmp_path): + for task in ("a", "b"): + _cell(tmp_path, task, "guidance:none", 1) + _cell(tmp_path, task, "guidance:raw", 1) + summary = compare([tmp_path], "guidance:none") + assert not any(r["measure"].startswith(("process:", "counter:")) for r in summary["comparisons"]) + # a null process (the trace could not be read) and a cell before process existed are left out, not zero + _with_process(tmp_path, lambda name: None if name.startswith("a__") else {"tool_calls": 5, "counters": None}) + summary = compare([tmp_path], "guidance:none") + assert [r["measure"] for r in summary["comparisons"]][3:] == ["process:tool_calls"] + assert _row(summary, "process:tool_calls")["n_tasks"] == 1 + assert {(u["task"], u["measure"]) for u in summary["unpaired"]} == {("a", "process:tool_calls")} + + +def test_report_shows_mean_process_counters(tmp_path): + for rep in (1, 2, 3): + _cell(tmp_path, "a", "guidance:raw", rep) + _cell(tmp_path, "b", "guidance:raw", 1) + counts = {"r1": 4, "r2": 8} # r3 has a null process: left out of the mean + _with_process(tmp_path, lambda name: None if not name.startswith("a__") or name.endswith("r3") else { + "tool_calls": counts[name[-2:]], "commands": 2, "failed_commands": 1, "denied_commands": 0, + "guidance_read": ["AGENTS.md", "docs/x.md"], "counters": {"bazel": 1, "bad": "x"}}) + rows = {r["task"]: r for r in summarize(tmp_path)["rows"]} + assert rows["a"]["process"] == {"process:tool_calls": 6.0, "process:commands": 2.0, + "process:failed_commands": 1.0, "process:denied_commands": 0.0, + "process:guidance_read": 2.0, "counter:bazel": 1.0} + assert rows["b"]["process"] == {} + text = format_report(summarize(tmp_path)) + assert "process tool_calls 6, commands 2, failed_commands 1, denied_commands 0, guidance_read 2, " \ + "counter:bazel 1" in text + assert text.count("process ") == 1 + + +def test_process_values_skip_what_is_not_a_count(): + assert process_values({}) == {} and process_values({"process": None}) == {} + got = process_values({"process": {"tool_calls": True, "commands": float("nan"), "failed_commands": "3", + "denied_commands": 1, "guidance_read": "AGENTS.md", + "counters": {"ok": 2, "bad": None}}}) + assert got == {"process:denied_commands": 1.0, "counter:ok": 2.0} From 621c00e887c2bb526e369b308e7ca35984fa02cd Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 11:40:50 -0700 Subject: [PATCH 2/3] fix(repo): a process count only one arm reports no longer floods compare's unpaired list When the baseline batch predates result.json process, every shared task became an unpaired entry for every process measure and hid the tasks that were really missing. A process measure an arm reports for no task is now left out. Pins the missing-value-is-not-zero mean and the selector scoping. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/compare.py | 3 ++ dokimi/tests/test_repo_process_counts.py | 36 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index a87d160e..c11f4d66 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -289,6 +289,9 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str], if b != t: # batches of different eval trees: the primary result of this task has no pair unpaired.append(_unpaired(treatment, model, task, None, f"{b} task in baseline, {t} in treatment")) for measure in measures: + if measure.startswith(("process:", "counter:")) and not all( + any(vals[(model, t)]["values"].get(measure) is not None for t in shared) for vals in (base_vals, treat_vals)): + continue # an arm that reports no such count at all (an older batch) leaves no task out of a pair pairs = [] for task in shared: base, treat = base_vals[(model, task)], treat_vals[(model, task)] diff --git a/dokimi/tests/test_repo_process_counts.py b/dokimi/tests/test_repo_process_counts.py index e1412ada..8d3a61e7 100644 --- a/dokimi/tests/test_repo_process_counts.py +++ b/dokimi/tests/test_repo_process_counts.py @@ -65,6 +65,42 @@ def test_process_absent_or_null_is_skipped(tmp_path): assert {(u["task"], u["measure"]) for u in summary["unpaired"]} == {("a", "process:tool_calls")} +def test_process_on_one_arm_only_does_not_flood_unpaired(tmp_path): + # The baseline batch predates `process`: each shared task would otherwise be unpaired once per measure. + for task in ("a", "b", "zz"): + _cell(tmp_path, task, "guidance:raw", 1) + if task != "zz": + _cell(tmp_path, task, "guidance:none", 1) + _with_process(tmp_path, lambda name: {"tool_calls": 3, "counters": {"bazel": 1}} + if "__guidance-raw__" in name else None) + summary = compare([tmp_path], "guidance:none") + assert not any(r["measure"].startswith(("process:", "counter:")) for r in summary["comparisons"]) + assert [(u["task"], u["measure"]) for u in summary["unpaired"]] == [("zz", None)] + + +def test_a_missing_process_value_in_a_task_is_not_zero(tmp_path): + for rep in (1, 2, 3): + _cell(tmp_path, "a", "guidance:none", rep) + _cell(tmp_path, "a", "guidance:raw", rep) + counts = {"guidance-none": 10, "guidance-raw": 4} + + def process(name): # rep 3 has no process at all + return None if name.endswith("r3") else {"tool_calls": counts[name.split("__")[1]] + int(name[-1])} + + _with_process(tmp_path, process) + row = _row(compare([tmp_path], "guidance:none"), "process:tool_calls") + assert (row["baseline_mean"], row["treatment_mean"]) == (11.5, 5.5) + + +def test_process_measures_come_only_from_the_compared_arms(tmp_path): + for arm in ("guidance:none", "guidance:raw", "guidance:other"): + _cell(tmp_path, "a", arm, 1) + _with_process(tmp_path, lambda name: {"tool_calls": 1, "counters": {"only_other": 1}} + if "__guidance-other__" in name else {"tool_calls": 1}) + summary = compare([tmp_path], "guidance:none", ["guidance:raw"]) + assert [r["measure"] for r in summary["comparisons"]][3:] == ["process:tool_calls"] + + def test_report_shows_mean_process_counters(tmp_path): for rep in (1, 2, 3): _cell(tmp_path, "a", "guidance:raw", rep) From f12b327180bb88a17ea9e34fa0f04c8a6c813983 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 01:26:44 -0700 Subject: [PATCH 3/3] test(repo): pin compare's one-arm process guard for an older treatment batch too The flood test covered only a baseline that predates process; a guard checking only the baseline arm passed every test. Parametrize over which arm lacks process. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_repo_process_counts.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dokimi/tests/test_repo_process_counts.py b/dokimi/tests/test_repo_process_counts.py index 8d3a61e7..308addec 100644 --- a/dokimi/tests/test_repo_process_counts.py +++ b/dokimi/tests/test_repo_process_counts.py @@ -65,14 +65,15 @@ def test_process_absent_or_null_is_skipped(tmp_path): assert {(u["task"], u["measure"]) for u in summary["unpaired"]} == {("a", "process:tool_calls")} -def test_process_on_one_arm_only_does_not_flood_unpaired(tmp_path): - # The baseline batch predates `process`: each shared task would otherwise be unpaired once per measure. +@pytest.mark.parametrize("arm_with_process", ["__guidance-raw__", "__guidance-none__"]) +def test_process_on_one_arm_only_does_not_flood_unpaired(tmp_path, arm_with_process): + # One arm's batch predates `process`: each shared task would otherwise be unpaired once per measure. for task in ("a", "b", "zz"): _cell(tmp_path, task, "guidance:raw", 1) if task != "zz": _cell(tmp_path, task, "guidance:none", 1) _with_process(tmp_path, lambda name: {"tool_calls": 3, "counters": {"bazel": 1}} - if "__guidance-raw__" in name else None) + if arm_with_process in name else None) summary = compare([tmp_path], "guidance:none") assert not any(r["measure"].startswith(("process:", "counter:")) for r in summary["comparisons"]) assert [(u["task"], u["measure"]) for u in summary["unpaired"]] == [("zz", None)]