feat(repo): show result.json process counts in repo report and repo compare

dokimi/changelog.d/20261021_160000_napetrov_process-in-report.md

@@ -0,0 +1,12 @@
+### Added
+
+- **`repo report` and `repo compare` show `result.json` `process` counts** —
+  `repo report` prints, under each (task, arm, model) row, the mean of each
+  `process` count over the cells that have one (`tool_calls`, `commands`,
+  `failed_commands`, `denied_commands`, `guidance_read` as a number of paths,
+  and `counter:<name>` for each `process_metrics` counter); the JSON row has
+  them under `process`. `repo compare` adds each count a compared cell
+  reports as a descriptive measure (`process:<key>`, `counter:<name>`): a
+  paired delta with a t interval and a raw p, outside Holm's family. Cells
+  whose `process` is absent or `null` are left out, so batches written before
+  `process` existed compare as before.

dokimi/docs/architecture.md

@@ -293,7 +293,8 @@ next release.
   interval, grader failures, missing cells, cost).
 - `dokimi/repos/compare.py` — `repo compare`: treatment arms against a
   baseline arm over one or more batches, one value per task and arm (gate
-  share, score, cost, turns, chosen `grade.json` metrics), with an interval
+  share, score, cost, turns, chosen `grade.json` metrics, `result.json`
+  `process` counts), with an interval
   over tasks (bootstrap for gate and score, t for the rest), a sign-flip
   permutation p and Holm's correction.
 - `dokimi/repos/selfcheck.py` — `repo tasks check`: on each task

dokimi/docs/repo-evaluation-contract.md

@@ -1001,12 +1001,17 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo
 - The report gives, per (task, arm, model), passes over finished cells with a
   Wilson 95% interval (score-only tasks: the mean of `score.metric`), grader
   failures (counted as failures too), missing cells, and mean cost and turns.
+  When cells carry `result.json` `process` (§4.1), a line under the row gives
+  the mean of each count over the cells that have it (`guidance_read` counted
+  as paths; `counter:<name>` for `process_metrics`); a `null` or absent
+  `process` is left out, not counted as zero.
 - `repo compare` reduces cells to one value per (task, arm, model) first:
   repetitions share a task's difficulty and are not paired across arms, so the
   task is the unit. Per treatment arm, model and measure (gate share, score,
-  cost, turns, each `--metric` from `grade.json`) it gives the mean per-task
+  cost, turns, each `--metric` from `grade.json`, and each `process` count a
+  compared cell reports as `process:<key>` or `counter:<name>`) it gives the mean per-task
   delta against the baseline with a 95% interval over tasks (a bootstrap for
-  gate and score, a t interval for cost, turns and metrics, `-` for a single
+  gate and score, a t interval for cost, turns, metrics and process counts, `-` for a single
   task), the tasks up, down and tied, and a two-sided sign-flip permutation p (exact up to 16
   changed tasks; at one cell per task and arm it equals McNemar's exact test).
   Holm's correction runs over the gate and score comparisons of one call; the

dokimi/dokimi/repos/compare.py

@@ -41,6 +41,10 @@
 measurement failed, not that the task had no room. Any other case has no flag: ``None``. The flag is a note;
 p and Holm are unchanged.
 
+Every ``result.json`` ``process`` count a cell of the compared arms reports
+(:mod:`.process_counts`: ``process:<key>``, ``counter:<name>``) is a measure
+too, the per-task mean over the cells that have it, descriptive like cost.
+
 Each selector's batches are reported with the stamp ``repo run`` pins a batch
 to (``tree``, ``eval_dir`` digest, ``image``) and the harness versions of its
 cells. A warning is printed, not an error, when a bare ``ARM`` pools cells of
@@ -70,6 +74,7 @@
 
 from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci
 
+from .process_counts import process_order, process_values
 from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, resolved_models, unmeasured_note,
                      uptake_counts)
 
@@ -141,6 +146,7 @@ def compare(batch_dirs: list[Path], baseline: str, treatments: list[str] | None
     base_vals = _task_values([c for c in cells if selector_matches(base_sel, c)], metrics)
     rows: list[dict[str, Any]] = []
     unpaired: list[dict[str, Any]] = []
+    measures += _process_measures(cells, [baseline, *treatments])
     for text in treatments:
         treat_vals = _task_values([c for c in cells if selector_matches(parse_selector(text), c)], metrics)
         unpaired += _one_arm_models(text, base_vals, treat_vals, models)
@@ -253,6 +259,13 @@ def _built_from_tree(stamp: dict[str, Any]) -> bool:
     return repo.startswith("dokimi/") and (tag.startswith("worktree-") or bool(revision) and tag == revision[:12])
 
 
+def _process_measures(cells: list[dict[str, Any]], selectors: list[str]) -> list[str]:
+    """The ``process`` counters any cell of the compared arms reports; none when no cell has ``process``."""
+    sels = [parse_selector(text) for text in selectors]
+    found = {k for c in cells if any(selector_matches(s, c) for s in sels) for k in process_values(c)}
+    return sorted(found, key=process_order)
+
+
 def _check_metrics(metrics: tuple[str, ...], cells: list[dict[str, Any]]) -> None:
     """A ``--metric`` that no finished cell reports is a typo or not a number: refuse it, not print nothing."""
     unknown = [m for m in metrics if not any(m in c["metrics"] for c in cells)]
@@ -289,28 +302,42 @@ def _compare_model(treatment: str, model: str, base_vals: dict[tuple[str, str],
         if b != t:  # batches of different eval trees: the primary result of this task has no pair
             unpaired.append(_unpaired(treatment, model, task, None, f"{b} task in baseline, {t} in treatment"))
     for measure in measures:
-        pairs = []
-        for task in shared:
-            base, treat = base_vals[(model, task)], treat_vals[(model, task)]
-            if measure in PRIMARY and not base["kind"] == treat["kind"] == measure:
-                continue  # a task of the other kind, or one listed above
-            pair = _pair(task, base, treat, measure)
-            if pair is None:
-                lacking = [arm for arm, v in (("baseline", base), ("treatment", treat))
-                           if v["values"].get(measure) is None]
-                unpaired.append(_unpaired(treatment, model, task, measure,
-                                          f"no {measure} in {' or '.join(lacking)}"))
-            else:
-                pairs.append(pair)
-        if pairs:
-            flag = None
-            if measure in PRIMARY:
-                tops = [(base_vals[(model, p["task"])]["top"], treat_vals[(model, p["task"])]["top"]) for p in pairs]
-                failed = [base_vals[(model, p["task"])]["grader_failed"]
-                          or treat_vals[(model, p["task"])]["grader_failed"] for p in pairs]
-                flag = headroom(pairs, tops, failed)
-            rows.append({"treatment": treatment, "model": model, "measure": measure,
-                         **paired_summary(pairs, measure, seed=seed), "headroom": flag})
+        if measure.startswith(("process:", "counter:")) and not all(
+                any(vals[(model, t)]["values"].get(measure) is not None for t in shared) for vals in (base_vals, treat_vals)):
+            continue  # an arm that reports no such count at all (an older batch) leaves no task out of a pair
+        row = _compare_measure(treatment, model, measure, shared, base_vals, treat_vals, seed, unpaired)
+        if row is not None:
+            rows.append(row)
+
+
+def _compare_measure(treatment: str, model: str, measure: str, shared: list[str],
+                     base_vals: dict[tuple[str, str], dict[str, Any]],
+                     treat_vals: dict[tuple[str, str], dict[str, Any]], seed: int,
+                     unpaired: list[dict[str, Any]]) -> dict[str, Any] | None:
+    """The row of one measure over the shared tasks of one (treatment, model), or None with no pair."""
+    pairs = []
+    for task in shared:
+        base, treat = base_vals[(model, task)], treat_vals[(model, task)]
+        if measure in PRIMARY and not base["kind"] == treat["kind"] == measure:
+            continue  # a task of the other kind, or one listed above
+        pair = _pair(task, base, treat, measure)
+        if pair is None:
+            lacking = [arm for arm, v in (("baseline", base), ("treatment", treat))
+                       if v["values"].get(measure) is None]
+            unpaired.append(_unpaired(treatment, model, task, measure,
+                                      f"no {measure} in {' or '.join(lacking)}"))
+        else:
+            pairs.append(pair)
+    if not pairs:
+        return None
+    flag = None
+    if measure in PRIMARY:
+        tops = [(base_vals[(model, p["task"])]["top"], treat_vals[(model, p["task"])]["top"]) for p in pairs]
+        failed = [base_vals[(model, p["task"])]["grader_failed"]
+                  or treat_vals[(model, p["task"])]["grader_failed"] for p in pairs]
+        flag = headroom(pairs, tops, failed)
+    return {"treatment": treatment, "model": model, "measure": measure,
+            **paired_summary(pairs, measure, seed=seed), "headroom": flag}
 
 
 def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | None]],
@@ -504,6 +531,7 @@ def _task_values(cells: list[dict[str, Any]], metrics: tuple[str, ...]) -> dict[
         found = {"cost_usd": [r["agent"].get("cost_usd") for r in recs],
                  "turns": [r["agent"].get("num_turns") for r in recs],
                  **{f"metric:{name}": [r["metrics"].get(name) for r in recs] for name in metrics}}
+        found.update(_process_lists(recs))
         if score_task:
             found["score"] = [0.0 if r["grade"]["status"] == "grader_failure" else (r["grade"]["score"] or 0.0)
                               for r in recs]
@@ -560,6 +588,12 @@ def _finite(values: list[Any]) -> list[float]:
     return [float(v) for v in values if isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v)]
 
 
+def _process_lists(recs: list[dict[str, Any]]) -> dict[str, list[float | None]]:
+    """Each process measure any of ``recs`` reports, with every cell's value (``None`` where it has none)."""
+    values = [process_values(r) for r in recs]
+    return {name: [v.get(name) for v in values] for name in {k for v in values for k in v}}
+
+
 def _mean(values: list[Any]) -> float | None:
     nums = _finite(values)
     return sum(nums) / len(nums) if nums else None

dokimi/dokimi/repos/process_counts.py

@@ -0,0 +1,68 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""``result.json`` ``process`` counts (contract §4.1) as measures of ``repo report`` and ``repo compare``.
+
+Each count is a number per cell: ``process:<key>`` for the framework's counts
+(``guidance_read``, a list of paths, is counted) and ``counter:<name>`` for
+each manifest ``process_metrics`` counter. A cell whose ``process`` is absent
+(written before it existed) or ``null`` (its trace could not be read) has none
+and is left out of a mean, not counted as zero; so is a value that is not a
+finite number.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+from typing import Any
+
+
+#: ``result.json`` ``process`` keys that are counts (``guidance_read`` is a list of paths, counted).
+PROCESS_KEYS = ("tool_calls", "commands", "failed_commands", "denied_commands", "guidance_read")
+
+
+def process_values(rec: dict[str, Any]) -> dict[str, float]:
+    """A cell's ``process`` counters as numbers: ``process:<key>`` and ``counter:<name>``; ``{}`` without one."""
+    found = rec.get("process")
+    if not isinstance(found, dict):
+        return {}
+    out = {}
+    for key in PROCESS_KEYS:
+        value = found.get(key)
+        count = len(value) if key == "guidance_read" and isinstance(value, list) else value
+        if (num := _count(count)) is not None:
+            out[f"process:{key}"] = num
+    counters = found.get("counters")
+    for name, value in (counters.items() if isinstance(counters, dict) else ()):
+        if (num := _count(value)) is not None:
+            out[f"counter:{name}"] = num
+    return out
+
+
+def process_means(recs: list[dict[str, Any]]) -> dict[str, float]:
+    """Per process measure, the mean over the cells that report it, in a stable order."""
+    values: dict[str, list[float]] = defaultdict(list)
+    for rec in recs:
+        for key, value in process_values(rec).items():
+            values[key].append(value)
+    return {key: round(sum(v) / len(v), 4) for key, v in sorted(values.items(), key=lambda kv: process_order(kv[0]))}
+
+
+def process_order(key: str) -> tuple[int, str]:
+    """Sort key of process measures: the framework's counts in contract order, then the counters by name."""
+    name = key.removeprefix("process:")
+    return (PROCESS_KEYS.index(name), "") if name in PROCESS_KEYS else (len(PROCESS_KEYS), key)
+
+
+def _count(value: Any) -> float | None:
+    """A finite number as a float; ``None`` for anything else (a bool, a string, NaN)."""
+    ok = isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
+    return float(value) if ok else None
+
+
+def process_line(means: dict[str, float] | None, width: int) -> list[str]:
+    """The row's process means on one indented line; none for a row without them."""
+    if not means:
+        return []
+    return [" " * (width + 4) + "process " + ", ".join(f"{k.removeprefix('process:')} {v:g}" for k, v in means.items())]

dokimi/dokimi/repos/report.py

@@ -20,6 +20,9 @@
 
 A warning is printed when the trace says a model other than the requested one
 answered a cell (``model_resolved``): a harness alias can pin an older model.
+
+A row's ``process`` holds the mean ``result.json`` ``process`` counts of its
+cells (:mod:`.process_counts`), printed on a line of its own under the row.
 """
 
 from __future__ import annotations
@@ -32,6 +35,7 @@
 
 import math
 
+from .process_counts import process_line, process_means
 from .runner import UNSUPPORTED_FILE, slug
 
 
@@ -91,6 +95,7 @@ def summarize(batch_dir: Path) -> dict[str, Any]:
                                "cost_usd": _mean([r["agent"]["cost_usd"] for r in recs]),
                                "total_cost_usd": round(sum(r["agent"]["cost_usd"] or 0 for r in recs), 6),
                                "turns": _mean([r["agent"]["num_turns"] for r in recs]),
+                               "process": process_means(recs),
                                "networks": sorted({str(r["agent"].get("network")) for r in recs}),
                                "model_resolved": sorted({m for r in recs for m in resolved_models(r)})}
         uptake = uptake_counts(recs)
@@ -136,6 +141,7 @@ def format_report(summary: dict[str, Any]) -> str:
                          + unmeasured_note(r["uptake"]))
         if notes:
             lines.append(" " * (width + 2) + "  " + ", ".join(notes))
+        lines += process_line(r.get("process"), width)
     if summary["missing"]:
         shown = summary["missing"][:10]
         lines.append("missing (no result.json; rerun the batch): " + ", ".join(shown)