From afdc62c289012c0513441cad6ea03e377319a45e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 16:08:55 -0700 Subject: [PATCH 1/6] feat(repo): record whether the agent used what a patch: arm added (#41 D4) A patch: arm that adds a tool can move a score through the guidance line that names it, with the tool never called. Prep now records the files the patch added that are not guidance, and the servers of an added .mcp.json. The runner counts commands that name those files, reads of them and calls of the servers' tools into result.json.uptake. report shows used/cells per arm. compare shows it per treatment and warns when a treatment used none of what its arm added. Signed-off-by: Nikolay Petrov --- ...0261022_090000_napetrov_repo-arm-uptake.md | 10 ++ dokimi/docs/repo-evaluation-contract.md | 15 ++- dokimi/dokimi/repos/compare.py | 19 ++- dokimi/dokimi/repos/report.py | 19 +++ dokimi/dokimi/repos/runner.py | 16 ++- dokimi/dokimi/repos/uptake.py | 82 ++++++++++++ dokimi/dokimi/repos/workspace.py | 14 +- dokimi/tests/repo_eval_fixture.py | 7 +- dokimi/tests/test_repo_arms.py | 3 +- dokimi/tests/test_repo_uptake.py | 121 ++++++++++++++++++ 10 files changed, 293 insertions(+), 13 deletions(-) create mode 100644 dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md create mode 100644 dokimi/dokimi/repos/uptake.py create mode 100644 dokimi/tests/test_repo_uptake.py diff --git a/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md b/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md new file mode 100644 index 00000000..a468d648 --- /dev/null +++ b/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md @@ -0,0 +1,10 @@ +### Added + +- **repo run: uptake of what a `patch:` arm added** — `meta.json` `arm_patch` + lists the files the patch `added` (guidance files left out) and the + `mcp_servers` of a `.mcp.json` it wrote, and `result.json` gets an `uptake` + object: the agent's commands naming an added file, reads of one, and calls of + those servers' tools. `repo report` shows the cells that used the arm's + additions per row; `repo compare` lists uptake per treatment and warns when a + treatment's cells used none of it, since its deltas then measure the patch's + text, not the tool (#41 D4). diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5f31e13d..edca8175 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -320,7 +320,9 @@ grader diffs the agent's changes from `start_sha`. With the arm come `arm_files`, the files it deleted, wrote or changed; for `guidance:rev:` an `arm_rev` object (`rev` as given, its `sha`, and `removed`: the base's guidance files with no counterpart at ``); for `patch:` an `arm_patch` object (the -file's absolute `path` and full `sha256`). +file's absolute `path` and full `sha256`, `added`: the files the patch added +that the guidance globs do not match, and `mcp_servers`: the servers of a +`.mcp.json` at the root the patch added or changed, read before the agent runs). Step 3 removes history only. A fix that landed upstream before a model's training cutoff may still be known to the model, so reports split results by @@ -372,6 +374,17 @@ Code's `modelUsage`), `main` when the harness reported the parent's only, and `null` when it reported none. `subagent_calls` counts the parent's and subagents' `Agent`/`Task` calls; `null` when the harness wrote no result record. +For a `patch:` arm with `added` files or `mcp_servers`, `result.json` has an +`uptake` object: whether the agent used what the arm added. `commands` counts +command events that name an added file as a workspace path (`tools/fmt`, +`./tools/fmt`, `/eval/run/workspace/tools/fmt`, not `cd tools && ./fmt`), `reads` +the read events of one, `mcp_calls` the calls of the servers' tools +(`mcp____`); `used` is whether any is non-zero, and `measured` is +`false` when the agent left no trace to count from. `repo report` shows the +cells that used it per row, and `repo compare` warns about a treatment whose +cells used none of it: its deltas then measure the patch's text, such as a +guidance line, not the tool. + ### 4.2 `grade.json` (written by `grade` to `{run_dir}/grade.json`) ```json diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 59b2db71..0ff249e7 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -39,6 +39,10 @@ than it asked for is warned about as in ``repo report``. An image ``repo run`` tagged from the tree's own revision follows the tree, so it is compared only against an ``--image`` given explicitly. + +A treatment arm that added a file or an MCP server (``result.json`` +``uptake``) whose agents used it in none of their cells is warned about too: its +deltas measure the patch's text (a guidance line, say), not the tool. """ from __future__ import annotations @@ -53,7 +57,7 @@ from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci -from .report import load_cells, mismatch_warnings, model_mismatch, model_matches, resolved_models +from .report import load_cells, mismatch_warnings, model_mismatch, model_matches, resolved_models, uptake_counts #: Largest number of non-zero deltas whose sign-flip distribution is enumerated (2**16 patterns). EXACT_MAX = 16 @@ -150,7 +154,8 @@ def _one_arm_models(treatment: str, base_vals: dict[tuple[str, str], dict[str, A def _provenance(batch_dirs: list[Path], cells: list[dict[str, Any]], baseline: str, treatments: list[str], models: list[str] | None = None) -> dict[str, Any]: - """``sources`` (each selector's batches), ``stamps`` (each batch's pin and harness versions), ``warnings``.""" + """``sources`` (each selector's batches), ``stamps`` (each batch's pin and harness versions), ``uptake`` (each + treatment's cells that used what its arm added), ``warnings``.""" names = [d.resolve().name for d in batch_dirs] stamps = {name: {**batch_stamp(d), "harness_versions": sorted({str(c.get("harness_version")) for c in cells if c["batch"] == name}), @@ -158,7 +163,9 @@ def _provenance(batch_dirs: list[Path], cells: list[dict[str, Any]], baseline: s for d, name in zip(batch_dirs, names, strict=True)} sources = {text: sorted({c["batch"] for c in cells if selector_matches(parse_selector(text), c)}) for text in [baseline, *treatments]} - return {"sources": sources, "stamps": stamps, + uptake = {text: uptake_counts([c for c in cells if selector_matches(parse_selector(text), c) + and (not models or c["model"] in models)]) for text in treatments} + return {"sources": sources, "stamps": stamps, "uptake": {k: v for k, v in uptake.items() if v is not None}, "warnings": _provenance_warnings(cells, baseline, treatments, sources, stamps, models)} @@ -215,6 +222,10 @@ def _provenance_warnings(cells: list[dict[str, Any]], baseline: str, treatments: if base and treat and not _models_equivalent(base, treat): out.append(f"{text} vs {baseline}: model {model} was answered by {', '.join(sorted(base))} in " f"baseline, {', '.join(sorted(treat))} in treatment") + uptake = uptake_counts(arms[1]) + if uptake and uptake["cells"] and not uptake["used"]: + out.append(f"{text}: the agent used none of what the arm added in {uptake['cells']} cell(s); " + "a delta measures the patch's text, not the tool") compared = [c for c in cells if (not models or c["model"] in models) and any(selector_matches(parse_selector(sel), c) for sel in (baseline, *treatments))] return out + mismatch_warnings(model_mismatch(compared)) @@ -404,6 +415,8 @@ def format_compare(summary: dict[str, Any]) -> str: + (f" and {len(summary['unpaired']) - len(shown)} more" if len(summary["unpaired"]) > 10 else "")) if summary["missing"]: lines.append(f"{len(summary['missing'])} cells missing (no result.json): `repo report` lists them") + lines += [f"uptake: {text} used what its arm added in {u['used']}/{u['cells']} cell(s)" + for text, u in summary.get("uptake", {}).items()] lines += [f"warning: {w}" for w in summary["warnings"]] return "\n".join(lines) diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index d6490b11..fc633e39 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -14,6 +14,9 @@ ``host`` requirements (``unsupported.json``, with the reason). They are neither failures nor missing. - cost and turns are means over finished cells, as the harness reported them. +- **uptake**: for an arm that added a file or an MCP server, the cells whose + agent used it (``result.json`` ``uptake``, see :mod:`.uptake`) over the + cells with a trace to count from. A warning is printed when the trace says a model other than the requested one answered a cell (``model_resolved``): a harness alias can pin an older model. @@ -90,6 +93,9 @@ def summarize(batch_dir: Path) -> dict[str, Any]: "turns": _mean([r["agent"]["num_turns"] for r in recs]), "networks": sorted({str(r["agent"].get("network")) for r in recs}), "model_resolved": sorted({m for r in recs for m in resolved_models(r)})} + uptake = uptake_counts(recs) + if uptake is not None: + row["uptake"] = uptake if score_only: scores = [0.0 if r["grade"]["status"] == "grader_failure" else r["grade"]["score"] for r in recs] row.update(kind="score", score=_mean([0.0 if s is None else s for s in scores])) @@ -125,6 +131,8 @@ def format_report(summary: dict[str, Any]) -> str: f"{result:<22} {r['grader_failures']:>11} {r['missing']:>7} {cost:>8} {turns:>6}") noted = ("agent_timeouts", "agent_errors", "over_budget", "cost_unknown") notes = [f"{r[k]} {k.replace('_', ' ')}" for k in noted if r[k]] + if "uptake" in r: + notes.append(f"arm's additions used in {r['uptake']['used']}/{r['uptake']['cells']} cell(s)") if notes: lines.append(" " * (width + 2) + " " + ", ".join(notes)) if summary["missing"]: @@ -147,6 +155,17 @@ def _unsupported_lines(cells: list[dict[str, Any]]) -> list[str]: for (task, reason), n in sorted(by_task.items())] +def uptake_counts(recs: list[dict[str, Any]]) -> dict[str, int] | None: + """``cells`` with a measured ``uptake`` and how many of them ``used`` what the arm added; ``None`` when no + cell's arm added anything to use.""" + found = [r["uptake"] for r in recs if isinstance(r.get("uptake"), dict)] + if not found: + return None + measured = [u for u in found if u.get("measured")] + return {"cells": len(measured), "used": sum(bool(u.get("used")) for u in measured), + "unmeasured": len(found) - len(measured)} + + def resolved_models(rec: dict[str, Any]) -> list[str]: """The models a cell's trace says answered it (``model_resolved``); empty when the trace could not tell.""" found = rec.get("model_resolved") diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index d6f3dfc1..ad7750ea 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -60,6 +60,7 @@ docker_network, ) from .events import convert +from .uptake import arm_uptake from .grading import EvalContext, failed_outcome, outcome, run_entry from .harness_profile import HarnessProfile from .host import probe_host, unmet, validate_tasks @@ -192,6 +193,9 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, "input_tokens": None, "output_tokens": None, "cache_read_tokens": None, "cache_write_tokens": None, "usage_scope": None, "subagent_calls": None, "plugins": [], "over_budget": False, "caps": caps, "network": network, "trace_errors": []} + added = meta.get("arm_patch") or {} + uptake_of = (added.get("added") or [], added.get("mcp_servers") or []) + events = None # the trace's events once it parsed; none: the uptake is unmeasured try: raw = (rd / "agent.stdout").read_text(encoding="utf-8", errors="replace") transcript = convert(opts.profile.trace, raw, WORKSPACE) @@ -201,6 +205,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, meta["harness_version"] = transcript.harness_version or opts.harness_version meta["model_resolved"] = transcript.model_resolved() # an alias can map `model` to another one _write_json(rd / "meta.json", meta) + if transcript.events or transcript.num_turns is not None: # else the agent never started + events = transcript.events cost = transcript.cost_usd agent.update( cost_usd=cost, charged_usd=cost if cost is not None else caps["budget_usd"], @@ -217,10 +223,12 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, # Excluding the run would favour arms that break workspaces (contract §4.2): it fails the task. what = "grade run failed" if isinstance(exc, ContainerError) else f"host error after the agent ran ({type(exc).__name__})" grade = failed_outcome(f"{what}: {str(exc)[:300]}") - record = {"task": task.id, "family": task.family, "arm": arm, "model": opts.model, - "model_resolved": meta.get("model_resolved"), "rep": rep, - "harness": opts.profile.name, "harness_version": meta["harness_version"], - "base_date": meta["base_date"], "score_task": bool(task.score), "agent": agent, "grade": grade} + record: dict[str, Any] = {"task": task.id, "family": task.family, "arm": arm, "model": opts.model, + "model_resolved": meta.get("model_resolved"), "rep": rep, + "harness": opts.profile.name, "harness_version": meta["harness_version"], + "base_date": meta["base_date"], "score_task": bool(task.score), "agent": agent, "grade": grade} + if any(uptake_of): # an arm that added something to use: did the agent use it (#41 D4) + record["uptake"] = arm_uptake(events, *uptake_of, WORKSPACE) try: _own_dir(rd) # the grader had the run dir read-write, and may have taken write access away _write_json(rd / "result.json", record) diff --git a/dokimi/dokimi/repos/uptake.py b/dokimi/dokimi/repos/uptake.py new file mode 100644 index 00000000..345ade48 --- /dev/null +++ b/dokimi/dokimi/repos/uptake.py @@ -0,0 +1,82 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Uptake: whether the agent of a cell used what its arm added (#41 D4). + +An arm that adds a tool (a script, a document, an MCP server) can move a score +through the guidance line that names the tool even if the tool is never +called. ``result.json`` therefore records, for a ``patch:`` arm that adds +something an agent can use, how often the agent used it: + +- **files**: the files the patch added that the guidance globs do not match + (a guidance file is loaded, not used); +- **commands**: command events whose text names one of those files, as a path + relative to the workspace (``tools/fmt``, ``./tools/fmt``, ``bash tools/fmt``) + or under the workspace's absolute path. A command run as ``cd tools && ./fmt`` + is not counted; +- **reads**: read events of one of those files; +- **mcp_servers** / **mcp_calls**: the servers of a ``.mcp.json`` the patch + added or changed, and the calls of their tools (``mcp____``). + Whether the harness starts them is the profile's business: a profile that + starts none gives 0 calls, which is what the warning is for. + +``used`` is whether any count is non-zero; ``measured`` is false when the agent +left no trace to count from (it never started, or its trace did not parse). +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +MCP_CONFIG = ".mcp.json" + + +def mcp_servers(ws: Path, arm_files: list[str]) -> list[str]: + """The server names of the ``.mcp.json`` at the workspace root, when the arm wrote it; read at prep, before + the agent can change it. A file that does not parse names no server.""" + if MCP_CONFIG not in arm_files: + return [] + try: + servers = json.loads((ws / MCP_CONFIG).read_text(encoding="utf-8")).get("mcpServers") + except (OSError, ValueError, AttributeError): + return [] + return sorted(str(k) for k in servers) if isinstance(servers, dict) else [] + + +def arm_uptake(events: list[dict[str, Any]] | None, files: list[str], servers: list[str], + workdir: str) -> dict[str, Any]: + """The ``uptake`` of ``result.json`` from the cell's normalized events (``None``: no trace).""" + out: dict[str, Any] = {"files": sorted(files), "mcp_servers": sorted(servers), "measured": events is not None, + "commands": 0, "reads": 0, "mcp_calls": 0, "used": False} + if events is None: + return out + named = _path_pattern(files, workdir) + prefixes = tuple(f"mcp__{_mcp_name(s)}__" for s in servers) + wanted = set(files) + for ev in events: + kind = ev.get("kind") + if kind == "command" and named is not None and named.search(str(ev.get("text", ""))): + out["commands"] += 1 + elif kind == "read" and ev.get("path") in wanted: + out["reads"] += 1 + elif kind == "tool" and prefixes and str(ev.get("name", "")).startswith(prefixes): + out["mcp_calls"] += 1 + out["used"] = bool(out["commands"] or out["reads"] or out["mcp_calls"]) + return out + + +def _path_pattern(files: list[str], workdir: str) -> re.Pattern[str] | None: + """One of ``files`` as a whole word of a command: relative (optionally ``./``) or under ``workdir``.""" + if not files: + return None + paths = "|".join(re.escape(f) for f in sorted(files, key=len, reverse=True)) + root = re.escape(workdir.rstrip("/") + "/") + return re.compile(rf"(?:^|(?<=[\s'\"`=;&|(<>])|(?<=\./)|(?<={root}))(?:{paths})(?=$|[\s'\"`;&|)<>:])") + + +def _mcp_name(server: str) -> str: + """A server name as Claude Code spells it in tool names: characters outside ``[A-Za-z0-9_-]`` become ``_``.""" + return re.sub(r"[^A-Za-z0-9_-]", "_", server) diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index 548a0a28..7d437710 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -52,6 +52,7 @@ from .harness_profile import HarnessProfile from .manifest import RepoManifest, _is_within, match_glob from .source import GitTree, RepoSource, RepoTree, SourceError, cache_path, default_cache_dir, resolve_rev +from .uptake import mcp_servers ARMS = ("guidance:none", "guidance:raw", "guidance:bridge") REV_ARM, PATCH_ARM = "guidance:rev:", "patch:" @@ -468,15 +469,22 @@ def _apply_patch(ws: Path, env: dict[str, str], manifest: RepoManifest, arm: Arm except PrepError as exc: raise PrepError(f"arm {arm}: patch {arm.patch_path} does not apply: {exc}") from exc _git(ws, env, "add", "-A") - changed = sorted(p for p in _git(ws, env, "diff", "--cached", "--name-only", "--no-renames", "-z", before).split("\0") - if p) + status = _git(ws, env, "diff", "--cached", "--name-status", "--no-renames", "-z", before).split("\0") + changed = sorted(p for p in status[1::2] if p) eval_dir = manifest.run_paths()[0] # the patch's own paths too: a file .gitignore hides is written but never staged inside = sorted(p for p in set(changed) | touched if _is_within(p, eval_dir)) if inside: # the agent must not see an eval_dir, not even a patched-in one raise PrepError(f"arm {arm}: patch {arm.patch_path} writes into eval_dir {eval_dir}: {', '.join(inside)}") # a written file .gitignore hides is in the workspace, not in start_sha: still a file the arm wrote - return sorted(set(changed) | touched), {"arm_patch": {"path": arm.patch_path, "sha256": arm.patch_sha256}} + files = sorted(set(changed) | touched) + # -z output ends in a NUL, so the split has one trailing "" the pairing drops + new = {p for k, p in zip(status[::2], status[1::2], strict=False) if k == "A"} + new |= {p for p in touched - set(changed) if (ws / p).is_file()} + # what an agent can use (uptake): a file the patch added, not a guidance file; the servers of its .mcp.json + added = sorted(p for p in new if not any(match_glob(g, p) for g in manifest.guidance_globs)) + return files, {"arm_patch": {"path": arm.patch_path, "sha256": arm.patch_sha256, "added": added, + "mcp_servers": mcp_servers(ws, files)}} def _patch_paths(numstat: str) -> set[str]: diff --git a/dokimi/tests/repo_eval_fixture.py b/dokimi/tests/repo_eval_fixture.py index 987079fc..6afa0c02 100644 --- a/dokimi/tests/repo_eval_fixture.py +++ b/dokimi/tests/repo_eval_fixture.py @@ -102,6 +102,10 @@ lines.append({"type": "assistant", "message": {"model": answered, "content": [ {"type": "tool_use", "id": "t1", "name": "Edit", "input": {"file_path": os.environ.get("FAKE_CONTAINER_CWD", str(cwd)) + "/src/calc.py"}}]}}) +if os.environ.get("FAKE_TOOL_USES"): # more tool calls, as a JSON list of [name, input] + lines.append({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": f"u{i}", "name": n, "input": a} + for i, (n, a) in enumerate(json.loads(os.environ["FAKE_TOOL_USES"]))]}}) result = {"type": "result", "subtype": "success", "result": "done: src/calc.py", "total_cost_usd": 0.01, "num_turns": 3, "is_error": False} if os.environ.get("FAKE_USAGE"): # a subagent (Agent tool) and the token counts of the whole tree @@ -120,7 +124,8 @@ "invoke": ["{bin}", "-p", "{prompt}", "--model", "{model}", "--max-turns", "{max_turns}", "--max-budget-usd", "{budget}", "{tools}"], "tools": {"none": ["--tools", ""], "read": ["--tools", "Read"], "all": []}, - "config_dir_env": "FAKE_CONFIG_DIR", "env_allow": ["FAKE_ACTION", "FAKE_ANSWER_MODEL", "FAKE_MARKER", "FAKE_USAGE", "PATH"], + "config_dir_env": "FAKE_CONFIG_DIR", + "env_allow": ["FAKE_ACTION", "FAKE_ANSWER_MODEL", "FAKE_MARKER", "FAKE_USAGE", "FAKE_TOOL_USES", "PATH"], "egress": ["api.example.com"], "trace": "claude-stream-json", "prompt_suffix": "Stay in the foreground.", "config_files": {"settings.json": '{"hook": "python3 {support_dir}/no_background.py"}'}, diff --git a/dokimi/tests/test_repo_arms.py b/dokimi/tests/test_repo_arms.py index dbc32d47..0aa73872 100644 --- a/dokimi/tests/test_repo_arms.py +++ b/dokimi/tests/test_repo_arms.py @@ -171,7 +171,8 @@ def test_patch_arm_applies_on_raw_guidance_and_records_the_file(toy): patch.write_text("changed after the batch read it\n") # every cell applies the bytes read once meta, ws = _prep(toy, arm) assert meta["arm"] == str(arm) and meta["arm_files"] == ["AGENTS.md", "Makefile"] - assert meta["arm_patch"] == {"path": str(patch.resolve()), "sha256": digest} + assert meta["arm_patch"] == {"path": str(patch.resolve()), "sha256": digest, "added": ["Makefile"], + "mcp_servers": []} assert (ws / "AGENTS.md").read_text() == "# guidance\nRun `make check`.\n" assert (ws / "Makefile").is_file() and (ws / "sub/AGENTS.md").is_file() # the rest of guidance:raw assert git(ws, "status", "--porcelain") == "" and git(ws, "rev-list", "--count", "HEAD") == "1" diff --git a/dokimi/tests/test_repo_uptake.py b/dokimi/tests/test_repo_uptake.py new file mode 100644 index 00000000..eb4fa451 --- /dev/null +++ b/dokimi/tests/test_repo_uptake.py @@ -0,0 +1,121 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Uptake of what a `patch:` arm added (#41 D4): prep records it, the runner counts it, report and compare show it. +No docker, no LLM.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos.compare import compare, format_compare +from dokimi.repos.report import format_report, summarize +from dokimi.repos.uptake import arm_uptake +from dokimi.repos.workspace import parse_arm +from test_repo_arms import setup, toy, _prep # noqa: F401 # fixtures + +KIT_PATCH = """diff --git a/tools/fmt b/tools/fmt +new file mode 100755 +--- /dev/null ++++ b/tools/fmt +@@ -0,0 +1,2 @@ ++#!/bin/sh ++exec clang-format -i "$@" +diff --git a/.mcp.json b/.mcp.json +new file mode 100644 +--- /dev/null ++++ b/.mcp.json +@@ -0,0 +1 @@ ++{"mcpServers": {"code.graph": {"command": "cg-mcp"}}} +diff --git a/sub/AGENTS.md b/sub/AGENTS.md +--- a/sub/AGENTS.md ++++ b/sub/AGENTS.md +@@ -1 +1,2 @@ + # sub guidance ++Format with `tools/fmt`; ask the code.graph tools who calls a function. +diff --git a/src/AGENTS.md b/src/AGENTS.md +new file mode 100644 +--- /dev/null ++++ b/src/AGENTS.md +@@ -0,0 +1 @@ ++# src guidance +""" + +WS = "/eval/run/workspace" + + +def _cmd(text): + return {"kind": "command", "text": text, "rc": 0} + + +@pytest.mark.parametrize("text", ["tools/fmt src/a.cpp", "./tools/fmt src/a.cpp", "sh tools/fmt", "x=1; tools/fmt", + f"{WS}/tools/fmt a", "bash -c 'tools/fmt a'", "(tools/fmt)"]) +def test_a_command_that_names_an_added_file_is_a_use(text): + assert arm_uptake([_cmd(text)], ["tools/fmt"], [], WS)["commands"] == 1 + + +@pytest.mark.parametrize("text", ["tools/fmt2 a", "mytools/fmt a", "cd tools && ./fmt a", "tools/fmt.bak", + "/elsewhere/tools/fmt"]) +def test_a_command_that_does_not_name_an_added_file_is_not(text): + assert arm_uptake([_cmd(text)], ["tools/fmt"], [], WS)["commands"] == 0 + + +def test_reads_and_calls_of_the_arms_mcp_servers_count_others_do_not(): + events = [{"kind": "read", "path": "tools/fmt"}, {"kind": "read", "path": "src/a.cpp"}, + {"kind": "tool", "name": "mcp__code_graph__callers"}, {"kind": "tool", "name": "mcp__other__x"}, + {"kind": "tool", "name": "Grep"}] + got = arm_uptake(events, ["tools/fmt"], ["code.graph"], WS) + assert (got["reads"], got["mcp_calls"], got["commands"], got["used"], got["measured"]) == (1, 1, 0, True, True) + assert arm_uptake([], ["tools/fmt"], [], WS)["used"] is False + unmeasured = arm_uptake(None, ["tools/fmt"], [], WS) + assert unmeasured["measured"] is False and unmeasured["used"] is False + + +def test_prep_records_the_added_files_that_are_not_guidance_and_the_mcp_servers(toy): # noqa: F811 + patch = toy["tmp"] / "kit.diff" + patch.write_text(KIT_PATCH) + meta, ws = _prep(toy, parse_arm(f"patch:{patch}")) + assert meta["arm_files"] == [".mcp.json", "src/AGENTS.md", "sub/AGENTS.md", "tools/fmt"] + # a changed guidance file and an added one are loaded, not used; the rest is what the agent can use + assert meta["arm_patch"]["added"] == [".mcp.json", "tools/fmt"] + assert meta["arm_patch"]["mcp_servers"] == ["code.graph"] + + +def _run(s, batch, *arms): + argv = ["repo", "run", f"{s['root']}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--harness", + str(s["profile"]), "--bin", str(s["exe"]), "--model", "m-1", "--batch", batch, "--out", str(s["out"]), + "--cache-root", str(s["out"] / "caches"), "--agent-network", "host", "--task", "fix_add", + "--reps", "2", *[x for a in arms for x in ("--arm", a)]] + args = build_parser().parse_args(argv) + args.func(args) + + +def test_repo_run_records_uptake_and_compare_warns_when_nothing_was_used(setup, monkeypatch): # noqa: F811 + patch = setup["tmp"] / "kit.diff" + patch.write_text(KIT_PATCH) + arm = str(parse_arm(f"patch:{patch}")) + _run(setup, "unused", "guidance:raw", f"patch:{patch}") + monkeypatch.setenv("FAKE_TOOL_USES", json.dumps([["Bash", {"command": "./tools/fmt src/calc.py"}], + ["mcp__code_graph__callers", {"symbol": "add"}]])) + _run(setup, "used", "guidance:raw", f"patch:{patch}") + + used = setup["out"] / "used" + result = json.loads(next(used.glob("*patch*__r1/result.json")).read_text()) + assert result["uptake"] == {"files": [".mcp.json", "tools/fmt"], "mcp_servers": ["code.graph"], "measured": True, + "commands": 1, "reads": 0, "mcp_calls": 1, "used": True} + assert "uptake" not in json.loads(next(used.glob("*guidance-raw__*__r1/result.json")).read_text()) + assert "arm's additions used in 2/2 cell(s)" in format_report(summarize(used)) + + unused = setup["out"] / "unused" + assert "arm's additions used in 0/2 cell(s)" in format_report(summarize(unused)) + quiet = compare([used], "guidance:raw", [arm]) + assert quiet["uptake"] == {arm: {"cells": 2, "used": 2, "unmeasured": 0}} + assert not [w for w in quiet["warnings"] if "used none" in w] + loud = compare([unused], "guidance:raw", [arm]) + assert [w for w in loud["warnings"] if "used none" in w] == [ + f"{arm}: the agent used none of what the arm added in 2 cell(s); a delta measures the patch's text, " + "not the tool"] + assert f"uptake: {arm} used what its arm added in 0/2 cell(s)" in format_compare(loud) From cb94f92b05122647159f08a0dfb03b08d8e2b660 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 21 Oct 2026 20:47:20 -0700 Subject: [PATCH 2/6] fix(repo): read .mcp.json for uptake only inside the workspace A patch: arm can add .mcp.json as a symlink. mcp_servers followed it, so prep read a host file outside the workspace into meta.json, and a link to /dev/zero read without end. Resolve the path first and read it only when it stays in the workspace and is a regular file; a link loop names none. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/repos/uptake.py | 10 +++++++--- dokimi/tests/test_repo_uptake.py | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/uptake.py b/dokimi/dokimi/repos/uptake.py index 345ade48..9965d897 100644 --- a/dokimi/dokimi/repos/uptake.py +++ b/dokimi/dokimi/repos/uptake.py @@ -36,12 +36,16 @@ def mcp_servers(ws: Path, arm_files: list[str]) -> list[str]: """The server names of the ``.mcp.json`` at the workspace root, when the arm wrote it; read at prep, before - the agent can change it. A file that does not parse names no server.""" + the agent can change it. A file that does not parse names no server, nor does a link out of the workspace + or to anything but a regular file: it is not read.""" if MCP_CONFIG not in arm_files: return [] try: - servers = json.loads((ws / MCP_CONFIG).read_text(encoding="utf-8")).get("mcpServers") - except (OSError, ValueError, AttributeError): + path = (ws / MCP_CONFIG).resolve() + if not path.is_relative_to(ws.resolve()) or not path.is_file(): + return [] + servers = json.loads(path.read_text(encoding="utf-8")).get("mcpServers") + except (OSError, RuntimeError, ValueError, AttributeError): # RuntimeError: a link loop, before Python 3.13 return [] return sorted(str(k) for k in servers) if isinstance(servers, dict) else [] diff --git a/dokimi/tests/test_repo_uptake.py b/dokimi/tests/test_repo_uptake.py index eb4fa451..b3d5eb7b 100644 --- a/dokimi/tests/test_repo_uptake.py +++ b/dokimi/tests/test_repo_uptake.py @@ -84,6 +84,30 @@ def test_prep_records_the_added_files_that_are_not_guidance_and_the_mcp_servers( assert meta["arm_patch"]["mcp_servers"] == ["code.graph"] +def _mcp_link_patch(target, extra=""): + return ("diff --git a/.mcp.json b/.mcp.json\nnew file mode 120000\n--- /dev/null\n+++ b/.mcp.json\n" + f"@@ -0,0 +1 @@\n+{target}\n\\ No newline at end of file\n{extra}") + + +@pytest.mark.parametrize("where", ["host", "loop"]) +def test_a_mcp_json_linked_out_of_the_workspace_or_in_a_loop_is_not_read(toy, where): # noqa: F811 + host = toy["tmp"] / "host.json" # outside the workspace: what the agent's harness would not find + host.write_text('{"mcpServers": {"host-only": {}}}') + patch = toy["tmp"] / "link.diff" + patch.write_text(_mcp_link_patch(host if where == "host" else ".mcp.json")) + meta, ws = _prep(toy, parse_arm(f"patch:{patch}")) + assert (ws / ".mcp.json").is_symlink() and meta["arm_patch"]["mcp_servers"] == [] + + +def test_a_mcp_json_linked_inside_the_workspace_names_its_servers(toy): # noqa: F811 + patch = toy["tmp"] / "link.diff" + patch.write_text(_mcp_link_patch("tools/mcp.json", "diff --git a/tools/mcp.json b/tools/mcp.json\n" + "new file mode 100644\n--- /dev/null\n+++ b/tools/mcp.json\n@@ -0,0 +1 @@\n" + '+{"mcpServers": {"code.graph": {}}}\n')) + meta, _ = _prep(toy, parse_arm(f"patch:{patch}")) + assert meta["arm_patch"]["mcp_servers"] == ["code.graph"] + + def _run(s, batch, *arms): argv = ["repo", "run", f"{s['root']}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--harness", str(s["profile"]), "--bin", str(s["exe"]), "--model", "m-1", "--batch", batch, "--out", str(s["out"]), From 179faec83aeaef3cbd0f3ead53b830678bab183d Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 21 Oct 2026 20:50:36 -0700 Subject: [PATCH 3/6] fix(repo): a call the harness denied is not uptake arm_uptake counted command, read and tool events whatever their denied flag, so a call a PreToolUse hook or permission_denials refused gave used: true and hid the "used none" warning. Skip denied events. Signed-off-by: Alexander Rybkin --- ...0261022_090000_napetrov_repo-arm-uptake.md | 3 ++- dokimi/docs/repo-evaluation-contract.md | 3 ++- dokimi/dokimi/repos/uptake.py | 7 +++++-- dokimi/tests/test_repo_uptake.py | 20 +++++++++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md b/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md index a468d648..273520ba 100644 --- a/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md +++ b/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md @@ -4,7 +4,8 @@ lists the files the patch `added` (guidance files left out) and the `mcp_servers` of a `.mcp.json` it wrote, and `result.json` gets an `uptake` object: the agent's commands naming an added file, reads of one, and calls of - those servers' tools. `repo report` shows the cells that used the arm's + those servers' tools, leaving out calls the harness denied. + `repo report` shows the cells that used the arm's additions per row; `repo compare` lists uptake per treatment and warns when a treatment's cells used none of it, since its deltas then measure the patch's text, not the tool (#41 D4). diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index edca8175..ec13a9de 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -379,7 +379,8 @@ For a `patch:` arm with `added` files or `mcp_servers`, `result.json` has an command events that name an added file as a workspace path (`tools/fmt`, `./tools/fmt`, `/eval/run/workspace/tools/fmt`, not `cd tools && ./fmt`), `reads` the read events of one, `mcp_calls` the calls of the servers' tools -(`mcp____`); `used` is whether any is non-zero, and `measured` is +(`mcp____`), none of them a call the harness denied; `used` +is whether any is non-zero, and `measured` is `false` when the agent left no trace to count from. `repo report` shows the cells that used it per row, and `repo compare` warns about a treatment whose cells used none of it: its deltas then measure the patch's text, such as a diff --git a/dokimi/dokimi/repos/uptake.py b/dokimi/dokimi/repos/uptake.py index 9965d897..19381f7a 100644 --- a/dokimi/dokimi/repos/uptake.py +++ b/dokimi/dokimi/repos/uptake.py @@ -20,8 +20,9 @@ Whether the harness starts them is the profile's business: a profile that starts none gives 0 calls, which is what the warning is for. -``used`` is whether any count is non-zero; ``measured`` is false when the agent -left no trace to count from (it never started, or its trace did not parse). +A call the harness denied (``denied``) is not counted. ``used`` is whether any +count is non-zero; ``measured`` is false when the agent left no trace to count +from (it never started, or its trace did not parse). """ from __future__ import annotations @@ -61,6 +62,8 @@ def arm_uptake(events: list[dict[str, Any]] | None, files: list[str], servers: l prefixes = tuple(f"mcp__{_mcp_name(s)}__" for s in servers) wanted = set(files) for ev in events: + if ev.get("denied"): # the harness refused the call: the tool never ran + continue kind = ev.get("kind") if kind == "command" and named is not None and named.search(str(ev.get("text", ""))): out["commands"] += 1 diff --git a/dokimi/tests/test_repo_uptake.py b/dokimi/tests/test_repo_uptake.py index b3d5eb7b..97189af5 100644 --- a/dokimi/tests/test_repo_uptake.py +++ b/dokimi/tests/test_repo_uptake.py @@ -12,6 +12,7 @@ from dokimi.cli import build_parser from dokimi.repos.compare import compare, format_compare +from dokimi.repos.events import convert from dokimi.repos.report import format_report, summarize from dokimi.repos.uptake import arm_uptake from dokimi.repos.workspace import parse_arm @@ -74,6 +75,25 @@ def test_reads_and_calls_of_the_arms_mcp_servers_count_others_do_not(): assert unmeasured["measured"] is False and unmeasured["used"] is False +def test_a_call_the_harness_denied_is_not_a_use(): + def use(i, name, args): + return {"type": "assistant", "message": {"content": [{"type": "tool_use", "id": i, "name": name, "input": args}]}} + hook = "PreToolUse:Bash hook error: [python3 /eval/support/no_background.py]: run_in_background is refused" + stream = [{"type": "system", "subtype": "init", "model": "m-1"}, + use("b1", "Bash", {"command": "./tools/fmt src/calc.py", "run_in_background": True}), + {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "b1", "is_error": True, + "content": hook}]}}, + use("m1", "mcp__code_graph__callers", {"symbol": "add"}), + {"type": "result", "subtype": "success", "result": "done", "num_turns": 2, + "permission_denials": [{"tool_use_id": "m1", "tool_name": "mcp__code_graph__callers"}]}] + events = convert("claude-stream-json", "\n".join(json.dumps(r) for r in stream), WS).events + assert [e.get("denied") for e in events] == [True, True] + got = arm_uptake(events, ["tools/fmt"], ["code.graph"], WS) + assert (got["commands"], got["mcp_calls"], got["used"], got["measured"]) == (0, 0, False, True) + retried = arm_uptake([*events, _cmd("./tools/fmt src/calc.py")], ["tools/fmt"], ["code.graph"], WS) + assert (retried["commands"], retried["used"]) == (1, True) + + def test_prep_records_the_added_files_that_are_not_guidance_and_the_mcp_servers(toy): # noqa: F811 patch = toy["tmp"] / "kit.diff" patch.write_text(KIT_PATCH) From 416a6ed490df7d2d7cda9b774a83ebaa5be8158c Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 21 Oct 2026 20:52:09 -0700 Subject: [PATCH 4/6] test(repo): pin which cells uptake measures and which files count as added No test failed when the runner's measured gate, report's measured filter, compare's cells guard or --model filter, the A-only added rule, the gitignored added file or the .mcp.json-in-arm-files guard was removed. The fake harness gains a silent and an idle action for the gate. Signed-off-by: Alexander Rybkin --- dokimi/tests/repo_eval_fixture.py | 9 ++-- dokimi/tests/test_repo_uptake.py | 76 +++++++++++++++++++++++++++++-- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/dokimi/tests/repo_eval_fixture.py b/dokimi/tests/repo_eval_fixture.py index 6afa0c02..e5186c3a 100644 --- a/dokimi/tests/repo_eval_fixture.py +++ b/dokimi/tests/repo_eval_fixture.py @@ -71,6 +71,8 @@ assert os.environ.get("FAKE_CONFIG_DIR"), "config dir not set" assert "SECRET_NOT_ALLOWED" not in os.environ, "environment not scrubbed" action = os.environ.get("FAKE_ACTION", "fix") +if action == "silent": # the agent dies before it writes a line + sys.exit(1) cwd = Path.cwd() # Guard against --model missing or being the last arg try: @@ -99,9 +101,10 @@ hook.chmod(0o755) with open(cwd / ".git" / "config", "a") as f: f.write("[core]\n\tfsmonitor = " + str(hook) + "\n") -lines.append({"type": "assistant", "message": {"model": answered, "content": [ - {"type": "tool_use", "id": "t1", "name": "Edit", - "input": {"file_path": os.environ.get("FAKE_CONTAINER_CWD", str(cwd)) + "/src/calc.py"}}]}}) +if action != "idle": # idle: the agent starts and ends without a tool call + lines.append({"type": "assistant", "message": {"model": answered, "content": [ + {"type": "tool_use", "id": "t1", "name": "Edit", + "input": {"file_path": os.environ.get("FAKE_CONTAINER_CWD", str(cwd)) + "/src/calc.py"}}]}}) if os.environ.get("FAKE_TOOL_USES"): # more tool calls, as a JSON list of [name, input] lines.append({"type": "assistant", "message": {"content": [ {"type": "tool_use", "id": f"u{i}", "name": n, "input": a} diff --git a/dokimi/tests/test_repo_uptake.py b/dokimi/tests/test_repo_uptake.py index 97189af5..8f92b019 100644 --- a/dokimi/tests/test_repo_uptake.py +++ b/dokimi/tests/test_repo_uptake.py @@ -14,7 +14,7 @@ from dokimi.repos.compare import compare, format_compare from dokimi.repos.events import convert from dokimi.repos.report import format_report, summarize -from dokimi.repos.uptake import arm_uptake +from dokimi.repos.uptake import arm_uptake, mcp_servers from dokimi.repos.workspace import parse_arm from test_repo_arms import setup, toy, _prep # noqa: F401 # fixtures @@ -128,9 +128,44 @@ def test_a_mcp_json_linked_inside_the_workspace_names_its_servers(toy): # noqa: assert meta["arm_patch"]["mcp_servers"] == ["code.graph"] -def _run(s, batch, *arms): +MIXED_PATCH = """diff --git a/src/calc.py b/src/calc.py +--- a/src/calc.py ++++ b/src/calc.py +@@ -1,2 +1,3 @@ ++# checked + def add(a, b): + return a - b +diff --git a/.gitignore b/.gitignore +new file mode 100644 +--- /dev/null ++++ b/.gitignore +@@ -0,0 +1 @@ ++gen.sh +diff --git a/gen.sh b/gen.sh +new file mode 100755 +--- /dev/null ++++ b/gen.sh +@@ -0,0 +1 @@ ++#!/bin/sh +""" + + +def test_prep_records_as_added_a_file_gitignore_hides_not_a_file_the_patch_changed(toy): # noqa: F811 + patch = toy["tmp"] / "mixed.diff" + patch.write_text(MIXED_PATCH) + meta, ws = _prep(toy, parse_arm(f"patch:{patch}")) + assert meta["arm_files"] == [".gitignore", "gen.sh", "src/calc.py"] and (ws / "gen.sh").is_file() + assert meta["arm_patch"]["added"] == [".gitignore", "gen.sh"] + + +def test_a_mcp_json_the_patch_did_not_write_names_no_server(tmp_path): + (tmp_path / ".mcp.json").write_text('{"mcpServers": {"from-base": {}}}') + assert mcp_servers(tmp_path, ["tools/fmt"]) == [] and mcp_servers(tmp_path, [".mcp.json"]) == ["from-base"] + + +def _run(s, batch, *arms, model="m-1"): argv = ["repo", "run", f"{s['root']}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--harness", - str(s["profile"]), "--bin", str(s["exe"]), "--model", "m-1", "--batch", batch, "--out", str(s["out"]), + str(s["profile"]), "--bin", str(s["exe"]), "--model", model, "--batch", batch, "--out", str(s["out"]), "--cache-root", str(s["out"] / "caches"), "--agent-network", "host", "--task", "fix_add", "--reps", "2", *[x for a in arms for x in ("--arm", a)]] args = build_parser().parse_args(argv) @@ -163,3 +198,38 @@ def test_repo_run_records_uptake_and_compare_warns_when_nothing_was_used(setup, f"{arm}: the agent used none of what the arm added in 2 cell(s); a delta measures the patch's text, " "not the tool"] assert f"uptake: {arm} used what its arm added in 0/2 cell(s)" in format_compare(loud) + + +def _used_none(summary): + return [w for w in summary["warnings"] if "used none" in w] + + +def test_an_agent_that_left_no_trace_is_unmeasured_one_that_called_no_tool_is_measured(setup, monkeypatch): # noqa: F811 + patch = setup["tmp"] / "kit.diff" + patch.write_text(KIT_PATCH) + arm = str(parse_arm(f"patch:{patch}")) + for action in ("silent", "idle"): + monkeypatch.setenv("FAKE_ACTION", action) + _run(setup, action, "guidance:raw", f"patch:{patch}") + silent, idle = setup["out"] / "silent", setup["out"] / "idle" + assert json.loads(next(silent.glob("*patch*__r1/result.json")).read_text())["uptake"]["measured"] is False + assert json.loads(next(idle.glob("*patch*__r1/result.json")).read_text())["uptake"]["measured"] is True + unmeasured = compare([silent], "guidance:raw", [arm]) + assert unmeasured["uptake"] == {arm: {"cells": 0, "used": 0, "unmeasured": 2}} and not _used_none(unmeasured) + measured = compare([idle], "guidance:raw", [arm]) + assert measured["uptake"] == {arm: {"cells": 2, "used": 0, "unmeasured": 0}} and _used_none(measured) + + +def test_compare_counts_the_uptake_of_the_models_it_compares(setup, monkeypatch): # noqa: F811 + patch = setup["tmp"] / "kit.diff" + patch.write_text(KIT_PATCH) + arm = str(parse_arm(f"patch:{patch}")) + monkeypatch.setenv("FAKE_TOOL_USES", json.dumps([["Bash", {"command": "./tools/fmt src/calc.py"}]])) + _run(setup, "m1", "guidance:raw", f"patch:{patch}", model="m-1") + monkeypatch.delenv("FAKE_TOOL_USES") + _run(setup, "m2", "guidance:raw", f"patch:{patch}", model="m-2") + batches = [setup["out"] / "m1", setup["out"] / "m2"] + both = compare(batches, "guidance:raw", [arm]) + assert both["uptake"] == {arm: {"cells": 4, "used": 2, "unmeasured": 0}} and not _used_none(both) + m2 = compare(batches, "guidance:raw", [arm], models=["m-2"]) + assert m2["uptake"] == {arm: {"cells": 2, "used": 0, "unmeasured": 0}} and _used_none(m2) From 073963efb0a3a16e660f4b10f3f243d1d349e225 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 13:37:41 +0000 Subject: [PATCH 5/6] docs(repo): name the other uptake fields and compare's uptake summary Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index ec13a9de..cbc8df4b 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -375,16 +375,19 @@ Code's `modelUsage`), `main` when the harness reported the parent's only, and subagents' `Agent`/`Task` calls; `null` when the harness wrote no result record. For a `patch:` arm with `added` files or `mcp_servers`, `result.json` has an -`uptake` object: whether the agent used what the arm added. `commands` counts +`uptake` object: whether the agent used what the arm added. `files` and +`mcp_servers` list what it added (sorted); `commands` counts command events that name an added file as a workspace path (`tools/fmt`, `./tools/fmt`, `/eval/run/workspace/tools/fmt`, not `cd tools && ./fmt`), `reads` the read events of one, `mcp_calls` the calls of the servers' tools (`mcp____`), none of them a call the harness denied; `used` is whether any is non-zero, and `measured` is `false` when the agent left no trace to count from. `repo report` shows the -cells that used it per row, and `repo compare` warns about a treatment whose -cells used none of it: its deltas then measure the patch's text, such as a -guidance line, not the tool. +cells that used it per row. `repo compare` puts an `uptake` entry per treatment +in its summary (`{cells, used, unmeasured}`: the measured cells, how many of +them used it, and how many could not be measured), prints it as an `uptake: ...` +line, and warns about a treatment whose cells used none of it: its deltas then +measure the patch's text, such as a guidance line, not the tool. ### 4.2 `grade.json` (written by `grade` to `{run_dir}/grade.json`) From 3117e662785d5e13158f38496145ad79b7674a6c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 08:10:09 -0700 Subject: [PATCH 6/6] fix(repo): uptake counts only servers the arm added and names root-level files strictly .mcp.json is no longer an added file, mcp_servers excludes servers the base declared, a slash-less added file matches only as ./name, under the workspace path, first in a command or after an interpreter, and the text report and compare show cells that left no trace to measure. Signed-off-by: Nikolay Petrov --- ...0261022_090000_napetrov_repo-arm-uptake.md | 4 +- dokimi/docs/repo-evaluation-contract.md | 12 +++-- dokimi/dokimi/repos/compare.py | 5 +- dokimi/dokimi/repos/report.py | 8 ++- dokimi/dokimi/repos/uptake.py | 51 ++++++++++++++----- dokimi/dokimi/repos/workspace.py | 11 ++-- dokimi/tests/test_repo_uptake.py | 46 +++++++++++++++-- 7 files changed, 109 insertions(+), 28 deletions(-) diff --git a/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md b/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md index 273520ba..ac08f0bf 100644 --- a/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md +++ b/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md @@ -1,8 +1,8 @@ ### Added - **repo run: uptake of what a `patch:` arm added** — `meta.json` `arm_patch` - lists the files the patch `added` (guidance files left out) and the - `mcp_servers` of a `.mcp.json` it wrote, and `result.json` gets an `uptake` + lists the files the patch `added` (guidance files and `.mcp.json` left out) and the + `mcp_servers` it added to a `.mcp.json`, and `result.json` gets an `uptake` object: the agent's commands naming an added file, reads of one, and calls of those servers' tools, leaving out calls the harness denied. `repo report` shows the cells that used the arm's diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index cbc8df4b..a376567f 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -321,8 +321,8 @@ grader diffs the agent's changes from `start_sha`. With the arm come `arm_rev` object (`rev` as given, its `sha`, and `removed`: the base's guidance files with no counterpart at ``); for `patch:` an `arm_patch` object (the file's absolute `path` and full `sha256`, `added`: the files the patch added -that the guidance globs do not match, and `mcp_servers`: the servers of a -`.mcp.json` at the root the patch added or changed, read before the agent runs). +that neither the guidance globs nor `.mcp.json` match, and `mcp_servers`: the servers +the patch added to the `.mcp.json` at the root, read before the agent runs). Step 3 removes history only. A fix that landed upstream before a model's training cutoff may still be known to the model, so reports split results by @@ -378,14 +378,16 @@ For a `patch:` arm with `added` files or `mcp_servers`, `result.json` has an `uptake` object: whether the agent used what the arm added. `files` and `mcp_servers` list what it added (sorted); `commands` counts command events that name an added file as a workspace path (`tools/fmt`, -`./tools/fmt`, `/eval/run/workspace/tools/fmt`, not `cd tools && ./fmt`), `reads` -the read events of one, `mcp_calls` the calls of the servers' tools +`./tools/fmt`, `/eval/run/workspace/tools/fmt`, not `cd tools && ./fmt`; a file +with no `/` only as `./name`, under the workspace path, first in a command or after an interpreter, so +`npm run lint` does not name a `lint` file), `reads` the read events of one (a +read of an added script counts as use), `mcp_calls` the calls of the servers' tools (`mcp____`), none of them a call the harness denied; `used` is whether any is non-zero, and `measured` is `false` when the agent left no trace to count from. `repo report` shows the cells that used it per row. `repo compare` puts an `uptake` entry per treatment in its summary (`{cells, used, unmeasured}`: the measured cells, how many of -them used it, and how many could not be measured), prints it as an `uptake: ...` +them used it, and how many could not be measured; the text lines say so), prints it as an `uptake: ...` line, and warns about a treatment whose cells used none of it: its deltas then measure the patch's text, such as a guidance line, not the tool. diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 0ff249e7..2207868f 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -57,7 +57,8 @@ from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci -from .report import load_cells, mismatch_warnings, model_mismatch, model_matches, resolved_models, uptake_counts +from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, resolved_models, unmeasured_note, + uptake_counts) #: Largest number of non-zero deltas whose sign-flip distribution is enumerated (2**16 patterns). EXACT_MAX = 16 @@ -415,7 +416,7 @@ def format_compare(summary: dict[str, Any]) -> str: + (f" and {len(summary['unpaired']) - len(shown)} more" if len(summary["unpaired"]) > 10 else "")) if summary["missing"]: lines.append(f"{len(summary['missing'])} cells missing (no result.json): `repo report` lists them") - lines += [f"uptake: {text} used what its arm added in {u['used']}/{u['cells']} cell(s)" + lines += [f"uptake: {text} used what its arm added in {u['used']}/{u['cells']} cell(s){unmeasured_note(u)}" for text, u in summary.get("uptake", {}).items()] lines += [f"warning: {w}" for w in summary["warnings"]] return "\n".join(lines) diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index fc633e39..576bb28d 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -132,7 +132,8 @@ def format_report(summary: dict[str, Any]) -> str: noted = ("agent_timeouts", "agent_errors", "over_budget", "cost_unknown") notes = [f"{r[k]} {k.replace('_', ' ')}" for k in noted if r[k]] if "uptake" in r: - notes.append(f"arm's additions used in {r['uptake']['used']}/{r['uptake']['cells']} cell(s)") + notes.append(f"arm's additions used in {r['uptake']['used']}/{r['uptake']['cells']} cell(s)" + + unmeasured_note(r["uptake"])) if notes: lines.append(" " * (width + 2) + " " + ", ".join(notes)) if summary["missing"]: @@ -155,6 +156,11 @@ def _unsupported_lines(cells: list[dict[str, Any]]) -> list[str]: for (task, reason), n in sorted(by_task.items())] +def unmeasured_note(u: dict[str, int]) -> str: + """What to append to a ``used/cells`` count when some cells left no trace to measure.""" + return f" (warning: {u['unmeasured']} more cell(s) left no trace to measure)" if u.get("unmeasured") else "" + + def uptake_counts(recs: list[dict[str, Any]]) -> dict[str, int] | None: """``cells`` with a measured ``uptake`` and how many of them ``used`` what the arm added; ``None`` when no cell's arm added anything to use.""" diff --git a/dokimi/dokimi/repos/uptake.py b/dokimi/dokimi/repos/uptake.py index 19381f7a..9be565d0 100644 --- a/dokimi/dokimi/repos/uptake.py +++ b/dokimi/dokimi/repos/uptake.py @@ -9,14 +9,17 @@ something an agent can use, how often the agent used it: - **files**: the files the patch added that the guidance globs do not match - (a guidance file is loaded, not used); + (a guidance file is loaded, not used; ``.mcp.json`` is a config whose servers are counted by their calls); - **commands**: command events whose text names one of those files, as a path relative to the workspace (``tools/fmt``, ``./tools/fmt``, ``bash tools/fmt``) - or under the workspace's absolute path. A command run as ``cd tools && ./fmt`` + or under the workspace's absolute path. A file at the workspace root is named only as ``./name``, under the + absolute path, as the first word of a command (or of a ``;``/``&&``/``|`` part) or after a shell or + interpreter (``sh run.sh``): ``npm run lint`` does not name a ``lint`` file. A command run as ``cd tools && ./fmt`` is not counted; -- **reads**: read events of one of those files; -- **mcp_servers** / **mcp_calls**: the servers of a ``.mcp.json`` the patch - added or changed, and the calls of their tools (``mcp____``). +- **reads**: read events of one of those files (reading a script's text counts: it is the agent engaging with + what the arm added, not only running it); +- **mcp_servers** / **mcp_calls**: the servers the patch added to the ``.mcp.json`` + (those the file already declared at the base are not the arm's), and the calls of their tools (``mcp____``). Whether the harness starts them is the profile's business: a profile that starts none gives 0 calls, which is what the warning is for. @@ -35,9 +38,18 @@ MCP_CONFIG = ".mcp.json" -def mcp_servers(ws: Path, arm_files: list[str]) -> list[str]: - """The server names of the ``.mcp.json`` at the workspace root, when the arm wrote it; read at prep, before - the agent can change it. A file that does not parse names no server, nor does a link out of the workspace +def base_servers(text: str) -> set[str]: + """The server names a ``.mcp.json`` text declares; none when it does not parse.""" + try: + servers = json.loads(text).get("mcpServers") + except (ValueError, AttributeError): + return set() + return {str(k) for k in servers} if isinstance(servers, dict) else set() + + +def mcp_servers(ws: Path, arm_files: list[str], base: set[str] | frozenset[str] = frozenset()) -> list[str]: + """The server names of the ``.mcp.json`` at the workspace root that the arm added: those it wrote, less the + ``base`` ones the file already declared before the patch; read at prep, before the agent can change it. A file that does not parse names no server, nor does a link out of the workspace or to anything but a regular file: it is not read.""" if MCP_CONFIG not in arm_files: return [] @@ -48,7 +60,7 @@ def mcp_servers(ws: Path, arm_files: list[str]) -> list[str]: servers = json.loads(path.read_text(encoding="utf-8")).get("mcpServers") except (OSError, RuntimeError, ValueError, AttributeError): # RuntimeError: a link loop, before Python 3.13 return [] - return sorted(str(k) for k in servers) if isinstance(servers, dict) else [] + return sorted(str(k) for k in servers if str(k) not in base) if isinstance(servers, dict) else [] def arm_uptake(events: list[dict[str, Any]] | None, files: list[str], servers: list[str], @@ -75,13 +87,28 @@ def arm_uptake(events: list[dict[str, Any]] | None, files: list[str], servers: l return out +_INTERPRETERS = r"(?:sh|bash|zsh|dash|python3?|node|ruby|perl)" + + def _path_pattern(files: list[str], workdir: str) -> re.Pattern[str] | None: - """One of ``files`` as a whole word of a command: relative (optionally ``./``) or under ``workdir``.""" + """One of ``files`` as a whole word of a command: relative (optionally ``./``) or under ``workdir``. + A file with no ``/`` is a common word, so it must be ``./``-prefixed, under ``workdir``, first in a command + or run by an interpreter.""" if not files: return None - paths = "|".join(re.escape(f) for f in sorted(files, key=len, reverse=True)) root = re.escape(workdir.rstrip("/") + "/") - return re.compile(rf"(?:^|(?<=[\s'\"`=;&|(<>])|(?<=\./)|(?<={root}))(?:{paths})(?=$|[\s'\"`;&|)<>:])") + end = r"(?=$|[\s'\"`;&|)<>:])" + parts = [] + nested = sorted((f for f in files if "/" in f), key=len, reverse=True) + flat = sorted((f for f in files if "/" not in f), key=len, reverse=True) + if nested: + paths = "|".join(re.escape(f) for f in nested) + parts.append(rf"(?:^|(?<=[\s'\"`=;&|(<>])|(?<=\./)|(?<={root}))(?:{paths}){end}") + if flat: + names = "|".join(re.escape(f) for f in flat) + parts.append(rf"(?:^\s*(?:\./)?|[;&|(]\s*(?:\./)?|(?:^|(?<=\s)){_INTERPRETERS}\s+|(?<=[\s'\"`=;&|(<>])\./|(?<={root}))" + rf"(?:{names}){end}") + return re.compile("|".join(parts)) def _mcp_name(server: str) -> str: diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index 7d437710..704dc916 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -52,7 +52,7 @@ from .harness_profile import HarnessProfile from .manifest import RepoManifest, _is_within, match_glob from .source import GitTree, RepoSource, RepoTree, SourceError, cache_path, default_cache_dir, resolve_rev -from .uptake import mcp_servers +from .uptake import MCP_CONFIG, base_servers, mcp_servers ARMS = ("guidance:none", "guidance:raw", "guidance:bridge") REV_ARM, PATCH_ARM = "guidance:rev:", "patch:" @@ -482,9 +482,14 @@ def _apply_patch(ws: Path, env: dict[str, str], manifest: RepoManifest, arm: Arm new = {p for k, p in zip(status[::2], status[1::2], strict=False) if k == "A"} new |= {p for p in touched - set(changed) if (ws / p).is_file()} # what an agent can use (uptake): a file the patch added, not a guidance file; the servers of its .mcp.json - added = sorted(p for p in new if not any(match_glob(g, p) for g in manifest.guidance_globs)) + # .mcp.json is not one: reading the config is not using a server, whose calls are counted as mcp_calls + added = sorted(p for p in new if p != MCP_CONFIG and not any(match_glob(g, p) for g in manifest.guidance_globs)) + try: # the servers the base already declared are not the arm's + base = base_servers(_git(ws, env, "show", f"{before}:{MCP_CONFIG}")) + except PrepError: + base = set() return files, {"arm_patch": {"path": arm.patch_path, "sha256": arm.patch_sha256, "added": added, - "mcp_servers": mcp_servers(ws, files)}} + "mcp_servers": mcp_servers(ws, files, base)}} def _patch_paths(numstat: str) -> set[str]: diff --git a/dokimi/tests/test_repo_uptake.py b/dokimi/tests/test_repo_uptake.py index 8f92b019..d9ab8c63 100644 --- a/dokimi/tests/test_repo_uptake.py +++ b/dokimi/tests/test_repo_uptake.py @@ -13,7 +13,7 @@ from dokimi.cli import build_parser from dokimi.repos.compare import compare, format_compare from dokimi.repos.events import convert -from dokimi.repos.report import format_report, summarize +from dokimi.repos.report import format_report, summarize, unmeasured_note from dokimi.repos.uptake import arm_uptake, mcp_servers from dokimi.repos.workspace import parse_arm from test_repo_arms import setup, toy, _prep # noqa: F401 # fixtures @@ -100,7 +100,7 @@ def test_prep_records_the_added_files_that_are_not_guidance_and_the_mcp_servers( meta, ws = _prep(toy, parse_arm(f"patch:{patch}")) assert meta["arm_files"] == [".mcp.json", "src/AGENTS.md", "sub/AGENTS.md", "tools/fmt"] # a changed guidance file and an added one are loaded, not used; the rest is what the agent can use - assert meta["arm_patch"]["added"] == [".mcp.json", "tools/fmt"] + assert meta["arm_patch"]["added"] == ["tools/fmt"] # not .mcp.json: reading the config is not using a server assert meta["arm_patch"]["mcp_servers"] == ["code.graph"] @@ -183,7 +183,7 @@ def test_repo_run_records_uptake_and_compare_warns_when_nothing_was_used(setup, used = setup["out"] / "used" result = json.loads(next(used.glob("*patch*__r1/result.json")).read_text()) - assert result["uptake"] == {"files": [".mcp.json", "tools/fmt"], "mcp_servers": ["code.graph"], "measured": True, + assert result["uptake"] == {"files": ["tools/fmt"], "mcp_servers": ["code.graph"], "measured": True, "commands": 1, "reads": 0, "mcp_calls": 1, "used": True} assert "uptake" not in json.loads(next(used.glob("*guidance-raw__*__r1/result.json")).read_text()) assert "arm's additions used in 2/2 cell(s)" in format_report(summarize(used)) @@ -233,3 +233,43 @@ def test_compare_counts_the_uptake_of_the_models_it_compares(setup, monkeypatch) assert both["uptake"] == {arm: {"cells": 4, "used": 2, "unmeasured": 0}} and not _used_none(both) m2 = compare(batches, "guidance:raw", [arm], models=["m-2"]) assert m2["uptake"] == {arm: {"cells": 2, "used": 0, "unmeasured": 0}} and _used_none(m2) + + +@pytest.mark.parametrize("text", ["npm run lint", "docker run img", "git log -- build", "cat lint", "make build"]) +def test_a_root_level_added_file_is_not_named_by_a_common_word_of_a_command(text): + for name in ("lint", "run", "build"): + assert arm_uptake([_cmd(text)], [name], [], WS)["commands"] == (1 if text.split()[0] == name else 0) + + +@pytest.mark.parametrize("text", ["./lint x", "lint x", f"{WS}/lint", "cd d && lint", "x; ./lint", "sh lint", "bash ./lint a", + "python3 lint"]) +def test_a_root_level_added_file_is_named_by_a_path_or_as_a_command(text): + assert arm_uptake([_cmd(text)], ["lint"], [], WS)["commands"] == 1 + + +def test_a_read_alone_or_an_mcp_call_alone_is_a_use(): + read = arm_uptake([{"kind": "read", "path": "tools/fmt"}], ["tools/fmt"], [], WS) + assert (read["reads"], read["mcp_calls"], read["commands"], read["used"]) == (1, 0, 0, True) + call = arm_uptake([{"kind": "tool", "name": "mcp__code_graph__x"}], [], ["code.graph"], WS) + assert (call["mcp_calls"], call["reads"], call["commands"], call["used"]) == (1, 0, 0, True) + other = arm_uptake([{"kind": "tool", "name": "mcp__code_graph2__x"}], [], ["code.graph"], WS) + assert (other["mcp_calls"], other["used"]) == (0, False) + + +def test_a_mcp_json_the_patch_changed_names_only_the_servers_it_added(toy): # noqa: F811 + from dokimi.repos.workspace import _apply_patch, _git, _git_env + + ws, env = toy["tmp"] / "w", _git_env(("t", "t@example.com")) + ws.mkdir() + (ws / ".mcp.json").write_text('{"mcpServers": {"github": {}}}\n') + _git(ws, env, "init", "-q") + patch = toy["tmp"] / "mcp.diff" + patch.write_text("diff --git a/.mcp.json b/.mcp.json\n--- a/.mcp.json\n+++ b/.mcp.json\n@@ -1 +1 @@\n" + '-{"mcpServers": {"github": {}}}\n+{"mcpServers": {"github": {}, "code.graph": {}}}\n') + _, meta = _apply_patch(ws, env, toy["manifest"], parse_arm(f"patch:{patch}")) + assert meta["arm_patch"]["mcp_servers"] == ["code.graph"] and meta["arm_patch"]["added"] == [] + + +def test_the_text_report_and_compare_show_the_cells_that_could_not_be_measured(): + assert unmeasured_note({"cells": 0, "used": 0, "unmeasured": 2}) == " (warning: 2 more cell(s) left no trace to measure)" + assert unmeasured_note({"cells": 2, "used": 1, "unmeasured": 0}) == ""