diff --git a/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md b/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md new file mode 100644 index 00000000..ac08f0bf --- /dev/null +++ b/dokimi/changelog.d/20261022_090000_napetrov_repo-arm-uptake.md @@ -0,0 +1,11 @@ +### Added + +- **repo run: uptake of what a `patch:` arm added** — `meta.json` `arm_patch` + lists the files the patch `added` (guidance files and `.mcp.json` left out) and the + `mcp_servers` it added to a `.mcp.json`, and `result.json` gets an `uptake` + object: the agent's commands naming an added file, reads of one, and calls of + those servers' tools, leaving out calls the harness denied. + `repo report` shows the cells that used the arm's + additions per row; `repo compare` lists uptake per treatment and warns when a + treatment's cells used none of it, since its deltas then measure the patch's + text, not the tool (#41 D4). diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5f31e13d..a376567f 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -320,7 +320,9 @@ grader diffs the agent's changes from `start_sha`. With the arm come `arm_files`, the files it deleted, wrote or changed; for `guidance:rev:` an `arm_rev` object (`rev` as given, its `sha`, and `removed`: the base's guidance files with no counterpart at ``); for `patch:` an `arm_patch` object (the -file's absolute `path` and full `sha256`). +file's absolute `path` and full `sha256`, `added`: the files the patch added +that neither the guidance globs nor `.mcp.json` match, and `mcp_servers`: the servers +the patch added to the `.mcp.json` at the root, read before the agent runs). Step 3 removes history only. A fix that landed upstream before a model's training cutoff may still be known to the model, so reports split results by @@ -372,6 +374,23 @@ Code's `modelUsage`), `main` when the harness reported the parent's only, and `null` when it reported none. `subagent_calls` counts the parent's and subagents' `Agent`/`Task` calls; `null` when the harness wrote no result record. +For a `patch:` arm with `added` files or `mcp_servers`, `result.json` has an +`uptake` object: whether the agent used what the arm added. `files` and +`mcp_servers` list what it added (sorted); `commands` counts +command events that name an added file as a workspace path (`tools/fmt`, +`./tools/fmt`, `/eval/run/workspace/tools/fmt`, not `cd tools && ./fmt`; a file +with no `/` only as `./name`, under the workspace path, first in a command or after an interpreter, so +`npm run lint` does not name a `lint` file), `reads` the read events of one (a +read of an added script counts as use), `mcp_calls` the calls of the servers' tools +(`mcp____`), none of them a call the harness denied; `used` +is whether any is non-zero, and `measured` is +`false` when the agent left no trace to count from. `repo report` shows the +cells that used it per row. `repo compare` puts an `uptake` entry per treatment +in its summary (`{cells, used, unmeasured}`: the measured cells, how many of +them used it, and how many could not be measured; the text lines say so), prints it as an `uptake: ...` +line, and warns about a treatment whose cells used none of it: its deltas then +measure the patch's text, such as a guidance line, not the tool. + ### 4.2 `grade.json` (written by `grade` to `{run_dir}/grade.json`) ```json diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 59b2db71..2207868f 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -39,6 +39,10 @@ than it asked for is warned about as in ``repo report``. An image ``repo run`` tagged from the tree's own revision follows the tree, so it is compared only against an ``--image`` given explicitly. + +A treatment arm that added a file or an MCP server (``result.json`` +``uptake``) whose agents used it in none of their cells is warned about too: its +deltas measure the patch's text (a guidance line, say), not the tool. """ from __future__ import annotations @@ -53,7 +57,8 @@ from dokimi.eval.stats import adjust_pvalues, bootstrap_ci, mean_ci -from .report import load_cells, mismatch_warnings, model_mismatch, model_matches, resolved_models +from .report import (load_cells, mismatch_warnings, model_matches, model_mismatch, resolved_models, unmeasured_note, + uptake_counts) #: Largest number of non-zero deltas whose sign-flip distribution is enumerated (2**16 patterns). EXACT_MAX = 16 @@ -150,7 +155,8 @@ def _one_arm_models(treatment: str, base_vals: dict[tuple[str, str], dict[str, A def _provenance(batch_dirs: list[Path], cells: list[dict[str, Any]], baseline: str, treatments: list[str], models: list[str] | None = None) -> dict[str, Any]: - """``sources`` (each selector's batches), ``stamps`` (each batch's pin and harness versions), ``warnings``.""" + """``sources`` (each selector's batches), ``stamps`` (each batch's pin and harness versions), ``uptake`` (each + treatment's cells that used what its arm added), ``warnings``.""" names = [d.resolve().name for d in batch_dirs] stamps = {name: {**batch_stamp(d), "harness_versions": sorted({str(c.get("harness_version")) for c in cells if c["batch"] == name}), @@ -158,7 +164,9 @@ def _provenance(batch_dirs: list[Path], cells: list[dict[str, Any]], baseline: s for d, name in zip(batch_dirs, names, strict=True)} sources = {text: sorted({c["batch"] for c in cells if selector_matches(parse_selector(text), c)}) for text in [baseline, *treatments]} - return {"sources": sources, "stamps": stamps, + uptake = {text: uptake_counts([c for c in cells if selector_matches(parse_selector(text), c) + and (not models or c["model"] in models)]) for text in treatments} + return {"sources": sources, "stamps": stamps, "uptake": {k: v for k, v in uptake.items() if v is not None}, "warnings": _provenance_warnings(cells, baseline, treatments, sources, stamps, models)} @@ -215,6 +223,10 @@ def _provenance_warnings(cells: list[dict[str, Any]], baseline: str, treatments: if base and treat and not _models_equivalent(base, treat): out.append(f"{text} vs {baseline}: model {model} was answered by {', '.join(sorted(base))} in " f"baseline, {', '.join(sorted(treat))} in treatment") + uptake = uptake_counts(arms[1]) + if uptake and uptake["cells"] and not uptake["used"]: + out.append(f"{text}: the agent used none of what the arm added in {uptake['cells']} cell(s); " + "a delta measures the patch's text, not the tool") compared = [c for c in cells if (not models or c["model"] in models) and any(selector_matches(parse_selector(sel), c) for sel in (baseline, *treatments))] return out + mismatch_warnings(model_mismatch(compared)) @@ -404,6 +416,8 @@ def format_compare(summary: dict[str, Any]) -> str: + (f" and {len(summary['unpaired']) - len(shown)} more" if len(summary["unpaired"]) > 10 else "")) if summary["missing"]: lines.append(f"{len(summary['missing'])} cells missing (no result.json): `repo report` lists them") + lines += [f"uptake: {text} used what its arm added in {u['used']}/{u['cells']} cell(s){unmeasured_note(u)}" + for text, u in summary.get("uptake", {}).items()] lines += [f"warning: {w}" for w in summary["warnings"]] return "\n".join(lines) diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index d6490b11..576bb28d 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -14,6 +14,9 @@ ``host`` requirements (``unsupported.json``, with the reason). They are neither failures nor missing. - cost and turns are means over finished cells, as the harness reported them. +- **uptake**: for an arm that added a file or an MCP server, the cells whose + agent used it (``result.json`` ``uptake``, see :mod:`.uptake`) over the + cells with a trace to count from. A warning is printed when the trace says a model other than the requested one answered a cell (``model_resolved``): a harness alias can pin an older model. @@ -90,6 +93,9 @@ def summarize(batch_dir: Path) -> dict[str, Any]: "turns": _mean([r["agent"]["num_turns"] for r in recs]), "networks": sorted({str(r["agent"].get("network")) for r in recs}), "model_resolved": sorted({m for r in recs for m in resolved_models(r)})} + uptake = uptake_counts(recs) + if uptake is not None: + row["uptake"] = uptake if score_only: scores = [0.0 if r["grade"]["status"] == "grader_failure" else r["grade"]["score"] for r in recs] row.update(kind="score", score=_mean([0.0 if s is None else s for s in scores])) @@ -125,6 +131,9 @@ def format_report(summary: dict[str, Any]) -> str: f"{result:<22} {r['grader_failures']:>11} {r['missing']:>7} {cost:>8} {turns:>6}") noted = ("agent_timeouts", "agent_errors", "over_budget", "cost_unknown") notes = [f"{r[k]} {k.replace('_', ' ')}" for k in noted if r[k]] + if "uptake" in r: + notes.append(f"arm's additions used in {r['uptake']['used']}/{r['uptake']['cells']} cell(s)" + + unmeasured_note(r["uptake"])) if notes: lines.append(" " * (width + 2) + " " + ", ".join(notes)) if summary["missing"]: @@ -147,6 +156,22 @@ def _unsupported_lines(cells: list[dict[str, Any]]) -> list[str]: for (task, reason), n in sorted(by_task.items())] +def unmeasured_note(u: dict[str, int]) -> str: + """What to append to a ``used/cells`` count when some cells left no trace to measure.""" + return f" (warning: {u['unmeasured']} more cell(s) left no trace to measure)" if u.get("unmeasured") else "" + + +def uptake_counts(recs: list[dict[str, Any]]) -> dict[str, int] | None: + """``cells`` with a measured ``uptake`` and how many of them ``used`` what the arm added; ``None`` when no + cell's arm added anything to use.""" + found = [r["uptake"] for r in recs if isinstance(r.get("uptake"), dict)] + if not found: + return None + measured = [u for u in found if u.get("measured")] + return {"cells": len(measured), "used": sum(bool(u.get("used")) for u in measured), + "unmeasured": len(found) - len(measured)} + + def resolved_models(rec: dict[str, Any]) -> list[str]: """The models a cell's trace says answered it (``model_resolved``); empty when the trace could not tell.""" found = rec.get("model_resolved") diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index d6f3dfc1..ad7750ea 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -60,6 +60,7 @@ docker_network, ) from .events import convert +from .uptake import arm_uptake from .grading import EvalContext, failed_outcome, outcome, run_entry from .harness_profile import HarnessProfile from .host import probe_host, unmet, validate_tasks @@ -192,6 +193,9 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, "input_tokens": None, "output_tokens": None, "cache_read_tokens": None, "cache_write_tokens": None, "usage_scope": None, "subagent_calls": None, "plugins": [], "over_budget": False, "caps": caps, "network": network, "trace_errors": []} + added = meta.get("arm_patch") or {} + uptake_of = (added.get("added") or [], added.get("mcp_servers") or []) + events = None # the trace's events once it parsed; none: the uptake is unmeasured try: raw = (rd / "agent.stdout").read_text(encoding="utf-8", errors="replace") transcript = convert(opts.profile.trace, raw, WORKSPACE) @@ -201,6 +205,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, meta["harness_version"] = transcript.harness_version or opts.harness_version meta["model_resolved"] = transcript.model_resolved() # an alias can map `model` to another one _write_json(rd / "meta.json", meta) + if transcript.events or transcript.num_turns is not None: # else the agent never started + events = transcript.events cost = transcript.cost_usd agent.update( cost_usd=cost, charged_usd=cost if cost is not None else caps["budget_usd"], @@ -217,10 +223,12 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, # Excluding the run would favour arms that break workspaces (contract §4.2): it fails the task. what = "grade run failed" if isinstance(exc, ContainerError) else f"host error after the agent ran ({type(exc).__name__})" grade = failed_outcome(f"{what}: {str(exc)[:300]}") - record = {"task": task.id, "family": task.family, "arm": arm, "model": opts.model, - "model_resolved": meta.get("model_resolved"), "rep": rep, - "harness": opts.profile.name, "harness_version": meta["harness_version"], - "base_date": meta["base_date"], "score_task": bool(task.score), "agent": agent, "grade": grade} + record: dict[str, Any] = {"task": task.id, "family": task.family, "arm": arm, "model": opts.model, + "model_resolved": meta.get("model_resolved"), "rep": rep, + "harness": opts.profile.name, "harness_version": meta["harness_version"], + "base_date": meta["base_date"], "score_task": bool(task.score), "agent": agent, "grade": grade} + if any(uptake_of): # an arm that added something to use: did the agent use it (#41 D4) + record["uptake"] = arm_uptake(events, *uptake_of, WORKSPACE) try: _own_dir(rd) # the grader had the run dir read-write, and may have taken write access away _write_json(rd / "result.json", record) diff --git a/dokimi/dokimi/repos/uptake.py b/dokimi/dokimi/repos/uptake.py new file mode 100644 index 00000000..9be565d0 --- /dev/null +++ b/dokimi/dokimi/repos/uptake.py @@ -0,0 +1,116 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Uptake: whether the agent of a cell used what its arm added (#41 D4). + +An arm that adds a tool (a script, a document, an MCP server) can move a score +through the guidance line that names the tool even if the tool is never +called. ``result.json`` therefore records, for a ``patch:`` arm that adds +something an agent can use, how often the agent used it: + +- **files**: the files the patch added that the guidance globs do not match + (a guidance file is loaded, not used; ``.mcp.json`` is a config whose servers are counted by their calls); +- **commands**: command events whose text names one of those files, as a path + relative to the workspace (``tools/fmt``, ``./tools/fmt``, ``bash tools/fmt``) + or under the workspace's absolute path. A file at the workspace root is named only as ``./name``, under the + absolute path, as the first word of a command (or of a ``;``/``&&``/``|`` part) or after a shell or + interpreter (``sh run.sh``): ``npm run lint`` does not name a ``lint`` file. A command run as ``cd tools && ./fmt`` + is not counted; +- **reads**: read events of one of those files (reading a script's text counts: it is the agent engaging with + what the arm added, not only running it); +- **mcp_servers** / **mcp_calls**: the servers the patch added to the ``.mcp.json`` + (those the file already declared at the base are not the arm's), and the calls of their tools (``mcp____``). + Whether the harness starts them is the profile's business: a profile that + starts none gives 0 calls, which is what the warning is for. + +A call the harness denied (``denied``) is not counted. ``used`` is whether any +count is non-zero; ``measured`` is false when the agent left no trace to count +from (it never started, or its trace did not parse). +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +MCP_CONFIG = ".mcp.json" + + +def base_servers(text: str) -> set[str]: + """The server names a ``.mcp.json`` text declares; none when it does not parse.""" + try: + servers = json.loads(text).get("mcpServers") + except (ValueError, AttributeError): + return set() + return {str(k) for k in servers} if isinstance(servers, dict) else set() + + +def mcp_servers(ws: Path, arm_files: list[str], base: set[str] | frozenset[str] = frozenset()) -> list[str]: + """The server names of the ``.mcp.json`` at the workspace root that the arm added: those it wrote, less the + ``base`` ones the file already declared before the patch; read at prep, before the agent can change it. A file that does not parse names no server, nor does a link out of the workspace + or to anything but a regular file: it is not read.""" + if MCP_CONFIG not in arm_files: + return [] + try: + path = (ws / MCP_CONFIG).resolve() + if not path.is_relative_to(ws.resolve()) or not path.is_file(): + return [] + servers = json.loads(path.read_text(encoding="utf-8")).get("mcpServers") + except (OSError, RuntimeError, ValueError, AttributeError): # RuntimeError: a link loop, before Python 3.13 + return [] + return sorted(str(k) for k in servers if str(k) not in base) if isinstance(servers, dict) else [] + + +def arm_uptake(events: list[dict[str, Any]] | None, files: list[str], servers: list[str], + workdir: str) -> dict[str, Any]: + """The ``uptake`` of ``result.json`` from the cell's normalized events (``None``: no trace).""" + out: dict[str, Any] = {"files": sorted(files), "mcp_servers": sorted(servers), "measured": events is not None, + "commands": 0, "reads": 0, "mcp_calls": 0, "used": False} + if events is None: + return out + named = _path_pattern(files, workdir) + prefixes = tuple(f"mcp__{_mcp_name(s)}__" for s in servers) + wanted = set(files) + for ev in events: + if ev.get("denied"): # the harness refused the call: the tool never ran + continue + kind = ev.get("kind") + if kind == "command" and named is not None and named.search(str(ev.get("text", ""))): + out["commands"] += 1 + elif kind == "read" and ev.get("path") in wanted: + out["reads"] += 1 + elif kind == "tool" and prefixes and str(ev.get("name", "")).startswith(prefixes): + out["mcp_calls"] += 1 + out["used"] = bool(out["commands"] or out["reads"] or out["mcp_calls"]) + return out + + +_INTERPRETERS = r"(?:sh|bash|zsh|dash|python3?|node|ruby|perl)" + + +def _path_pattern(files: list[str], workdir: str) -> re.Pattern[str] | None: + """One of ``files`` as a whole word of a command: relative (optionally ``./``) or under ``workdir``. + A file with no ``/`` is a common word, so it must be ``./``-prefixed, under ``workdir``, first in a command + or run by an interpreter.""" + if not files: + return None + root = re.escape(workdir.rstrip("/") + "/") + end = r"(?=$|[\s'\"`;&|)<>:])" + parts = [] + nested = sorted((f for f in files if "/" in f), key=len, reverse=True) + flat = sorted((f for f in files if "/" not in f), key=len, reverse=True) + if nested: + paths = "|".join(re.escape(f) for f in nested) + parts.append(rf"(?:^|(?<=[\s'\"`=;&|(<>])|(?<=\./)|(?<={root}))(?:{paths}){end}") + if flat: + names = "|".join(re.escape(f) for f in flat) + parts.append(rf"(?:^\s*(?:\./)?|[;&|(]\s*(?:\./)?|(?:^|(?<=\s)){_INTERPRETERS}\s+|(?<=[\s'\"`=;&|(<>])\./|(?<={root}))" + rf"(?:{names}){end}") + return re.compile("|".join(parts)) + + +def _mcp_name(server: str) -> str: + """A server name as Claude Code spells it in tool names: characters outside ``[A-Za-z0-9_-]`` become ``_``.""" + return re.sub(r"[^A-Za-z0-9_-]", "_", server) diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index 548a0a28..704dc916 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -52,6 +52,7 @@ from .harness_profile import HarnessProfile from .manifest import RepoManifest, _is_within, match_glob from .source import GitTree, RepoSource, RepoTree, SourceError, cache_path, default_cache_dir, resolve_rev +from .uptake import MCP_CONFIG, base_servers, mcp_servers ARMS = ("guidance:none", "guidance:raw", "guidance:bridge") REV_ARM, PATCH_ARM = "guidance:rev:", "patch:" @@ -468,15 +469,27 @@ def _apply_patch(ws: Path, env: dict[str, str], manifest: RepoManifest, arm: Arm except PrepError as exc: raise PrepError(f"arm {arm}: patch {arm.patch_path} does not apply: {exc}") from exc _git(ws, env, "add", "-A") - changed = sorted(p for p in _git(ws, env, "diff", "--cached", "--name-only", "--no-renames", "-z", before).split("\0") - if p) + status = _git(ws, env, "diff", "--cached", "--name-status", "--no-renames", "-z", before).split("\0") + changed = sorted(p for p in status[1::2] if p) eval_dir = manifest.run_paths()[0] # the patch's own paths too: a file .gitignore hides is written but never staged inside = sorted(p for p in set(changed) | touched if _is_within(p, eval_dir)) if inside: # the agent must not see an eval_dir, not even a patched-in one raise PrepError(f"arm {arm}: patch {arm.patch_path} writes into eval_dir {eval_dir}: {', '.join(inside)}") # a written file .gitignore hides is in the workspace, not in start_sha: still a file the arm wrote - return sorted(set(changed) | touched), {"arm_patch": {"path": arm.patch_path, "sha256": arm.patch_sha256}} + files = sorted(set(changed) | touched) + # -z output ends in a NUL, so the split has one trailing "" the pairing drops + new = {p for k, p in zip(status[::2], status[1::2], strict=False) if k == "A"} + new |= {p for p in touched - set(changed) if (ws / p).is_file()} + # what an agent can use (uptake): a file the patch added, not a guidance file; the servers of its .mcp.json + # .mcp.json is not one: reading the config is not using a server, whose calls are counted as mcp_calls + added = sorted(p for p in new if p != MCP_CONFIG and not any(match_glob(g, p) for g in manifest.guidance_globs)) + try: # the servers the base already declared are not the arm's + base = base_servers(_git(ws, env, "show", f"{before}:{MCP_CONFIG}")) + except PrepError: + base = set() + return files, {"arm_patch": {"path": arm.patch_path, "sha256": arm.patch_sha256, "added": added, + "mcp_servers": mcp_servers(ws, files, base)}} def _patch_paths(numstat: str) -> set[str]: diff --git a/dokimi/tests/repo_eval_fixture.py b/dokimi/tests/repo_eval_fixture.py index 987079fc..e5186c3a 100644 --- a/dokimi/tests/repo_eval_fixture.py +++ b/dokimi/tests/repo_eval_fixture.py @@ -71,6 +71,8 @@ assert os.environ.get("FAKE_CONFIG_DIR"), "config dir not set" assert "SECRET_NOT_ALLOWED" not in os.environ, "environment not scrubbed" action = os.environ.get("FAKE_ACTION", "fix") +if action == "silent": # the agent dies before it writes a line + sys.exit(1) cwd = Path.cwd() # Guard against --model missing or being the last arg try: @@ -99,9 +101,14 @@ hook.chmod(0o755) with open(cwd / ".git" / "config", "a") as f: f.write("[core]\n\tfsmonitor = " + str(hook) + "\n") -lines.append({"type": "assistant", "message": {"model": answered, "content": [ - {"type": "tool_use", "id": "t1", "name": "Edit", - "input": {"file_path": os.environ.get("FAKE_CONTAINER_CWD", str(cwd)) + "/src/calc.py"}}]}}) +if action != "idle": # idle: the agent starts and ends without a tool call + lines.append({"type": "assistant", "message": {"model": answered, "content": [ + {"type": "tool_use", "id": "t1", "name": "Edit", + "input": {"file_path": os.environ.get("FAKE_CONTAINER_CWD", str(cwd)) + "/src/calc.py"}}]}}) +if os.environ.get("FAKE_TOOL_USES"): # more tool calls, as a JSON list of [name, input] + lines.append({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": f"u{i}", "name": n, "input": a} + for i, (n, a) in enumerate(json.loads(os.environ["FAKE_TOOL_USES"]))]}}) result = {"type": "result", "subtype": "success", "result": "done: src/calc.py", "total_cost_usd": 0.01, "num_turns": 3, "is_error": False} if os.environ.get("FAKE_USAGE"): # a subagent (Agent tool) and the token counts of the whole tree @@ -120,7 +127,8 @@ "invoke": ["{bin}", "-p", "{prompt}", "--model", "{model}", "--max-turns", "{max_turns}", "--max-budget-usd", "{budget}", "{tools}"], "tools": {"none": ["--tools", ""], "read": ["--tools", "Read"], "all": []}, - "config_dir_env": "FAKE_CONFIG_DIR", "env_allow": ["FAKE_ACTION", "FAKE_ANSWER_MODEL", "FAKE_MARKER", "FAKE_USAGE", "PATH"], + "config_dir_env": "FAKE_CONFIG_DIR", + "env_allow": ["FAKE_ACTION", "FAKE_ANSWER_MODEL", "FAKE_MARKER", "FAKE_USAGE", "FAKE_TOOL_USES", "PATH"], "egress": ["api.example.com"], "trace": "claude-stream-json", "prompt_suffix": "Stay in the foreground.", "config_files": {"settings.json": '{"hook": "python3 {support_dir}/no_background.py"}'}, diff --git a/dokimi/tests/test_repo_arms.py b/dokimi/tests/test_repo_arms.py index dbc32d47..0aa73872 100644 --- a/dokimi/tests/test_repo_arms.py +++ b/dokimi/tests/test_repo_arms.py @@ -171,7 +171,8 @@ def test_patch_arm_applies_on_raw_guidance_and_records_the_file(toy): patch.write_text("changed after the batch read it\n") # every cell applies the bytes read once meta, ws = _prep(toy, arm) assert meta["arm"] == str(arm) and meta["arm_files"] == ["AGENTS.md", "Makefile"] - assert meta["arm_patch"] == {"path": str(patch.resolve()), "sha256": digest} + assert meta["arm_patch"] == {"path": str(patch.resolve()), "sha256": digest, "added": ["Makefile"], + "mcp_servers": []} assert (ws / "AGENTS.md").read_text() == "# guidance\nRun `make check`.\n" assert (ws / "Makefile").is_file() and (ws / "sub/AGENTS.md").is_file() # the rest of guidance:raw assert git(ws, "status", "--porcelain") == "" and git(ws, "rev-list", "--count", "HEAD") == "1" diff --git a/dokimi/tests/test_repo_uptake.py b/dokimi/tests/test_repo_uptake.py new file mode 100644 index 00000000..d9ab8c63 --- /dev/null +++ b/dokimi/tests/test_repo_uptake.py @@ -0,0 +1,275 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Uptake of what a `patch:` arm added (#41 D4): prep records it, the runner counts it, report and compare show it. +No docker, no LLM.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos.compare import compare, format_compare +from dokimi.repos.events import convert +from dokimi.repos.report import format_report, summarize, unmeasured_note +from dokimi.repos.uptake import arm_uptake, mcp_servers +from dokimi.repos.workspace import parse_arm +from test_repo_arms import setup, toy, _prep # noqa: F401 # fixtures + +KIT_PATCH = """diff --git a/tools/fmt b/tools/fmt +new file mode 100755 +--- /dev/null ++++ b/tools/fmt +@@ -0,0 +1,2 @@ ++#!/bin/sh ++exec clang-format -i "$@" +diff --git a/.mcp.json b/.mcp.json +new file mode 100644 +--- /dev/null ++++ b/.mcp.json +@@ -0,0 +1 @@ ++{"mcpServers": {"code.graph": {"command": "cg-mcp"}}} +diff --git a/sub/AGENTS.md b/sub/AGENTS.md +--- a/sub/AGENTS.md ++++ b/sub/AGENTS.md +@@ -1 +1,2 @@ + # sub guidance ++Format with `tools/fmt`; ask the code.graph tools who calls a function. +diff --git a/src/AGENTS.md b/src/AGENTS.md +new file mode 100644 +--- /dev/null ++++ b/src/AGENTS.md +@@ -0,0 +1 @@ ++# src guidance +""" + +WS = "/eval/run/workspace" + + +def _cmd(text): + return {"kind": "command", "text": text, "rc": 0} + + +@pytest.mark.parametrize("text", ["tools/fmt src/a.cpp", "./tools/fmt src/a.cpp", "sh tools/fmt", "x=1; tools/fmt", + f"{WS}/tools/fmt a", "bash -c 'tools/fmt a'", "(tools/fmt)"]) +def test_a_command_that_names_an_added_file_is_a_use(text): + assert arm_uptake([_cmd(text)], ["tools/fmt"], [], WS)["commands"] == 1 + + +@pytest.mark.parametrize("text", ["tools/fmt2 a", "mytools/fmt a", "cd tools && ./fmt a", "tools/fmt.bak", + "/elsewhere/tools/fmt"]) +def test_a_command_that_does_not_name_an_added_file_is_not(text): + assert arm_uptake([_cmd(text)], ["tools/fmt"], [], WS)["commands"] == 0 + + +def test_reads_and_calls_of_the_arms_mcp_servers_count_others_do_not(): + events = [{"kind": "read", "path": "tools/fmt"}, {"kind": "read", "path": "src/a.cpp"}, + {"kind": "tool", "name": "mcp__code_graph__callers"}, {"kind": "tool", "name": "mcp__other__x"}, + {"kind": "tool", "name": "Grep"}] + got = arm_uptake(events, ["tools/fmt"], ["code.graph"], WS) + assert (got["reads"], got["mcp_calls"], got["commands"], got["used"], got["measured"]) == (1, 1, 0, True, True) + assert arm_uptake([], ["tools/fmt"], [], WS)["used"] is False + unmeasured = arm_uptake(None, ["tools/fmt"], [], WS) + assert unmeasured["measured"] is False and unmeasured["used"] is False + + +def test_a_call_the_harness_denied_is_not_a_use(): + def use(i, name, args): + return {"type": "assistant", "message": {"content": [{"type": "tool_use", "id": i, "name": name, "input": args}]}} + hook = "PreToolUse:Bash hook error: [python3 /eval/support/no_background.py]: run_in_background is refused" + stream = [{"type": "system", "subtype": "init", "model": "m-1"}, + use("b1", "Bash", {"command": "./tools/fmt src/calc.py", "run_in_background": True}), + {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "b1", "is_error": True, + "content": hook}]}}, + use("m1", "mcp__code_graph__callers", {"symbol": "add"}), + {"type": "result", "subtype": "success", "result": "done", "num_turns": 2, + "permission_denials": [{"tool_use_id": "m1", "tool_name": "mcp__code_graph__callers"}]}] + events = convert("claude-stream-json", "\n".join(json.dumps(r) for r in stream), WS).events + assert [e.get("denied") for e in events] == [True, True] + got = arm_uptake(events, ["tools/fmt"], ["code.graph"], WS) + assert (got["commands"], got["mcp_calls"], got["used"], got["measured"]) == (0, 0, False, True) + retried = arm_uptake([*events, _cmd("./tools/fmt src/calc.py")], ["tools/fmt"], ["code.graph"], WS) + assert (retried["commands"], retried["used"]) == (1, True) + + +def test_prep_records_the_added_files_that_are_not_guidance_and_the_mcp_servers(toy): # noqa: F811 + patch = toy["tmp"] / "kit.diff" + patch.write_text(KIT_PATCH) + meta, ws = _prep(toy, parse_arm(f"patch:{patch}")) + assert meta["arm_files"] == [".mcp.json", "src/AGENTS.md", "sub/AGENTS.md", "tools/fmt"] + # a changed guidance file and an added one are loaded, not used; the rest is what the agent can use + assert meta["arm_patch"]["added"] == ["tools/fmt"] # not .mcp.json: reading the config is not using a server + assert meta["arm_patch"]["mcp_servers"] == ["code.graph"] + + +def _mcp_link_patch(target, extra=""): + return ("diff --git a/.mcp.json b/.mcp.json\nnew file mode 120000\n--- /dev/null\n+++ b/.mcp.json\n" + f"@@ -0,0 +1 @@\n+{target}\n\\ No newline at end of file\n{extra}") + + +@pytest.mark.parametrize("where", ["host", "loop"]) +def test_a_mcp_json_linked_out_of_the_workspace_or_in_a_loop_is_not_read(toy, where): # noqa: F811 + host = toy["tmp"] / "host.json" # outside the workspace: what the agent's harness would not find + host.write_text('{"mcpServers": {"host-only": {}}}') + patch = toy["tmp"] / "link.diff" + patch.write_text(_mcp_link_patch(host if where == "host" else ".mcp.json")) + meta, ws = _prep(toy, parse_arm(f"patch:{patch}")) + assert (ws / ".mcp.json").is_symlink() and meta["arm_patch"]["mcp_servers"] == [] + + +def test_a_mcp_json_linked_inside_the_workspace_names_its_servers(toy): # noqa: F811 + patch = toy["tmp"] / "link.diff" + patch.write_text(_mcp_link_patch("tools/mcp.json", "diff --git a/tools/mcp.json b/tools/mcp.json\n" + "new file mode 100644\n--- /dev/null\n+++ b/tools/mcp.json\n@@ -0,0 +1 @@\n" + '+{"mcpServers": {"code.graph": {}}}\n')) + meta, _ = _prep(toy, parse_arm(f"patch:{patch}")) + assert meta["arm_patch"]["mcp_servers"] == ["code.graph"] + + +MIXED_PATCH = """diff --git a/src/calc.py b/src/calc.py +--- a/src/calc.py ++++ b/src/calc.py +@@ -1,2 +1,3 @@ ++# checked + def add(a, b): + return a - b +diff --git a/.gitignore b/.gitignore +new file mode 100644 +--- /dev/null ++++ b/.gitignore +@@ -0,0 +1 @@ ++gen.sh +diff --git a/gen.sh b/gen.sh +new file mode 100755 +--- /dev/null ++++ b/gen.sh +@@ -0,0 +1 @@ ++#!/bin/sh +""" + + +def test_prep_records_as_added_a_file_gitignore_hides_not_a_file_the_patch_changed(toy): # noqa: F811 + patch = toy["tmp"] / "mixed.diff" + patch.write_text(MIXED_PATCH) + meta, ws = _prep(toy, parse_arm(f"patch:{patch}")) + assert meta["arm_files"] == [".gitignore", "gen.sh", "src/calc.py"] and (ws / "gen.sh").is_file() + assert meta["arm_patch"]["added"] == [".gitignore", "gen.sh"] + + +def test_a_mcp_json_the_patch_did_not_write_names_no_server(tmp_path): + (tmp_path / ".mcp.json").write_text('{"mcpServers": {"from-base": {}}}') + assert mcp_servers(tmp_path, ["tools/fmt"]) == [] and mcp_servers(tmp_path, [".mcp.json"]) == ["from-base"] + + +def _run(s, batch, *arms, model="m-1"): + argv = ["repo", "run", f"{s['root']}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--harness", + str(s["profile"]), "--bin", str(s["exe"]), "--model", model, "--batch", batch, "--out", str(s["out"]), + "--cache-root", str(s["out"] / "caches"), "--agent-network", "host", "--task", "fix_add", + "--reps", "2", *[x for a in arms for x in ("--arm", a)]] + args = build_parser().parse_args(argv) + args.func(args) + + +def test_repo_run_records_uptake_and_compare_warns_when_nothing_was_used(setup, monkeypatch): # noqa: F811 + patch = setup["tmp"] / "kit.diff" + patch.write_text(KIT_PATCH) + arm = str(parse_arm(f"patch:{patch}")) + _run(setup, "unused", "guidance:raw", f"patch:{patch}") + monkeypatch.setenv("FAKE_TOOL_USES", json.dumps([["Bash", {"command": "./tools/fmt src/calc.py"}], + ["mcp__code_graph__callers", {"symbol": "add"}]])) + _run(setup, "used", "guidance:raw", f"patch:{patch}") + + used = setup["out"] / "used" + result = json.loads(next(used.glob("*patch*__r1/result.json")).read_text()) + assert result["uptake"] == {"files": ["tools/fmt"], "mcp_servers": ["code.graph"], "measured": True, + "commands": 1, "reads": 0, "mcp_calls": 1, "used": True} + assert "uptake" not in json.loads(next(used.glob("*guidance-raw__*__r1/result.json")).read_text()) + assert "arm's additions used in 2/2 cell(s)" in format_report(summarize(used)) + + unused = setup["out"] / "unused" + assert "arm's additions used in 0/2 cell(s)" in format_report(summarize(unused)) + quiet = compare([used], "guidance:raw", [arm]) + assert quiet["uptake"] == {arm: {"cells": 2, "used": 2, "unmeasured": 0}} + assert not [w for w in quiet["warnings"] if "used none" in w] + loud = compare([unused], "guidance:raw", [arm]) + assert [w for w in loud["warnings"] if "used none" in w] == [ + f"{arm}: the agent used none of what the arm added in 2 cell(s); a delta measures the patch's text, " + "not the tool"] + assert f"uptake: {arm} used what its arm added in 0/2 cell(s)" in format_compare(loud) + + +def _used_none(summary): + return [w for w in summary["warnings"] if "used none" in w] + + +def test_an_agent_that_left_no_trace_is_unmeasured_one_that_called_no_tool_is_measured(setup, monkeypatch): # noqa: F811 + patch = setup["tmp"] / "kit.diff" + patch.write_text(KIT_PATCH) + arm = str(parse_arm(f"patch:{patch}")) + for action in ("silent", "idle"): + monkeypatch.setenv("FAKE_ACTION", action) + _run(setup, action, "guidance:raw", f"patch:{patch}") + silent, idle = setup["out"] / "silent", setup["out"] / "idle" + assert json.loads(next(silent.glob("*patch*__r1/result.json")).read_text())["uptake"]["measured"] is False + assert json.loads(next(idle.glob("*patch*__r1/result.json")).read_text())["uptake"]["measured"] is True + unmeasured = compare([silent], "guidance:raw", [arm]) + assert unmeasured["uptake"] == {arm: {"cells": 0, "used": 0, "unmeasured": 2}} and not _used_none(unmeasured) + measured = compare([idle], "guidance:raw", [arm]) + assert measured["uptake"] == {arm: {"cells": 2, "used": 0, "unmeasured": 0}} and _used_none(measured) + + +def test_compare_counts_the_uptake_of_the_models_it_compares(setup, monkeypatch): # noqa: F811 + patch = setup["tmp"] / "kit.diff" + patch.write_text(KIT_PATCH) + arm = str(parse_arm(f"patch:{patch}")) + monkeypatch.setenv("FAKE_TOOL_USES", json.dumps([["Bash", {"command": "./tools/fmt src/calc.py"}]])) + _run(setup, "m1", "guidance:raw", f"patch:{patch}", model="m-1") + monkeypatch.delenv("FAKE_TOOL_USES") + _run(setup, "m2", "guidance:raw", f"patch:{patch}", model="m-2") + batches = [setup["out"] / "m1", setup["out"] / "m2"] + both = compare(batches, "guidance:raw", [arm]) + assert both["uptake"] == {arm: {"cells": 4, "used": 2, "unmeasured": 0}} and not _used_none(both) + m2 = compare(batches, "guidance:raw", [arm], models=["m-2"]) + assert m2["uptake"] == {arm: {"cells": 2, "used": 0, "unmeasured": 0}} and _used_none(m2) + + +@pytest.mark.parametrize("text", ["npm run lint", "docker run img", "git log -- build", "cat lint", "make build"]) +def test_a_root_level_added_file_is_not_named_by_a_common_word_of_a_command(text): + for name in ("lint", "run", "build"): + assert arm_uptake([_cmd(text)], [name], [], WS)["commands"] == (1 if text.split()[0] == name else 0) + + +@pytest.mark.parametrize("text", ["./lint x", "lint x", f"{WS}/lint", "cd d && lint", "x; ./lint", "sh lint", "bash ./lint a", + "python3 lint"]) +def test_a_root_level_added_file_is_named_by_a_path_or_as_a_command(text): + assert arm_uptake([_cmd(text)], ["lint"], [], WS)["commands"] == 1 + + +def test_a_read_alone_or_an_mcp_call_alone_is_a_use(): + read = arm_uptake([{"kind": "read", "path": "tools/fmt"}], ["tools/fmt"], [], WS) + assert (read["reads"], read["mcp_calls"], read["commands"], read["used"]) == (1, 0, 0, True) + call = arm_uptake([{"kind": "tool", "name": "mcp__code_graph__x"}], [], ["code.graph"], WS) + assert (call["mcp_calls"], call["reads"], call["commands"], call["used"]) == (1, 0, 0, True) + other = arm_uptake([{"kind": "tool", "name": "mcp__code_graph2__x"}], [], ["code.graph"], WS) + assert (other["mcp_calls"], other["used"]) == (0, False) + + +def test_a_mcp_json_the_patch_changed_names_only_the_servers_it_added(toy): # noqa: F811 + from dokimi.repos.workspace import _apply_patch, _git, _git_env + + ws, env = toy["tmp"] / "w", _git_env(("t", "t@example.com")) + ws.mkdir() + (ws / ".mcp.json").write_text('{"mcpServers": {"github": {}}}\n') + _git(ws, env, "init", "-q") + patch = toy["tmp"] / "mcp.diff" + patch.write_text("diff --git a/.mcp.json b/.mcp.json\n--- a/.mcp.json\n+++ b/.mcp.json\n@@ -1 +1 @@\n" + '-{"mcpServers": {"github": {}}}\n+{"mcpServers": {"github": {}, "code.graph": {}}}\n') + _, meta = _apply_patch(ws, env, toy["manifest"], parse_arm(f"patch:{patch}")) + assert meta["arm_patch"]["mcp_servers"] == ["code.graph"] and meta["arm_patch"]["added"] == [] + + +def test_the_text_report_and_compare_show_the_cells_that_could_not_be_measured(): + assert unmeasured_note({"cells": 0, "used": 0, "unmeasured": 2}) == " (warning: 2 more cell(s) left no trace to measure)" + assert unmeasured_note({"cells": 2, "used": 1, "unmeasured": 0}) == ""