Repository navigation
feat(repo): record whether the agent used what a patch: arm added (#41 D4) #153
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| ### Added | ||
|
|
||
| - **repo run: uptake of what a `patch:` arm added** — `meta.json` `arm_patch` | ||
| lists the files the patch `added` (guidance files and `.mcp.json` left out) and the | ||
| `mcp_servers` it added to a `.mcp.json`, and `result.json` gets an `uptake` | ||
| object: the agent's commands naming an added file, reads of one, and calls of | ||
| those servers' tools, leaving out calls the harness denied. | ||
| `repo report` shows the cells that used the arm's | ||
| additions per row; `repo compare` lists uptake per treatment and warns when a | ||
| treatment's cells used none of it, since its deltas then measure the patch's | ||
| text, not the tool (#41 D4). |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| # Copyright (C) 2026 Intel Corporation | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| """Uptake: whether the agent of a cell used what its arm added (#41 D4). | ||
|
|
||
| An arm that adds a tool (a script, a document, an MCP server) can move a score | ||
| through the guidance line that names the tool even if the tool is never | ||
| called. ``result.json`` therefore records, for a ``patch:`` arm that adds | ||
| something an agent can use, how often the agent used it: | ||
|
|
||
| - **files**: the files the patch added that the guidance globs do not match | ||
| (a guidance file is loaded, not used; ``.mcp.json`` is a config whose servers are counted by their calls); | ||
| - **commands**: command events whose text names one of those files, as a path | ||
| relative to the workspace (``tools/fmt``, ``./tools/fmt``, ``bash tools/fmt``) | ||
| or under the workspace's absolute path. A file at the workspace root is named only as ``./name``, under the | ||
| absolute path, as the first word of a command (or of a ``;``/``&&``/``|`` part) or after a shell or | ||
| interpreter (``sh run.sh``): ``npm run lint`` does not name a ``lint`` file. A command run as ``cd tools && ./fmt`` | ||
| is not counted; | ||
| - **reads**: read events of one of those files (reading a script's text counts: it is the agent engaging with | ||
| what the arm added, not only running it); | ||
| - **mcp_servers** / **mcp_calls**: the servers the patch added to the ``.mcp.json`` | ||
| (those the file already declared at the base are not the arm's), and the calls of their tools (``mcp__<server>__<tool>``). | ||
| Whether the harness starts them is the profile's business: a profile that | ||
| starts none gives 0 calls, which is what the warning is for. | ||
|
|
||
| A call the harness denied (``denied``) is not counted. ``used`` is whether any | ||
| count is non-zero; ``measured`` is false when the agent left no trace to count | ||
| from (it never started, or its trace did not parse). | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import json | ||
| import re | ||
| from pathlib import Path | ||
| from typing import Any | ||
|
|
||
| MCP_CONFIG = ".mcp.json" | ||
|
|
||
|
|
||
| def base_servers(text: str) -> set[str]: | ||
| """The server names a ``.mcp.json`` text declares; none when it does not parse.""" | ||
| try: | ||
| servers = json.loads(text).get("mcpServers") | ||
| except (ValueError, AttributeError): | ||
| return set() | ||
| return {str(k) for k in servers} if isinstance(servers, dict) else set() | ||
|
|
||
|
|
||
| def mcp_servers(ws: Path, arm_files: list[str], base: set[str] | frozenset[str] = frozenset()) -> list[str]: | ||
| """The server names of the ``.mcp.json`` at the workspace root that the arm added: those it wrote, less the | ||
| ``base`` ones the file already declared before the patch; read at prep, before the agent can change it. A file that does not parse names no server, nor does a link out of the workspace | ||
| or to anything but a regular file: it is not read.""" | ||
| if MCP_CONFIG not in arm_files: | ||
| return [] | ||
| try: | ||
| path = (ws / MCP_CONFIG).resolve() | ||
| if not path.is_relative_to(ws.resolve()) or not path.is_file(): | ||
| return [] | ||
| servers = json.loads(path.read_text(encoding="utf-8")).get("mcpServers") | ||
| except (OSError, RuntimeError, ValueError, AttributeError): # RuntimeError: a link loop, before Python 3.13 | ||
| return [] | ||
| return sorted(str(k) for k in servers if str(k) not in base) if isinstance(servers, dict) else [] | ||
|
|
||
|
|
||
| def arm_uptake(events: list[dict[str, Any]] | None, files: list[str], servers: list[str], | ||
| workdir: str) -> dict[str, Any]: | ||
| """The ``uptake`` of ``result.json`` from the cell's normalized events (``None``: no trace).""" | ||
| out: dict[str, Any] = {"files": sorted(files), "mcp_servers": sorted(servers), "measured": events is not None, | ||
| "commands": 0, "reads": 0, "mcp_calls": 0, "used": False} | ||
| if events is None: | ||
| return out | ||
| named = _path_pattern(files, workdir) | ||
| prefixes = tuple(f"mcp__{_mcp_name(s)}__" for s in servers) | ||
| wanted = set(files) | ||
| for ev in events: | ||
| if ev.get("denied"): # the harness refused the call: the tool never ran | ||
| continue | ||
| kind = ev.get("kind") | ||
| if kind == "command" and named is not None and named.search(str(ev.get("text", ""))): | ||
| out["commands"] += 1 | ||
| elif kind == "read" and ev.get("path") in wanted: | ||
|
Contributor
There was a problem hiding this comment. Question: Is a Read of an added script (or uptake.py:62-64 counts every
Contributor
Author
There was a problem hiding this comment. Intended: a Read of an added file counts as uptake, as the contract lists reads (repo-evaluation-contract.md). The signal is 'the agent engaged with what the arm added', not only 'executed it'. 3117e66 states this in the uptake docstring and the contract; I did not split reads from commands, since the warning is about an arm whose additions the agent ignored entirely. |
||
| out["reads"] += 1 | ||
| elif kind == "tool" and prefixes and str(ev.get("name", "")).startswith(prefixes): | ||
| out["mcp_calls"] += 1 | ||
| out["used"] = bool(out["commands"] or out["reads"] or out["mcp_calls"]) | ||
|
Contributor
There was a problem hiding this comment. The tests do not pin that a read alone, or an MCP call alone, makes Mutation check: dropping Tests to add:
Contributor
Author
There was a problem hiding this comment. Added in 3117e66: test_a_read_alone_or_an_mcp_call_alone_is_a_use (read alone, MCP call alone, and mcp__code_graph2__x gives mcp_calls == 0). I did not add a run_cell case for an MCP-only patch or a files-only/servers-only gating test. |
||
| return out | ||
|
|
||
|
|
||
| _INTERPRETERS = r"(?:sh|bash|zsh|dash|python3?|node|ruby|perl)" | ||
|
|
||
|
|
||
| def _path_pattern(files: list[str], workdir: str) -> re.Pattern[str] | None: | ||
| """One of ``files`` as a whole word of a command: relative (optionally ``./``) or under ``workdir``. | ||
| A file with no ``/`` is a common word, so it must be ``./``-prefixed, under ``workdir``, first in a command | ||
| or run by an interpreter.""" | ||
| if not files: | ||
| return None | ||
| root = re.escape(workdir.rstrip("/") + "/") | ||
| end = r"(?=$|[\s'\"`;&|)<>:])" | ||
| parts = [] | ||
| nested = sorted((f for f in files if "/" in f), key=len, reverse=True) | ||
| flat = sorted((f for f in files if "/" not in f), key=len, reverse=True) | ||
| if nested: | ||
| paths = "|".join(re.escape(f) for f in nested) | ||
| parts.append(rf"(?:^|(?<=[\s'\"`=;&|(<>])|(?<=\./)|(?<={root}))(?:{paths}){end}") | ||
| if flat: | ||
| names = "|".join(re.escape(f) for f in flat) | ||
| parts.append(rf"(?:^\s*(?:\./)?|[;&|(]\s*(?:\./)?|(?:^|(?<=\s)){_INTERPRETERS}\s+|(?<=[\s'\"`=;&|(<>])\./|(?<={root}))" | ||
|
Contributor
There was a problem hiding this comment. Question: Is The flat-name branch of
Contributor
Author
There was a problem hiding this comment. Not meant to: a bare
Contributor
Author
|
||
| rf"(?:{names}){end}") | ||
| return re.compile("|".join(parts)) | ||
|
|
||
|
|
||
| def _mcp_name(server: str) -> str: | ||
| """A server name as Claude Code spells it in tool names: characters outside ``[A-Za-z0-9_-]`` become ``_``.""" | ||
| return re.sub(r"[^A-Za-z0-9_-]", "_", server) | ||
Uh oh!
There was an error while loading. Please reload this page.