diff --git a/dokimi/changelog.d/20261021_140000_napetrov_d4-process-metrics.md b/dokimi/changelog.d/20261021_140000_napetrov_d4-process-metrics.md new file mode 100644 index 00000000..a277bd75 --- /dev/null +++ b/dokimi/changelog.d/20261021_140000_napetrov_d4-process-metrics.md @@ -0,0 +1,15 @@ +### Added + +- **Process metrics in `result.json`** — every `repo run` cell's `result.json` has a + `process` object counted from `events.jsonl`: `tool_calls`, `commands`, + `failed_commands` (not denied, `rc` neither 0 nor null), `denied_commands`, and `guidance_read` (the + workspace paths read, not denied, that the manifest's guidance globs match). A manifest's new `process_metrics:` maps + a name to a regex, searched in each command's text, or its `output` (`match: output|any`), + and `process.counters` holds each count. `repo validate` refuses a regex that does not + compile, and a name that is not a string (a bare YAML `404:` or `on:`). + +### Changed + +- **`entry_points.grade` is optional** when every task's grader is a built-in `dokimi:` one; + `repo validate`, `repo run` and `repo tasks check` name the tasks that need it otherwise, + the last two before any agent runs. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 00aec556..76dc0ff9 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -191,6 +191,10 @@ tasks: dev/agent_evals/tasks explore_tasks: null # optional ExploreBench YAML directory build_systems: [bazel, make] # selects L0 claim-check adapters + +process_metrics: # optional repository counters, result.json `process.counters` (§4.1) + bazel_cmds: '\bbazel(?:isk)?\b' # a plain regex searches the command text + icpx_missing: {regex: "icpx is not found", match: output} # match: text (default), output or any ``` Rules: @@ -199,7 +203,17 @@ Rules: - `eval_dir`, `tasks` and `entry_points` come together or not at all. Without them the manifest is *guidance-only*: `scan` and `reach` run on it and `validate` checks its globs, while `tasks check` and `run` refuse it. `image` and `explore_tasks` need - `eval_dir`. + `eval_dir`, and so does `process_metrics`. +- `entry_points.grade` is required only when some task's `grader` does not + start with `dokimi:` (a grader built into the framework); `repo validate` + names the tasks that need it. +- `process_metrics` maps a name (letters, digits, `_`; not + starting with a digit) to a Python regex, + given as a string or as `{regex: , match: text|output|any}`. For each + name the framework counts the `command` events of the run whose `text` + (default), `output` or either contains a match (`re.search`); denied commands + count too, since the agent asked for them. `repo validate` refuses a regex + that does not compile. - `eval_dir` is a proper subdirectory, and `tasks` and `explore_tasks` sit inside it: it is the one directory deleted from every workspace (§3). - `{eval_dir}`, `{source_dir}`, `{run_dir}`, `{out}` and `{task}` are the only @@ -384,6 +398,20 @@ Code's `modelUsage`), `main` when the harness reported the parent's only, and `null` when it reported none. `subagent_calls` counts the parent's and subagents' `Agent`/`Task` calls; `null` when the harness wrote no result record. +`result.json` `process` holds what the framework counts in the events, the +parent's and subagents' together, so a grader need not: + +| Key | Meaning | +|---|---| +| `tool_calls` | every event: each tool call the agent made | +| `commands` | `command` events | +| `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | +| `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match | +| `counters` | each `process_metrics` name (§2) and its count | + +`process` is `null` when the trace could not be read, or the agent left none (no event and no result record). + For a `patch:` arm with `added` files or `mcp_servers`, `result.json` has an `uptake` object: whether the agent used what the arm added. `files` and `mcp_servers` list what it added (sorted); `commands` counts @@ -524,8 +552,8 @@ A review task with `pass: null` is scored, not gated: give it `score`. A `gate` on its metrics is not applied (`success` is null), and without `score` the run is a grader failure. A clean-diff control (no seeds) has `recall: null`, so it takes no `score`; its gate is the default `["pass"]`. The manifest's -`entry_points.grade` is still required (`repo_manifest.v1`), even when every -task uses a built-in grader. +`entry_points.grade` is optional when every task's `grader` starts with +`dokimi:`. Both review graders read the answer the same way. Its last parseable ` ```json ` fence (one nesting arrays or objects more than 100 deep counts diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index ef165f37..9533e496 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -49,6 +49,7 @@ def _context(args: argparse.Namespace, out: Path, pin: bool = False): content = out / "_eval" / "content" # a pinned batch keeps its content until the pin is checked staged = export_eval_dir(tree, manifest, content.with_name("content.next") if pin else content) + _require_grade(staged, manifest, args.task) # before the pin and the image build runner = DockerRunner(args.docker) image, ctx_dir = args.image, None if not image: @@ -109,6 +110,21 @@ def context_digest(root: Path) -> str: return h.hexdigest() +def _require_grade(content: Path, manifest: RepoManifest, ids: list[str] | None) -> None: + """Refuse the selected tasks when one needs ``entry_points.grade`` and the manifest has none.""" + from dokimi.repos.manifest import ManifestError, TaskSummary, grade_problems + from dokimi.repos.workspace import PrepError, load_tasks + + try: + tasks = load_tasks(content, manifest, ids) + except (OSError, ValueError, PrepError): + return # reported where the run loads its tasks + graders = [(t.id, str(t.raw.get("grader") or "") if isinstance(t.raw, dict) else "") for t in tasks] + problems = grade_problems(manifest, [TaskSummary(i, "", "", g, ()) for i, g in graders]) + if problems: + raise ManifestError(manifest.path, problems) + + def _uses_builtin_graders(content: Path, manifest: RepoManifest) -> bool: """Whether a task of the exported eval dir names a ``dokimi:`` grader; an unreadable task.json counts.""" from dokimi.repos.graders import builtin_name diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index ea8e2eda..3a8a8bb8 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -114,6 +114,28 @@ def _claude_stream_json(raw: str, workdir: str | None) -> Transcript: return out +def _apply_result(block: dict[str, Any], by_id: dict[str, dict[str, Any]]) -> None: + """Attach one tool_result to the event of its tool_use.""" + result_of = by_id.get(block.get("tool_use_id", "")) + if result_of is None: + return + output = _result_text(block.get("content")) + # A PreToolUse hook that refuses the call (exit 2) comes back as an error result + # naming the hook; the command never ran, so it has no rc. + denied = bool(block.get("is_error")) and bool(_HOOK_DENIAL.match(output or "")) + if denied: + result_of["denied"] = True + if result_of["kind"] == "read" and not denied: + # A read that failed (the file does not exist) shows nothing of the file. + result_of["rc"] = 1 if block.get("is_error") else 0 + if result_of["kind"] == "command": + # Claude Code reports success or failure only, with stdout and stderr merged into one text. + if not denied: + result_of["rc"] = 1 if block.get("is_error") else 0 + if output is not None: + result_of["output"], result_of["output_truncated"] = clip_output(output) + + def _apply(out: Transcript, rec: dict[str, Any], t: float | None, by_id: dict[str, dict[str, Any]], workdir: str | None) -> None: kind = rec.get("type") @@ -141,22 +163,7 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, elif kind == "user": for block in _content(rec): if block.get("type") == "tool_result": - result_of = by_id.get(block.get("tool_use_id", "")) - if result_of is None: - continue - output = _result_text(block.get("content")) - # A PreToolUse hook that refuses the call (exit 2) comes back as an error result - # naming the hook; the command never ran, so it has no rc. - denied = bool(block.get("is_error")) and bool(_HOOK_DENIAL.match(output or "")) - if denied: - result_of["denied"] = True - if result_of["kind"] == "command": - # Claude Code reports success or failure only, with stdout - # and stderr merged into one text. - if not denied: - result_of["rc"] = 1 if block.get("is_error") else 0 - if output is not None: - result_of["output"], result_of["output_truncated"] = clip_output(output) + _apply_result(block, by_id) elif kind == "result": out.answer = str(rec.get("result") or "").strip() out.cost_usd = rec.get("total_cost_usd") diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py index 28cfbc75..40cbbb05 100644 --- a/dokimi/dokimi/repos/manifest.py +++ b/dokimi/dokimi/repos/manifest.py @@ -33,6 +33,7 @@ from .graders import DATA_FILES, NAMES as BUILTIN_NAMES, PREFIX as BUILTIN_PREFIX from .host_spec import HostSpecError, parse_host +from .process_metrics import Counter, parse_counter from .source import RepoTree SCHEMA_VERSION = "repo_manifest.v1" @@ -53,7 +54,6 @@ TASK_REQUIRED = ("id", "family", "base", "prompt", "grader") _PLACEHOLDER_RE = re.compile(r"\{([^{}]*)\}") - # Run-dir entries the framework creates itself and the agent's container does not mount. An `image.env` # scratch directory with one of these names would hand the agent the framework's own entry. The # mounted ones (workspace, home, config) are not scratch and stay allowed: nothing new is mounted. @@ -131,6 +131,7 @@ class RepoManifest: image: ImageSpec | None = None explore_tasks: str | None = None build_systems: tuple[str, ...] = () + process_metrics: dict[str, Counter] = field(default_factory=dict) @property def runnable(self) -> bool: @@ -167,6 +168,8 @@ def to_dict(self) -> dict[str, Any]: "tasks": self.tasks, "explore_tasks": self.explore_tasks, "build_systems": list(self.build_systems), + "process_metrics": {name: {"regex": c.pattern.pattern, "match": c.field} + for name, c in self.process_metrics.items()}, } @@ -236,7 +239,7 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: missing = [k for k in _RUN_KEYS if k not in raw] raise ManifestError(path, [f"{', '.join(run_keys)} without {', '.join(missing)}: a manifest " "that runs tasks needs all of eval_dir, tasks and entry_points"]) - needs_eval = [k for k in ("image", "explore_tasks") + needs_eval = [k for k in ("image", "explore_tasks", "process_metrics") if isinstance(raw, dict) and k in raw and not run_keys] if needs_eval: raise ManifestError(path, [f"{', '.join(needs_eval)} without eval_dir, tasks and entry_points: " @@ -272,6 +275,7 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: duplicates = sorted({n for n in cache_names if cache_names.count(n) > 1}) if duplicates: errors.append(f"image/caches: duplicate cache name(s) {duplicates}") + counters = _parse_counters(raw.get("process_metrics") or {}, errors) if errors: raise ManifestError(path, errors) @@ -292,9 +296,24 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: ), explore_tasks=_norm(raw["explore_tasks"]) if raw.get("explore_tasks") else None, build_systems=tuple(raw.get("build_systems") or ()), + process_metrics=counters, ) +def _parse_counters(raw: dict[Any, Any], errors: list[str]) -> dict[str, Counter]: + """``process_metrics`` compiled; each one that does not compile adds to ``errors``.""" + counters: dict[str, Counter] = {} + for name, value in raw.items(): + if not isinstance(name, str): # the schema's name pattern skips a non-string key + errors.append(f"process_metrics/{name!r}: a name is a string; YAML reads a bare 404, on or yes as another type") + continue + try: + counters[name] = parse_counter(name, value) + except ValueError as exc: + errors.append(str(exc)) + return counters + + def check_repo(tree: RepoTree, manifest: RepoManifest, open_base: Callable[[str], RepoTree] | None = None) -> RepoCheckReport: """Check ``manifest`` against the snapshot in ``tree``. @@ -336,6 +355,7 @@ def check_repo(tree: RepoTree, manifest: RepoManifest, if tree.is_dir(manifest.tasks): _check_tasks(tree, manifest, files, report) + report.errors.extend(grade_problems(manifest, report.tasks)) else: report.errors.append(f"tasks directory does not exist: {manifest.tasks}") if open_base is None: @@ -347,6 +367,17 @@ def check_repo(tree: RepoTree, manifest: RepoManifest, return report +def grade_problems(manifest: RepoManifest, tasks: list[TaskSummary]) -> list[str]: + """``entry_points.grade`` is required only when some task's grader is not a built-in ``dokimi:`` one.""" + if manifest.entry_points.get("grade"): + return [] + own = sorted(t.id for t in tasks if not t.grader.startswith(BUILTIN_PREFIX)) + if not own: + return [] + return [f"entry_points/grade: required, since these tasks use the repository's own grader " + f"(not {BUILTIN_PREFIX}*): {', '.join(own)}"] + + def require_runnable(manifest: RepoManifest) -> None: """Refuse a guidance-only manifest where tasks are about to be prepared or run.""" manifest.run_paths() diff --git a/dokimi/dokimi/repos/process_metrics.py b/dokimi/dokimi/repos/process_metrics.py new file mode 100644 index 00000000..7d69489a --- /dev/null +++ b/dokimi/dokimi/repos/process_metrics.py @@ -0,0 +1,88 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Process metrics of one run, from its ``events.jsonl`` events, for ``result.json`` ``process`` (contract §4.1). + +The framework counts what every repository asks of a trace: + +- ``tool_calls``: every event, so every tool call of the parent and its subagents; +- ``commands``: the ``command`` events; +- ``failed_commands``: commands not denied whose ``rc`` is neither ``0`` nor ``null`` (``null`` means + no result came back, which is not a failure; a permission denial carries ``rc`` 1 but never ran); +- ``denied_commands``: commands a hook or the harness's permissions refused (``denied``); +- ``guidance_read``: the sorted workspace paths of ``read`` events not denied that the manifest's guidance + globs match; a path outside the workspace (absolute, or with ``..``) is no repository file. + +``counters`` holds the manifest's ``process_metrics``: for each name, how many ``command`` events +its regex finds a match in (``re.search``). A counter matches the command text unless it names +``output`` (the command's output, as clipped in the event) or ``any`` (either). +""" + +from __future__ import annotations + +import posixpath +import re +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Any + +COUNTER_FIELDS = ("text", "output", "any") + + +@dataclass(frozen=True) +class Counter: + """One ``process_metrics`` entry: a compiled regex and the command field it searches.""" + + pattern: re.Pattern[str] + field: str = "text" + + def hits(self, event: Mapping[str, Any]) -> bool: + fields = ("text", "output") if self.field == "any" else (self.field,) + return any(self.pattern.search(str(event.get(f) or "")) for f in fields) + + +def parse_counter(name: str, value: Any) -> Counter: + """A ``process_metrics`` value, ```` or ``{regex: , match: text|output|any}``. + + Raises ``ValueError`` naming the metric when the regex does not compile or the shape is wrong.""" + regex, where = (value, "text") if isinstance(value, str) else ( + (value.get("regex"), value.get("match", "text")) if isinstance(value, dict) else (None, None)) + if not isinstance(regex, str) or not regex: + raise ValueError(f"process_metrics/{name}: give a regex, or {{regex: , match: text|output|any}}") + if where not in COUNTER_FIELDS: + raise ValueError(f"process_metrics/{name}: match must be one of {', '.join(COUNTER_FIELDS)}, got {where!r}") + try: + return Counter(re.compile(regex), where) + except re.error as exc: + raise ValueError(f"process_metrics/{name}: not a valid regex: {exc}") from exc + + +def _workspace_path(path: str) -> str | None: + """``path`` normalized (``./a``, ``a//b``, ``a/./b``), or None when it is not inside the workspace. + + A ``..`` that stays inside (``src/../AGENTS.md``) is resolved; ``~`` is a home directory, not the repository.""" + if path.startswith("~"): + return None + p = PurePosixPath(posixpath.normpath(path)) + if p.is_absolute() or ".." in p.parts: + return None + return str(p) + + +def process_metrics(events: Iterable[Mapping[str, Any]], is_guidance: Callable[[str], bool], + counters: Mapping[str, Counter] | None = None) -> dict[str, Any]: + """The ``process`` object of ``result.json`` for one run's events; ``is_guidance`` tells a guidance path.""" + events = list(events) + commands = [e for e in events if e.get("kind") == "command"] + # A read that failed (rc 1: the file does not exist) read no guidance. + reads = {w for e in events if e.get("kind") == "read" and e.get("path") and not e.get("denied") + and e.get("rc") in (0, None) and (w := _workspace_path(str(e["path"]))) is not None} + return { + "tool_calls": len(events), + "commands": len(commands), + "failed_commands": sum(e.get("rc") not in (0, None) and not e.get("denied") for e in commands), + "denied_commands": sum(bool(e.get("denied")) for e in commands), + "guidance_read": sorted(p for p in reads if is_guidance(p)), + "counters": {name: sum(c.hits(e) for e in commands) for name, c in (counters or {}).items()}, + } diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index 1fb14685..e0c9ba7c 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -64,7 +64,8 @@ from .grading import EvalContext, failed_outcome, outcome, run_entry from .harness_profile import HarnessProfile from .host import probe_host, unmet, validate_tasks -from .manifest import scratch_name +from .manifest import match_glob, scratch_name +from .process_metrics import process_metrics from .workspace import PrepError, Task, prepare_workspace SUPPORT_HOST = Path(__file__).resolve().parent / "support" @@ -193,6 +194,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, "input_tokens": None, "output_tokens": None, "cache_read_tokens": None, "cache_write_tokens": None, "usage_scope": None, "subagent_calls": None, "plugins": [], "over_budget": False, "caps": caps, "network": network, "trace_errors": []} + process: dict[str, Any] | None = None # null when the trace could not be read added = meta.get("arm_patch") or {} uptake_of = (added.get("added") or [], added.get("mcp_servers") or []) events = None # the trace's events once it parsed; none: the uptake is unmeasured @@ -207,6 +209,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, _write_json(rd / "meta.json", meta) if transcript.events or transcript.num_turns is not None: # else the agent never started events = transcript.events + process = run_process_metrics(ctx, events) cost = transcript.cost_usd agent.update( cost_usd=cost, charged_usd=cost if cost is not None else caps["budget_usd"], @@ -227,7 +230,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, "model_resolved": meta.get("model_resolved"), "rep": rep, "harness": opts.profile.name, "harness_version": meta["harness_version"], "base_date": meta["base_date"], "score_task": bool(task.score), - "score_max": task.score["max"] if task.score else None, "agent": agent, "grade": grade} + "score_max": task.score["max"] if task.score else None, "agent": agent, + "process": process, "grade": grade} if any(uptake_of): # an arm that added something to use: did the agent use it (#41 D4) record["uptake"] = arm_uptake(events, *uptake_of, WORKSPACE) try: @@ -238,6 +242,13 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, return record +def run_process_metrics(ctx: EvalContext, events: list[dict[str, Any]]) -> dict[str, Any]: + """``result.json`` ``process``: the framework's counts and the manifest's ``process_metrics`` (contract §4.1).""" + globs = ctx.manifest.guidance_globs + return process_metrics(events, lambda path: any(match_glob(g, path) for g in globs), + ctx.manifest.process_metrics) + + class CellCrash(Exception): """A cell whose agent ran but whose ``result.json`` could not be written: its cost still counts.""" diff --git a/dokimi/dokimi/schemas/repo_manifest.v1.json b/dokimi/dokimi/schemas/repo_manifest.v1.json index f5ba0a82..6490c5a4 100644 --- a/dokimi/dokimi/schemas/repo_manifest.v1.json +++ b/dokimi/dokimi/schemas/repo_manifest.v1.json @@ -62,7 +62,6 @@ "entry_points": { "type": "object", "additionalProperties": false, - "required": ["grade"], "properties": { "grade": {"type": "string", "minLength": 1}, "oracle": {"type": "string", "minLength": 1}, @@ -75,6 +74,24 @@ "type": "array", "uniqueItems": true, "items": {"enum": ["bazel", "make", "cmake", "pytest", "pre-commit"]} + }, + "process_metrics": { + "type": "object", + "propertyNames": {"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, + "additionalProperties": { + "oneOf": [ + {"type": "string", "minLength": 1}, + { + "type": "object", + "additionalProperties": false, + "required": ["regex"], + "properties": { + "regex": {"type": "string", "minLength": 1}, + "match": {"enum": ["text", "output", "any"]} + } + } + ] + } } } } diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 974dd0e6..823180fd 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -84,7 +84,7 @@ "dokimi/repos/environment/risks.py::_runs_repository_code": {"complexity": 12, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/environment/risks.py::_segments": {"complexity": 29, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/environment/risks.py::detect_capabilities": {"complexity": 16, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/repos/events.py::_apply": {"complexity": 22, "rationale": "pre-existing when the gate was introduced"}, + "dokimi/repos/events.py::_apply": {"complexity": 17, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/harness_profile.py::parse_profile": {"complexity": 11}, "dokimi/repos/l0/adapters.py::_build_names": {"complexity": 15, "rationale": "grew on main in #115 (L0 rules for static_check parity) before the ratchet landed; split it in its own PR"}, "dokimi/repos/l0/adapters.py::_make": {"complexity": 11}, diff --git a/dokimi/tests/repo_eval_fixture.py b/dokimi/tests/repo_eval_fixture.py index e5186c3a..916cffe8 100644 --- a/dokimi/tests/repo_eval_fixture.py +++ b/dokimi/tests/repo_eval_fixture.py @@ -105,6 +105,13 @@ lines.append({"type": "assistant", "message": {"model": answered, "content": [ {"type": "tool_use", "id": "t1", "name": "Edit", "input": {"file_path": os.environ.get("FAKE_CONTAINER_CWD", str(cwd)) + "/src/calc.py"}}]}}) +if os.environ.get("FAKE_PROCESS"): # a guidance read and one build command, for the process metrics + lines.append({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": "t3", "name": "Read", + "input": {"file_path": os.environ.get("FAKE_CONTAINER_CWD", str(cwd)) + "/AGENTS.md"}}, + {"type": "tool_use", "id": "t4", "name": "Bash", "input": {"command": "bazel build //x"}}]}}) + lines.append({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "t4", "is_error": True, "content": "ERROR: icpx is not found"}]}}) if os.environ.get("FAKE_TOOL_USES"): # more tool calls, as a JSON list of [name, input] lines.append({"type": "assistant", "message": {"content": [ {"type": "tool_use", "id": f"u{i}", "name": n, "input": a} @@ -128,7 +135,7 @@ "--max-budget-usd", "{budget}", "{tools}"], "tools": {"none": ["--tools", ""], "read": ["--tools", "Read"], "all": []}, "config_dir_env": "FAKE_CONFIG_DIR", - "env_allow": ["FAKE_ACTION", "FAKE_ANSWER_MODEL", "FAKE_MARKER", "FAKE_USAGE", "FAKE_TOOL_USES", "PATH"], + "env_allow": ["FAKE_ACTION", "FAKE_ANSWER_MODEL", "FAKE_MARKER", "FAKE_USAGE", "FAKE_PROCESS", "FAKE_TOOL_USES", "PATH"], "egress": ["api.example.com"], "trace": "claude-stream-json", "prompt_suffix": "Stay in the foreground.", "config_files": {"settings.json": '{"hook": "python3 {support_dir}/no_background.py"}'}, @@ -139,6 +146,7 @@ "schema": "repo_manifest.v1", "id": "toy", "source": {"url": "https://example.invalid/toy"}, "eval_dir": ".agent-eval", "guidance": {"globs": ["**/AGENTS.md"]}, + "process_metrics": {"bazel": r"\bbazel\b", "icpx_missing": {"regex": "icpx is not found", "match": "output"}}, "image": {"dockerfile": ".agent-eval/Dockerfile", "caches": [{"name": "build-cache", "path": "/cache/build"}], "env": {"TOY_STATE": "{run_dir}/state"}, "network": "none"}, diff --git a/dokimi/tests/test_repo_builtin_graders.py b/dokimi/tests/test_repo_builtin_graders.py index c2f3e7a3..0df69838 100644 --- a/dokimi/tests/test_repo_builtin_graders.py +++ b/dokimi/tests/test_repo_builtin_graders.py @@ -11,6 +11,7 @@ from pathlib import Path import pytest +import yaml from dokimi.cli import build_parser from dokimi.repos.container import CONTENT_DIR, GRADERS_DIR, SOURCE_DIR, ContainerError, ContainerResult @@ -583,3 +584,46 @@ def test_a_replay_window_is_checked_only_for_replay_and_defaults_to_five(tmp_pat out = capsys.readouterr().out errors = json.loads(out).get("errors", []) if out.strip() else [] assert not any(t in e for e in errors for t in ("replay_nowin", "replay_win3", "seeds_strwin")), errors + + +def test_a_cell_graded_by_a_builtin_grader_needs_no_grade_entry_point(tmp_path, monkeypatch): + # `repo validate` accepts a manifest without entry_points.grade when every grader is `dokimi:`; + # `repo run` must then grade with the built-in one instead of failing every cell. + root = _repo(tmp_path) + manifest = {**MANIFEST, "entry_points": {}} + write(root, ".agent-eval/repo-eval.yaml", yaml.safe_dump(manifest)) + git(root, "add", "-A") + git(root, "commit", "-qm", "no grade entry point") + exe, profile = fake_harness(tmp_path) + runner = FakeRunner() + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: runner) + monkeypatch.setenv("FAKE_ACTION", "fix") + out = tmp_path / "runs" + args = build_parser().parse_args( + ["repo", "run", f"{root}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--harness", str(profile), + "--bin", str(exe), "--model", "m-1", "--batch", "b1", "--out", str(out), "--cache-root", str(out / "caches"), + "--agent-network", "host", "--arm", "guidance:raw", "--task", "seeds_mul"]) + args.func(args) + rec = json.loads((out / "b1/seeds_mul__guidance-raw__m-1__r1/result.json").read_text()) + assert rec["grade"]["status"] == "graded", rec["grade"]["reasons"] + assert any(f"{GRADERS_DIR}/main.py" in s.argv for s in runner.specs) + + +def test_a_repository_grader_without_a_grade_entry_point_is_refused_before_any_agent_runs(tmp_path, monkeypatch, capsys): + # Without the check every cell would run, be paid for, and grade as grader_failure. + root = _repo(tmp_path) + write(root, ".agent-eval/repo-eval.yaml", yaml.safe_dump({**MANIFEST, "entry_points": {}})) + git(root, "add", "-A") + git(root, "commit", "-qm", "no grade entry point") + exe, profile = fake_harness(tmp_path) + runner = FakeRunner() + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: runner) + out = tmp_path / "runs" + args = build_parser().parse_args( + ["repo", "run", f"{root}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--harness", str(profile), + "--bin", str(exe), "--model", "m-1", "--batch", "b1", "--out", str(out), "--cache-root", str(out / "caches"), + "--agent-network", "host", "--arm", "guidance:raw", "--task", "fix_add"]) + with pytest.raises(SystemExit): + args.func(args) + assert runner.specs == [] and not (out / "b1/_eval/batch.json").exists() + assert "entry_points/grade: required" in capsys.readouterr().err diff --git a/dokimi/tests/test_repo_manifest.py b/dokimi/tests/test_repo_manifest.py index da1eb1e8..d5713f59 100644 --- a/dokimi/tests/test_repo_manifest.py +++ b/dokimi/tests/test_repo_manifest.py @@ -115,7 +115,10 @@ def test_valid_repo_passes_and_reports_tasks_and_guidance(tmp_path): (lambda m: m.update(schema="repo_manifest.v2"), "schema"), (lambda m: m.update(surprise=1), "surprise"), (lambda m: m.pop("entry_points"), "entry_points"), - (lambda m: m["entry_points"].pop("grade"), "grade"), + (lambda m: m.update(process_metrics={"bazel": "bazel("}), "process_metrics/bazel: not a valid regex"), + (lambda m: m.update(process_metrics={"icpx": {"regex": "icpx", "match": "stderr"}}), "match"), + (lambda m: m.update(process_metrics={"bad name": "x"}), "process_metrics"), + (lambda m: m.update(process_metrics={"icpx": {"regex": "icpx", "mach": "output"}}), "mach"), (lambda m: m.update(id="Bad Id"), "id"), (lambda m: m.update(build_systems=["scons"]), "build_systems"), (lambda m: m["image"].update(network="host"), "network"), diff --git a/dokimi/tests/test_repo_process_metrics.py b/dokimi/tests/test_repo_process_metrics.py new file mode 100644 index 00000000..64d2985e --- /dev/null +++ b/dokimi/tests/test_repo_process_metrics.py @@ -0,0 +1,220 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Process metrics from ``events.jsonl`` (contract §4.1), and the manifest rules around them (§2).""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +from dokimi.repos import WorkTree, check_repo, load_manifest, match_glob, parse_manifest +from dokimi.repos.manifest import TaskSummary, grade_problems +from dokimi.repos.process_metrics import parse_counter, process_metrics +from dokimi.repos.runner import run_process_metrics + +from test_repo_manifest import MANIFEST, _repo, _task + +EVENTS = [ + {"t": 1.0, "kind": "read", "path": "AGENTS.md"}, + {"t": 1.5, "kind": "read", "path": "src/AGENTS.md"}, + {"t": 1.6, "kind": "read", "path": "AGENTS.md"}, + {"t": 2.0, "kind": "read", "path": "src/lib.c"}, + {"t": 3.0, "kind": "command", "text": "bazel build //x", "rc": 0}, + {"t": 4.0, "kind": "command", "text": "bazelisk test //x", "rc": 1, "output": "ERROR: icpx is not found"}, + {"t": 5.0, "kind": "command", "text": "make -j8 &", "rc": None, "denied": True}, + {"t": 6.0, "kind": "command", "text": "ls", "rc": None}, + {"t": 7.0, "kind": "edit", "path": "src/lib.c"}, + {"t": 8.0, "kind": "tool", "name": "WebFetch", "denied": True}, +] +GLOBS = ("**/AGENTS.md", ".github/instructions/*.md") + + +def _guidance(path: str) -> bool: + return any(match_glob(g, path) for g in GLOBS) + + +def _counters(**raw): + return {name: parse_counter(name, value) for name, value in raw.items()} + + +def test_built_in_counts(): + got = process_metrics(EVENTS, _guidance) + assert got["tool_calls"] == 10 + assert got["commands"] == 4 + assert got["failed_commands"] == 1 # rc null (no result, or denied) is not a failure + assert got["denied_commands"] == 1 # the denied WebFetch is a tool call, not a command + assert got["guidance_read"] == ["AGENTS.md", "src/AGENTS.md"] # matched by the globs, once each + assert got["counters"] == {} + + +def test_no_events_counts_zero(): + assert process_metrics([], _guidance) == {"tool_calls": 0, "commands": 0, "failed_commands": 0, + "denied_commands": 0, "guidance_read": [], "counters": {}} + + +def test_counters_search_text_output_or_both(): + counters = _counters(bazel=r"\bbazel(?:isk)?\b", icpx={"regex": "icpx is not found", "match": "output"}, + icpx_in_text={"regex": "icpx"}, x_anywhere={"regex": r"//x|icpx", "match": "any"}, + make=r"\bmake\b") + got = process_metrics(EVENTS, _guidance, counters)["counters"] + # a denied command still counts: the agent asked for it + assert got == {"bazel": 2, "icpx": 1, "icpx_in_text": 0, "x_anywhere": 2, "make": 1} + + +@pytest.mark.parametrize("value, needle", [ + ("bazel(", "not a valid regex"), + ({"regex": "[a-"}, "not a valid regex"), + ({"regex": "x", "match": "stderr"}, "match must be one of"), + ({"match": "text"}, "give a regex"), + ("", "give a regex"), + (3, "give a regex"), +]) +def test_parse_counter_refuses(value, needle): + with pytest.raises(ValueError, match=needle) as exc: + parse_counter("m", value) + assert str(exc.value).startswith("process_metrics/m: ") + + +def test_manifest_keeps_counters_and_reports_every_bad_regex(): + raw = json.loads(json.dumps(MANIFEST)) + raw["process_metrics"] = {"bazel": r"\bbazel\b", "icpx": {"regex": "icpx", "match": "output"}} + m = parse_manifest(raw) + assert m.to_dict()["process_metrics"] == {"bazel": {"regex": r"\bbazel\b", "match": "text"}, + "icpx": {"regex": "icpx", "match": "output"}} + raw["process_metrics"] = {"a": "(", "b": "[", "ok": "x"} + with pytest.raises(Exception) as exc: + parse_manifest(raw) + assert "process_metrics/a" in str(exc.value) and "process_metrics/b" in str(exc.value) + + +def test_guidance_read_is_the_workspace_guidance_files_the_agent_read(): + # globs are root-relative: a home-dir or ../ AGENTS.md is no repository guidance, a denied read read nothing. + events = [{"kind": "read", "path": p} for p in + ("AGENTS.md", "src/lib.c", "/home/agent/.claude/AGENTS.md", "../AGENTS.md")] + events.append({"kind": "read", "path": "src/AGENTS.md", "denied": True}) + ctx = SimpleNamespace(manifest=parse_manifest(MANIFEST)) + assert run_process_metrics(ctx, events)["guidance_read"] == ["AGENTS.md"] + + +def test_a_command_the_permissions_denied_is_denied_not_failed(): + # A permission denial comes back as an is_error result (rc 1); a hook denial has rc null. Neither ran. + events = [{"kind": "command", "text": "rm -rf build", "rc": 1, "denied": True}, + {"kind": "command", "text": "make &", "rc": None, "denied": True}, + {"kind": "command", "text": "make", "rc": 2}] + m = process_metrics(events, lambda path: False) + assert (m["failed_commands"], m["denied_commands"]) == (1, 2) + + +def test_a_counter_name_that_yaml_reads_as_a_number_or_bool_is_refused(): + # Such a key passed validation, then broke the sorted result.json write after the agent had run. + raw = json.loads(json.dumps(MANIFEST)) + raw["process_metrics"] = {404: "Not Found", True: "x", "ok": "y"} + with pytest.raises(Exception) as exc: + parse_manifest(raw) + assert "process_metrics/404" in str(exc.value) and "process_metrics/True" in str(exc.value) + + +def test_process_metrics_needs_a_runnable_manifest(): + raw = {"schema": "repo_manifest.v1", "id": "toy", "guidance": {"globs": ["AGENTS.md"]}, + "process_metrics": {"x": "x"}} + with pytest.raises(Exception, match="process_metrics without eval_dir"): + parse_manifest(raw) + + +def _summary(task_id: str, grader: str) -> TaskSummary: + return TaskSummary(id=task_id, family="fix", base="0" * 40, grader=grader, gate=("pass",)) + + +def test_grade_is_required_only_for_a_repository_grader(): + raw = json.loads(json.dumps(MANIFEST)) + raw["entry_points"].pop("grade") + m = parse_manifest(raw) # the schema no longer requires it + assert grade_problems(m, [_summary("a", "dokimi:review_replay"), _summary("b", "dokimi:x")]) == [] + problems = grade_problems(m, [_summary("a", "dokimi:review_replay"), _summary("b", "tests")]) + assert len(problems) == 1 and problems[0].startswith("entry_points/grade: required") and problems[0].endswith(": b") + assert grade_problems(parse_manifest(MANIFEST), [_summary("b", "tests")]) == [] + + +def test_validate_reports_a_missing_grade(tmp_path): + raw = json.loads(json.dumps(MANIFEST)) + raw["entry_points"].pop("grade") + root = _repo(tmp_path, raw) + tree = WorkTree(root) + report = check_repo(tree, load_manifest(tree, "dev/evals/repo-eval.yaml")) + assert any("entry_points/grade: required" in e and "fix_one, review_two" in e for e in report.errors) + + +def test_validate_reports_a_bad_task_host(tmp_path): + root = _repo(tmp_path) + _task(root, "needs_cpus", host={"min_cpus": "four"}) + tree = WorkTree(root) + report = check_repo(tree, load_manifest(tree, "dev/evals/repo-eval.yaml")) + assert any(e.startswith("dev/evals/tasks/needs_cpus/task.json:") and "min_cpus" in e for e in report.errors) + + +def test_any_matches_the_output_when_the_text_does_not(): + counters = _counters(out_only={"regex": "is not found", "match": "any"}, + text_only={"regex": "is not found", "match": "text"}) + got = process_metrics(EVENTS, _guidance, counters)["counters"] + assert got == {"out_only": 1, "text_only": 0} + + +def test_counters_search_command_events_only(): + events = [{"kind": "read", "path": "a", "text": "bazel", "output": "bazel"}, + {"kind": "tool", "name": "WebFetch", "text": "bazel", "output": "bazel"}, + {"kind": "command", "text": "bazel build", "rc": 0}] + counters = _counters(bazel={"regex": "bazel", "match": "any"}) + assert process_metrics(events, _guidance, counters)["counters"] == {"bazel": 1} + + +@pytest.mark.parametrize("rc, failed", [(1, 1), (2, 1), (-9, 1), (137, 1), (0, 0), (None, 0)]) +def test_failed_commands_counts_every_non_zero_rc(rc, failed): + events = [{"kind": "command", "text": "x", "rc": rc}] + assert process_metrics(events, _guidance)["failed_commands"] == failed + + +def test_a_grader_that_only_resembles_a_built_in_one_needs_grade(): + raw = json.loads(json.dumps(MANIFEST)) + raw["entry_points"].pop("grade") + m = parse_manifest(raw) + problems = grade_problems(m, [_summary("a", "dokimi_tests"), _summary("b", "x-dokimi:y"), _summary("c", "dokimi")]) + assert len(problems) == 1 and problems[0].endswith(": a, b, c") + + +def test_only_a_read_counts_as_reading_guidance(): + got = process_metrics([{"kind": "edit", "path": "AGENTS.md"}], _guidance) + assert got["guidance_read"] == [] + + +@pytest.mark.parametrize("path, read", [ + ("./AGENTS.md", ["AGENTS.md"]), ("src/./AGENTS.md", ["src/AGENTS.md"]), ("src//AGENTS.md", ["src/AGENTS.md"]), + ("src/../AGENTS.md", ["AGENTS.md"]), ("a/../../x/AGENTS.md", []), ("../AGENTS.md", []), + ("~/AGENTS.md", []), ("/home/a/AGENTS.md", [])]) +def test_guidance_paths_are_normalized_before_the_workspace_test(path, read): + assert process_metrics([{"kind": "read", "path": path}], _guidance)["guidance_read"] == read + + +def test_a_read_that_failed_read_no_guidance(): + events = [{"kind": "read", "path": "AGENTS.md", "rc": 1}, {"kind": "read", "path": "src/AGENTS.md", "rc": 0}] + assert process_metrics(events, _guidance)["guidance_read"] == ["src/AGENTS.md"] + + +def test_a_read_error_in_the_stream_carries_a_failure(tmp_path): + from dokimi.repos.events import convert + def line(rec): + return json.dumps(rec) + raw = "\n".join([ + line({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": "r1", "name": "Read", "input": {"file_path": "/workspace/AGENTS.md"}}]}}), + line({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "r1", "is_error": True, "content": "File does not exist."}]}}), + line({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": "r2", "name": "Read", "input": {"file_path": "/workspace/src/AGENTS.md"}}]}}), + line({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "r2", "content": "ok"}]}})]) + events = convert("claude-stream-json", raw, "/workspace").events + assert [e["rc"] for e in events] == [1, 0] + assert process_metrics(events, _guidance)["guidance_read"] == ["src/AGENTS.md"] diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index d5828406..c44a32d9 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -62,6 +62,9 @@ def test_run_isolates_the_agent_and_grades_each_cell(setup, capsys): assert result["grade"]["status"] == "graded" and result["grade"]["success"] is True assert result["agent"]["cost_usd"] == 0.01 and result["agent"]["num_turns"] == 3 assert result["model_resolved"] == "m-1" + assert result["process"] == {"tool_calls": 1, "commands": 0, "failed_commands": 0, "denied_commands": 0, + "guidance_read": [], + "counters": {"bazel": 0, "icpx_missing": 0}} agents = [s for s in setup["runner"].specs if s.argv and s.argv[0] == f"{HARNESS_DIR}/fake-agent" and "-p" in s.argv] @@ -89,6 +92,27 @@ def test_run_isolates_the_agent_and_grades_each_cell(setup, capsys): assert raw["model_resolved"] == ["m-1"] and summary["model_mismatch"] == {} +def test_process_metrics_reach_result_json(setup, monkeypatch): + # guidance_read and counters come from the manifest (guidance globs, process_metrics) and the trace. + monkeypatch.setenv("FAKE_PROCESS", "1") + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + rec = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").read_text()) + assert rec["process"] == {"tool_calls": 3, "commands": 1, "failed_commands": 1, "denied_commands": 0, + "guidance_read": ["AGENTS.md"], + "counters": {"bazel": 1, "icpx_missing": 1}} + + +def test_an_unreadable_trace_leaves_process_null_and_still_grades(setup, monkeypatch): + def refuse(*args, **kwargs): + raise ValueError("unreadable trace") + + monkeypatch.setattr("dokimi.repos.runner.convert", refuse) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + rec = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").read_text()) + assert rec["process"] is None + assert rec["grade"]["status"] == "grader_failure" + + def test_a_run_answered_by_another_model_is_recorded_and_reported(setup, monkeypatch, capsys): # The record must carry what the harness reported, not the request, or the mismatch warning never fires. monkeypatch.setenv("FAKE_ANSWER_MODEL", "m-0") diff --git a/dokimi/tests/test_repo_uptake.py b/dokimi/tests/test_repo_uptake.py index b9f13aa8..18ae7c15 100644 --- a/dokimi/tests/test_repo_uptake.py +++ b/dokimi/tests/test_repo_uptake.py @@ -231,6 +231,9 @@ def test_an_agent_that_left_no_trace_is_unmeasured_one_that_called_no_tool_is_me _run(setup, action, "guidance:raw", f"patch:{patch}") silent, idle = setup["out"] / "silent", setup["out"] / "idle" assert json.loads(next(silent.glob("*patch*__r1/result.json")).read_text())["uptake"]["measured"] is False + # an agent that left no trace has no process counts; a run that started and made no call has zeros + assert json.loads(next(silent.glob("*patch*__r1/result.json")).read_text())["process"] is None + assert json.loads(next(idle.glob("*patch*__r1/result.json")).read_text())["process"]["tool_calls"] == 0 assert json.loads(next(idle.glob("*patch*__r1/result.json")).read_text())["uptake"]["measured"] is True unmeasured = compare([silent], "guidance:raw", [arm]) assert unmeasured["uptake"] == {arm: {"cells": 0, "used": 0, "unmeasured": 2}} and not _used_none(unmeasured)