diff --git a/dokimi/changelog.d/20261018_120100_napetrov_repo-execution-identity.md b/dokimi/changelog.d/20261018_120100_napetrov_repo-execution-identity.md new file mode 100644 index 00000000..9cf535f1 --- /dev/null +++ b/dokimi/changelog.d/20261018_120100_napetrov_repo-execution-identity.md @@ -0,0 +1,30 @@ +### Added + +- **`repo run --delegation none` for a single-agent baseline.** Under + `--tools all` Claude Code could start subagents through its `Agent` tool, so + a `repo run` cell was not a single-agent run. `--delegation none` runs the + harness profile's new `delegation.none` tool arguments instead (for + `claude-code`: `Agent`, `Task` and `Workflow` added to + `--disallowedTools`); a profile without them refuses the flag, and so does a tool mode that + `delegation.none` does not list (an allow-list, no fallback to the native + arguments; `claude-code` lists `none`, `read` and `all`). A cell records the + delegation whose arguments ran: `native` for a tool mode whose arguments are + the same either way. The default, + `--delegation native`, keeps today's behaviour. + +### Changed + +- **A `repo run` cell is reused only by the same execution.** `meta.json` and + `result.json` gain `execution` (`id`, `tools`, `delegation`); the `id` + hashes the profile's name, command template, tool arguments, environment + (`env_set`, `env_allow`), config (`config_dir_env`, `config_files`), prompt + suffix, `egress`, `bridge`, the SHA-256 of the harness executable and harness + version (the executable's content, not its path). Rerunning a batch with + another `--tools`, `--delegation`, an edited profile field listed above, harness binary, or harness + upgrade no longer counts cells from + the old one as done: the run is refused before any agent starts, and `--force` or a + new `--batch` runs them. A batch dir that already holds cells of another + execution is refused before any agent starts (unless `--force`), and so is a + result from earlier versions, which carries no `execution`: it is not + reused. +- `--force` is refused when the batch dir holds finished cells outside the run's plan, or of a task the host cannot run (which `--force` keeps), whose execution id differs, so one report row never pools two executions. `--delegation none` does not stop a skill with `context: fork` from starting a subagent. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 00aec556..b7b6f379 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -622,6 +622,11 @@ tools: # what the "{tools}" element expands to none: ["--tools", ""] read: ["--tools", "Read"] all: ["--disallowedTools", "WebFetch", "WebSearch"] +delegation: # --delegation none: these replace the tool mode's arguments + none: + none: ["--tools", ""] # list every tool mode a run may use, even if unchanged + read: ["--tools", "Read"] + all: ["--disallowedTools", "WebFetch", "WebSearch", "Agent", "Task", "Workflow"] config_dir_env: CLAUDE_CONFIG_DIR # a fresh directory per run env_allow: [PATH, HOME, AWS_REGION, AWS_BEARER_TOKEN_BEDROCK, ANTHROPIC_API_KEY, CLAUDE_CODE_USE_BEDROCK, https_proxy, no_proxy, ...] @@ -650,6 +655,18 @@ bridge: {for: AGENTS.md, write: CLAUDE.md, content: "@AGENTS.md"} directory, with `{config_dir}` and `{support_dir}` (the framework's helper scripts) as the agent's container sees them. The built-in `claude-code` profile uses them for a hook that refuses backgrounded shell commands. +- Optional `delegation.none` maps a tool mode to the arguments `repo run + --delegation none` uses instead, so the agent cannot start a subagent. It + is an allow-list: list every tool mode the run may use (for `none` and + `read` the arguments are the native ones); a tool mode it leaves out is + refused. A profile without `delegation.none` refuses `--delegation none` + before anything runs. The default, `--delegation native`, runs the tool mode + as written: under `all`, Claude Code may start subagents. + `--delegation none` limits the tools the agent is given, not the skills it + loads: a skill whose frontmatter says `context: fork` can still start a + subagent under `none`. Do not read a `none` cell as subagent-free when the + run's skills include one (tracked in intel/dokimi, "repo run: block + fork-context skills under --delegation none"). Profiles for other harnesses (Codex, Gemini CLI, Copilot CLI, Cursor agent, Aider, OpenCode, in-house agents) follow the same shape. Their `guidance` @@ -976,9 +993,26 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo - Each cell holds the ยง4 inputs plus `agent.stdout` (the raw trace) and `result.json`, the framework's verdict. A cell with `result.json` is skipped on a rerun unless `--force`; a cell without one is reported as missing, not - failed, and the next run of the batch fills it. A batch stays on the eval - tree it started with: its revision, or for a working tree the digests of - its `eval_dir` and of the image build context. A batch with a task that + failed, and the next run of the batch fills it. `meta.json` and + `result.json` record `execution`: the tool mode, the delegation mode (the one + whose arguments ran; a skill with `context: fork` can still start a subagent + under `none`) and an + `id` hashing the profile's name, `invoke`, tool arguments, `env_set`, + `env_allow`, `config_dir_env`, `config_files`, `prompt_suffix`, `egress`, + `bridge`, the SHA-256 of the harness executable (`--bin`) and harness version. + A rerun whose `id` differs (another `--tools`, a `--delegation` that changes the tool arguments, profile, + harness binary or harness version, or an edited `invoke`, `env_set`, `env_allow`, `config_dir_env`, `config_files`, `prompt_suffix`, `egress` or `bridge`) is refused before any agent starts, and + `--force` or another `--batch` runs it. `--force` replaces the cells of the + run's plan only: if the batch dir holds finished cells outside that plan + whose `id` differs from this run's (e.g. `--reps 2`, then `--reps 1 + --delegation none --force`), the run is refused before any agent starts, because + the report would pool both executions in one row; use another `--batch`. The + same goes for the finished cells of a task this host cannot run, which + `--force` keeps; that refusal comes after the host probe, before any agent starts. A result from before `execution` was + recorded is refused as `unrecorded`, not counted as native; `--force` or + another `--batch` runs it. A batch stays on the eval tree it + started with: its revision, or for a working tree the digests of its `eval_dir` + and of the image build context. A batch with a task that names a `dokimi:` grader (or a task directory that cannot be read) also stays on the built-in graders' sources. - The agent's container mounts the workspace, the per-run `home`, `config` and scratch directories (each `image.env` value under `{run_dir}/`), the caches, diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index ef165f37..c3e71979 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -18,6 +18,8 @@ from dokimi.repos.manifest import RepoManifest from dokimi.repos.workspace import Arm +from dokimi.repos.harness_profile import DELEGATION_MODES + def _context(args: argparse.Namespace, out: Path, pin: bool = False): """Source, manifest, exported eval dir, image and caches shared by `tasks check` and `run`. @@ -204,6 +206,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock try: parsed = [parse_arm(a) for a in arms] # before the image build: a typo or an unreadable patch fails now profile = load_profile(args.harness) + profile.tool_args(args.tools, args.delegation) # a profile that cannot turn subagents off: refuse now harness_bin = _harness_bin(args.bin or profile.bin) source = RepoSource.parse(args.source) # a misspelled revision fails before the build and the pin too arms = _pin_arms(parsed, source_git_dir(source, open_source(source, _cache_dir(args)), _cache_dir(args)), @@ -213,8 +216,8 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock tasks = load_tasks(ctx.content_dir, ctx.manifest, args.task) validate_tasks(tasks) # refuse malformed host requirements before containers start options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network, - tools=args.tools, timeout_s=args.timeout, max_turns=args.max_turns, - budget_usd=args.budget) for m in args.model] + tools=args.tools, delegation=args.delegation, timeout_s=args.timeout, + max_turns=args.max_turns, budget_usd=args.budget) for m in args.model] for task in tasks: # refuse before anything runs, not cell by cell task_caps(task, options[0]) agent_network(options[0], ctx, task) @@ -381,6 +384,9 @@ def register(repo_sub, positive_int, source_help: str, cache_help: str) -> None: help="REQUIRED while the egress proxy is not implemented: the agent's container gets " "this network, so its commands can reach any host. Recorded in meta.json") run_p.add_argument("--tools", default="all", help="Profile tool mode (default: all)") + run_p.add_argument("--delegation", choices=DELEGATION_MODES, default="native", + help="native: the agent may start subagents (default); none: the profile's tool " + "arguments without the subagent tools, for a single-agent baseline") run_p.add_argument("--timeout", type=positive_int, default=2700, help="Seconds per agent run unless task.json sets timeout_s (default 2700)") run_p.add_argument("--max-turns", type=positive_int, default=200, dest="max_turns", diff --git a/dokimi/dokimi/repos/harness_profile.py b/dokimi/dokimi/repos/harness_profile.py index c4ac84fe..acf619cd 100644 --- a/dokimi/dokimi/repos/harness_profile.py +++ b/dokimi/dokimi/repos/harness_profile.py @@ -6,7 +6,8 @@ A profile is data, not code. ``invoke`` is an argv template, and every element becomes exactly one argument after placeholder substitution, so a prompt with quotes or spaces never passes through a shell. The element ``"{tools}"`` is -replaced by the argument list of the selected tool mode. The child process +replaced by the argument list of the selected tool mode, or by its +``delegation`` override when subagents are turned off. The child process gets a scrubbed environment: only ``env_allow`` names are inherited, then ``env_set`` is applied, then ``config_dir_env`` points at a per-run directory so the user's own settings and memory cannot leak into a run. @@ -14,6 +15,8 @@ from __future__ import annotations +import hashlib +import json import re from dataclasses import dataclass, field from pathlib import Path @@ -28,6 +31,7 @@ PLACEHOLDERS = frozenset({"bin", "prompt", "model", "max_turns", "budget"}) CONFIG_PLACEHOLDERS = frozenset({"config_dir", "support_dir"}) TOOLS_ELEMENT = "{tools}" +DELEGATION_MODES = ("native", "none") # native: the agent may start subagents; none: it cannot _PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}") _VERSION_RE = re.compile(r"\d+\.\d+\.\d+") @@ -52,6 +56,7 @@ class HarnessProfile: bridge: dict[str, str] | None = None prompt_suffix: str = "" config_files: dict[str, str] = field(default_factory=dict) + delegation: dict[str, dict[str, tuple[str, ...]]] = field(default_factory=dict) path: str = "" @property @@ -74,11 +79,10 @@ def argv( max_turns: int = 4, budget_usd: float = 0.25, bin_path: str | None = None, + delegation: str = "native", ) -> list[str]: """The command line for one run, without a shell.""" - if tools not in self.tools: - raise ProfileError(f"{self.path}: tool mode {tools!r} is not defined " - f"(have: {', '.join(sorted(self.tools))})") + tool_args = self.tool_args(tools, delegation) values = { "bin": bin_path or self.bin, "prompt": prompt, @@ -89,11 +93,54 @@ def argv( out: list[str] = [] for element in self.invoke: if element == TOOLS_ELEMENT: - out.extend(self.tools[tools]) + out.extend(tool_args) else: out.append(_substitute(element, values)) return out + def tool_args(self, tools: str, delegation: str = "native") -> tuple[str, ...]: + """What ``{tools}`` becomes: the tool mode's arguments, or its ``delegation`` override.""" + if tools not in self.tools: + raise ProfileError(f"{self.path}: tool mode {tools!r} is not defined " + f"(have: {', '.join(sorted(self.tools))})") + if delegation == "native": + return self.tools[tools] + if delegation not in DELEGATION_MODES: + raise ProfileError(f"unknown delegation mode {delegation!r} (known: {', '.join(DELEGATION_MODES)})") + if delegation not in self.delegation: + raise ProfileError(f"{self.path}: declares no `delegation: {delegation}`, so a run cannot be " + "kept from starting subagents") + listed = self.delegation[delegation] + if tools not in listed: # an allow-list: no silent fallback to the native arguments + raise ProfileError(f"{self.path}: `delegation: {delegation}` lists no arguments for tool mode " + f"{tools!r} (lists: {', '.join(sorted(listed)) or 'none'}); list every mode " + "it may run with") + return listed[tools] + + def effective_delegation(self, tools: str, delegation: str = "native") -> str: + """The delegation whose arguments the command line gets: ``native`` when the profile's arguments for + ``delegation`` are the native ones (the tool mode never starts subagents anyway).""" + if delegation != "native" and self.tool_args(tools, delegation) == self.tools[tools]: + return "native" + return delegation + + def execution_id(self, tools: str, delegation: str = "native", harness_sha256: str | None = None, + harness_version: str | None = None) -> str: + """Short hash of how the agent is started: the profile's ``invoke`` template with its tool arguments, + environment, config, prompt suffix, egress and bridge. + + Two runs with the same id ran the same command line up to the per-cell values (prompt, model, + caps); ``repo run`` reuses a finished cell only when its id matches. Includes the harness + executable's digest and version to detect changes to the effective harness, wherever it lives. + """ + key = {"profile": self.name, "invoke": list(self.invoke), "tools": list(self.tool_args(tools, delegation)), + "env_set": dict(sorted(self.env_set.items())), "env_allow": list(self.env_allow), + "config_dir_env": self.config_dir_env, "config_files": dict(sorted(self.config_files.items())), + "egress": list(self.egress), "bridge": self.bridge, + "prompt_suffix": self.prompt_suffix, "harness_sha256": harness_sha256, + "harness_version": harness_version} + return hashlib.sha256(json.dumps(key, sort_keys=True).encode("utf-8")).hexdigest()[:12] + def prompt(self, task_prompt: str) -> str: """The task prompt as the agent gets it: with ``prompt_suffix`` after a blank line.""" return f"{task_prompt}\n\n{self.prompt_suffix}" if self.prompt_suffix else task_prompt @@ -190,6 +237,8 @@ def parse_profile(raw: Any, path: str = "") -> HarnessProfile: bridge=raw.get("bridge"), prompt_suffix=raw.get("prompt_suffix") or "", config_files=dict(raw.get("config_files") or {}), + delegation={mode: {t: tuple(args) for t, args in by_tools.items()} + for mode, by_tools in (raw.get("delegation") or {}).items()}, path=path, ) diff --git a/dokimi/dokimi/repos/profiles/claude-code.yaml b/dokimi/dokimi/repos/profiles/claude-code.yaml index 6748aedb..1aae4193 100644 --- a/dokimi/dokimi/repos/profiles/claude-code.yaml +++ b/dokimi/dokimi/repos/profiles/claude-code.yaml @@ -29,6 +29,12 @@ tools: none: ["--tools", ""] read: ["--tools", "Read"] all: ["--disallowedTools", "WebFetch", "WebSearch"] +# --delegation none: the same tools without the ones that start subagents (Agent/Task, Workflow). +delegation: + none: + none: ["--tools", ""] + read: ["--tools", "Read"] + all: ["--disallowedTools", "WebFetch", "WebSearch", "Agent", "Task", "Workflow"] config_dir_env: CLAUDE_CONFIG_DIR env_allow: - PATH diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index 1fb14685..513ef650 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -31,6 +31,7 @@ from __future__ import annotations import contextlib +import hashlib import json import math import os @@ -82,6 +83,7 @@ class AgentOptions: agent_network: str | None = None # explicit opt-in while the egress proxy is missing harness_version: str | None = None tools: str = "all" + delegation: str = "native" # DELEGATION_MODES; "none" runs the profile's no-subagent tool arguments timeout_s: float = 2700 max_turns: int = 200 budget_usd: float = 2.0 @@ -95,6 +97,55 @@ def cell_name(task: str, arm: str, model: str, rep: int) -> str: return f"{task}__{slug(arm.replace(':', '-'))}__{slug(model)}__r{rep}" +def harness_sha256(path: Path) -> str: + """Digest of the harness executable's bytes: the same file gives the same id wherever it lives.""" + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def execution_record(opts: AgentOptions) -> dict[str, str]: + """How the cell's agent was started, as ``meta.json`` and ``result.json`` record it.""" + return {"id": opts.profile.execution_id(opts.tools, opts.delegation, harness_sha256(opts.harness_bin), + opts.harness_version), + "tools": opts.tools, "delegation": opts.profile.effective_delegation(opts.tools, opts.delegation)} + + +def recorded_execution(result: Any) -> dict[str, Any]: + """The ``execution`` record of a parsed ``result.json``; empty when there is none or it is not an object.""" + ran = result.get("execution") if isinstance(result, dict) else None + return ran if isinstance(ran, dict) else {} + + +class ExecutionMismatch(PrepError): + """A finished cell of the batch ran with another execution than the one asked for.""" + + +def check_batch_executions(batch_dir: Path, options: list[AgentOptions], replacing: set[str] | None = None) -> None: + """Refuse, before anything runs, a batch dir whose finished cells ran with another execution. + + ``replacing`` (the cells of a ``--force`` run) are skipped: ``--force`` rewrites those, but not the others, + which would stay in the report next to the new ones.""" + wanted = {execution_record(o)["id"] for o in options} + found: set[str] = set() + for path in sorted(batch_dir.glob("*/result.json")): + if replacing is not None and path.parent.name in replacing: + continue + try: + ran = recorded_execution(json.loads(path.read_text(encoding="utf-8"))) + except (OSError, ValueError): + continue + found.add(ran["id"] if isinstance(ran.get("id"), str) else "unrecorded") + other = found - wanted + if other: + raise ExecutionMismatch( + f"{batch_dir} holds cells of another execution ({', '.join(sorted(other))}) than this run " + f"({', '.join(sorted(wanted))}); use another --batch, or --force to replace them" + + (" (--force replaces only the cells of this run's plan)" if replacing is not None else "")) + + def agent_network(opts: AgentOptions, ctx: EvalContext, task: Task) -> str: """The agent container's network: offline only when nothing needs egress, else the explicit opt-in.""" needs = tuple(opts.profile.egress) + task.egress @@ -153,8 +204,17 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, ``host`` (the probed facts of :func:`.host.probe_host`) is recorded in ``meta.json``.""" rd = batch_dir / cell_name(task.id, arm, opts.model, rep) + execution = execution_record(opts) if (rd / "result.json").is_file() and not force: - return {**json.loads((rd / "result.json").read_text(encoding="utf-8")), "skipped": True} + done = json.loads((rd / "result.json").read_text(encoding="utf-8")) + ran = recorded_execution(done) + # A result from before execution ids carries none: nothing says how it ran, so it is not reused. + if ran.get("id") != execution["id"]: + raise ExecutionMismatch( + f"the cell finished with another execution ({ran.get('id', 'unrecorded')}, delegation " + f"{ran.get('delegation', 'native')}) than this run ({execution['id']}, delegation " + f"{execution['delegation']}); use another --batch, or --force to replace it") + return {**done, "skipped": True} network = agent_network(opts, ctx, task) grade_network = docker_network(ctx.manifest.image.network if ctx.manifest.image else "none", task.egress) caps = task_caps(task, opts) @@ -163,7 +223,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, rd.mkdir(parents=True) meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, arm, rd, opts.profile) meta.update(harness=opts.profile.name, harness_version=opts.harness_version, model=opts.model, rep=rep, - batch=batch_dir.name, agent_network=network, egress_proxy=False, host=host) + batch=batch_dir.name, agent_network=network, egress_proxy=False, execution=execution, host=host) _write_json(rd / "meta.json", meta) mounts, env = agent_layout(ctx, rd, task) @@ -178,7 +238,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, env["DOKIMI_HOOK_LOG"] = f"{CONFIG_DIR}/hook.log" opts.profile.write_config(rd / "config", {"config_dir": CONFIG_DIR, "support_dir": SUPPORT_DIR}) argv = opts.profile.argv(prompt=opts.profile.prompt(str(task.raw["prompt"])), model=opts.model, - tools=opts.tools, max_turns=int(caps["max_turns"]), budget_usd=caps["budget_usd"], + tools=opts.tools, delegation=opts.delegation, max_turns=int(caps["max_turns"]), budget_usd=caps["budget_usd"], bin_path=bin_in) spec = ContainerSpec(image=ctx.image, argv=argv, workdir=WORKSPACE, mounts=mounts, env=env, pass_env=passed, network=network, name=container_name(rd.name, "agent")) @@ -226,7 +286,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, record: dict[str, Any] = {"task": task.id, "family": task.family, "arm": arm, "model": opts.model, "model_resolved": meta.get("model_resolved"), "rep": rep, "harness": opts.profile.name, "harness_version": meta["harness_version"], - "base_date": meta["base_date"], "score_task": bool(task.score), + "base_date": meta["base_date"], "execution": execution, "score_task": bool(task.score), "score_max": task.score["max"] if task.score else None, "agent": agent, "grade": grade} if any(uptake_of): # an arm that added something to use: did the agent use it (#41 D4) record["uptake"] = arm_uptake(events, *uptake_of, WORKSPACE) @@ -252,6 +312,28 @@ def _own_dir(path: Path) -> None: os.chmod(path, stat.S_IMODE(mode) | stat.S_IRWXU) +def _log_cell(log, name: str, rec: dict[str, Any]) -> None: + g, a = rec["grade"], rec["agent"] + verdict = ("GRADER FAILURE" if g["status"] == "grader_failure" + else f"score {g['score']}" if g["score_only"] else "pass" if g["success"] else "fail") + log(f"{name}: {verdict}; ${a['cost_usd'] or 0:.4f}, {a['num_turns']} turns, {a['wall_s']}s" + + (" (agent timed out)" if a["timed_out"] else "")) + + +def _probe_failed(batch_dir: Path, cells: list, verdicts: dict) -> list[str]: + """Tasks whose host probe failed and that still have an unrun cell: a finished cell keeps its result.""" + return [task_id for task_id, (_, by) in verdicts.items() if by == "host-probe" + and any(not _done(batch_dir / cell_name(t.id, a, o.model, r)) + for t, a, o, r in cells if t.id == task_id)] + + +def _clear_unsupported_markers(batch_dir: Path, cells: list, unsupported: dict) -> None: + """This host meets the requirements of the other tasks: an old marker would hide a cell this run never reaches.""" + for t, a, o, r in cells: + if t.id not in unsupported: + (batch_dir / cell_name(t.id, a, o.model, r) / UNSUPPORTED_FILE).unlink(missing_ok=True) + + def run_batch(ctx: EvalContext, tasks: list[Task], arms: list[str], options: list[AgentOptions], reps: int, batch_dir: Path, jobs: int = 1, force: bool = False, max_batch_usd: float | None = None, log=print) -> dict[str, Any]: @@ -262,20 +344,19 @@ def run_batch(ctx: EvalContext, tasks: list[Task], arms: list[str], options: lis ``result.json`` (from a host that met them) keeps it, even with ``force``.""" validate_tasks(tasks) cells = [(t, a, o, r) for t in tasks for a in arms for o in options for r in range(1, reps + 1)] - _record_plan(batch_dir, [cell_name(t.id, a, o.model, r) for t, a, o, r in cells]) + check_batch_executions(batch_dir, options, {cell_name(t.id, a, o.model, r) for t, a, o, r in cells} if force else None) host = probe_host(ctx, batch_dir / "_eval") _write_json(batch_dir / "_eval" / "host.json", host) verdicts = {t.id: unmet(t, host) for t in tasks} unsupported = {task_id: why for task_id, (why, _) in verdicts.items() if why} - # A probe failure only costs the run when a cell of the task is left unrun: a finished cell keeps its result. - probe_failed = [task_id for task_id, (_, by) in verdicts.items() if by == "host-probe" - and any(not _done(batch_dir / cell_name(t.id, a, o.model, r)) - for t, a, o, r in cells if t.id == task_id)] + if force and unsupported: # --force keeps their finished cells, so those must match this run too + check_batch_executions(batch_dir, options, {cell_name(t.id, a, o.model, r) for t, a, o, r in cells + if t.id not in unsupported}) + _record_plan(batch_dir, [cell_name(t.id, a, o.model, r) for t, a, o, r in cells]) + probe_failed = _probe_failed(batch_dir, cells, verdicts) for task_id, why in unsupported.items(): log(f"{task_id}: UNSUPPORTED, its cells are not run: {why}") - for t, a, o, r in cells: # this host meets them: an old marker would hide a cell this run never reaches - if t.id not in unsupported: - (batch_dir / cell_name(t.id, a, o.model, r) / UNSUPPORTED_FILE).unlink(missing_ok=True) + _clear_unsupported_markers(batch_dir, cells, unsupported) lock = threading.Lock() state: dict[str, Any] = {"spent": 0.0, "ran": 0, "skipped": 0, "errors": [], "budget_stopped": 0, "unsupported": 0} @@ -313,11 +394,7 @@ def one(cell: tuple[Task, str, AgentOptions, int]) -> None: return state["ran"] += 1 state["spent"] += rec["agent"].get("charged_usd", rec["agent"]["cost_usd"]) or 0.0 - g, a = rec["grade"], rec["agent"] - verdict = ("GRADER FAILURE" if g["status"] == "grader_failure" - else f"score {g['score']}" if g["score_only"] else "pass" if g["success"] else "fail") - log(f"{name}: {verdict}; ${a['cost_usd'] or 0:.4f}, {a['num_turns']} turns, {a['wall_s']}s" - + (" (agent timed out)" if a["timed_out"] else "")) + _log_cell(log, name, rec) with ThreadPoolExecutor(max(1, jobs)) as pool: list(pool.map(one, cells)) diff --git a/dokimi/dokimi/schemas/harness_profile.v1.json b/dokimi/dokimi/schemas/harness_profile.v1.json index 20a8900d..040d8f0a 100644 --- a/dokimi/dokimi/schemas/harness_profile.v1.json +++ b/dokimi/dokimi/schemas/harness_profile.v1.json @@ -22,6 +22,15 @@ "all": {"type": "array", "items": {"type": "string"}} } }, + "delegation": { + "type": "object", + "additionalProperties": false, + "description": "How to run without subagents (`repo run --delegation none`): per tool mode, the argument list that replaces that mode's `tools` entry. A tool mode not listed here is refused (list every tool mode the profile may run with); a profile without `none` cannot run `--delegation none`. It restricts tools only: a skill with `context: fork` can still start a subagent.", + "properties": { + "none": {"type": "object", "propertyNames": {"enum": ["none", "read", "all"]}, + "additionalProperties": {"type": "array", "items": {"type": "string"}}} + } + }, "config_dir_env": {"type": ["string", "null"]}, "env_allow": {"type": "array", "items": {"type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}}, "env_set": { diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 974dd0e6..7cbcf845 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -91,7 +91,6 @@ "dokimi/repos/l0/core.py::Scan.resolve_path": {"complexity": 19}, "dokimi/repos/manifest.py::_check_task": {"complexity": 23, "rationale": "grew on main in #113 and again in #120 (built-in review graders) after #93 seeded this baseline; split it in its own PR"}, "dokimi/repos/manifest.py::parse_manifest": {"complexity": 15}, - "dokimi/repos/runner.py::run_batch": {"complexity": 11, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/source.py::GitTree.read_many": {"complexity": 14}, "dokimi/repos/workspace.py::_apply_rev": {"complexity": 11, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/workspace.py::_guidance_at": {"complexity": 11, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/test_repo_host.py b/dokimi/tests/test_repo_host.py index 964c1f3c..deba9197 100644 --- a/dokimi/tests/test_repo_host.py +++ b/dokimi/tests/test_repo_host.py @@ -301,6 +301,53 @@ def test_run_skips_unsupported_cells_and_the_report_shows_them(tmp_path, monkeyp assert len(load_cells(batch)[0]) == 4 and not list(batch.glob("*/unsupported.json")) +def test_force_refuses_to_keep_cells_of_another_execution_for_a_task_the_host_cannot_run( + tmp_path, monkeypatch, capsys): + root = _repo(tmp_path) + out = tmp_path / "runs" + _run(root, out, HostRunner(_probe_out(affinity=8)), monkeypatch, "--task", "fix_add", "--task", "fix_race") + capsys.readouterr() + small = HostRunner(_probe_out(affinity=2, cpuinfo=AMD_CPUINFO)) + plan = (out / "b1" / "_eval" / "plan.json").read_text() + with pytest.raises(SystemExit): # fix_race's cells would stay as they ran, next to a new fix_add + _run(root, out, small, monkeypatch, "--task", "fix_add", "--task", "fix_race", "--force", "--tools", "read", + "--reps", "4") + assert "holds cells of another execution" in capsys.readouterr().err + assert (out / "b1" / "_eval" / "plan.json").read_text() == plan # a refused run adds no cell to the plan + assert not [s for s in small.specs if "-p" in s.argv] + _run(root, out, HostRunner(_probe_out(affinity=2, cpuinfo=AMD_CPUINFO)), monkeypatch, + "--task", "fix_add", "--task", "fix_race", "--force") # the same execution: fix_add replaced, fix_race kept + assert "2 ran, 2 already done" in capsys.readouterr().out + + +def _restamp(batch, task_id, execution_id="old"): + """Make the finished cells of one task look like they ran with another execution.""" + for path in batch.glob(f"{task_id}__*/result.json"): + record = json.loads(path.read_text()) + record["execution"] = {"id": execution_id} + path.write_text(json.dumps(record)) + + +@pytest.mark.parametrize("stale,refused", [("fix_race", True), ("fix_add", False)]) +def test_force_checks_the_kept_cells_of_the_unsupported_task_and_not_the_replaced_ones( + tmp_path, monkeypatch, capsys, stale, refused): + root = _repo(tmp_path) + out = tmp_path / "runs" + _run(root, out, HostRunner(_probe_out(affinity=8)), monkeypatch, "--task", "fix_add", "--task", "fix_race") + capsys.readouterr() + _restamp(out / "b1", stale) # fix_race is kept as it ran on this host; fix_add is replaced + small = HostRunner(_probe_out(affinity=2, cpuinfo=AMD_CPUINFO)) + args = (root, out, small, monkeypatch, "--task", "fix_add", "--task", "fix_race", "--force") + if refused: + with pytest.raises(SystemExit): + _run(*args) + assert "holds cells of another execution" in capsys.readouterr().err + assert not [s for s in small.specs if "-p" in s.argv] + else: + _run(*args) + assert "2 ran, 2 already done" in capsys.readouterr().out + + def test_a_capable_run_that_stops_early_leaves_the_unreached_cells_missing(tmp_path, monkeypatch, capsys): root = _repo(tmp_path) out = tmp_path / "runs" diff --git a/dokimi/tests/test_repo_reach.py b/dokimi/tests/test_repo_reach.py index 3d6f8940..333674c2 100644 --- a/dokimi/tests/test_repo_reach.py +++ b/dokimi/tests/test_repo_reach.py @@ -193,6 +193,91 @@ def test_builtin_claude_code_profile_builds_argv_without_a_shell(tmp_path): profile.argv(prompt="p", model="m", tools="everything") +def test_delegation_none_drops_the_subagent_tools_and_changes_the_execution_id(): + profile = load_profile("claude-code") + native = profile.argv(prompt="p", model="m", tools="all") + solo = profile.argv(prompt="p", model="m", tools="all", delegation="none") + assert native[-3:] == ["--disallowedTools", "WebFetch", "WebSearch"] + assert solo[-6:] == ["--disallowedTools", "WebFetch", "WebSearch", "Agent", "Task", "Workflow"] + # Claude Code 2.1.304's Workflow tool starts subagents without going through Agent/Task. + assert profile.argv(prompt="p", model="m", tools="read", delegation="none")[-2:] == ["--tools", "Read"] + ids = {profile.execution_id("all"), profile.execution_id("all", "none"), profile.execution_id("read")} + assert len(ids) == 3 and profile.execution_id("all") == load_profile("claude-code").execution_id("all") + with pytest.raises(ProfileError, match="unknown delegation"): + profile.tool_args("all", "fork") + + +def test_harness_binary_and_version_change_the_execution_id(): + profile = load_profile("claude-code") + # Same settings without harness info: same id + id1 = profile.execution_id("all") + id2 = profile.execution_id("all") + assert id1 == id2 + # Different harness executable: different id + id3 = profile.execution_id("all", harness_sha256="a" * 64) + assert id3 != id1 + # Different harness version: different id + id4 = profile.execution_id("all", harness_sha256="a" * 64, harness_version="2.0.0") + id5 = profile.execution_id("all", harness_sha256="a" * 64, harness_version="2.1.0") + assert id4 != id5 and id4 != id1 + + +def test_execution_id_pinned_by_profile_name(tmp_path): + assert _ids_differ_only_by(tmp_path, {"name": "fake-a"}, {"name": "fake-b"}) + + +def test_execution_id_pinned_by_invoke(tmp_path): + assert _ids_differ_only_by(tmp_path, {}, {"invoke": ["{bin}", "--other", "-p", "{prompt}", "{tools}"]}) + + +def test_execution_id_pinned_by_env_allow_and_config_dir_env(tmp_path): + assert _ids_differ_only_by(tmp_path, {}, {"env_allow": ["PATH"]}) + assert _ids_differ_only_by(tmp_path, {}, {"config_dir_env": "OTHER_CONFIG_DIR"}) + + +def _ids_differ_only_by(tmp_path, first: dict, second: dict) -> bool: + """Load two profiles from the same directory, changing only the fields given, and compare their ids.""" + before = load_profile(_profile(tmp_path, **first)).execution_id("read") + after = load_profile(_profile(tmp_path, **second)).execution_id("read") + return before != after + + +def test_execution_id_pinned_by_env_set(tmp_path): + assert _ids_differ_only_by(tmp_path, {"env_set": {"VAR": "value1"}}, {"env_set": {"VAR": "value2"}}) + + +def test_execution_id_pinned_by_config_files(tmp_path): + assert _ids_differ_only_by(tmp_path, {"config_files": {"cfg.json": '{"a": 1}'}}, + {"config_files": {"cfg.json": '{"a": 2}'}}) + + +def test_execution_id_pinned_by_prompt_suffix(tmp_path): + assert _ids_differ_only_by(tmp_path, {"prompt_suffix": "suffix1"}, {"prompt_suffix": "suffix2"}) + + +def test_execution_id_pinned_by_bridge_and_egress(tmp_path): + bridge = {"for": "AGENTS.md", "write": "CLAUDE.md", "content": "@AGENTS.md"} + assert _ids_differ_only_by(tmp_path, {"bridge": bridge}, {"bridge": {**bridge, "content": "@OTHER.md"}}) + assert _ids_differ_only_by(tmp_path, {"egress": ["a.example"]}, {"egress": ["b.example"]}) + + +def test_a_profile_without_delegation_cannot_run_without_subagents(tmp_path): + with pytest.raises(ProfileError, match="declares no `delegation: none`"): + load_profile(_profile(tmp_path)).tool_args("read", "none") + + +def test_delegation_none_is_an_allow_list_of_tool_modes(tmp_path): + raw = yaml.safe_load(Path(_profile(tmp_path)).read_text()) + raw["delegation"] = {"none": {}} + path = tmp_path / "custom.yaml" + path.write_text(yaml.safe_dump(raw)) + with pytest.raises(ProfileError, match="lists no arguments for tool mode"): + load_profile(str(path)).tool_args("read", "none") + claude = load_profile("claude-code") + assert {m: claude.effective_delegation(m, "none") for m in ("none", "read", "all")} == { + "none": "native", "read": "native", "all": "none"} + + def test_profile_env_is_allow_listed_and_isolated(tmp_path): profile = load_profile("claude-code") env = profile.env(tmp_path / "cfg").resolve({"PATH": "/bin", "AWS_REGION": "us-east-1", diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index d5828406..e8339137 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -11,6 +11,7 @@ from pathlib import Path import pytest +import yaml from dokimi.cli import build_parser from dokimi.repos.container import CONTENT_DIR, HARNESS_DIR, RUN_DIR, SOURCE_DIR, WORKSPACE @@ -122,6 +123,197 @@ def test_already_graded_cells_are_skipped_unless_forced(setup, capsys): assert "1 ran, 0 already done" in capsys.readouterr().out +def test_a_cell_is_reused_only_by_the_same_execution(setup, capsys): + raw = yaml.safe_load(setup["profile"].read_text()) + raw["delegation"] = {"none": {"all": [*raw["tools"]["all"], "--no-agents"]}} + setup["profile"].write_text(yaml.safe_dump(raw)) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + capsys.readouterr() + rd = setup["out"] / "b1/fix_add__guidance-raw__m-1__r1" + ran = json.loads((rd / "result.json").read_text())["execution"] + assert ran["delegation"] == "native" and ran["tools"] == "all" + assert json.loads((rd / "meta.json").read_text())["execution"] == ran + with pytest.raises(SystemExit): # refused before any cell runs + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--delegation", "none") + assert "holds cells of another execution" in capsys.readouterr().err + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--delegation", "none", "--force") + assert "1 ran" in capsys.readouterr().out + rec = json.loads((rd / "result.json").read_text()) + assert rec["execution"]["delegation"] == "none" and rec["execution"]["id"] != ran["id"] + agent = [s for s in setup["runner"].specs if "-p" in s.argv][-1] + assert "--no-agents" in agent.argv + + +def test_force_does_not_pool_executions_across_the_plan(setup, capsys): + raw = yaml.safe_load(setup["profile"].read_text()) + raw["delegation"] = {"none": {"all": [*raw["tools"]["all"], "--no-agents"]}} + setup["profile"].write_text(yaml.safe_dump(raw)) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--reps", "2") + capsys.readouterr() + setup["runner"].specs.clear() + with pytest.raises(SystemExit): # r2 stays native next to a new r1 under none + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--reps", "1", "--delegation", "none", "--force") + assert "holds cells of another execution" in capsys.readouterr().err + assert not [s for s in setup["runner"].specs if "-p" in s.argv] + # the cells of the plan itself are still replaced + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--reps", "2", "--delegation", "none", "--force") + assert "2 ran" in capsys.readouterr().out + + +def test_a_result_from_before_execution_ids_is_not_reused(setup, capsys): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + rd = setup["out"] / "b1/fix_add__guidance-raw__m-1__r1" + rec = json.loads((rd / "result.json").read_text()) + del rec["execution"] # written by an earlier version + (rd / "result.json").write_text(json.dumps(rec)) + capsys.readouterr() + with pytest.raises(SystemExit): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + assert "unrecorded" in capsys.readouterr().err + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--force") + assert "1 ran" in capsys.readouterr().out + + +def test_a_cell_finished_with_another_execution_is_not_reused_even_if_the_batch_check_is_skipped( + setup, capsys, monkeypatch): + """``run_cell`` guards itself: the up-front batch check is not the only thing between a cell and its reuse.""" + raw = yaml.safe_load(setup["profile"].read_text()) + raw["delegation"] = {"none": {"all": [*raw["tools"]["all"], "--no-agents"]}} + setup["profile"].write_text(yaml.safe_dump(raw)) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + capsys.readouterr() + setup["runner"].specs.clear() + monkeypatch.setattr("dokimi.repos.runner.check_batch_executions", lambda *a, **k: None) + with pytest.raises(SystemExit): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--delegation", "none") + assert "finished with another execution" in capsys.readouterr().out + assert not [s for s in setup["runner"].specs if "-p" in s.argv] + + +def test_a_cell_finished_without_an_execution_is_not_reused_even_if_the_batch_check_is_skipped( + setup, capsys, monkeypatch): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + capsys.readouterr() + result = setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json" + done = json.loads(result.read_text()) + del done["execution"] + result.write_text(json.dumps(done)) + setup["runner"].specs.clear() + monkeypatch.setattr("dokimi.repos.runner.check_batch_executions", lambda *a, **k: None) + with pytest.raises(SystemExit): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + assert "unrecorded" in capsys.readouterr().out + assert not [s for s in setup["runner"].specs if "-p" in s.argv] + + +def test_the_same_harness_bytes_resume_from_any_path(setup, capsys, tmp_path): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + capsys.readouterr() + moved = tmp_path / "elsewhere" / "fake-harness" + moved.parent.mkdir() + moved.write_bytes(setup["exe"].read_bytes()) + moved.chmod(0o755) + setup["exe"].unlink() + setup["exe"].symlink_to(moved) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + assert "1 already done" in capsys.readouterr().out + + +def test_a_changed_harness_binary_is_refused_on_resume(setup, capsys): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + capsys.readouterr() + # Same name, same reported version, other bytes: the id is the executable's digest + setup["exe"].write_text(setup["exe"].read_text() + "\n# upgraded in place\n") + with pytest.raises(SystemExit): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + assert "holds cells of another execution" in capsys.readouterr().err + + +def test_a_batch_dir_with_cells_of_another_execution_is_refused_up_front(setup, capsys): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + capsys.readouterr() + setup["runner"].specs.clear() + with pytest.raises(SystemExit): + _run(setup, "--arm", "guidance:raw,guidance:none", "--task", "fix_add", "--tools", "read") + assert "holds cells of another execution" in capsys.readouterr().err + assert not [s for s in setup["runner"].specs if "-p" in s.argv] + + +def test_the_cell_records_the_tools_mode_it_ran_with(setup): + rd = setup["out"] / "b1/fix_add__guidance-raw__m-1__r1" + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--tools", "read") + for name in ("result.json", "meta.json"): + assert json.loads((rd / name).read_text())["execution"]["tools"] == "read", name + + +def test_an_unreadable_or_unparsable_result_does_not_stop_the_batch_check(setup, capsys): + from dokimi.repos.harness_profile import load_profile + from dokimi.repos.runner import AgentOptions, ExecutionMismatch, check_batch_executions + + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + capsys.readouterr() + batch = setup["out"] / "b1" + (batch / "broken").mkdir() + (batch / "broken" / "result.json").write_text("{not json", encoding="utf-8") # ValueError + (batch / "adir").mkdir() + (batch / "adir" / "result.json").mkdir() # OSError (a directory) + options = [AgentOptions(profile=load_profile(setup["profile"]), model="m-1", harness_bin=setup["exe"], + harness_version="1.2.3")] + check_batch_executions(batch, options) # skipped, not raised + (batch / "fix_add__guidance-raw__m-1__r1" / "result.json").unlink() + (batch / "other").mkdir() + (batch / "other" / "result.json").write_text("{}", encoding="utf-8") + with pytest.raises(ExecutionMismatch, match="unrecorded"): # a parsable one is still checked + check_batch_executions(batch, options) + + +def test_a_harness_reporting_another_version_with_the_same_bytes_is_another_execution(setup, monkeypatch, capsys): + """A wrapper script or launcher keeps its bytes while the agent behind it changes: the version is an input.""" + monkeypatch.setattr("dokimi.repos.runner.harness_version", lambda *a, **k: "1.2.3") + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + capsys.readouterr() + setup["runner"].specs.clear() + monkeypatch.setattr("dokimi.repos.runner.harness_version", lambda *a, **k: "1.2.4") + with pytest.raises(SystemExit): # refused before any cell runs + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + assert "holds cells of another execution" in capsys.readouterr().err + assert not [s for s in setup["runner"].specs if "-p" in s.argv] + + +def test_a_refused_delegation_none_starts_no_harness_version_probe(setup, monkeypatch, capsys): + probes = [] + monkeypatch.setattr("dokimi.repos.runner.harness_version", lambda *a, **k: probes.append(a) or "1") + with pytest.raises(SystemExit): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--delegation", "none") + capsys.readouterr() + assert probes == [] + + +def test_the_cell_records_the_delegation_whose_arguments_ran(setup, capsys): + raw = yaml.safe_load(setup["profile"].read_text()) + raw["delegation"] = {"none": {"all": [*raw["tools"]["all"], "--no-agents"], "none": raw["tools"]["none"]}} + setup["profile"].write_text(yaml.safe_dump(raw)) + rd = setup["out"] / "b1/fix_add__guidance-raw__m-1__r1" + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--tools", "none", "--delegation", "none") + assert json.loads((rd / "result.json").read_text())["execution"]["delegation"] == "native" + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--tools", "all", "--delegation", "none", "--force") + assert json.loads((rd / "result.json").read_text())["execution"]["delegation"] == "none" + + +def test_delegation_none_is_refused_before_anything_runs_when_the_profile_cannot(setup, capsys): + with pytest.raises(SystemExit): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--delegation", "none") + assert "declares no `delegation: none`" in capsys.readouterr().err + assert not [s for s in setup["runner"].specs if "-p" in s.argv] + + +def test_a_refused_delegation_none_leaves_no_batch_behind(setup, capsys): + with pytest.raises(SystemExit): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--delegation", "none") + capsys.readouterr() + assert not (setup["out"] / "b1").exists() # no pin, no eval content, no image build for a refused run + + @pytest.mark.outcome("cjm-repo-grader-crash") def test_a_grader_crash_is_a_failure_not_an_exclusion(setup, monkeypatch, capsys): monkeypatch.setenv("FAKE_ACTION", "break") # the agent deletes src/calc.py, the grader cannot import it