From 5afd0211ebbcb02652107095c28cf61a121d5b17 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 18 Oct 2026 22:11:25 -0700 Subject: [PATCH 01/36] feat(repo): Codex and Copilot CLI harness profiles, native review mode Add built-in harness_profile.v1 files codex, codex-review, copilot and copilot-review, so repo run and repo reach can compare guidance arms under codex exec, codex exec review --commit HEAD, copilot -p and Copilot's /review agent, not only Claude Code. - trace: codex-json and copilot-json convert the CLIs' JSON event streams to events.jsonl (cli_traces.py); Codex read events come from shell commands that only print files. - native_prompt: a profile may run the harness's own prompt; reach refuses it. - answer_format: codex-review appends a findings block parsed from Codex's review text (answers.py), so review graders read it unchanged. - bin_companions: helper files mounted beside the executable (Codex's codex-code-mode-host, without which no tool call runs in the container). - repo reach now writes the profile's config_files, as repo run does. Signed-off-by: Nikolay Petrov --- ...nikolay.a.petrov_codex_copilot_profiles.md | 7 + dokimi/docs/architecture.md | 10 +- dokimi/docs/repo-evaluation-contract.md | 55 +++- dokimi/dokimi/repos/answers.py | 58 ++++ dokimi/dokimi/repos/cli_traces.py | 210 ++++++++++++ dokimi/dokimi/repos/events.py | 6 +- dokimi/dokimi/repos/harness_profile.py | 16 +- .../dokimi/repos/profiles/codex-review.yaml | 60 ++++ dokimi/dokimi/repos/profiles/codex.yaml | 68 ++++ .../dokimi/repos/profiles/copilot-review.yaml | 80 +++++ dokimi/dokimi/repos/profiles/copilot.yaml | 77 +++++ dokimi/dokimi/repos/reach.py | 7 + dokimi/dokimi/repos/runner.py | 22 +- dokimi/dokimi/schemas/harness_profile.v1.json | 5 +- dokimi/tests/test_repo_cli_harnesses.py | 306 ++++++++++++++++++ dokimi/tests/test_repo_reach.py | 19 ++ 16 files changed, 991 insertions(+), 15 deletions(-) create mode 100644 dokimi/changelog.d/20261019_050000_nikolay.a.petrov_codex_copilot_profiles.md create mode 100644 dokimi/dokimi/repos/answers.py create mode 100644 dokimi/dokimi/repos/cli_traces.py create mode 100644 dokimi/dokimi/repos/profiles/codex-review.yaml create mode 100644 dokimi/dokimi/repos/profiles/codex.yaml create mode 100644 dokimi/dokimi/repos/profiles/copilot-review.yaml create mode 100644 dokimi/dokimi/repos/profiles/copilot.yaml create mode 100644 dokimi/tests/test_repo_cli_harnesses.py diff --git a/dokimi/changelog.d/20261019_050000_nikolay.a.petrov_codex_copilot_profiles.md b/dokimi/changelog.d/20261019_050000_nikolay.a.petrov_codex_copilot_profiles.md new file mode 100644 index 00000000..955c2ef5 --- /dev/null +++ b/dokimi/changelog.d/20261019_050000_nikolay.a.petrov_codex_copilot_profiles.md @@ -0,0 +1,7 @@ +### Added + +- **Codex CLI and Copilot CLI harness profiles for `repo run` and `repo reach`** — `--harness codex`, `codex-review`, `copilot` and `copilot-review` run oneDAL-style repository tasks under `codex exec`, Codex's native review mode (`codex exec review --commit HEAD`), `copilot -p` and Copilot's `/review` agent, so guidance arms can be compared per harness, not only under Claude Code. Their JSON traces convert to `events.jsonl` (`trace: codex-json`, `copilot-json`). A profile can now run the harness's own prompt (`native_prompt`) and convert its answer before grading (`answer_format: codex-review` appends a findings block parsed from Codex's review text); `repo reach` refuses a `native_prompt` profile. `bin_companions` mounts the helper files an executable needs beside it (Codex's `codex-code-mode-host`). + +### Fixed + +- **`repo reach` writes the profile's `config_files`** — only `repo run` wrote them, so a profile whose endpoint or hooks live in a config file (a Codex `config.toml` naming a provider) probed a different setup than it ran. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index d032b7ca..9a20657d 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -236,15 +236,19 @@ next release. it can see. A file comes back `loaded`, `truncated` (one token — the file was cut), `fetched` (seen only after the agent read it), `not_loaded`, or `error` (no usable answer). Harness behaviour is data, not code: - `dokimi/repos/profiles/claude-code.yaml` is a `harness_profile.v1` - file and any other harness is another YAML. + `dokimi/repos/profiles/` holds `harness_profile.v1` files for Claude + Code, the Codex CLI (`codex`, and its native review mode `codex-review`) and + the Copilot CLI (`copilot`, and its `/review` agent `copilot-review`); any + other harness is another YAML. - `dokimi/repos/static.py` — the `repo_static.v1` schema: what the repository's own `static` entry point writes, so its checks can be merged into L0. - `dokimi/repos/bases.py` — checks that need every task base: each `base` must resolve, and the guidance globs are evaluated at each base. - `dokimi/repos/events.py` — converts a harness trace to the - `events.jsonl` run trace. + `events.jsonl` run trace; `dokimi/repos/cli_traces.py` holds the Codex and + Copilot JSON converters, and `dokimi/repos/answers.py` the `answer_format` + conversions that turn a native review answer into a findings block. - `dokimi/repos/l0/` — `repo scan` (L0): checks the links, paths, identifiers, commands and build labels the guidance names against the same tree (`repo_readiness.v1`), with no LLM call. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 40069f9d..f83c3879 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -22,7 +22,7 @@ SPDX-License-Identifier: Apache-2.0 > Slice S3 adds harness profiles (§6) and the reach probe (§7): > > ```bash -> python cli.py repo reach --manifest --model [--harness claude-code] +> python cli.py repo reach --manifest --model [--harness claude-code|codex|copilot|] > [--bin ] [--scenarios root,cwd,read] [--only ] [--dry-run] > ``` > @@ -436,7 +436,7 @@ env_allow: [PATH, HOME, AWS_REGION, AWS_BEARER_TOKEN_BEDROCK, ANTHROPIC_API_KEY, env_set: {CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1", DISABLE_AUTOUPDATER: "1"} egress: ["bedrock-runtime.*.amazonaws.com", "api.anthropic.com"] # the only hosts the agent container reaches (§2) endpoint_env: ANTHROPIC_BASE_URL # used by payload capture, if set -trace: claude-stream-json # converter to events.jsonl; or: text +trace: claude-stream-json # converter to events.jsonl; or: codex-json, copilot-json, text guidance: # DECLARED; verified by the reach probe, never trusted names: [CLAUDE.md, CLAUDE.local.md, AGENTS.md] resolution: ancestors-of-cwd + lazy-on-subtree-access @@ -459,8 +459,55 @@ bridge: {for: AGENTS.md, write: CLAUDE.md, content: "@AGENTS.md"} scripts) as the agent's container sees them. The built-in `claude-code` profile uses them for a hook that refuses backgrounded shell commands. -Profiles for other harnesses (Codex, Gemini CLI, Copilot CLI, Cursor agent, -Aider, OpenCode, in-house agents) follow the same shape. Their `guidance` +- `trace` names the converter of the harness's stdout to `events.jsonl`: + `claude-stream-json`, `codex-json` (`codex exec --json`), `copilot-json` + (`copilot -p --output-format json`) or `text` (stdout is the answer, no + events). Codex has no read tool, so its `read` events come from shell + commands that only print files (`cat`, `head`, `tail`, `sed -n`, `nl`). + Neither Codex nor Copilot reports a cost: `cost_usd` is `null` and a batch + budget counts each cell's cap. +- `bin_companions` names files the executable starts from its own directory. + `repo run` mounts the executable alone at `/eval/harness/`, so each + companion is mounted read-only beside it, from beside `--bin`; a missing one + stops the run before any agent starts. Codex 0.160 needs + `codex-code-mode-host`: every tool call goes through it. +- `repo reach` writes `config_files` into each scenario's config directory as + `repo run` does, with `{config_dir}` and `{support_dir}` as host paths. +- `native_prompt: true` marks a harness that runs its own built-in prompt, such + as a native review mode. `invoke` then passes no `{prompt}` (and there is no + `prompt_suffix`), the agent never sees the task prompt, and `repo reach` + refuses the profile. Use such a profile only on tasks whose grader reads what + it answers, which `answer_format` provides. +- `answer_format` converts the final answer before grading. `codex-review` + keeps Codex's review text and appends a ```` ```json ```` block of findings + `{file, line, end_line, severity, priority, rule, message}` parsed from its + `- [P1] title — path:12-14` lines: `file` relative to the workspace, + `severity` `blocker` (P0), `major` (P1) or `minor` (P2, P3), `rule` the title + and `message` the body. + +Built-in profiles besides `claude-code`: + +| profile | runs | trace | notes | +|---|---|---|---| +| `codex` | `codex exec --json … -- {prompt}` | `codex-json` | `CODEX_HOME` is the per-run config dir; no turn or spend flag, the timeout bounds a run; `none`/`read` get Codex's read-only sandbox, which needs unprivileged user namespaces | +| `codex-review` | `codex exec review --commit HEAD --json` | `codex-json` | `native_prompt`, `answer_format: codex-review`; review tasks only (the change under review is HEAD) | +| `copilot` | `copilot -p {prompt} --output-format json --allow-all-tools …` | `copilot-json` | `COPILOT_HOME` per run; GitHub token, or BYOK through the `COPILOT_PROVIDER_*` variables; `none` names a tool that does not exist, because an empty `--available-tools` leaves every tool on | +| `copilot-review` | as `copilot`, with the prompt `/review {prompt}` | `copilot-json` | Copilot CLI's code-review agent; the task prompt still asks for its answer format | + +npm installs `codex` and `copilot` as Node launchers. A run mounts one file as +the harness, so `--bin` names the native binary the launcher starts +(`@openai/codex-/vendor//bin/codex`, +`@github/copilot-/copilot`). To route a CLI to another endpoint, +copy its profile: for Copilot, set the `COPILOT_PROVIDER_*` variables (and +`COPILOT_OFFLINE=true` to skip GitHub sign-in); for Codex, add a `config.toml` +under `config_files` that defines a `model_providers` entry with the endpoint's +`base_url` and `wire_api = "responses"` and selects it with `model_provider`. +Add the endpoint's host to `egress` in both cases. The hosted pull-request +reviewers (Copilot code review, `@codex review`, CodeRabbit) are not CLIs and +have no profile. + +Profiles for other harnesses (Gemini CLI, Cursor agent, Aider, OpenCode, +in-house agents) follow the same shape. Their `guidance` blocks start as the vendor's documented behaviour and are corrected by the reach probe. Loading changes between versions of one harness, so reach results are keyed by version. diff --git a/dokimi/dokimi/repos/answers.py b/dokimi/dokimi/repos/answers.py new file mode 100644 index 00000000..360fad86 --- /dev/null +++ b/dokimi/dokimi/repos/answers.py @@ -0,0 +1,58 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Final-answer conversions a harness profile names in ``answer_format`` (contract §6). + +A native review mode answers in its own format, not the ```` ```json ```` block +a review task asks for. The conversion keeps the harness's text and appends +that block, so a grader reading the last ```` ```json ```` block of the answer +reads the native findings; nothing is dropped from what the agent said. + +Findings are ``{file, line, end_line, severity, priority, rule, message}``: +``file`` relative to the workspace, ``line`` the first line of the range, +``severity`` ``blocker`` (P0), ``major`` (P1) or ``minor`` (P2, P3), ``rule`` +the finding's title and ``message`` its body. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Callable +from typing import Any + +from .events import _relative + +ANSWER_FORMATS = ("codex-review",) +_SEVERITY = {"P0": "blocker", "P1": "major", "P2": "minor", "P3": "minor"} +_CODEX_FINDING = re.compile(r"^- \[(P\d)\] (.+?) — (\S.*?):(\d+)(?:-(\d+))?\s*$", re.M) + + +def codex_review_findings(text: str, workdir: str | None = None) -> list[dict[str, Any]]: + """Findings of Codex's native review text: ``- [P1] title — /abs/path:12-14`` then an indented body.""" + found = list(_CODEX_FINDING.finditer(text)) + out = [] + for i, m in enumerate(found): + end = found[i + 1].start() if i + 1 < len(found) else len(text) + body = " ".join(line.strip() for line in text[m.end():end].splitlines() if line.strip()) + first = int(m.group(4)) + out.append({"file": _relative(m.group(3), workdir), "line": first, + "end_line": int(m.group(5)) if m.group(5) else first, + "severity": _SEVERITY.get(m.group(1), "minor"), "priority": m.group(1), + "rule": m.group(2).strip(), "message": body}) + return out + + +_CONVERTERS: dict[str, Callable[[str, str | None], list[dict[str, Any]]]] = { + "codex-review": codex_review_findings, +} + + +def convert_answer(fmt: str | None, answer: str, workdir: str | None = None) -> str: + """``answer`` with the findings block of ``fmt`` appended; unchanged when ``fmt`` is ``None``.""" + if fmt is None: + return answer + if fmt not in _CONVERTERS: + raise ValueError(f"unknown answer format {fmt!r} (known: {', '.join(ANSWER_FORMATS)})") + findings = _CONVERTERS[fmt](answer, workdir) + return f"{answer}\n\n```json\n{json.dumps(findings, indent=1)}\n```\n" diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py new file mode 100644 index 00000000..432744c7 --- /dev/null +++ b/dokimi/dokimi/repos/cli_traces.py @@ -0,0 +1,210 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Codex CLI and Copilot CLI JSON traces -> :class:`~dokimi.repos.events.Transcript` (contract §4.1). + +``codex-json`` is the stdout of ``codex exec --json`` (codex-cli 0.160): one +``item.*`` event per agent message, shell command, file change, MCP call or web +search, and ``turn.completed`` with token usage. Codex has no read tool, so a +``read`` event is derived from a shell command that only prints files +(``cat``, ``head``, ``tail``, ``sed -n``, ``nl``) and names them. + +``copilot-json`` is the stdout of ``copilot -p --output-format json`` (Copilot +CLI 1.0.91): ``tool.execution_start`` / ``tool.execution_complete`` pairs, +``assistant.message`` and a final ``result``. It reports no tokens and no cost. + +Neither CLI reports a cost, so ``cost_usd`` stays ``None`` and a batch budget +counts the cell's cap. +""" + +from __future__ import annotations + +import json +import re +import shlex +from typing import Any + +from .events import Transcript, _parse_time, _relative, clip_output + +_BASH_WRAP = re.compile(r"^(?:/\S*/)?(?:ba|z)?sh -lc (.+)$", re.S) +_PRINTERS = {"cat", "head", "tail", "nl", "sed"} +_SPLIT = re.compile(r"\s*(?:&&|\|\||;|\n)\s*") +_COPILOT_SHELL = {"bash", "powershell"} +_COPILOT_READ = {"view"} +_COPILOT_EDIT = {"edit", "create", "str_replace_editor", "write"} +_PATCH_FILE = re.compile(r"^\*\*\* (?:Add|Update|Delete) File: (.+)$", re.M) + + +def unwrap_shell(command: str) -> str: + """The script of ``/usr/bin/bash -lc '