From f4ce883951644e8fa1d17e8a5c0bdce0b67dd5ce Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 22:18:57 -0700 Subject: [PATCH 1/8] feat(tasks): detect eval-task formats with `tasks inspect` A probe registry (dokimi.task_formats) names the format of a file, a task directory or a tree, and reports kind, item count, present/missing parts, lint problems and which dokimi backends consume it today. Probes: harbor and task-md (wrapping harnesses.tasks), terminal-bench v1, SWE-bench, Inspect AI samples and @task files (text scan, never imported), OpenAI evals, skill-creator evals.json and trigger sets, skill-reach QuerySet, promptfoo and dokimi questions. Unknown paths get a best-effort structure report; malformed files are lint problems. Detection only: nothing new runs. Signed-off-by: Nikolay Petrov --- ...1023_120000_napetrov_task-format-detect.md | 11 + dokimi/docs/architecture.md | 2 +- dokimi/docs/coding-harnesses.md | 29 ++ dokimi/dokimi/__init__.py | 3 +- dokimi/dokimi/commands/tasks.py | 45 ++ dokimi/dokimi/task_formats/__init__.py | 16 + dokimi/dokimi/task_formats/configs.py | 100 +++++ dokimi/dokimi/task_formats/datasets.py | 336 +++++++++++++++ dokimi/dokimi/task_formats/detect.py | 158 +++++++ dokimi/dokimi/task_formats/dirs.py | 155 +++++++ dokimi/dokimi/task_formats/model.py | 91 ++++ dokimi/dokimi/task_formats/source.py | 113 +++++ dokimi/tests/test_import_architecture.py | 2 +- dokimi/tests/test_task_format_detect.py | 387 ++++++++++++++++++ 14 files changed, 1445 insertions(+), 3 deletions(-) create mode 100644 dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md create mode 100644 dokimi/dokimi/task_formats/__init__.py create mode 100644 dokimi/dokimi/task_formats/configs.py create mode 100644 dokimi/dokimi/task_formats/datasets.py create mode 100644 dokimi/dokimi/task_formats/detect.py create mode 100644 dokimi/dokimi/task_formats/dirs.py create mode 100644 dokimi/dokimi/task_formats/model.py create mode 100644 dokimi/dokimi/task_formats/source.py create mode 100644 dokimi/tests/test_task_format_detect.py diff --git a/dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md b/dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md new file mode 100644 index 00000000..deafaf0d --- /dev/null +++ b/dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md @@ -0,0 +1,11 @@ +### Added + +- **`tasks inspect [--json]`** — detects the eval-task format of a file, + a task directory or a whole tree and reports its structure: kind + (executable, answer-only, trigger), item count, present and missing parts, + lint problems and which dokimi backends consume it today. Recognises Harbor, + BenchFlow `task.md`, terminal-bench v1, SWE-bench instances, Inspect AI + samples and `@task` files (scanned as text, never imported), OpenAI evals + JSONL, skill-creator `evals.json` and trigger sets, skill-reach query sets, + promptfoo configs and dokimi question sets; anything else is reported as + `unknown` with its files. Detection only: no new format runs yet. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 4a37c85d..64d8fcc9 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -70,7 +70,7 @@ points up: | Layer | Modules | | --- | --- | | front end | `cli`, `commands`, `__main__` (the argparse front end; no layer below imports it) | -| tracks | `runner`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, and the checkers `config_check`, `schema_check`, `schema_compat` | +| tracks | `runner`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, `task_formats`, and the checkers `config_check`, `schema_check`, `schema_compat` | | evaluation | `eval`, `personas`, `questions`, `treatments`, `plugins` | | components | `ingest`, `metrics`, `gate`, `mcp`, `skills`, `agent_profiles` | | foundation | `__init__` (the `dokimi` package itself), `_version`, `artifacts`, `records`, `llm`, `process`, `caching`, `config`, `registry`, `intents`, `defaults`, `console`, `utils`, `metrics/usage.py` | diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index 91cb3a87..a482d123 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -78,6 +78,35 @@ refused (a few hundred bytes of nested aliases expand without bound), and frontmatter nested too deeply to parse is a parse error. YAML and TOML dates and times become ISO strings, and YAML sets sorted lists. +### Other eval formats: `tasks inspect` + +`python cli.py tasks inspect [--json]` names the eval-task format of a +file, a task directory, or every item under a tree, and reports its structure: +kind, item count, which parts are present or missing (instruction, +environment, oracle, verifier, metadata), lint problems, and what in dokimi +can consume it today. It only reads: nothing is run or converted, a Python +file is scanned as text and never imported, and a file that does not parse is +a problem on its report, not an error. A path no format claims is reported as +`unknown` with its files (or keys) and likely prompt and verifier files; when +two formats claim a path about equally, the report lists the other as +`ambiguous` with its evidence. + +| Format | Recognised by | Kind | Consumed by | +|---|---|---|---| +| `harbor` | directory with `task.toml` | executable | Harbor-backed harnesses (unless Harbor would refuse it), `docker-*` | +| `task-md` | directory with a BenchFlow `task.md` | executable | `benchflow*`, `docker-*` | +| `terminal-bench-v1` | directory with `task.yaml` + `run-tests.sh` / `Dockerfile` / `solution.sh` | executable | none yet | +| `swe-bench` | JSON/JSONL rows with `instance_id`, `repo`, `base_commit`, `problem_statement` (`FAIL_TO_PASS`) | executable | none yet | +| `inspect-samples` | rows with `input` and `target` | answer-only | none yet | +| `inspect-task` | `.py` importing `inspect_ai` with a `@task` function | answer-only (executable with a `sandbox=`) | none yet | +| `openai-evals` | JSONL rows with `input` (chat messages) and `ideal` | answer-only | none yet | +| `skill-creator-evals` | `evals.json` with `skill_name` and `evals[].prompt` | answer-only | none yet | +| `skill-creator-trigger` | rows with `query` and `should_trigger` | trigger | none yet | +| `skill-reach` | `QuerySet` (`queries[]` with `query_id`/`id`, `text`, `expected_skill`) | trigger | none yet | +| `dokimi-questions` | questions.v1/v2 JSON | answer-only (trigger with `expected_skill`) | questions track | + +`tasks list` and `tasks run` still read only the first two. + The `docker-*` harnesses mount the oracle at `/solution` and `/oracle` and the verifier at `/tests` and `/verifier`, because a script written for one layout refers to its siblings by that layout's path. The directory's contents are diff --git a/dokimi/dokimi/__init__.py b/dokimi/dokimi/__init__.py index 81f93aa5..eabdfaaa 100644 --- a/dokimi/dokimi/__init__.py +++ b/dokimi/dokimi/__init__.py @@ -22,7 +22,8 @@ [`dashboard`][dokimi.dashboard] aggregates results across libraries and [`orchestrator`][dokimi.orchestrator] runs the one-command pipeline. - Executable tasks: [`harnesses`][dokimi.harnesses] runs coding agents on task - directories. + directories; [`task_formats`][dokimi.task_formats] names the format of other + eval-task files and directories (detection only). - [`context`][dokimi.context] reads what a run's context window held. - Content novelty: [`novelty`][dokimi.novelty]. Repositories as subjects: [`repos`][dokimi.repos]. Subject scorecards, one per run: diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py index 0564c8fc..79779562 100644 --- a/dokimi/dokimi/commands/tasks.py +++ b/dokimi/dokimi/commands/tasks.py @@ -47,6 +47,43 @@ def cmd_tasks_list(args: argparse.Namespace) -> None: print(f"{task.name}\t{difficulty}\t{','.join(tags)}") +def _inspect_line(report: dict) -> str: + n = report["items"] + items = "" if n is None else f"\t{n} item{'' if n == 1 else 's'}" + return f"{report['path']}\t{report['format']}\t{report['kind']}{items}\t-> {', '.join(report['consumers'])}" + + +def _print_inspect(reports: list[dict]) -> None: + for r in reports: + print(_inspect_line(r)) + if r["present"] or r["missing"]: + print(f" parts: {', '.join(r['present']) or '-'}; missing: {', '.join(r['missing']) or '-'}") + for alt in r["alternatives"]: + print(f" ambiguous: also {alt['format']} ({alt['confidence']:.2f}: {', '.join(alt['evidence'])})") + for problem in r["problems"]: + print(f" problem: {problem}") + by_format: dict[str, int] = {} + for r in reports: + by_format[r["format"]] = by_format.get(r["format"], 0) + 1 + print(f"{len(reports)} item{'' if len(reports) == 1 else 's'}: " + ", ".join(f"{f} {n}" for f, n in sorted(by_format.items()))) + + +def cmd_tasks_inspect(args: argparse.Namespace) -> None: + """Detect the eval-task format of a path (or of everything under it) and report its structure.""" + import json + + from dokimi.task_formats import inspect_path + + try: + reports = [r.as_dict() for r in inspect_path(Path(args.path))] + except FileNotFoundError as exc: + raise SystemExit(f"error: {exc}") from None + if args.json: + print(json.dumps({"root": str(args.path), "items": reports}, indent=2, default=str)) + else: + _print_inspect(reports) + + def cmd_tasks_harnesses(args: argparse.Namespace) -> None: from dokimi.harnesses import known_harnesses @@ -185,6 +222,14 @@ def register(sub, positive_int) -> None: list_p.add_argument("--tasks-root", default=None, help=_TASKS_ROOT_HELP) list_p.set_defaults(func=cmd_tasks_list) + inspect_p = tasks_sub.add_parser( + "inspect", + help="Detect the eval-task format of a file or directory tree and report its structure (no run)", + ) + inspect_p.add_argument("path", help="A task directory, a dataset file, or a tree holding either") + inspect_p.add_argument("--json", action="store_true", help="Print the reports as JSON") + inspect_p.set_defaults(func=cmd_tasks_inspect) + harnesses_p = tasks_sub.add_parser("harnesses", help="List known coding harness aliases") harnesses_p.set_defaults(func=cmd_tasks_harnesses) diff --git a/dokimi/dokimi/task_formats/__init__.py b/dokimi/dokimi/task_formats/__init__.py new file mode 100644 index 00000000..12f9ab34 --- /dev/null +++ b/dokimi/dokimi/task_formats/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Eval-task format detection: which format is a path, and what does it hold? + +Detection and a structure report only: nothing here runs or converts a task. +``dokimi tasks inspect `` is the command line; :func:`inspect_path` the API. +""" + +from dokimi.task_formats.detect import DIR_PROBES, FILE_PROBES, PROBES, detect, inspect_path, unknown +from dokimi.task_formats.model import Fingerprint, Probe, Report + +__all__ = [ + "DIR_PROBES", "FILE_PROBES", "PROBES", "Fingerprint", "Probe", "Report", "detect", + "inspect_path", "unknown", +] diff --git a/dokimi/dokimi/task_formats/configs.py b/dokimi/dokimi/task_formats/configs.py new file mode 100644 index 00000000..75ff306d --- /dev/null +++ b/dokimi/dokimi/task_formats/configs.py @@ -0,0 +1,100 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probes for eval definitions that are config or code: promptfoo, Inspect AI tasks. + +An Inspect ``@task`` file is Python: it is scanned as text with regular +expressions and never imported or run. +""" + +from __future__ import annotations + +import re +from typing import Any + +from dokimi.task_formats.model import KIND_ANSWER_ONLY, KIND_EXECUTABLE, Fingerprint, Report +from dokimi.task_formats.source import FileSource + +# --- promptfoo (promptfoo.dev/docs/configuration/reference) ------------------- + +_PROMPTFOO_NAMES = ("promptfooconfig.yaml", "promptfooconfig.yml", "promptfooconfig.json") + + +class PromptfooProbe: + name = "promptfoo" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix not in (".yaml", ".yml", ".json"): + return None + named = source.path.name in _PROMPTFOO_NAMES + data = source.data + if not isinstance(data, dict): + return Fingerprint(0.6, ("file name",)) if named else None + keys = [k for k in ("prompts", "providers", "targets", "tests", "scenarios") if k in data] + has_target = "providers" in data or "targets" in data + if not named and not ("prompts" in data and has_target and ("tests" in data or "scenarios" in data)): + return None + return Fingerprint(min(1.0, 0.5 + 0.1 * named + 0.1 * len(keys)), (("file name",) if named else ()) + tuple(keys)) + + def describe(self, source: FileSource) -> Report: + data: dict[str, Any] = source.data if isinstance(source.data, dict) else {} + tests = data.get("tests") + report = Report(source.path, self.name, KIND_ANSWER_ONLY, + items=len(tests) if isinstance(tests, list) else None) + report.problems.extend(source.problems) + if source.data is not None and not data: + report.problems.append("not a mapping") + asserts = isinstance(data.get("defaultTest"), dict) and "assert" in data["defaultTest"] + asserts = asserts or any(isinstance(t, dict) and t.get("assert") for t in tests or [] if isinstance(tests, list)) + report.set_parts({ + "instruction": bool(data.get("prompts")), + "environment": bool(data.get("providers") or data.get("targets")), + "verifier": asserts, + "metadata": bool(data.get("description")), + }) + if isinstance(tests, str): + report.details["tests_from"] = tests + elif data and tests is None and "scenarios" not in data: + report.problems.append("no tests") + if data and not data.get("prompts"): + report.problems.append("no prompts") + return report + + +# --- Inspect AI @task files ---------------------------------------------------- + +_INSPECT_IMPORT = re.compile(r"^\s*(?:from\s+inspect_ai(?:\.\w+)*\s+import|import\s+inspect_ai)\b", re.M) +_TASK_DECORATOR = re.compile(r"^\s*@(?:inspect_ai\.)?task\b", re.M) +_SANDBOX = re.compile(r"\bsandbox\s*=") +_SCORER = re.compile(r"\bscorer\s*=") +_DATASET = re.compile(r"\bdataset\s*=|\bSample\s*\(|\b\w+_dataset\s*\(") +_SOLVER = re.compile(r"\b(?:solver|plan)\s*=") + + +class InspectTaskProbe: + name = "inspect-task" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix != ".py" or not source.text: + return None + if not _INSPECT_IMPORT.search(source.text) or not _TASK_DECORATOR.search(source.text): + return None + return Fingerprint(0.9, ("imports inspect_ai", "@task")) + + def describe(self, source: FileSource) -> Report: + text = source.text or "" + sandboxed = bool(_SANDBOX.search(text)) + report = Report(source.path, self.name, KIND_EXECUTABLE if sandboxed else KIND_ANSWER_ONLY, + items=len(_TASK_DECORATOR.findall(text))) + report.problems.extend(source.problems) + report.set_parts({ + "instruction": bool(_DATASET.search(text)), + "environment": sandboxed, + "verifier": bool(_SCORER.search(text)), + "metadata": False, + }) + report.details["solver"] = bool(_SOLVER.search(text)) + report.details["scan"] = "static text scan; the file was not imported" + return report diff --git a/dokimi/dokimi/task_formats/datasets.py b/dokimi/dokimi/task_formats/datasets.py new file mode 100644 index 00000000..e433ff1d --- /dev/null +++ b/dokimi/dokimi/task_formats/datasets.py @@ -0,0 +1,336 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probes for dataset files: one file, many items. + +Field names follow each upstream's own definition: + +- SWE-bench: ``swebench/types.py`` ``SWEbenchInstance`` (SWE-bench/SWE-bench). +- Inspect AI: ``Sample`` / ``FieldSpec`` defaults in ``inspect_ai/dataset/_dataset.py``. +- OpenAI evals: ``docs/build-eval.md`` (``input`` as chat messages, ``ideal``). +- anthropics/skills skill-creator: ``references/schemas.md`` (``evals.json``) and + the trigger eval set of its ``SKILL.md`` (``[{query, should_trigger}]``). +- google/skill-reach: ``Query`` / ``QuerySet`` in ``src/reach/models.py`` and + ``src/reach/queries.py`` (``id`` is the legacy spelling of ``query_id``). +""" + +from __future__ import annotations + +import json +from collections import Counter +from typing import Any + +from dokimi.task_formats.model import ( + BY_QUESTIONS, KIND_ANSWER_ONLY, KIND_EXECUTABLE, KIND_TRIGGER, Fingerprint, Report, +) +from dokimi.task_formats.source import FileSource, first_rows, key_share + +_ROW_SUFFIXES = (".json", ".jsonl", ".yaml", ".yml") + + +def _records(source: FileSource, marker: str) -> list[dict[str, Any]] | None: + """The file's rows; a single object holding *marker* is a one-row file.""" + if source.suffix not in _ROW_SUFFIXES: + return None + if source.rows is not None: + return source.rows + data = source.data + return [data] if isinstance(data, dict) and marker in data else None + + +def _without(rows: list[dict[str, Any]], key: str) -> int: + return sum(1 for r in rows if not r.get(key)) + + +def _lint_missing(report: Report, rows: list[dict[str, Any]], keys: tuple[str, ...]) -> None: + for key in keys: + n = _without(rows, key) + if n: + report.problems.append(f"{n} of {len(rows)} items have no {key}") + + +def _lint_duplicate_ids(report: Report, rows: list[dict[str, Any]], *keys: str) -> None: + ids = Counter(str(next((r[k] for k in keys if k in r), "")) for r in rows) + dupes = sorted(i for i, n in ids.items() if i and n > 1) + if dupes: + report.problems.append(f"duplicate ids: {', '.join(dupes[:5])}") + + +def _base(source: FileSource, name: str, kind: str, rows: list[dict[str, Any]]) -> Report: + report = Report(source.path, name, kind, items=len(rows)) + report.problems.extend(source.problems) + return report + + +# --- SWE-bench --------------------------------------------------------------- + +_SWE_REQUIRED = ("instance_id", "repo", "base_commit", "problem_statement") + + +def _test_list(value: Any) -> list[Any] | None: + """FAIL_TO_PASS / PASS_TO_PASS: a list, or (as the HF datasets ship it) its JSON string.""" + if isinstance(value, list): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except ValueError: + return None + return parsed if isinstance(parsed, list) else None + return None + + +class SweBenchProbe: + name = "swe-bench" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + rows = first_rows(_records(source, "instance_id")) + share = key_share(rows, *_SWE_REQUIRED) + if share < 0.5: + return None + tests = key_share(rows, "FAIL_TO_PASS", "PASS_TO_PASS") + return Fingerprint(0.6 * share + 0.4 * tests, (*_SWE_REQUIRED, "FAIL_TO_PASS/PASS_TO_PASS" if tests else "")) + + def describe(self, source: FileSource) -> Report: + rows = _records(source, "instance_id") or [] + report = _base(source, self.name, KIND_EXECUTABLE, rows) + report.set_parts({ + "instruction": key_share(rows, "problem_statement") == 1, + "environment": key_share(rows, "repo", "base_commit") == 1, + "oracle": key_share(rows, "patch") == 1, + "verifier": key_share(rows, "FAIL_TO_PASS") == 1, + "metadata": key_share(rows, "version") == 1, + }) + _lint_missing(report, rows, (*_SWE_REQUIRED, "FAIL_TO_PASS")) + bad = sum(1 for r in rows if "FAIL_TO_PASS" in r and _test_list(r["FAIL_TO_PASS"]) is None) + if bad: + report.problems.append(f"{bad} items have a FAIL_TO_PASS that is not a list of test ids") + _lint_duplicate_ids(report, rows, "instance_id") + report.details["repos"] = sorted({str(r.get("repo")) for r in rows if r.get("repo")})[:20] + return report + + +# --- Inspect AI samples and OpenAI evals -------------------------------------- + + +def _is_chat(value: Any) -> bool: + return isinstance(value, list) and bool(value) and all( + isinstance(m, dict) and "role" in m and "content" in m for m in value) + + +class InspectSamplesProbe: + name = "inspect-samples" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + rows = first_rows(_records(source, "input")) + with_target = key_share(rows, "input", "target") + if with_target >= 0.5: + return Fingerprint(0.85 * with_target, ("input", "target")) + if key_share(rows, "input") >= 0.8 and not key_share(rows, "ideal"): + return Fingerprint(0.55, ("input (no target)",)) + return None + + def describe(self, source: FileSource) -> Report: + rows = _records(source, "input") or [] + report = _base(source, self.name, KIND_ANSWER_ONLY, rows) + report.set_parts({ + "instruction": key_share(rows, "input") == 1, + "environment": any(k in r for r in rows for k in ("sandbox", "files", "setup")), + "oracle": key_share(rows, "target") == 1, + # The scorer is code in the @task file, not in the samples. + "verifier": False, + "metadata": key_share(rows, "metadata") == 1, + }) + _lint_missing(report, rows, ("input",)) + bad = sum(1 for r in rows if "input" in r and not isinstance(r["input"], str) and not _is_chat(r["input"])) + if bad: + report.problems.append(f"{bad} items have an input that is neither text nor chat messages") + _lint_duplicate_ids(report, rows, "id") + report.details["with_choices"] = sum(1 for r in rows if "choices" in r) + return report + + +class OpenAIEvalsProbe: + name = "openai-evals" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix not in (".jsonl", ".json"): + return None + rows = first_rows(_records(source, "ideal")) + share = key_share(rows, "input", "ideal") + if share < 0.5: + return None + chat = sum(_is_chat(r.get("input")) for r in rows) / len(rows) + return Fingerprint(0.7 * share + 0.25 * chat, ("input", "ideal", "chat messages" if chat else "")) + + def describe(self, source: FileSource) -> Report: + rows = _records(source, "ideal") or [] + report = _base(source, self.name, KIND_ANSWER_ONLY, rows) + report.set_parts({ + "instruction": key_share(rows, "input") == 1, + "oracle": key_share(rows, "ideal") == 1, + }) + _lint_missing(report, rows, ("input", "ideal")) + chat = sum(1 for r in rows if _is_chat(r.get("input"))) + report.details["chat_inputs"] = chat + return report + + +# --- skill-creator ----------------------------------------------------------- + + +class SkillCreatorEvalsProbe: + name = "skill-creator-evals" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + data = source.data if source.suffix == ".json" else None + if not isinstance(data, dict) or not isinstance(data.get("evals"), list): + return None + rows = [r for r in data["evals"] if isinstance(r, dict)] + score = 0.5 + 0.2 * ("skill_name" in data) + 0.3 * key_share(first_rows(rows), "prompt") + return Fingerprint(score, ("evals[]", "skill_name" if "skill_name" in data else "", "prompt")) + + def describe(self, source: FileSource) -> Report: + data = source.data if isinstance(source.data, dict) else {} + rows = [r for r in data.get("evals", []) if isinstance(r, dict)] + report = _base(source, self.name, KIND_ANSWER_ONLY, rows) + report.set_parts({ + "instruction": key_share(rows, "prompt") == 1 and bool(rows), + "environment": any(r.get("files") for r in rows), + "oracle": key_share(rows, "expected_output") == 1 and bool(rows), + "verifier": any(r.get("expectations") for r in rows), + "metadata": bool(data.get("skill_name")), + }) + _lint_missing(report, rows, ("prompt",)) + bad = sum(1 for r in rows if not isinstance(r.get("expectations", []), list)) + if bad: + report.problems.append(f"{bad} items have expectations that are not a list") + _lint_duplicate_ids(report, rows, "id") + report.details["skill_name"] = data.get("skill_name") + return report + + +class SkillCreatorTriggerProbe: + name = "skill-creator-trigger" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + rows = first_rows(source.rows if source.suffix in (".json", ".jsonl") else None) + share = key_share(rows, "query", "should_trigger") + return Fingerprint(0.95 * share, ("query", "should_trigger")) if share >= 0.5 else None + + def describe(self, source: FileSource) -> Report: + rows = source.rows or [] + report = _base(source, self.name, KIND_TRIGGER, rows) + report.set_parts({"instruction": key_share(rows, "query") == 1, + "oracle": key_share(rows, "should_trigger") == 1}) + _lint_missing(report, rows, ("query",)) + bad = sum(1 for r in rows if not isinstance(r.get("should_trigger"), bool)) + if bad: + report.problems.append(f"{bad} items have a should_trigger that is not true/false") + report.details["should_trigger"] = sum(1 for r in rows if r.get("should_trigger") is True) + report.details["should_not_trigger"] = sum(1 for r in rows if r.get("should_trigger") is False) + return report + + +# --- skill-reach ------------------------------------------------------------- + +_REACH_QUERY_KEYS = frozenset({"query_id", "id", "text", "kind", "expected_skill", "acceptable_skills", "notes"}) +_REACH_SET_KEYS = frozenset({"catalog_id", "notes", "provenance", "queries"}) + + +def _reach_rows(source: FileSource) -> list[dict[str, Any]] | None: + data = source.data + if isinstance(data, dict) and isinstance(data.get("queries"), list): + return [q for q in data["queries"] if isinstance(q, dict)] + return source.rows + + +def _lint_reach_query(q: dict[str, Any]) -> str | None: + qid = q.get("query_id", q.get("id", "?")) + unknown = sorted(set(q) - _REACH_QUERY_KEYS) + if unknown: + return f"{qid}: keys skill-reach refuses: {', '.join(unknown)}" + if (q.get("kind") == "out_of_scope") != (q.get("expected_skill") is None): + return f"{qid}: kind={q.get('kind')!r} disagrees with expected_skill={q.get('expected_skill')!r}" + return None + + +class SkillReachProbe: + name = "skill-reach" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix not in _ROW_SUFFIXES: + return None + rows = first_rows(_reach_rows(source)) + texts = key_share(rows, "text", "expected_skill") + if texts < 0.5: + return None + ids = max(key_share(rows, "query_id"), key_share(rows, "id")) + wrapped = isinstance(source.data, dict) and "queries" in source.data + evidence = ("queries[]" if wrapped else "rows", "text", "expected_skill", "query_id/id" if ids else "") + return Fingerprint(0.5 * texts + 0.3 * ids + 0.2 * wrapped, evidence) + + def describe(self, source: FileSource) -> Report: + rows = _reach_rows(source) or [] + report = _base(source, self.name, KIND_TRIGGER, rows) + data = source.data if isinstance(source.data, dict) else {} + report.set_parts({"instruction": key_share(rows, "text") == 1, + "oracle": key_share(rows, "expected_skill") == 1, + "metadata": bool(data.get("catalog_id") or data.get("provenance"))}) + _lint_missing(report, rows, ("text",)) + extra = sorted(set(data) - _REACH_SET_KEYS) + if extra: + report.problems.append(f"QuerySet keys skill-reach refuses: {', '.join(extra)}") + report.problems.extend(p for p in map(_lint_reach_query, rows) if p) + _lint_duplicate_ids(report, rows, "query_id", "id") + report.details["out_of_scope"] = sum(1 for r in rows if r.get("expected_skill") is None) + return report + + +# --- dokimi questions -------------------------------------------------------- + +_QUESTIONS_STAMPS = ("questions.v1", "questions.v2") + + +class QuestionsProbe: + name = "dokimi-questions" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + data = source.data if source.suffix == ".json" else None + if isinstance(data, dict) and data.get("schema_version") in _QUESTIONS_STAMPS: + return Fingerprint(1.0, (f"schema_version {data['schema_version']}",)) + if not isinstance(data, dict) or not isinstance(data.get("questions"), list): + return None + rows = first_rows([q for q in data["questions"] if isinstance(q, dict)]) + share = max(key_share(rows, "question"), key_share(rows, "text")) + return Fingerprint(0.4 + 0.4 * share, ("questions[]", "question/text")) if share else None + + def describe(self, source: FileSource) -> Report: + from dokimi.questions.artifact import load_questions + + data = source.data if isinstance(source.data, dict) else {} + rows = [q for q in data.get("questions", []) if isinstance(q, dict)] + kind = KIND_TRIGGER if any("expected_skill" in q for q in rows) else KIND_ANSWER_ONLY + report = _base(source, self.name, kind, rows) + has_text = all(q.get("question") or q.get("text") for q in rows) and bool(rows) + report.set_parts({ + "instruction": has_text, + "oracle": any(q.get("ground_truth_chunk") or q.get("expected_skill") for q in rows), + "verifier": any(q.get("expected_topics") for q in rows), + "metadata": bool(data.get("product") or data.get("library")), + }) + try: + load_questions(source.path) + except Exception as exc: # noqa: BLE001 -- any refusal is a lint finding, the scan goes on + report.problems.append(" ".join(str(exc).split())[:300]) + else: + report.consumers.append(BY_QUESTIONS) + report.details["schema_version"] = data.get("schema_version") + return report diff --git a/dokimi/dokimi/task_formats/detect.py b/dokimi/dokimi/task_formats/detect.py new file mode 100644 index 00000000..f373d7d7 --- /dev/null +++ b/dokimi/dokimi/task_formats/detect.py @@ -0,0 +1,158 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The probe registry and the scan: which eval-task format is each path? + +A directory a directory probe claims (a task) is one item and is not descended +into; any other directory is walked, and each data file in it is offered to the +file probes. When two probes claim a path about equally, the best is reported +and the others are listed as alternatives with their evidence. A path nothing +claims is reported as ``unknown`` with what it holds, never raised. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator, Sequence +from pathlib import Path +from typing import Any + +from dokimi.task_formats.configs import InspectTaskProbe, PromptfooProbe +from dokimi.task_formats.datasets import ( + InspectSamplesProbe, OpenAIEvalsProbe, QuestionsProbe, SkillCreatorEvalsProbe, + SkillCreatorTriggerProbe, SkillReachProbe, SweBenchProbe, +) +from dokimi.task_formats.dirs import HarborProbe, TaskMdProbe, TerminalBenchV1Probe +from dokimi.task_formats.model import ( + AMBIGUITY_MARGIN, FORMAT_UNKNOWN, KIND_UNKNOWN, MATCH_THRESHOLD, Fingerprint, Probe, Report, +) +from dokimi.task_formats.source import FileSource + +DIR_PROBES: tuple[Probe, ...] = (HarborProbe(), TaskMdProbe(), TerminalBenchV1Probe()) +FILE_PROBES: tuple[Probe, ...] = ( + SweBenchProbe(), InspectSamplesProbe(), OpenAIEvalsProbe(), SkillCreatorEvalsProbe(), + SkillCreatorTriggerProbe(), SkillReachProbe(), QuestionsProbe(), PromptfooProbe(), + InspectTaskProbe(), +) +PROBES: tuple[Probe, ...] = DIR_PROBES + FILE_PROBES + +#: Directories a scan never enters. +SKIP_DIRS = frozenset({".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv", "venv", + ".tox", ".mypy_cache", ".pytest_cache", ".ruff_cache"}) +#: Suffixes a scan offers to the file probes. +SCAN_SUFFIXES = frozenset({".json", ".jsonl", ".yaml", ".yml", ".py"}) +#: A scan stops after this many files and says so. +MAX_SCAN_FILES = 20000 + + +def _claims(probes: Sequence[Probe], subject: Any) -> list[tuple[Fingerprint, Probe]]: + found = [] + for probe in probes: + fp = probe.fingerprint(subject) + if fp is not None and fp.confidence >= MATCH_THRESHOLD: + found.append((fp, probe)) + return sorted(found, key=lambda c: -c[0].confidence) + + +def _evidence(fp: Fingerprint) -> list[str]: + return [e for e in fp.evidence if e] + + +def detect(path: Path, probes: Sequence[Probe] | None = None) -> Report | None: + """The report of the best probe claiming *path*, or ``None`` if none does.""" + path = Path(path) + if probes is None: + probes = DIR_PROBES if path.is_dir() else FILE_PROBES + subject: Any = path if path.is_dir() else FileSource(path) + claims = _claims(probes, subject) + if not claims: + return None + (best_fp, best), rest = claims[0], claims[1:] + report = best.describe(subject) + report.confidence = best_fp.confidence + report.evidence = _evidence(best_fp) + report.alternatives = [ + {"format": p.name, "confidence": round(fp.confidence, 2), "evidence": _evidence(fp)} + for fp, p in rest if round(best_fp.confidence - fp.confidence, 6) <= AMBIGUITY_MARGIN + ] + return report + + +def inspect_path(path: Path) -> list[Report]: + """Every item under *path* (a file, a task directory, or a tree), or one ``unknown``.""" + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"no such file or directory: {path}") + if path.is_file(): + return [detect(path) or unknown(path)] + scan = _Scan() + reports = list(scan.walk(path)) + if not reports: + reports = [unknown(path)] + if scan.truncated: + reports[-1].problems.append(f"scan stopped after {MAX_SCAN_FILES} files") + return reports + + +class _Scan: + def __init__(self) -> None: + self.files = 0 + self.truncated = False + + def walk(self, directory: Path) -> Iterator[Report]: + task = detect(directory, DIR_PROBES) + if task is not None: + yield task + return + try: + children = sorted(directory.iterdir()) + except OSError: + return + for child in children: + if self.truncated: + return + yield from self._child(child) + + def _child(self, child: Path) -> Iterator[Report]: + if child.is_symlink(): + return + if child.is_dir(): + if child.name not in SKIP_DIRS and not child.name.startswith("."): + yield from self.walk(child) + return + if child.suffix.lower() not in SCAN_SUFFIXES: + return + self.files += 1 + if self.files > MAX_SCAN_FILES: + self.truncated = True + return + report = detect(child, FILE_PROBES) + if report is not None: + yield report + + +# --- unknown ----------------------------------------------------------------- + +_PROMPT_NAMES = re.compile(r"^(instruction|prompt|task|problem|question)s?[\w-]*\.(md|txt|ya?ml|json)$", re.I) +_VERIFIER_NAMES = re.compile(r"^(test|tests|verify|verifier|run-tests|check|grade|grader)[\w-]*(\.\w+)?$", re.I) +_LISTED = 30 + + +def unknown(path: Path) -> Report: + """Best-effort structure of a path no probe claims.""" + report = Report(path, FORMAT_UNKNOWN, KIND_UNKNOWN) + if path.is_dir(): + names = sorted(p.name + ("/" if p.is_dir() else "") for p in path.iterdir()) + report.details["files"] = names[:_LISTED] + report.details["prompt_candidates"] = [n for n in names if _PROMPT_NAMES.match(n)] + report.details["verifier_candidates"] = [n for n in names if _VERIFIER_NAMES.match(n.rstrip("/"))] + return report + source = FileSource(path) + data = source.data + report.problems.extend(source.problems) + if isinstance(data, dict): + report.details["keys"] = sorted(map(str, data))[:_LISTED] + elif source.rows: + report.items = len(source.rows) + report.details["row_keys"] = sorted(map(str, source.rows[0]))[:_LISTED] + return report diff --git a/dokimi/dokimi/task_formats/dirs.py b/dokimi/dokimi/task_formats/dirs.py new file mode 100644 index 00000000..0f2e2ff3 --- /dev/null +++ b/dokimi/dokimi/task_formats/dirs.py @@ -0,0 +1,155 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probes for task directories: Harbor, BenchFlow task.md, terminal-bench v1. + +Harbor and task-md recognition is :func:`dokimi.harnesses.tasks.task_format` and +their checks are :func:`~dokimi.harnesses.tasks.load_task` and +:func:`~dokimi.harnesses.tasks.harbor_task_problem`: the probe reports what the +runners would do, it does not decide again. +""" + +from __future__ import annotations + +import warnings +from pathlib import Path +from typing import Any + +from dokimi.harnesses import tasks as task_layouts +from dokimi.task_formats.model import ( + BY_BENCHFLOW, BY_DOCKER, BY_HARBOR, KIND_EXECUTABLE, Fingerprint, Report, +) +from dokimi.task_formats.source import FileSource + + +def _any_file(path: Path, *names: str) -> bool: + return any((path / n).is_file() for n in names) + + +def _any_dir(path: Path, *names: str) -> bool: + return any((path / n).is_dir() for n in names) + + +def _load(path: Path, report: Report) -> task_layouts.TaskSpec | None: + """load_task, with its exception (and any warning it raises) as lint problems.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + spec = task_layouts.load_task(path) + except (ValueError, OSError) as exc: + report.problems.append(str(exc)) + spec = None + report.problems.extend(str(w.message) for w in caught) + return spec + + +class HarborProbe: + name = task_layouts.FORMAT_HARBOR + target = "dir" + + def fingerprint(self, path: Path) -> Fingerprint | None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fmt = task_layouts.task_format(path) + if fmt != task_layouts.FORMAT_HARBOR: + return None + return Fingerprint(1.0, ("task.toml",)) + + def describe(self, path: Path) -> Report: + report = Report(path, self.name, KIND_EXECUTABLE) + spec = _load(path, report) + report.set_parts({ + "instruction": (path / "instruction.md").is_file(), + "environment": (path / "environment").is_dir(), + "oracle": _any_dir(path, "solution", "oracle"), + "verifier": _any_dir(path, "tests", "verifier"), + "metadata": bool(spec and spec.metadata), + }) + if spec is None: + return report + report.details["metadata"] = _summary(spec.metadata) + problem = task_layouts.harbor_task_problem(spec) + if problem: + report.problems.append(f"Harbor would refuse it: {problem}") + else: + report.consumers.append(BY_HARBOR) + if _any_dir(path, "tests", "verifier"): + report.consumers.append(BY_DOCKER) + return report + + +class TaskMdProbe: + name = task_layouts.FORMAT_TASK_MD + target = "dir" + + def fingerprint(self, path: Path) -> Fingerprint | None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fmt = task_layouts.task_format(path) + if fmt != task_layouts.FORMAT_TASK_MD: + return None + return Fingerprint(1.0, ("task.md with BenchFlow frontmatter",)) + + def describe(self, path: Path) -> Report: + report = Report(path, self.name, KIND_EXECUTABLE) + spec = _load(path, report) + report.set_parts({ + "instruction": bool(spec and spec.instruction.strip()), + "environment": (path / "environment").is_dir(), + "oracle": _any_dir(path, "oracle", "solution"), + "verifier": _any_dir(path, "verifier", "tests"), + "metadata": bool(spec and spec.metadata), + }) + if spec is None: + return report + report.details["metadata"] = _summary(spec.metadata) + report.details["schema_version"] = spec.schema_version + report.consumers.append(BY_BENCHFLOW) + if _any_dir(path, "verifier", "tests"): + report.consumers.append(BY_DOCKER) + return report + + +# terminal-bench 0.x (laude-institute/terminal-bench, original-tasks/): task.yaml +# holds the instruction and metadata; the container is Dockerfile + +# docker-compose.yaml, the oracle solution.sh (or solution.yaml), the verifier +# run-tests.sh running tests/. +_TB1_SIGNALS = ("run-tests.sh", "docker-compose.yaml", "Dockerfile", "solution.sh", "solution.yaml") + + +class TerminalBenchV1Probe: + name = "terminal-bench-v1" + target = "dir" + + def fingerprint(self, path: Path) -> Fingerprint | None: + if not (path / "task.yaml").is_file(): + return None + seen = tuple(n for n in _TB1_SIGNALS if (path / n).is_file()) + if not seen: + return None + return Fingerprint(min(1.0, 0.5 + 0.1 * len(seen)), ("task.yaml", *seen)) + + def describe(self, path: Path) -> Report: + report = Report(path, self.name, KIND_EXECUTABLE) + source = FileSource(path / "task.yaml") + data = source.data + report.problems.extend(f"task.yaml: {p}" for p in source.problems) + config: dict[str, Any] = data if isinstance(data, dict) else {} + if data is not None and not config: + report.problems.append("task.yaml: not a mapping") + instruction = config.get("instruction") or config.get("descriptions") + report.set_parts({ + "instruction": bool(instruction), + "environment": _any_file(path, "Dockerfile", "docker-compose.yaml"), + "oracle": _any_file(path, "solution.sh", "solution.yaml"), + "verifier": (path / "run-tests.sh").is_file(), + "metadata": any(k in config for k in ("difficulty", "category", "tags")), + }) + if config and not instruction: + report.problems.append("task.yaml: no instruction") + report.details["metadata"] = _summary(config, ("difficulty", "category", "tags", "parser_name")) + return report + + +def _summary(table: dict[str, Any], keys: tuple[str, ...] = ("difficulty", "category", "tags")) -> dict[str, Any]: + return {k: table[k] for k in keys if k in table} diff --git a/dokimi/dokimi/task_formats/model.py b/dokimi/dokimi/task_formats/model.py new file mode 100644 index 00000000..7843aa06 --- /dev/null +++ b/dokimi/dokimi/task_formats/model.py @@ -0,0 +1,91 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""What a format probe returns: a fingerprint and a structure report.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Protocol + +KIND_EXECUTABLE = "executable" # has an environment and a verifier +KIND_ANSWER_ONLY = "answer-only" # prompt/target samples +KIND_TRIGGER = "trigger" # skill-selection queries +KIND_UNKNOWN = "unknown" + +FORMAT_UNKNOWN = "unknown" + +#: The parts a report says are present or missing, in display order. +PARTS = ("instruction", "environment", "oracle", "verifier", "metadata") + +# Labels for what can consume a format today. +BY_HARBOR = "harbor" +BY_DOCKER = "docker-*" +BY_BENCHFLOW = "benchflow" +BY_QUESTIONS = "questions track" +NONE_YET = "none-yet" + +#: A probe at or above this confidence claims the path. +MATCH_THRESHOLD = 0.5 +#: Other claims within this margin of the best one are reported as ambiguous. +AMBIGUITY_MARGIN = 0.15 + + +@dataclass(frozen=True) +class Fingerprint: + """How sure a probe is that a path is its format, and why.""" + + confidence: float + evidence: tuple[str, ...] = () + + +@dataclass +class Report: + """Structure of one detected item (a task directory or a dataset file).""" + + path: Path + format: str + kind: str + confidence: float = 0.0 + evidence: list[str] = field(default_factory=list) + items: int | None = None + present: list[str] = field(default_factory=list) + missing: list[str] = field(default_factory=list) + problems: list[str] = field(default_factory=list) + consumers: list[str] = field(default_factory=list) + alternatives: list[dict[str, Any]] = field(default_factory=list) + details: dict[str, Any] = field(default_factory=dict) + + def set_parts(self, parts: dict[str, bool]) -> None: + """Record which of :data:`PARTS` are present; a part not named is missing.""" + self.present = [p for p in PARTS if parts.get(p)] + self.missing = [p for p in PARTS if not parts.get(p)] + + def as_dict(self) -> dict[str, Any]: + return { + "path": str(self.path), + "format": self.format, + "kind": self.kind, + "confidence": round(self.confidence, 2), + "evidence": list(self.evidence), + "items": self.items, + "present": list(self.present), + "missing": list(self.missing), + "problems": list(self.problems), + "consumers": list(self.consumers) or [NONE_YET], + "alternatives": list(self.alternatives), + "details": dict(self.details), + } + + +class Probe(Protocol): + """One format: recognise it (:meth:`fingerprint`) and describe it (:meth:`describe`).""" + + name: str + #: ``"dir"`` probes look at directories, ``"file"`` probes at files. + target: str + + def fingerprint(self, source: Any) -> Fingerprint | None: ... + + def describe(self, source: Any) -> Report: ... diff --git a/dokimi/dokimi/task_formats/source.py b/dokimi/dokimi/task_formats/source.py new file mode 100644 index 00000000..d239dfa6 --- /dev/null +++ b/dokimi/dokimi/task_formats/source.py @@ -0,0 +1,113 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A file read once for every probe: parsed data, rows, and what failed to parse. + +Files are data here, never code: JSON, JSONL, YAML and TOML are parsed with safe +loaders (YAML refuses aliases), and a ``.py`` file is only read as text. A parse +failure is a lint problem on the report, not an exception. +""" + +from __future__ import annotations + +import json +import tomllib +from functools import cached_property +from pathlib import Path +from typing import Any + +import yaml + +from dokimi.harnesses.tasks import _load_yaml + +#: Files larger than this are not parsed (their structure is reported as unread). +MAX_BYTES = 64 * 1024 * 1024 + + +class FileSource: + """One candidate file. Parsing is lazy and happens at most once.""" + + def __init__(self, path: Path) -> None: + self.path = Path(path) + self.problems: list[str] = [] + + @property + def suffix(self) -> str: + return self.path.suffix.lower() + + @cached_property + def text(self) -> str | None: + try: + if self.path.stat().st_size > MAX_BYTES: + self.problems.append(f"larger than {MAX_BYTES // (1024 * 1024)} MiB: not read") + return None + return self.path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + self.problems.append(f"cannot read: {exc}") + return None + + @cached_property + def data(self) -> Any: + """The parsed document (a JSONL file parses to its list of rows), or ``None``.""" + text = self.text + if text is None or self.suffix == ".py": + return None + try: + return _parse(self.suffix, text, self.problems) + except (ValueError, yaml.YAMLError, RecursionError) as exc: + self.problems.append(f"not valid {_LANG.get(self.suffix, 'data')}: {_one_line(exc)}") + return None + + @cached_property + def rows(self) -> list[dict[str, Any]] | None: + """The records of a list-shaped document (JSONL, a JSON/YAML list), else ``None``.""" + data = self.data + if not isinstance(data, list): + return None + return [r for r in data if isinstance(r, dict)] + + +_LANG = {".json": "JSON", ".jsonl": "JSONL", ".yaml": "YAML", ".yml": "YAML", ".toml": "TOML"} + + +def _one_line(exc: BaseException) -> str: + return " ".join(str(exc).split())[:200] + + +def _parse(suffix: str, text: str, problems: list[str]) -> Any: + if suffix == ".jsonl": + return _parse_jsonl(text, problems) + if suffix == ".json": + return json.loads(text) + if suffix == ".toml": + return tomllib.loads(text) + return _load_yaml(text) + + +def _parse_jsonl(text: str, problems: list[str]) -> list[Any]: + rows: list[Any] = [] + bad = 0 + for number, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except ValueError as exc: + bad += 1 + if bad <= 3: + problems.append(f"line {number}: not valid JSON: {_one_line(exc)}") + if bad > 3: + problems.append(f"{bad - 3} more lines are not valid JSON") + return rows + + +def first_rows(rows: list[dict[str, Any]] | None, n: int = 50) -> list[dict[str, Any]]: + """The rows a fingerprint looks at: the first *n*, so a large file costs no more.""" + return (rows or [])[:n] + + +def key_share(rows: list[dict[str, Any]], *keys: str) -> float: + """Share of *rows* that hold every one of *keys*.""" + if not rows: + return 0.0 + return sum(all(k in r for k in keys) for r in rows) / len(rows) diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index 5df0aac9..978567bb 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -52,7 +52,7 @@ "dokimi.datasets_export", "dokimi.explore", "dokimi.external", "dokimi.harnesses", "dokimi.novelty", "dokimi.orchestrator", "dokimi.report", "dokimi.repos", "dokimi.runner", "dokimi.schema_check", "dokimi.schema_compat", "dokimi.subjects", - "dokimi.tool_preference", + "dokimi.task_formats", "dokimi.tool_preference", )), # The LLM evaluation engine: answering, judging, panels, arms, questions, personas. ("evaluation", ( diff --git a/dokimi/tests/test_task_format_detect.py b/dokimi/tests/test_task_format_detect.py new file mode 100644 index 00000000..c5c3d77f --- /dev/null +++ b/dokimi/tests/test_task_format_detect.py @@ -0,0 +1,387 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for eval-task format detection (``dokimi.task_formats``, ``tasks inspect``). + +Each fixture is hand-written from the upstream shape it imitates: + +- terminal-bench v1: laude-institute/terminal-bench ``original-tasks/hello-world``. +- SWE-bench: ``SWEbenchInstance`` in SWE-bench/SWE-bench ``swebench/types.py`` + (``FAIL_TO_PASS`` as the JSON string the HF datasets ship). +- Inspect AI: ``Sample`` fields (``input``, ``target``, ``id``, ``sandbox``) and an + ``examples/hello_world.py``-style ``@task`` file. +- OpenAI evals: ``evals/registry/data/coqa/match.jsonl`` (chat ``input``, ``ideal``). +- skill-creator: ``references/schemas.md`` ``evals.json`` and the ``SKILL.md`` trigger set. +- skill-reach: ``QuerySet`` in google/skill-reach ``src/reach/queries.py``. +- promptfoo: ``examples/*/promptfooconfig.yaml``. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.external.trigger_set import TriggerQuery, to_reach, to_skill_creator +from dokimi.harnesses import discover_tasks +from dokimi.task_formats import detect, inspect_path + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _jsonl(path: Path, rows: list) -> Path: + return _write(path, "".join(json.dumps(r) + "\n" for r in rows)) + + +def _harbor(root: Path, name: str = "hb") -> Path: + task = root / name + _write(task / "task.toml", '[metadata]\ndifficulty = "easy"\ntags = ["x"]\n') + _write(task / "instruction.md", "Fix it.") + (task / "environment").mkdir() + _write(task / "solution" / "solve.sh", "#!/bin/bash\n") + _write(task / "tests" / "test.sh", "#!/bin/bash\n") + return task + + +def _task_md(root: Path, name: str = "tm") -> Path: + task = root / name + _write(task / "task.md", "---\nschema_version: '1.3'\nmetadata:\n difficulty: medium\n" + "verifier:\n timeout_sec: 60\n---\n\nDo the thing.\n") + _write(task / "oracle" / "solve.sh", "#!/bin/bash\n") + _write(task / "verifier" / "test.sh", "#!/bin/bash\n") + return task + + +def _tb1(root: Path, name: str = "hello-world") -> Path: + task = root / name + _write(task / "task.yaml", "instruction: |\n Create hello.txt with 'Hello, world!'.\n" + "author_name: a\ndifficulty: easy\ncategory: file-operations\n" + "tags:\n - file-operations\nparser_name: pytest\n") + _write(task / "Dockerfile", "FROM ubuntu:24.04\nWORKDIR /app\n") + _write(task / "docker-compose.yaml", "services: {}\n") + _write(task / "run-tests.sh", "#!/bin/bash\npytest $TEST_DIR/test_outputs.py\n") + _write(task / "solution.sh", "echo 'Hello, world!' > hello.txt\n") + _write(task / "tests" / "test_outputs.py", "def test_hello():\n pass\n") + return task + + +_SWE_ROW = { + "repo": "astropy/astropy", "instance_id": "astropy__astropy-12907", + "base_commit": "d16bfe05a744909de4b27f5875fe0d4ed41ce607", "patch": "diff --git a/x b/x\n", + "test_patch": "diff --git a/t b/t\n", "problem_statement": "separability_matrix is wrong", + "hints_text": "", "created_at": "2022-03-03T15:14:54Z", "version": "4.3", + "FAIL_TO_PASS": '["astropy/modeling/tests/test_separable.py::test_separable[compound_model6]"]', + "PASS_TO_PASS": "[]", "environment_setup_commit": "298ccb478e6bf092953bca67a3d29dc6c35f6752", +} +_CHAT = [{"role": "system", "content": "Answer briefly."}, {"role": "user", "content": "2+2?"}] + + +def _one(path: Path): + reports = inspect_path(path) + assert len(reports) == 1, [r.as_dict() for r in reports] + return reports[0] + + +# ── one probe per format ────────────────────────────────────────────────────── + + +def test_harbor_task_is_executable_and_consumable_by_harbor_and_docker(tmp_path): + r = _one(_harbor(tmp_path)) + assert (r.format, r.kind, r.consumers) == ("harbor", "executable", ["harbor", "docker-*"]) + assert r.missing == [] and r.problems == [] + + +def test_harbor_task_without_environment_reports_why_harbor_refuses_it(tmp_path): + task = _harbor(tmp_path) + (task / "environment").rmdir() + r = _one(task) + assert "environment" in r.missing + assert r.consumers == ["docker-*"] + assert any("Harbor would refuse it: no environment/" in p for p in r.problems) + + +def test_task_md_task_is_consumable_by_benchflow(tmp_path): + r = _one(_task_md(tmp_path)) + assert (r.format, r.kind) == ("task-md", "executable") + assert r.consumers == ["benchflow", "docker-*"] + assert r.details["schema_version"] == "1.3" + + +def test_terminal_bench_v1_task_has_every_part_and_no_consumer_yet(tmp_path): + r = _one(_tb1(tmp_path)) + assert (r.format, r.kind, r.missing) == ("terminal-bench-v1", "executable", []) + assert r.as_dict()["consumers"] == ["none-yet"] + assert r.details["metadata"]["difficulty"] == "easy" + + +def test_swe_bench_jsonl_counts_instances_and_parses_string_test_lists(tmp_path): + other = {**_SWE_ROW, "instance_id": "astropy__astropy-14182"} + r = _one(_jsonl(tmp_path / "lite.jsonl", [_SWE_ROW, other])) + assert (r.format, r.kind, r.items) == ("swe-bench", "executable", 2) + assert r.missing == [] and r.problems == [] + assert r.details["repos"] == ["astropy/astropy"] + + +def test_swe_bench_lints_missing_fields_and_duplicate_ids(tmp_path): + broken = {k: v for k, v in _SWE_ROW.items() if k != "FAIL_TO_PASS"} + r = _one(_jsonl(tmp_path / "bad.jsonl", [_SWE_ROW, broken, {**_SWE_ROW, "FAIL_TO_PASS": "nope"}])) + assert r.format == "swe-bench" + assert "1 of 3 items have no FAIL_TO_PASS" in r.problems + assert "1 items have a FAIL_TO_PASS that is not a list of test ids" in r.problems + assert "duplicate ids: astropy__astropy-12907" in r.problems + + +def test_inspect_samples_jsonl_is_answer_only(tmp_path): + rows = [{"id": 1, "input": "Capital of France?", "target": "Paris"}, + {"id": 2, "input": _CHAT, "target": "4", "sandbox": "docker"}] + r = _one(_jsonl(tmp_path / "samples.jsonl", rows)) + assert (r.format, r.kind, r.items) == ("inspect-samples", "answer-only", 2) + assert "environment" in r.present and "verifier" in r.missing + + +def test_inspect_task_file_is_scanned_not_imported(tmp_path): + marker = tmp_path / "imported" + src = ( + "from inspect_ai import Task, task\nfrom inspect_ai.dataset import Sample\n" + f"open({str(marker)!r}, 'w').write('ran')\n\n" + "@task\ndef hello():\n return Task(dataset=[Sample(input='hi', target='hi')],\n" + " solver=[generate()], scorer=exact(), sandbox='docker')\n" + ) + r = _one(_write(tmp_path / "hello.py", src)) + assert (r.format, r.kind, r.items) == ("inspect-task", "executable", 1) + assert {"instruction", "environment", "verifier"} <= set(r.present) + assert not marker.exists() + + +def test_python_without_inspect_import_is_not_an_inspect_task(tmp_path): + r = _one(_write(tmp_path / "x.py", "@task\ndef f():\n pass\n")) + assert r.format == "unknown" + + +def test_openai_evals_chat_input_with_ideal(tmp_path): + r = _one(_jsonl(tmp_path / "match.jsonl", [{"input": _CHAT, "ideal": "4"}] * 3)) + assert (r.format, r.kind, r.items) == ("openai-evals", "answer-only", 3) + assert r.details["chat_inputs"] == 3 + assert r.alternatives == [] + + +def test_skill_creator_evals_json(tmp_path): + data = {"skill_name": "pdf", "evals": [ + {"id": 1, "prompt": "Extract the tables", "expected_output": "a CSV", + "files": ["evals/files/a.pdf"], "expectations": ["The output includes X"]}, + {"id": 1, "prompt": "Split it", "expectations": "not a list"}, + ]} + r = _one(_write(tmp_path / "evals" / "evals.json", json.dumps(data))) + assert (r.format, r.kind, r.items) == ("skill-creator-evals", "answer-only", 2) + assert r.details["skill_name"] == "pdf" + assert "1 items have expectations that are not a list" in r.problems + assert "duplicate ids: 1" in r.problems + + +def _queries() -> list[TriggerQuery]: + return [TriggerQuery(id="q1", text="load my pdf", expected_skill="pdf"), + TriggerQuery(id="q2", text="what's the weather", expected_skill=None)] + + +def test_skill_creator_trigger_set_written_by_dokimi_is_detected(tmp_path): + path = _write(tmp_path / "trigger.json", json.dumps(to_skill_creator(_queries(), "pdf"))) + r = _one(path) + assert (r.format, r.kind, r.items) == ("skill-creator-trigger", "trigger", 2) + assert (r.details["should_trigger"], r.details["should_not_trigger"]) == (1, 1) + + +def test_skill_reach_queryset_written_by_dokimi_is_detected_and_clean(tmp_path): + path = _write(tmp_path / "queries.json", json.dumps(to_reach(_queries(), "cat"))) + r = _one(path) + assert (r.format, r.kind, r.items) == ("skill-reach", "trigger", 2) + assert r.problems == [] + assert r.details["out_of_scope"] == 1 + + +def test_skill_reach_lints_what_reach_would_refuse(tmp_path): + data = {"queries": [{"query_id": "a", "text": "t", "expected_skill": None}, + {"query_id": "a", "text": "u", "expected_skill": "s", "extra": 1}], + "owner": "me"} + r = _one(_write(tmp_path / "q.json", json.dumps(data))) + assert r.format == "skill-reach" + assert "QuerySet keys skill-reach refuses: owner" in r.problems + assert "a: kind=None disagrees with expected_skill=None" in r.problems + assert "a: keys skill-reach refuses: extra" in r.problems + assert "duplicate ids: a" in r.problems + + +def test_promptfoo_config(tmp_path): + cfg = ("description: suite\nprompts:\n - 'Answer: {{query}}'\nproviders:\n - openai:gpt-4o\n" + "tests:\n - vars: {query: hi}\n assert:\n - type: contains\n value: hi\n" + " - vars: {query: yo}\n") + r = _one(_write(tmp_path / "promptfooconfig.yaml", cfg)) + assert (r.format, r.kind, r.items) == ("promptfoo", "answer-only", 2) + assert r.present == ["instruction", "environment", "verifier", "metadata"] + + +def test_promptfoo_tests_from_a_file_reference(tmp_path): + cfg = "prompts: [p.txt]\nproviders: [echo]\ntests: file://tests.csv\n" + r = _one(_write(tmp_path / "eval.yaml", cfg)) + assert (r.format, r.items, r.details["tests_from"]) == ("promptfoo", None, "file://tests.csv") + + +def test_dokimi_questions_v2_is_consumable_by_the_questions_track(tmp_path): + data = {"schema_version": "questions.v2", "product": "p", + "questions": [{"id": "q1", "question": "How?", "expected_topics": ["a"]}]} + r = _one(_write(tmp_path / "golden.json", json.dumps(data))) + assert (r.format, r.kind, r.items, r.consumers) == ("dokimi-questions", "answer-only", 1, ["questions track"]) + assert r.problems == [] + + +def test_dokimi_questions_with_expected_skill_are_trigger_and_v1_loads(tmp_path): + data = {"questions": [{"id": "q1", "text": "How?", "expected_skill": "pdf"}]} + r = _one(_write(tmp_path / "v1.json", json.dumps(data))) + assert (r.format, r.kind, r.consumers) == ("dokimi-questions", "trigger", ["questions track"]) + + +def test_dokimi_questions_the_loader_refuses_is_a_problem_not_a_consumer(tmp_path): + data = {"schema_version": "questions.v2", "questions": [{"id": "q1", "question": ""}]} + r = _one(_write(tmp_path / "bad.json", json.dumps(data))) + assert r.format == "dokimi-questions" + assert r.consumers == [] and r.problems + + +# ── ambiguity, unknown, malformed ───────────────────────────────────────────── + + +def test_rows_with_target_and_ideal_list_the_other_format_as_ambiguous(tmp_path): + rows = [{"input": _CHAT, "target": "4", "ideal": "4"}] * 2 + r = _one(_jsonl(tmp_path / "both.jsonl", rows)) + assert r.format == "openai-evals" + assert [a["format"] for a in r.alternatives] == ["inspect-samples"] + assert r.alternatives[0]["evidence"] == ["input", "target"] + + +def test_directory_with_task_toml_and_task_yaml_lists_terminal_bench_as_alternative(tmp_path): + task = _harbor(tmp_path) + _write(task / "task.yaml", "instruction: hi\n") + _write(task / "run-tests.sh", "#!/bin/bash\n") + r = _one(task) + assert r.format == "harbor" + assert [a["format"] for a in r.alternatives] == [] # 0.6 is not within the margin of 1.0 + assert detect(task).format == "harbor" + + +def test_unknown_directory_lists_prompt_and_verifier_candidates(tmp_path): + _write(tmp_path / "prompt.md", "do x") + _write(tmp_path / "check.sh", "exit 0") + _write(tmp_path / "notes.txt", "n") + r = _one(tmp_path) + assert (r.format, r.kind) == ("unknown", "unknown") + assert r.details["prompt_candidates"] == ["prompt.md"] + assert r.details["verifier_candidates"] == ["check.sh"] + + +def test_unknown_jsonl_reports_row_keys(tmp_path): + r = _one(_jsonl(tmp_path / "qa.jsonl", [{"question": "q", "answer": "a"}])) + assert (r.format, r.items, r.details["row_keys"]) == ("unknown", 1, ["answer", "question"]) + + +@pytest.mark.parametrize("name, text, problem", [ + ("bad.json", "{not json", "not valid JSON"), + ("bad.yaml", "a: [1, 2\n", "not valid YAML"), + ("alias.yaml", "a: &x [1]\nb: *x\n", "not valid YAML"), + ("bad.jsonl", '{"input": "a", "target": "b"}\n{oops\n', "line 2: not valid JSON"), +]) +def test_malformed_data_file_is_a_lint_problem_not_a_crash(tmp_path, name, text, problem): + r = _one(_write(tmp_path / name, text)) + assert any(problem in p for p in r.problems), r.problems + + +def test_malformed_jsonl_keeps_its_good_rows(tmp_path): + r = _one(_write(tmp_path / "s.jsonl", '{"input": "a", "target": "b"}\n{oops\n')) + assert (r.format, r.items) == ("inspect-samples", 1) + + +def test_malformed_task_toml_is_a_problem(tmp_path): + task = _harbor(tmp_path) + _write(task / "task.toml", "[metadata\n") + r = _one(task) + assert r.format == "harbor" + assert any("not valid TOML" in p for p in r.problems) + + +def test_malformed_tb1_task_yaml_is_a_problem(tmp_path): + task = _tb1(tmp_path) + _write(task / "task.yaml", "instruction: [\n") + r = _one(task) + assert r.format == "terminal-bench-v1" + assert any(p.startswith("task.yaml: not valid YAML") for p in r.problems) + + +# ── scan ────────────────────────────────────────────────────────────────────── + + +def test_scan_of_a_mixed_tree_reports_each_item_once(tmp_path): + _harbor(tmp_path / "tasks", "a") + _task_md(tmp_path / "tasks", "b") + _tb1(tmp_path / "tb") + _jsonl(tmp_path / "data" / "swe.jsonl", [_SWE_ROW]) + _write(tmp_path / "data" / "package.json", '{"name": "x"}') # not an eval: skipped + _write(tmp_path / ".git" / "q.json", json.dumps(to_reach(_queries()))) # never entered + found = {(r.path.relative_to(tmp_path).as_posix(), r.format) for r in inspect_path(tmp_path)} + assert found == {("tasks/a", "harbor"), ("tasks/b", "task-md"), + ("tb/hello-world", "terminal-bench-v1"), ("data/swe.jsonl", "swe-bench")} + + +def test_inspect_missing_path_raises(tmp_path): + with pytest.raises(FileNotFoundError): + inspect_path(tmp_path / "nope") + + +def test_existing_discovery_is_unchanged_by_new_formats_in_the_root(tmp_path): + _harbor(tmp_path, "a") + _task_md(tmp_path, "b") + _tb1(tmp_path, "c") + assert [(t.name, t.format) for t in discover_tasks(tmp_path)] == [("a", "harbor"), ("b", "task-md")] + + +# ── CLI ─────────────────────────────────────────────────────────────────────── + + +def _run(argv: list[str]) -> None: + args = build_parser().parse_args(argv) + args.func(args) + + +def test_cli_inspect_prints_a_table_line_per_item(tmp_path, capsys): + _harbor(tmp_path) + _jsonl(tmp_path / "m.jsonl", [{"input": _CHAT, "ideal": "4"}]) + _run(["tasks", "inspect", str(tmp_path)]) + out = capsys.readouterr().out.splitlines() + assert out[0] == f"{tmp_path / 'hb'}\tharbor\texecutable\t-> harbor, docker-*" + assert f"{tmp_path / 'm.jsonl'}\topenai-evals\tanswer-only\t1 item\t-> none-yet" in out + assert out[-1] == "2 items: harbor 1, openai-evals 1" + + +def test_cli_inspect_json(tmp_path, capsys): + _tb1(tmp_path) + _run(["tasks", "inspect", str(tmp_path), "--json"]) + payload = json.loads(capsys.readouterr().out) + assert [i["format"] for i in payload["items"]] == ["terminal-bench-v1"] + assert payload["items"][0]["consumers"] == ["none-yet"] + + +def test_cli_inspect_shows_ambiguity_and_problems(tmp_path, capsys): + _write(tmp_path / "x.jsonl", '{"input": "a", "target": "b", "ideal": "b"}\n{bad\n') + _run(["tasks", "inspect", str(tmp_path / "x.jsonl")]) + out = capsys.readouterr().out + assert " ambiguous: also openai-evals" in out + assert " problem: line 2: not valid JSON" in out + + +def test_cli_inspect_missing_path_exits_with_a_message(tmp_path): + with pytest.raises(SystemExit) as excinfo: + _run(["tasks", "inspect", str(tmp_path / "nope")]) + assert "no such file or directory" in str(excinfo.value) From f7abf7c192b9fe51705c30b7c69187939a693a86 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 24 Oct 2026 09:54:55 -0700 Subject: [PATCH 2/8] feat(catalog): read trigger sets back and list skill package formats Readers for the three trigger-set formats dokimi already writes (skill-reach QuerySet, skill-creator / agentskills.io eval queries, questions with expected_skill) into TriggerQuery, held to the writers by round-trip tests. catalog trigger-set --trigger-set relints an imported set against the catalog and hands it to the skill-catalog: arm; arms run points external sets at it. catalog formats names the skill/rule packages of other ecosystems (Cursor rules, Copilot instructions, Gemini extensions, Claude plugins) next to agentskills.io SKILL.md, with spec lint and the selecting mechanism. Signed-off-by: Nikolay Petrov --- ...0261024_120000_napetrov_trigger_formats.md | 4 + dokimi/dokimi/commands/arms.py | 25 ++ dokimi/dokimi/commands/catalog.py | 111 ++++++++- dokimi/dokimi/external/trigger_set.py | 147 +++++++++++ dokimi/dokimi/skills/formats.py | 228 ++++++++++++++++++ dokimi/dokimi/task_formats/datasets.py | 6 +- dokimi/dokimi/task_formats/model.py | 2 + dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_skill_formats.py | 107 ++++++++ dokimi/tests/test_trigger_set_readers.py | 169 +++++++++++++ 10 files changed, 786 insertions(+), 15 deletions(-) create mode 100644 dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md create mode 100644 dokimi/dokimi/skills/formats.py create mode 100644 dokimi/tests/test_skill_formats.py create mode 100644 dokimi/tests/test_trigger_set_readers.py diff --git a/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md new file mode 100644 index 00000000..80294970 --- /dev/null +++ b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md @@ -0,0 +1,4 @@ +### Added + +- **Read trigger sets back** — `dokimi catalog trigger-set --trigger-set PATH[=TARGET]` reads a google/skill-reach QuerySet, an anthropics skill-creator / agentskills.io `[{query, should_trigger}]` set (TARGET names its skill), or a questions set with `expected_skill`, relints it against `--catalog`, and writes any of the three formats, so an imported set runs on the `skill-catalog:` arm. `arms run` given a skill-reach or skill-creator file names the conversion command; `tasks inspect` lists `catalog trigger-set` as the consumer of both formats. +- **Skill package formats** — `dokimi catalog formats --root DIR [--json]` lists Agent Skills `SKILL.md` packages (with the host read from the path, the agentskills.io name/description limits linted, and Codex `agents/openai.yaml` noted), Cursor `.mdc` rules, Copilot `*.instructions.md`, Gemini CLI `gemini-extension.json` and Claude Code `.claude-plugin/plugin.json` bundles, with how each is selected (description, globs, always, install). diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index e07da1f8..e540b827 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -450,6 +450,7 @@ def _load_questions(path: str) -> list[dict]: """ from dokimi.questions.artifact import load_questions + _refuse_external_trigger_set(path) questions = load_questions(path).questions ids = [question_id(q) for q in questions] for i, qid in enumerate(ids): @@ -461,6 +462,30 @@ def _load_questions(path: str) -> list[dict]: return questions +def _refuse_external_trigger_set(path: str) -> None: + """Point a skill-reach or skill-creator trigger set at the command that reads it. + + Those sets are read through ``catalog trigger-set --trigger-set``, which also + lints them against the catalog the arm will offer; a questions loader that + guessed at them would drop that lint and, for skill-creator, the target skill. + """ + import json + + from dokimi.external.trigger_set import FORMAT_QUESTIONS, detect_trigger_format + + try: + data = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return # the questions loader reports it + fmt = detect_trigger_format(data) + if fmt and fmt != FORMAT_QUESTIONS: + target = "=" if fmt == "skill-creator" else "" + raise ValueError( + f"{path} is a {fmt} trigger set, not a questions set; convert it with " + f"`dokimi catalog trigger-set --catalog --trigger-set {path}{target} " + f"--format questions --out q.json`, then run the skill-catalog: arm on q.json") + + def _refuse_duplicate_ids(ids: list, path: str) -> None: """Raise on a repeated id.""" counts = Counter(qid for qid in ids if qid is not None) diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index a365cb21..75fdcef4 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -67,7 +67,6 @@ def cmd_catalog_trigger_set(args: argparse.Namespace) -> None: from dokimi.external import trigger_set as ts from dokimi.skills import discover_skills - from dokimi.utils import read_json_input catalog_root = Path(args.catalog) if not catalog_root.is_dir(): @@ -80,16 +79,11 @@ def cmd_catalog_trigger_set(args: argparse.Namespace) -> None: if args.target not in {s.name for s in catalog}: raise SystemExit(f"--target {args.target!r} is not a skill in --catalog " f"(have: {', '.join(sorted(s.name for s in catalog))})") - sources = [] - for spec in args.questions: - path, _, rule = spec.rpartition("=") # labels never contain '=', paths may - if not path or not rule: - raise SystemExit(f"--questions needs PATH=LABEL (a skill name, 'none' or '@field'): {spec}") - data = read_json_input(path, flag="--questions") - questions = data.get("questions", data) if isinstance(data, dict) else data - if not isinstance(questions, list) or not all(isinstance(q, dict) for q in questions): - raise SystemExit(f"{path}: expected a list of question objects") - sources.append((questions, rule)) + sources = _question_sources(args.questions or []) + sources += _trigger_set_sources(args.trigger_set or [], catalog) + if not sources: + raise SystemExit("give --questions PATH=LABEL or --trigger-set PATH[=TARGET]") + specs = list(args.questions or []) + list(args.trigger_set or []) rewrite = None if args.rewrite: @@ -107,7 +101,7 @@ def rewrite(text: str) -> str: # An @field rule that resolves for no question in its source is a typo in the # path, not a set of negatives; refuse it rather than write "nothing loads". start = 0 - for (qs, rule), spec in zip(sources, args.questions, strict=True): + for (qs, rule), spec in zip(sources, specs, strict=True): chunk = queries[start:start + len(qs)] start += len(qs) if rule.startswith("@") and chunk and not any(q.labelled for q in chunk): @@ -143,6 +137,82 @@ def rewrite(text: str) -> str: print(f"wrote {args.out} ({args.format}, {n_out} queries, catalog of {len(catalog)})") +def cmd_catalog_formats(args: argparse.Namespace) -> None: + import json + + from dokimi.skills import formats + + root = Path(args.root) + if not root.is_dir(): + raise SystemExit(f"--root is not a directory: {root}") + packages = formats.detect_skill_packages(root) + summary = formats.summarize(packages) + if args.json: + print(json.dumps({"root": str(root), "summary": summary, + "packages": [p.to_dict(root) for p in packages]}, indent=2)) + return + for p in packages: + problems = "; ".join(p.problems) or "ok" + print(f"{p.format}\t{p.name or '-'}\t{p.selected_by}\t{p.host or '-'}\t" + f"{p.path.relative_to(root).as_posix()}\t{problems}") + print() + for fmt, row in summary.items(): + print(f"# {fmt}: {row['n']} ({row['loadable']} loadable as skills, " + f"{row['with_problems']} with problems); selected by {row['selected_by']}") + + +def _question_sources(specs: list) -> list: + """Each ``--questions PATH=LABEL`` as a (questions, rule) source.""" + from dokimi.utils import read_json_input + + sources = [] + for spec in specs: + path, _, rule = spec.rpartition("=") # labels never contain '=', paths may + if not path or not rule: + raise SystemExit(f"--questions needs PATH=LABEL (a skill name, 'none' or '@field'): {spec}") + data = read_json_input(path, flag="--questions") + questions = data.get("questions", data) if isinstance(data, dict) else data + if not isinstance(questions, list) or not all(isinstance(q, dict) for q in questions): + raise SystemExit(f"{path}: expected a list of question objects") + sources.append((questions, rule)) + return sources + + +def _trigger_set_sources(specs: list, catalog) -> list: + """Each ``--trigger-set PATH[=TARGET]`` read back as a (questions, rule) source. + + The set is read into the canonical trigger model and handed on as questions + labelled by their own ``expected_skill``, so it is linted (and rewritten) like + any other source. TARGET names the skill a skill-creator set was written for. + """ + from dokimi.external import trigger_set as ts + from dokimi.utils import read_json_input + + names = {s.name for s in catalog} + out = [] + for spec in specs: + path, target = spec, None + if not Path(spec).exists() and "=" in spec: + path, _, target = spec.rpartition("=") + data = read_json_input(path, flag="--trigger-set") + try: + fmt, queries = ts.read_trigger_set(data, target=target) + except ValueError as exc: + raise SystemExit(f"{path}: {exc}") from exc + if fmt == ts.FORMAT_SKILL_CREATOR: + if target not in names: + raise SystemExit(f"{spec}: skill-creator target {target!r} is not a skill in " + f"--catalog (have: {', '.join(sorted(names))})") + if len(names) > 1: + print(f"note: {path}: skill-creator negatives are read as 'no skill loads'; in a " + f"catalog of {len(names)} a rival load counts as a false trigger", file=sys.stderr) + elif target: + raise SystemExit(f"{spec}: =TARGET applies to skill-creator sets only, this is {fmt}") + print(f"read {path} as {fmt} ({len(queries)} queries)", file=sys.stderr) + out.append((ts.to_questions(queries)["questions"], "@expected_skill")) + return out + + def register(sub, positive_int) -> None: """Add the `catalog` subcommand group.""" @@ -167,16 +237,31 @@ def register(sub, positive_int) -> None: ) scan_p.set_defaults(func=cmd_catalog_scan) + fm_p = catalog_sub.add_parser( + "formats", + help="List the skill/rule packages under --root by ecosystem (Agent Skills, Cursor " + "rules, Copilot instructions, Gemini extensions, Claude plugins)", + ) + fm_p.add_argument("--root", required=True, help="Local checkout to scan") + fm_p.add_argument("--json", action="store_true", help="Print JSON instead of a table") + fm_p.set_defaults(func=cmd_catalog_formats) + ts_p = catalog_sub.add_parser( "trigger-set", help="Build a skill-selection query set (labels + leakage/framing lint) from question sets", ) ts_p.add_argument("--catalog", required=True, help="Directory of the skills the agent will see") ts_p.add_argument( - "--questions", action="append", required=True, metavar="PATH=LABEL", + "--questions", action="append", metavar="PATH=LABEL", help="Question file and its label: a skill name, 'none' (no skill should load), or " "'@field.path' read per question (e.g. @metadata.skill_under_test). Repeatable.", ) + ts_p.add_argument( + "--trigger-set", action="append", dest="trigger_set", metavar="PATH[=TARGET]", + help="An existing trigger set to read back: a skill-reach QuerySet, a skill-creator / " + "agentskills.io [{query, should_trigger}] set (=TARGET names its skill), or questions " + "with expected_skill. Repeatable; combines with --questions.", + ) ts_p.add_argument("--format", choices=["reach", "skill-creator", "questions"], default="reach", help="reach: google/skill-reach QuerySet; skill-creator: one target's " "eval set; questions: a questions artifact for the skill-catalog: arm") diff --git a/dokimi/dokimi/external/trigger_set.py b/dokimi/dokimi/external/trigger_set.py index 839fdfe7..441c6d19 100644 --- a/dokimi/dokimi/external/trigger_set.py +++ b/dokimi/dokimi/external/trigger_set.py @@ -24,6 +24,10 @@ (``QuerySet`` JSON), anthropics/skills skill-creator (``[{query, should_trigger}]`` for one target skill), and this repo's questions artifact (for the ``skill-catalog:`` arm; saved with :func:`~dokimi.questions.artifact.save_questions_artifact`). + +Readers take the same three formats back to :class:`TriggerQuery` (the +canonical model), so a set authored for one tool can be linted against a +catalog and run by the ``skill-catalog:`` arm, or re-written for another tool. """ from __future__ import annotations @@ -257,3 +261,146 @@ def to_questions(queries: Sequence[TriggerQuery]) -> Dict[str, Any]: row["source_text"] = q.source_text rows.append(row) return {"questions": rows} + + +# --- readers ----------------------------------------------------------------- + +FORMAT_REACH = "reach" +FORMAT_SKILL_CREATOR = "skill-creator" +FORMAT_QUESTIONS = "questions" +TRIGGER_FORMATS = (FORMAT_REACH, FORMAT_SKILL_CREATOR, FORMAT_QUESTIONS) + + +def detect_trigger_format(data: Any) -> Optional[str]: + """Which trigger-set format *data* (parsed JSON) is, or ``None``. + + By shape only: a ``{"queries": [...]}`` object is a skill-reach QuerySet, a list + of ``{query, should_trigger}`` rows is a skill-creator (and agentskills.io + ``eval_queries.json``) set, and ``{"questions": [...]}`` or a bare list of + ``question`` rows is a dokimi questions set. + """ + if isinstance(data, dict): + if isinstance(data.get("queries"), list): + return FORMAT_REACH + if isinstance(data.get("questions"), list): + return FORMAT_QUESTIONS + return None + if isinstance(data, list) and data and all(isinstance(r, dict) for r in data): + if all("query" in r and "should_trigger" in r for r in data): + return FORMAT_SKILL_CREATOR + if all("question" in r or "text" in r for r in data): + return FORMAT_QUESTIONS + return None + + +def _label(value: Any, where: str) -> ExpectedSkill: + if value is None: + return None + if isinstance(value, str) and value: + return value + if isinstance(value, list) and all(isinstance(v, str) and v for v in value): + return list(value) or None + raise ValueError(f"{where}: expected_skill must be a skill name, a list of names, or null; " + f"got {value!r}") + + +def _text(row: Dict[str, Any], keys: Sequence[str], where: str) -> str: + for key in keys: + value = row.get(key) + if isinstance(value, str) and value.strip(): + return value + raise ValueError(f"{where}: no query text (looked in {', '.join(keys)})") + + +def from_reach(data: Dict[str, Any]) -> List[TriggerQuery]: + """A google/skill-reach ``QuerySet``; ``acceptable_skills`` join ``expected_skill``. + + skill-reach credits only ``expected_skill`` and treats ``acceptable_skills`` as + neutral; dokimi's selection score credits any of them, so a query with + acceptable siblings scores at least as high here. An id-less query gets + skill-reach's own default, ``q-001`` by position. + """ + rows = data.get("queries") + if not isinstance(rows, list): + raise ValueError("skill-reach QuerySet has no queries list") + out = [] + for i, q in enumerate(rows, start=1): + if not isinstance(q, dict): + raise ValueError(f"queries[{i - 1}] is not an object") + qid = str(q.get("id") or q.get("query_id") or f"q-{i:03d}") + first = _label(q.get("expected_skill"), qid) + if isinstance(first, list): + raise ValueError(f"{qid}: skill-reach expected_skill is one name; use acceptable_skills") + extra = q.get("acceptable_skills") or [] + if not isinstance(extra, list) or not all(isinstance(v, str) and v for v in extra): + raise ValueError(f"{qid}: acceptable_skills must be a list of skill names") + if first is None and extra: + raise ValueError(f"{qid}: acceptable_skills without an expected_skill") + names = [first, *[v for v in extra if v != first]] if first else [] + label: ExpectedSkill = names if len(names) > 1 else (first or None) + out.append(TriggerQuery(id=qid, text=_text(q, ("text",), qid), expected_skill=label)) + return out + + +def from_skill_creator(rows: List[Dict[str, Any]], target: str) -> List[TriggerQuery]: + """An anthropics skill-creator / agentskills.io set ``[{query, should_trigger}]`` for *target*. + + The format has no ids, so they are positional (``sc-001``). A negative is + read as "no skill should load": exact for a catalog of the target alone, + stricter than the source in a larger one, where loading a rival was not + something the set's author ruled out. + """ + if not target: + raise ValueError("a skill-creator trigger set needs the target skill it was written for") + out = [] + for i, r in enumerate(rows, start=1): + qid = f"sc-{i:03d}" + if not isinstance(r, dict): + raise ValueError(f"{qid}: row is not an object") + flag = r.get("should_trigger") + if not isinstance(flag, bool): + raise ValueError(f"{qid}: should_trigger must be true or false, got {flag!r}") + out.append(TriggerQuery(id=qid, text=_text(r, ("query",), qid), + expected_skill=target if flag else None)) + return out + + +def from_questions(data: Any) -> List[TriggerQuery]: + """A dokimi questions set; a question without ``expected_skill`` stays unlabelled.""" + rows = data.get("questions") if isinstance(data, dict) else data + if not isinstance(rows, list): + raise ValueError("questions set has no questions list") + out = [] + for i, q in enumerate(rows): + if not isinstance(q, dict): + raise ValueError(f"questions[{i}] is not an object") + raw = question_id(q) + qid = str(f"q{i + 1}" if raw is None else raw) + text = _text(q, ("question", "text", "prompt", "question_text"), qid) + tq = TriggerQuery(id=qid, text=text, expected_skill=None, labelled="expected_skill" in q) + if tq.labelled: + tq.expected_skill = _label(q["expected_skill"], qid) + if isinstance(q.get("source_text"), str): + tq.source_text = q["source_text"] + if isinstance(q.get("trigger_lint"), dict): + tq.lint = dict(q["trigger_lint"]) + out.append(tq) + return out + + +def read_trigger_set(data: Any, target: Optional[str] = None, + fmt: Optional[str] = None) -> tuple: + """Read a parsed trigger set as ``(format, [TriggerQuery, ...])``. + + *fmt* overrides detection; *target* is required for a skill-creator set, + which names no skill of its own. + """ + fmt = fmt or detect_trigger_format(data) + if fmt == FORMAT_REACH: + return fmt, from_reach(data) + if fmt == FORMAT_SKILL_CREATOR: + return fmt, from_skill_creator(data, target or "") + if fmt == FORMAT_QUESTIONS: + return fmt, from_questions(data) + raise ValueError("not a trigger set this dokimi reads (skill-reach QuerySet, " + "skill-creator [{query, should_trigger}], or questions with expected_skill)") diff --git a/dokimi/dokimi/skills/formats.py b/dokimi/dokimi/skills/formats.py new file mode 100644 index 00000000..b1ee0344 --- /dev/null +++ b/dokimi/dokimi/skills/formats.py @@ -0,0 +1,228 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Detect the skill and rule package formats of other agent ecosystems. + +The loader reads one format, the Agent Skills ``SKILL.md`` package +(agentskills.io), which Claude Code, OpenAI Codex, Gemini CLI and GitHub Copilot +all read. Other ecosystems ship guidance in shapes the loader does not load; +this module names them so a catalog scan can say what a checkout holds and +which part of it dokimi can evaluate. Nothing here loads or runs a package. + +Formats, and what selects each one at run time: + +- ``agent-skill`` -- ``SKILL.md`` with ``name``/``description`` frontmatter; the + description triggers it. Linted against the agentskills.io field limits. The + host is read from the path (``.claude/skills``, ``.agents/skills``, ...); a + Codex skill may add ``agents/openai.yaml`` (display metadata only); +- ``cursor-rule`` -- ``.cursor/rules/*.mdc``; ``alwaysApply``, ``globs`` or (an + "agent requested" rule) its ``description`` selects it; +- ``copilot-instructions`` -- ``*.instructions.md`` with ``applyTo`` globs, and + the repo-wide ``.github/copilot-instructions.md``; file globs select them; +- ``gemini-extension`` -- ``gemini-extension.json``: a bundle (context file, MCP + servers, commands, and skills under ``skills/``), installed, not triggered; +- ``claude-plugin`` -- ``.claude-plugin/plugin.json``: a bundle whose + ``skills/`` hold agent skills; installed, not triggered. + +Only ``agent-skill`` is loadable (``discover_skills``); the other rows are +reported so a reader sees what a description-trigger eval does not cover. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + +from dokimi.utils import parse_frontmatter + +FORMAT_AGENT_SKILL = "agent-skill" +FORMAT_CURSOR_RULE = "cursor-rule" +FORMAT_COPILOT = "copilot-instructions" +FORMAT_GEMINI_EXTENSION = "gemini-extension" +FORMAT_CLAUDE_PLUGIN = "claude-plugin" + +# agentskills.io/specification: name 1-64 chars of [a-z0-9-], no leading, +# trailing or doubled hyphen, equal to the directory name; description 1-1024. +_SPEC_NAME = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +SPEC_NAME_MAX = 64 +SPEC_DESCRIPTION_MAX = 1024 + +# Path markers of the hosts that read SKILL.md, first match wins. +_HOSTS = ( + (".claude/skills", "claude-code"), + (".codex/skills", "codex"), + (".gemini/skills", "gemini-cli"), + (".github/skills", "copilot"), + (".agents/skills", "agents"), # the cross-tool alias Codex and Gemini CLI read + ("environment/skills", "task-bundled"), # a SkillsBench / Harbor task's own skills +) +_SKIP_DIRS = frozenset({".git", "node_modules", "__pycache__", ".venv"}) + + +@dataclass +class SkillPackage: + """One package found under a root.""" + + path: Path + format: str + name: str = "" + description: str = "" + selected_by: str = "" # description | globs | always | install + loadable: bool = False + host: Optional[str] = None + problems: List[str] = field(default_factory=list) + details: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self, root: Optional[Path] = None) -> Dict[str, Any]: + path = self.path.relative_to(root) if root else self.path + return {"path": path.as_posix(), "format": self.format, "name": self.name, + "selected_by": self.selected_by, "loadable": self.loadable, "host": self.host, + "description_chars": len(self.description), "problems": list(self.problems), + "details": dict(self.details)} + + +def _frontmatter(text: str) -> Dict[str, Any]: + """Frontmatter, or flat ``key: value`` lines where YAML refuses it. + + Cursor rules write ``globs: **/*``, which YAML reads as an alias and rejects. + """ + meta, _ = parse_frontmatter(text) + if meta or not text.startswith("---"): + return meta + flat: Dict[str, Any] = {} + for line in text.splitlines()[1:]: + if line.strip() == "---": + break + key, sep, value = line.partition(":") + if sep and key.strip() and not key.startswith(" "): + flat[key.strip()] = value.strip().strip("'\"") + return flat + + +def _host(path: Path) -> Optional[str]: + posix = path.as_posix() + for marker, host in _HOSTS: + if f"{marker}/" in posix: + return host + return None + + +def lint_agent_skill(meta: Dict[str, Any], skill_file: Path) -> List[str]: + """Where a SKILL.md breaks the agentskills.io field limits.""" + problems = [] + name = str(meta.get("name") or "") + desc = str(meta.get("description") or "") + if not name: + problems.append("no name") + elif len(name) > SPEC_NAME_MAX or not _SPEC_NAME.match(name): + problems.append(f"name {name!r} is not 1-{SPEC_NAME_MAX} chars of a-z, 0-9, single hyphens") + elif name != skill_file.parent.name: + problems.append(f"name {name!r} differs from its directory {skill_file.parent.name!r}") + if not desc.strip(): + problems.append("no description") + elif len(desc) > SPEC_DESCRIPTION_MAX: + problems.append(f"description is {len(desc)} chars, over {SPEC_DESCRIPTION_MAX}") + return problems + + +def _agent_skill(path: Path) -> SkillPackage: + meta = _frontmatter(path.read_text(encoding="utf-8", errors="replace")) + pkg = SkillPackage(path, FORMAT_AGENT_SKILL, str(meta.get("name") or path.parent.name), + str(meta.get("description") or ""), "description", + loadable=bool(str(meta.get("description") or "").strip()), host=_host(path)) + pkg.problems = lint_agent_skill(meta, path) + if (path.parent / "agents" / "openai.yaml").is_file(): + pkg.details["codex_metadata"] = "agents/openai.yaml" + return pkg + + +def _cursor_rule(path: Path) -> SkillPackage: + meta = _frontmatter(path.read_text(encoding="utf-8", errors="replace")) + always = str(meta.get("alwaysApply", "")).lower() == "true" + globs = meta.get("globs") + desc = str(meta.get("description") or "") + selected = "always" if always else "globs" if globs else "description" if desc else "manual" + pkg = SkillPackage(path, FORMAT_CURSOR_RULE, path.stem, desc, selected) + if globs: + pkg.details["globs"] = globs if isinstance(globs, (str, list)) else str(globs) + return pkg + + +def _copilot(path: Path) -> SkillPackage: + meta = _frontmatter(path.read_text(encoding="utf-8", errors="replace")) + name = path.name.removesuffix(".instructions.md").removesuffix(".md") + pkg = SkillPackage(path, FORMAT_COPILOT, name, str(meta.get("description") or ""), + "globs" if meta.get("applyTo") else "always") + if meta.get("applyTo"): + pkg.details["applyTo"] = str(meta["applyTo"]) + return pkg + + +def _manifest(path: Path, fmt: str) -> SkillPackage: + pkg = SkillPackage(path, fmt, selected_by="install") + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + pkg.problems.append(f"unreadable JSON: {exc}") + return pkg + if not isinstance(data, dict): + pkg.problems.append("manifest is not an object") + return pkg + pkg.name, pkg.description = str(data.get("name") or ""), str(data.get("description") or "") + if not pkg.name: + pkg.problems.append("no name") + bundle = path.parent if fmt == FORMAT_GEMINI_EXTENSION else path.parent.parent + pkg.details["skills"] = len(list((bundle / "skills").glob("*/SKILL.md"))) + if fmt == FORMAT_GEMINI_EXTENSION: + pkg.details["context_file"] = data.get("contextFileName") or ( + "GEMINI.md" if (bundle / "GEMINI.md").is_file() else None) + pkg.details["mcp_servers"] = len(data.get("mcpServers") or {}) + return pkg + + +def classify(path: Path) -> Optional[SkillPackage]: + """The package *path* is, or ``None``.""" + name = path.name + if name == "SKILL.md": + return _agent_skill(path) + if name.endswith(".mdc"): # Cursor's own extension + return _cursor_rule(path) + if name.endswith(".instructions.md") or ( + name == "copilot-instructions.md" and path.parent.name == ".github"): + return _copilot(path) + if name == "gemini-extension.json": + return _manifest(path, FORMAT_GEMINI_EXTENSION) + if name == "plugin.json" and path.parent.name == ".claude-plugin": + return _manifest(path, FORMAT_CLAUDE_PLUGIN) + return None + + +def detect_skill_packages(root) -> List[SkillPackage]: + """Every package under *root*, in path order (VCS and dependency dirs skipped).""" + root = Path(root) + found = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or _SKIP_DIRS.intersection(path.relative_to(root).parts): + continue + pkg = classify(path) + if pkg is not None: + found.append(pkg) + return found + + +def summarize(packages: List[SkillPackage]) -> Dict[str, Any]: + """Counts per format, how each is selected, and spec problems.""" + out: Dict[str, Any] = {} + for p in packages: + row = out.setdefault(p.format, {"n": 0, "loadable": 0, "with_problems": 0, + "selected_by": {}, "hosts": {}}) + row["n"] += 1 + row["loadable"] += p.loadable + row["with_problems"] += bool(p.problems) + row["selected_by"][p.selected_by] = row["selected_by"].get(p.selected_by, 0) + 1 + if p.host: + row["hosts"][p.host] = row["hosts"].get(p.host, 0) + 1 + return out diff --git a/dokimi/dokimi/task_formats/datasets.py b/dokimi/dokimi/task_formats/datasets.py index e433ff1d..951bf33b 100644 --- a/dokimi/dokimi/task_formats/datasets.py +++ b/dokimi/dokimi/task_formats/datasets.py @@ -21,7 +21,7 @@ from typing import Any from dokimi.task_formats.model import ( - BY_QUESTIONS, KIND_ANSWER_ONLY, KIND_EXECUTABLE, KIND_TRIGGER, Fingerprint, Report, + BY_QUESTIONS, BY_TRIGGER_SET, KIND_ANSWER_ONLY, KIND_EXECUTABLE, KIND_TRIGGER, Fingerprint, Report, ) from dokimi.task_formats.source import FileSource, first_rows, key_share @@ -234,6 +234,8 @@ def describe(self, source: FileSource) -> Report: report.problems.append(f"{bad} items have a should_trigger that is not true/false") report.details["should_trigger"] = sum(1 for r in rows if r.get("should_trigger") is True) report.details["should_not_trigger"] = sum(1 for r in rows if r.get("should_trigger") is False) + if not report.problems: + report.consumers.append(BY_TRIGGER_SET) return report @@ -290,6 +292,8 @@ def describe(self, source: FileSource) -> Report: report.problems.extend(p for p in map(_lint_reach_query, rows) if p) _lint_duplicate_ids(report, rows, "query_id", "id") report.details["out_of_scope"] = sum(1 for r in rows if r.get("expected_skill") is None) + if not report.problems: + report.consumers.append(BY_TRIGGER_SET) return report diff --git a/dokimi/dokimi/task_formats/model.py b/dokimi/dokimi/task_formats/model.py index 7843aa06..c02bec38 100644 --- a/dokimi/dokimi/task_formats/model.py +++ b/dokimi/dokimi/task_formats/model.py @@ -24,6 +24,8 @@ BY_DOCKER = "docker-*" BY_BENCHFLOW = "benchflow" BY_QUESTIONS = "questions track" +# Read back by `catalog trigger-set --trigger-set`, then run by the skill-catalog: arm. +BY_TRIGGER_SET = "catalog trigger-set" NONE_YET = "none-yet" #: A probe at or above this confidence claims the path. diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 13f91674..7023a3b7 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -21,7 +21,7 @@ "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 22}, "dokimi/commands/catalog.py::cmd_catalog_scan": {"complexity": 12}, - "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 18}, + "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 16}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"}, "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 12}, diff --git a/dokimi/tests/test_skill_formats.py b/dokimi/tests/test_skill_formats.py new file mode 100644 index 00000000..bbea2f36 --- /dev/null +++ b/dokimi/tests/test_skill_formats.py @@ -0,0 +1,107 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Skill/rule package detection across agent ecosystems (``dokimi.skills.formats``). + +The fixtures copy the shapes of public samples: openai/skills (``agents/openai.yaml``), +google-gemini/gemini-cli (``gemini-extension.json``, ``.gemini/skills``), +PatrickJS/awesome-cursorrules (``.mdc`` with ``globs: **/*``), github/awesome-copilot +(``*.instructions.md`` with ``applyTo``), anthropics/claude-plugins-official +(``.claude-plugin/plugin.json``) and benchflow-ai/skillsbench (``environment/skills``). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from dokimi.cli import build_parser +from dokimi.skills import discover_skills +from dokimi.skills import formats as sf + + +def _w(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _skill(d: Path, name: str, desc: str = "Use when profiling.") -> Path: + return _w(d / "SKILL.md", f"---\nname: {name}\ndescription: {desc}\n---\nbody\n") + + +def _tree(root: Path) -> Path: + _skill(root / ".codex/skills/imagegen", "imagegen") + _w(root / ".codex/skills/imagegen/agents/openai.yaml", "interface:\n display_name: Image Gen\n") + _skill(root / ".gemini/skills/code_explorer", "code_explorer") + _skill(root / "tasks/t1/environment/skills/xlsx", "xlsx", "x" * 1100) + _w(root / ".cursor/rules/ts.mdc", + "---\ndescription: \"TypeScript rules\"\nglobs: **/*.ts\nalwaysApply: false\n---\n# TS\n") + _w(root / ".cursor/rules/always.mdc", "---\nalwaysApply: true\n---\nbe terse\n") + _w(root / ".github/instructions/swift.instructions.md", + "---\ndescription: 'Swift MCP'\napplyTo: \"**/*.swift\"\n---\n# Swift\n") + _w(root / ".github/copilot-instructions.md", "Repo-wide rules\n") + ext = root / "ext/policy" + _w(ext / "gemini-extension.json", + json.dumps({"name": "policy", "version": "1", "mcpServers": {"s": {}}})) + _w(ext / "GEMINI.md", "context\n") + _skill(ext / "skills/audit", "audit") + plugin = root / "plugins/hookify" + _w(plugin / ".claude-plugin/plugin.json", json.dumps({"name": "hookify", "description": "d"})) + _skill(plugin / "skills/writing-rules", "writing-hookify-rules") + _w(root / "node_modules/x/SKILL.md", "---\nname: x\ndescription: d\n---\n") + return root + + +def test_each_ecosystem_is_named_with_how_it_is_selected(tmp_path): + root = _tree(tmp_path) + by_path = {p.path.relative_to(root).as_posix(): p for p in sf.detect_skill_packages(root)} + assert "node_modules/x/SKILL.md" not in by_path + codex = by_path[".codex/skills/imagegen/SKILL.md"] + assert (codex.format, codex.host, codex.loadable, codex.problems) == ( + "agent-skill", "codex", True, []) + assert codex.details == {"codex_metadata": "agents/openai.yaml"} + assert by_path[".gemini/skills/code_explorer/SKILL.md"].problems[0].startswith( + "name 'code_explorer' is not") + bundled = by_path["tasks/t1/environment/skills/xlsx/SKILL.md"] + assert bundled.host == "task-bundled" and "over 1024" in bundled.problems[0] + ts_rule = by_path[".cursor/rules/ts.mdc"] + assert (ts_rule.format, ts_rule.selected_by, ts_rule.description) == ( + "cursor-rule", "globs", "TypeScript rules") # YAML refuses `**/*.ts`; read anyway + assert by_path[".cursor/rules/always.mdc"].selected_by == "always" + assert by_path[".github/instructions/swift.instructions.md"].details == {"applyTo": "**/*.swift"} + assert by_path[".github/copilot-instructions.md"].selected_by == "always" + ext = by_path["ext/policy/gemini-extension.json"] + assert (ext.selected_by, ext.details) == ( + "install", {"skills": 1, "context_file": "GEMINI.md", "mcp_servers": 1}) + plugin = by_path["plugins/hookify/.claude-plugin/plugin.json"] + assert (plugin.format, plugin.details["skills"]) == ("claude-plugin", 1) + assert "differs from its directory" in by_path[ + "plugins/hookify/skills/writing-rules/SKILL.md"].problems[0] + + +def test_every_loadable_package_is_one_the_skill_loader_loads(tmp_path): + root = _tree(tmp_path) + loadable = {p.name for p in sf.detect_skill_packages(root) if p.loadable} + # The loader does not skip node_modules/ (a vendored copy loads as a skill); the scan does. + assert loadable == {s.name for s in discover_skills(root)} - {"x"} + + +def test_spec_lint_rules(): + f = Path("a-b/SKILL.md") + assert sf.lint_agent_skill({"name": "a-b", "description": "d"}, f) == [] + for name in ("A-b", "-ab", "a--b", "a" * 65): + assert "is not" in sf.lint_agent_skill({"name": name, "description": "d"}, Path(f"{name}/SKILL.md"))[0] + assert sf.lint_agent_skill({}, f) == ["no name", "no description"] + + +def test_catalog_formats_cli(tmp_path, capsys): + root = _tree(tmp_path / "r") + args = build_parser().parse_args(["catalog", "formats", "--root", str(root), "--json"]) + args.func(args) + data = json.loads(capsys.readouterr().out) + assert data["summary"]["agent-skill"]["n"] == 5 + assert data["summary"]["cursor-rule"]["selected_by"] == {"globs": 1, "always": 1} + args = build_parser().parse_args(["catalog", "formats", "--root", str(root)]) + args.func(args) + assert "# gemini-extension: 1" in capsys.readouterr().out diff --git a/dokimi/tests/test_trigger_set_readers.py b/dokimi/tests/test_trigger_set_readers.py new file mode 100644 index 00000000..6e70f6a2 --- /dev/null +++ b/dokimi/tests/test_trigger_set_readers.py @@ -0,0 +1,169 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Reading trigger sets back: skill-reach, skill-creator / agentskills.io, dokimi questions. + +Each reader is held to the writer of the same format in ``dokimi.external.trigger_set``: +what dokimi writes, it reads back to the same labels. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.external import trigger_set as ts +from dokimi.questions.artifact import load_questions +from dokimi.skills import discover_skills +from dokimi.task_formats import detect + + +def _write_skill(root: Path, name: str, description: str) -> None: + d = root / name + d.mkdir(parents=True) + (d / "SKILL.md").write_text(f"---\nname: {name}\ndescription: \"{description}\"\n---\n# {name}\n", + encoding="utf-8") + + +@pytest.fixture +def catalog(tmp_path: Path) -> Path: + root = tmp_path / "skills" + _write_skill(root, "linux-perf", "Profile Linux programs with perf: hotspots, IPC, cache misses.") + _write_skill(root, "dpnp-io", "Read and write dpnp arrays to files on Intel GPUs.") + _write_skill(root, "dpnp-memory", "Place dpnp arrays on devices and size memory.") + return root + + +@pytest.fixture +def queries(catalog): + return ts.build_trigger_set([ + ([{"id": "p", "text": "my program is slow, where is the time going"}], "linux-perf"), + ([{"id": "f", "text": "save this array to disk", "skills": ["dpnp-io", "dpnp-memory"]}], + "@skills"), + ([{"id": "n", "text": "write a haiku"}], "none"), + ([{"id": "u", "text": "unlabelled", "metadata": {}}], "@metadata.skill"), + ], discover_skills(catalog)) + + +def _labels(qs): + return [(q.id, q.text, q.expected_skill, q.labelled) for q in qs] + + +def test_reach_round_trip_keeps_ids_texts_and_acceptable_sets(queries): + back = ts.from_reach(ts.to_reach(queries, catalog_id="c")) + assert _labels(back) == _labels(ts.labelled(queries)) + assert ts.detect_trigger_format(ts.to_reach(queries)) == ts.FORMAT_REACH + + +def test_skill_creator_round_trip_is_identity_on_its_own_rows(queries): + rows = ts.to_skill_creator(queries, "linux-perf") + back = ts.from_skill_creator(rows, "linux-perf") + assert ts.to_skill_creator(back, "linux-perf") == rows + assert [q.expected_skill for q in back] == ["linux-perf", None, None] + assert [q.id for q in back] == ["sc-001", "sc-002", "sc-003"] + assert ts.detect_trigger_format(rows) == ts.FORMAT_SKILL_CREATOR + + +def test_questions_round_trip_keeps_unlabelled_and_lint(queries, tmp_path): + payload = ts.to_questions(queries) + back = ts.from_questions(payload) + assert _labels(back) == _labels(queries) + assert back[0].lint == queries[0].lint + # Through the stamped artifact the arms command loads, too. + from dokimi.questions.artifact import save_questions_artifact + + path = tmp_path / "q.json" + save_questions_artifact(payload, str(path)) + stamped = json.loads(path.read_text(encoding="utf-8")) + assert _labels(ts.from_questions(stamped)) == _labels(queries) + assert len(load_questions(path).questions) == 4 + + +def test_reach_reader_matches_skill_reach_defaults_and_refuses_bad_rows(): + qs = ts.from_reach({"queries": [ + {"text": "a", "expected_skill": "x", "acceptable_skills": ["y", "x"]}, + {"query_id": "k", "text": "b", "expected_skill": None, "kind": "out_of_scope"}, + ]}) + assert [(q.id, q.expected_skill) for q in qs] == [("q-001", ["x", "y"]), ("k", None)] + for bad, msg in [({"text": "a", "expected_skill": None, "acceptable_skills": ["y"]}, "without"), + ({"text": "", "expected_skill": "x"}, "no query text"), + ({"text": "a", "expected_skill": 3}, "skill name"), + ({"text": "a", "expected_skill": ["x"]}, "one name")]: + with pytest.raises(ValueError, match=msg): + ts.from_reach({"queries": [bad]}) + + +def test_skill_creator_reader_needs_a_target_and_booleans(): + with pytest.raises(ValueError, match="target"): + ts.read_trigger_set([{"query": "a", "should_trigger": True}]) + with pytest.raises(ValueError, match="true or false"): + ts.from_skill_creator([{"query": "a", "should_trigger": "yes"}], "x") + + +def test_detect_refuses_what_is_not_a_trigger_set(): + for data in ({"evals": []}, [], [{"prompt": "a"}], "text", {"queries": "x"}): + assert ts.detect_trigger_format(data) is None + with pytest.raises(ValueError, match="not a trigger set"): + ts.read_trigger_set({"evals": []}) + + +def _run(argv): + args = build_parser().parse_args(argv) + args.func(args) + + +def test_cli_reads_each_format_back_and_relints_against_the_catalog(tmp_path, catalog, queries, + capsys): + reach = tmp_path / "reach.json" + reach.write_text(json.dumps(ts.to_reach(queries)), encoding="utf-8") + sc = tmp_path / "sc.json" + sc.write_text(json.dumps([{"query": "use linux-perf on it", "should_trigger": True}]), + encoding="utf-8") + out = tmp_path / "q.json" + _run(["catalog", "trigger-set", "--catalog", str(catalog), "--format", "questions", + "--trigger-set", str(reach), "--trigger-set", f"{sc}=linux-perf", "--out", str(out)]) + rows = load_questions(out).questions + assert [r.get("expected_skill") for r in rows] == [ + "linux-perf", ["dpnp-io", "dpnp-memory"], None, "linux-perf"] + assert rows[3]["trigger_lint"]["names_skill"] == ["linux-perf"] # linted on import + err = capsys.readouterr().err + assert "as reach (3 queries)" in err and "rival load counts as a false trigger" in err + + +def test_cli_refuses_bad_trigger_set_specs(tmp_path, catalog): + sc = tmp_path / "sc.json" + sc.write_text(json.dumps([{"query": "a", "should_trigger": True}]), encoding="utf-8") + reach = tmp_path / "r.json" + reach.write_text(json.dumps({"queries": [{"id": "1", "text": "a", "expected_skill": None}]}), + encoding="utf-8") + base = ["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(tmp_path / "o.json")] + with pytest.raises(SystemExit, match="not a skill in --catalog"): + _run(base + ["--trigger-set", f"{sc}=nope"]) + with pytest.raises(SystemExit, match="needs the target"): + _run(base + ["--trigger-set", str(sc)]) + with pytest.raises(SystemExit, match="skill-creator sets only"): + _run(base + ["--trigger-set", f"{reach}=linux-perf"]) + with pytest.raises(SystemExit, match="--questions PATH=LABEL or --trigger-set"): + _run(base) + + +def test_arms_points_an_external_trigger_set_at_the_converter(tmp_path): + from dokimi.commands.arms import _load_questions + + sc = tmp_path / "sc.json" + sc.write_text(json.dumps([{"query": "a", "should_trigger": True}]), encoding="utf-8") + with pytest.raises(ValueError, match=r"skill-creator trigger set.*--trigger-set .*="): + _load_questions(str(sc)) + + +def test_task_formats_name_the_reader_as_consumer(tmp_path, queries): + reach = tmp_path / "reach.json" + reach.write_text(json.dumps(ts.to_reach(queries)), encoding="utf-8") + sc = tmp_path / "sc.json" + sc.write_text(json.dumps(ts.to_skill_creator(queries, "linux-perf")), encoding="utf-8") + for path in (reach, sc): + report = detect(path) + assert report.consumers == ["catalog trigger-set"], report From 19822bab06088e77f5070399651318f467f4ceef Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 24 Oct 2026 10:57:20 -0700 Subject: [PATCH 3/8] feat(arms): precision in skill_selection_summary Hits over the questions where the arm loaded any skill, strict on multi-load like recall; rendered in the arms report's Skill selection table. Replaces the ad-hoc calculation used for the #211 evidence table. Signed-off-by: Nikolay Petrov --- .../20261024_120000_napetrov_trigger_formats.md | 1 + dokimi/dokimi/eval/arm_summary.py | 5 ++++- dokimi/dokimi/report/arms_report.py | 6 +++--- dokimi/tests/test_skill_selection.py | 14 ++++++++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md index 80294970..1590aaa3 100644 --- a/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md +++ b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md @@ -2,3 +2,4 @@ - **Read trigger sets back** — `dokimi catalog trigger-set --trigger-set PATH[=TARGET]` reads a google/skill-reach QuerySet, an anthropics skill-creator / agentskills.io `[{query, should_trigger}]` set (TARGET names its skill), or a questions set with `expected_skill`, relints it against `--catalog`, and writes any of the three formats, so an imported set runs on the `skill-catalog:` arm. `arms run` given a skill-reach or skill-creator file names the conversion command; `tasks inspect` lists `catalog trigger-set` as the consumer of both formats. - **Skill package formats** — `dokimi catalog formats --root DIR [--json]` lists Agent Skills `SKILL.md` packages (with the host read from the path, the agentskills.io name/description limits linted, and Codex `agents/openai.yaml` noted), Cursor `.mdc` rules, Copilot `*.instructions.md`, Gemini CLI `gemini-extension.json` and Claude Code `.claude-plugin/plugin.json` bundles, with how each is selected (description, globs, always, install). +- **Skill selection precision** — `skill_selection_summary` and the arms report's *Skill selection* table carry `precision`: hits over the questions where the arm loaded any skill, so misroutes and false triggers count against it. diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index 968cbf2b..cf109ae5 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -130,7 +130,7 @@ def summarize_skill_selection(records: Iterable[RecordLike]) -> Dict[str, Any]: or ``None`` — no skill should load. Arms that offered no skill are skipped. The skills loaded in the first turn that loaded any are one choice. ``recall`` - is strict: a hit needs every one of them acceptable. ``recall_any`` (one + and ``precision`` (hits over questions where any skill loaded) are strict: a hit needs every one of them acceptable. ``recall_any`` (one acceptable skill among them), ``multi_load_rate`` and ``mean_first_turn_loads`` show how far the strict figure depends on multi-loading. """ @@ -192,6 +192,9 @@ def rate(a: int, b: int) -> Any: "n_failed": s["failed"], "selection_accuracy": rate(s["correct"], s["n"]), "recall": rate(s["hit"], s["pos"]), + # Of the questions where the arm loaded a skill, the share where every + # loaded skill was acceptable (misroutes and false triggers count against). + "precision": rate(s["hit"], s["loaded_n"]), # Lenient: the right skill was among those loaded in the first turn. "recall_any": rate(s["hit_any"], s["pos"]), "multi_load_rate": rate(s["multi"], s["loaded_n"]), diff --git a/dokimi/dokimi/report/arms_report.py b/dokimi/dokimi/report/arms_report.py index 702bf6de..da298f0e 100644 --- a/dokimi/dokimi/report/arms_report.py +++ b/dokimi/dokimi/report/arms_report.py @@ -265,13 +265,13 @@ def pct(v: Any) -> str: "harness does." ) lines.append("") - lines.append("| Arm | Accuracy | Recall | Recall (any) | Multi-load | Misroute | False trigger " + lines.append("| Arm | Accuracy | Recall | Precision | Recall (any) | Multi-load | Misroute | False trigger " "| +/− | n | Failed |") - lines.append("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") + lines.append("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") for arm_name, s in selection.items(): lines.append( f"| `{arm_name}` | {pct(s.get('selection_accuracy'))} | {pct(s.get('recall'))} | " - f"{pct(s.get('recall_any'))} | {pct(s.get('multi_load_rate'))} | " + f"{pct(s.get('precision'))} | {pct(s.get('recall_any'))} | {pct(s.get('multi_load_rate'))} | " f"{pct(s.get('misroute_rate'))} | {pct(s.get('false_trigger_rate'))} | " f"{s.get('n_positive', 0)}/{s.get('n_negative', 0)} | {s.get('n', 0)} | " f"{s.get('n_failed', 0)} |" diff --git a/dokimi/tests/test_skill_selection.py b/dokimi/tests/test_skill_selection.py index ae704b4a..eceed720 100644 --- a/dokimi/tests/test_skill_selection.py +++ b/dokimi/tests/test_skill_selection.py @@ -101,6 +101,8 @@ def test_selection_summary_scores_hits_misroutes_and_false_triggers(): assert s["recall"] == 0.5 assert s["misroute_rate"] == 0.25 assert s["false_trigger_rate"] == 0.5 + # Loaded on 4 of 6: two hits, one misroute, one false trigger. + assert s["precision"] == 0.5 assert s["selection_accuracy"] == round(3 / 6, 3) assert s["confusion"]["linux-perf"] == {"linux-perf": 1, "onetbb-quickstart": 1, "(none)": 1} assert s["confusion"]["(none)"] == {"(none)": 1, "linux-perf": 1} @@ -117,7 +119,7 @@ def test_report_renders_selection_table(): "skill_selection_summary": summarize_skill_selection([_rec("a", "a"), _rec(None, "a")]), }) assert "## Skill selection" in md - assert "| `cat` | 50% | 100% | 100% | 0% | 0% | 100% | 1/1 | 2 | 0 |" in md + assert "| `cat` | 50% | 100% | 50% | 100% | 0% | 0% | 100% | 1/1 | 2 | 0 |" in md # ── trigger set ────────────────────────────────────────────────────────── @@ -428,7 +430,7 @@ def test_combined_arm_rejects_duplicate_tools(catalog): def test_report_renders_dash_for_missing_rates(): md = render_arms_report({"arms": ["cat"], "answers": [], "skill_selection_summary": summarize_skill_selection([_rec(None, None)])}) - assert "| `cat` | 100% | — | — | — | — | 0% | 0/1 | 1 | 0 |" in md + assert "| `cat` | 100% | — | — | — | — | — | 0% | 0/1 | 1 | 0 |" in md def _qfile(tmp_path, name, rows): @@ -726,3 +728,11 @@ def test_loading_the_whole_catalog_in_one_turn_is_not_a_hit(catalog): assert (s["recall"], s["misroute_rate"], s["recall_any"]) == (0.0, 1.0, 1.0) assert s["mean_first_turn_loads"] == 3.0 assert s["confusion"] == {"linux-perf": {"dpnp-io+linux-perf+onetbb-quickstart": 1}} + + +def test_selection_precision_is_none_when_nothing_loaded_and_strict_on_multi_load(): + assert summarize_skill_selection([_rec("a", None), _rec(None, None)])["cat"]["precision"] is None + rec = _rec("a", "a") + rec["arms"]["cat"]["tool_usage"]["skill"]["first_turn_loaded"] = ["a", "b"] + s = summarize_skill_selection([rec, _rec("a", "a")])["cat"] + assert (s["precision"], s["recall_any"]) == (0.5, 1.0) From 5de519914b11aff56b2c2b6bda113b4c8e3b60ca Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 24 Oct 2026 11:13:22 -0700 Subject: [PATCH 4/8] feat(catalog): trigger-probe runs each query in real Claude Code; payload leak audit catalog trigger-probe runs every trigger-set query in a fresh claude -p: a new git repo with the catalog committed in .claude/skills, scratch HOME and config, the query verbatim, bundled skills on. Skill calls are scored by the same selection summary as the skill-catalog arm, one column per trial. payload_audit reports dokimi artifacts (arm names, label keys, question ids, grading words, task paths) in a captured request, argv or env, with operator content blanked and the stock harness's own text subtracted. Signed-off-by: Nikolay Petrov --- ...261024_120000_napetrov_trigger-fidelity.md | 10 + dokimi/docs/evaluating-treatments.md | 13 ++ dokimi/dokimi/commands/catalog.py | 4 + dokimi/dokimi/commands/catalog_probe.py | 93 ++++++++ dokimi/dokimi/harnesses/payload_audit.py | 134 +++++++++++ dokimi/dokimi/harnesses/trigger_probe.py | 214 ++++++++++++++++++ dokimi/tests/test_payload_audit.py | 117 ++++++++++ dokimi/tests/test_trigger_probe.py | 131 +++++++++++ 8 files changed, 716 insertions(+) create mode 100644 dokimi/changelog.d/20261024_120000_napetrov_trigger-fidelity.md create mode 100644 dokimi/dokimi/commands/catalog_probe.py create mode 100644 dokimi/dokimi/harnesses/payload_audit.py create mode 100644 dokimi/dokimi/harnesses/trigger_probe.py create mode 100644 dokimi/tests/test_payload_audit.py create mode 100644 dokimi/tests/test_trigger_probe.py diff --git a/dokimi/changelog.d/20261024_120000_napetrov_trigger-fidelity.md b/dokimi/changelog.d/20261024_120000_napetrov_trigger-fidelity.md new file mode 100644 index 00000000..061b1053 --- /dev/null +++ b/dokimi/changelog.d/20261024_120000_napetrov_trigger-fidelity.md @@ -0,0 +1,10 @@ +### Added + +- **`catalog trigger-probe`** — runs each trigger-set query in a fresh, unannounced + `claude -p` session with the catalog committed as project skills, a scratch `HOME` + and config, and the query sent verbatim; scores the `Skill` calls with the same + selection summary as the `skill-catalog:` arm, one column per trial. +- **`dokimi.harnesses.payload_audit`** — `audit_payload` / `new_leaks` report dokimi + artifacts (arm names, label keys, question ids, grading words, task paths) in a + captured request, argv or env, with operator content blanked and the stock harness's + own text subtracted. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 493bd782..e817e393 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -582,6 +582,19 @@ uv run reach eval --skills /skills --queries trigger_set.json \ uv run reach diff --vary description control.json treatment.json # A/B vs noise floor ``` +**Real harness, in dokimi: `catalog trigger-probe`.** Runs each query in a fresh +`claude -p`: a new git repository with the catalog committed in `.claude/skills/`, a +scratch `HOME` and `CLAUDE_CONFIG_DIR` (your own skills, settings and `CLAUDE.md` never +reach it), the query sent verbatim with no notice or task framing, and bundled skills +left on as a user has them (`--sealed` turns them off). The skill chosen is read from the +transcript's `Skill` calls, and the output's `skill_selection_summary` is scored by the +same code as the `skill-catalog:` arm, one column per trial: + +```bash +python cli.py catalog trigger-probe --catalog /skills \ + --trigger-set trigger_set.json --model --trials 3 --out probe.json +``` + Two things decide whether such a number means anything: - **Listing budget.** Claude Code shows skill descriptions only up to a budget (a diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index 75fdcef4..6419430c 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -274,3 +274,7 @@ def register(sub, positive_int) -> None: help="Exit 1 if any query names a skill, keeps exam framing, or has an unresolved @field label") ts_p.add_argument("--out", required=True, help="Output JSON path") ts_p.set_defaults(func=cmd_catalog_trigger_set) + + from dokimi.commands import catalog_probe + + catalog_probe.register(catalog_sub, positive_int) diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py new file mode 100644 index 00000000..d55177e8 --- /dev/null +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -0,0 +1,93 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``catalog trigger-probe``: run a trigger set through real Claude Code, one fresh run per query.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + + +def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: + from dokimi.eval.arm_summary import summarize_skill_selection + from dokimi.external import trigger_set as ts + from dokimi.harnesses import trigger_probe as tp + + catalog = Path(args.catalog) + if not catalog.is_dir(): + raise SystemExit(f"--catalog is not a directory: {catalog}") + if shutil.which(args.claude) is None: + raise SystemExit(f"{args.claude!r} is not on PATH") + try: + data = json.loads(Path(args.trigger_set).read_text(encoding="utf-8")) + _fmt, queries = ts.read_trigger_set(data, target=args.target) + except (OSError, ValueError) as exc: + raise SystemExit(f"--trigger-set: {exc}") from exc + queries = [q for q in queries if q.labelled][: args.limit or None] + cfg = tp.ProbeConfig(model=args.model, max_turns=args.max_turns, timeout_sec=args.timeout, + stock=not args.sealed, listing_budget_fraction=args.listing_budget_fraction, + claude=args.claude) + + def one(q): + rec = tp.probe_trigger_set([q], catalog, cfg, trials=args.trials)[0] + print(f"{q.id}: " + ", ".join( + f"{a}={'+'.join(v['tool_usage']['skill']['first_turn_loaded']) or '-'}" + + (f" ({v['error']})" if v["error"] else "") for a, v in rec["arms"].items()), + file=sys.stderr) + return rec + + with ThreadPoolExecutor(max(1, args.concurrency)) as pool: + records = list(pool.map(one, queries)) + cost = sum(v["metrics"]["cost_usd"] or 0 for r in records for v in r["arms"].values()) + out = { + "harness": "claude-code", + "mode": "sealed" if args.sealed else "stock", + "model": args.model, + "catalog": str(catalog), + "trials": args.trials, + "max_turns": args.max_turns, + "listing_budget_fraction": args.listing_budget_fraction, + "claude_code_version": next((v["tool_usage"]["skill"]["claude_code_version"] for r in records + for v in r["arms"].values() + if v["tool_usage"]["skill"]["claude_code_version"]), None), + "cost_usd": round(cost, 4), + "answers": records, + "skill_selection_summary": summarize_skill_selection(records), + } + Path(args.out).write_text(json.dumps(out, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"cost_usd": out["cost_usd"], "claude_code_version": out["claude_code_version"], + "summary": {a: {k: s[k] for k in ("recall", "false_trigger_rate", "misroute_rate", "n_failed")} + for a, s in out["skill_selection_summary"].items()}}, indent=2)) + print(f"wrote {args.out}") + + +def register(catalog_sub, positive_int) -> None: + """Add ``catalog trigger-probe`` to the ``catalog`` group.""" + p = catalog_sub.add_parser( + "trigger-probe", + help="Run each trigger-set query in a fresh, unannounced Claude Code session with the " + "catalog installed as project skills; score which skill it invokes", + ) + p.add_argument("--catalog", required=True, help="Directory of skills, installed as .claude/skills") + p.add_argument("--trigger-set", required=True, dest="trigger_set", + help="skill-reach QuerySet, skill-creator set (with --target) or questions with expected_skill") + p.add_argument("--target", help="Target skill of a skill-creator set") + p.add_argument("--model", help="--model passed to claude (default: the harness default)") + p.add_argument("--trials", type=positive_int, default=1, help="Fresh runs per query (default: 1)") + p.add_argument("--max-turns", type=positive_int, default=3, dest="max_turns", + help="claude --max-turns (default: 3)") + p.add_argument("--timeout", type=float, default=300.0, help="Seconds per run (default: 300)") + p.add_argument("--concurrency", type=positive_int, default=4, help="Queries run at once (default: 4)") + p.add_argument("--limit", type=positive_int, help="Only the first N labelled queries") + p.add_argument("--listing-budget-fraction", type=float, dest="listing_budget_fraction", + help="skillListingBudgetFraction for the run (default: the harness's own, 0.01)") + p.add_argument("--sealed", action="store_true", + help="Switch off bundled skills as the docker-claude arms do (default: stock harness)") + p.add_argument("--claude", default="claude", help="claude executable (default: claude)") + p.add_argument("--out", required=True, help="Output JSON path") + p.set_defaults(func=cmd_catalog_trigger_probe) diff --git a/dokimi/dokimi/harnesses/payload_audit.py b/dokimi/dokimi/harnesses/payload_audit.py new file mode 100644 index 00000000..d83c1e4e --- /dev/null +++ b/dokimi/dokimi/harnesses/payload_audit.py @@ -0,0 +1,134 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Leak audit: does what an agent is sent carry dokimi's own artifacts? + +A measured run is meant to reproduce what a user's agent sees. Anything dokimi adds +that names the experiment (an arm name, the ``expected_skill`` key, a question id, +judge or rubric text, a task path that says what is graded) can change what the model +does, and makes the number measure dokimi rather than the harness. + +:func:`audit_payload` walks a payload (a captured request body, an argv, an env map: +any JSON-like value) and reports every marker it finds. Text the operator supplied +on purpose (the query, the skills' own descriptions and bodies) is passed as +``allowed`` and blanked out before matching: a skill whose description says +"baseline" is content, not a leak. What is left is what dokimi wrote. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any, Iterable, Iterator, List, Optional, Sequence, Tuple + +#: Words and keys that name an experiment rather than a user's task. Matched whole-word, +#: case-insensitive, after the allowed texts are blanked. +DEFAULT_MARKERS: Tuple[str, ...] = ( + "dokimi", "expected_skill", "acceptable_skills", "should_trigger", "trigger_set", + "skill_catalog", "skill-catalog", "skill_selection", "treatment", "baseline", + "with_docs", "without_docs", "judge", "rubric", "golden", "grader", "verifier", + "reward", "oracle", "eval", "evals", "evaluation", "benchmark", "arm_kind", + "abskill", "test_outputs", +) + + +@dataclass(frozen=True) +class Leak: + """One marker found: where (a JSON path), which marker, and the text around it.""" + + path: str + marker: str + excerpt: str + + def as_dict(self) -> dict: + return {"path": self.path, "marker": self.marker, "excerpt": self.excerpt} + + +def markers_for(*, question_ids: Iterable[str] = (), arm_names: Iterable[str] = (), + paths: Iterable[str] = (), extra: Iterable[str] = (), + base: Sequence[str] = DEFAULT_MARKERS) -> List[str]: + """The default markers plus this run's own identifiers. + + *question_ids* and *arm_names* come from the run; *paths* are host paths that + reveal intent (a task directory, an output dir). Empty values are dropped. + """ + out = list(base) + for group in (question_ids, arm_names, paths, extra): + out += [str(v) for v in group if str(v).strip()] + return out + + +def _strings(value: Any, path: str = "$") -> Iterator[Tuple[str, str]]: + """Every string in *value* (keys included) with its JSON path.""" + if isinstance(value, str): + yield path, value + elif isinstance(value, dict): + for k, v in value.items(): + yield f"{path}.{k}#key", str(k) + yield from _strings(v, f"{path}.{k}") + elif isinstance(value, (list, tuple)): + for i, v in enumerate(value): + yield from _strings(v, f"{path}[{i}]") + + +def _pattern(marker: str) -> "re.Pattern[str]": + # Identifier-ish markers match between non-alphanumerics, so `dokimi-run-x` and + # `eval_queries` are caught while `evaluate` is not; anything else is a substring. + if re.fullmatch(r"[\w-]+", marker): + return re.compile(rf"(? str: + for a in allowed: + if a and a in text: + text = text.replace(a, " " * len(a)) + return text + + +def audit_payload(payload: Any, markers: Optional[Iterable[str]] = None, + allowed: Iterable[str] = ()) -> List[Leak]: + """Markers found in *payload* outside the *allowed* texts, one :class:`Leak` each. + + *allowed* is longest-first blanked, so a skill body containing its description + blanks as a whole. An empty result means the payload carries none of the markers. + """ + pats = [(m, _pattern(m)) for m in (DEFAULT_MARKERS if markers is None else markers) if m] + keep = sorted({a for a in allowed if a and a.strip()}, key=len, reverse=True) + leaks: List[Leak] = [] + for path, text in _strings(payload): + visible = _blank(text, keep) + for marker, pat in pats: + for m in pat.finditer(visible): + lo, hi = max(0, m.start() - 40), min(len(text), m.end() + 40) + leaks.append(Leak(path, marker, text[lo:hi].replace("\n", " "))) + return leaks + + +def added_framing(prompt: str, query: str) -> str: + """What a harness put in the prompt around the user's *query* (``""`` if verbatim). + + The framing is not a leak by the marker test, but it is an influence: an + out-of-the-box run sends the query alone. + """ + if prompt == query: + return "" + if query and query in prompt: + head, _, tail = prompt.partition(query) + return (head + tail).strip() + return prompt + + +def new_leaks(payload: Any, reference: Any, markers: Optional[Iterable[str]] = None, + allowed: Iterable[str] = ()) -> List[Leak]: + """Leaks in *payload* that a *reference* capture does not have. + + The reference is the same harness run out of the box (no dokimi in the loop): its + own system prompt says "judge edge cases" and "golden path", which is the + harness, not the experiment. A marker counts as new when its excerpt is not + among the reference's. + """ + marks = list(DEFAULT_MARKERS if markers is None else markers) + allowed = list(allowed) + seen = {(lk.marker, lk.excerpt) for lk in audit_payload(reference, marks, allowed)} + return [lk for lk in audit_payload(payload, marks, allowed) if (lk.marker, lk.excerpt) not in seen] diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py new file mode 100644 index 00000000..bce1b005 --- /dev/null +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -0,0 +1,214 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Real-harness skill triggering: one fresh ``claude -p`` per query, no notice. + +The ``skill-catalog:`` arm is a tool-calling emulation: one tool per skill, every +description in full, no system prompt, no other tools. Claude Code instead sends one +``Skill`` tool, a listing it cuts to a budget, its own system prompt and a dozen other +tools, and the model may look around before it loads anything. This module measures +that harness as shipped: + +* each query runs in a new git repository under a temp dir (no ancestor ``CLAUDE.md``, + no checkout to walk), with the catalog committed in ``.claude/skills/``, where + Claude Code looks for project skills, so ``git status`` shows nothing unusual; +* ``HOME`` and ``CLAUDE_CONFIG_DIR`` are fresh and the env is the sandbox allowlist, + so the operator's skills, settings and ``CLAUDE.md`` never reach the run; +* the prompt is the query verbatim: no treatment notice, no task framing; +* in ``stock`` mode (the default) the bundled skills and tools stay on, as a user + has them; ``sealed`` switches them off like the docker-claude arms do. + +The skills invoked are read from the ``stream-json`` transcript (``Skill`` tool uses); +the first assistant turn that invokes any is the choice, as for the emulated arm. +Records come out in the answers shape :func:`dokimi.eval.arm_summary.summarize_skill_selection` +reads, so both arms are scored by the same code. +""" + +from __future__ import annotations + +import json +import shutil +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence + +from dokimi import process +from dokimi.harnesses.agent_sandbox import _SEAL_ENV, AgentWorkspace, agent_env, operator_source_env +from dokimi.harnesses.claude_json import result_event + +#: Fixed identity and dates for the staging commit, so the git context the harness +#: shows the model is the same on every run. +_GIT_IDENT = { + "GIT_AUTHOR_NAME": "user", "GIT_AUTHOR_EMAIL": "user@example.com", + "GIT_COMMITTER_NAME": "user", "GIT_COMMITTER_EMAIL": "user@example.com", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z", "GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z", +} +_COMMIT_MESSAGE = "Initial commit" + + +@dataclass +class ProbeConfig: + """How each query is run. ``listing_budget_fraction`` ``None`` keeps the harness default.""" + + model: Optional[str] = None + max_turns: int = 3 + timeout_sec: float = 300.0 + stock: bool = True + listing_budget_fraction: Optional[float] = None + claude: str = "claude" + extra_args: List[str] = field(default_factory=list) + + +#: Prefix of each run's temp root. Claude Code shows the model its working directory, +#: so the name must not say what the run is (``AgentWorkspace.create`` uses ``dokimi-run-``). +WORKSPACE_PREFIX = "project-" + + +def new_workspace() -> AgentWorkspace: + """A scratch ``{work,home,config}`` root whose path names no experiment.""" + root = Path(tempfile.mkdtemp(prefix=WORKSPACE_PREFIX)) + for sub in ("work", "home", "config"): + (root / sub).mkdir() + return AgentWorkspace(root=root) + + +def stage_workspace(catalog: Path, ws: AgentWorkspace, runner: process.Runner = process.run) -> Path: + """Copy *catalog* to ``work/.claude/skills`` and commit it; return the work dir.""" + work = ws.work + shutil.copytree(catalog, work / ".claude" / "skills", symlinks=False) + env = process.Env.only(("PATH",), {**_GIT_IDENT, "HOME": str(ws.root / "home"), + "GIT_CONFIG_NOSYSTEM": "1", "GIT_TERMINAL_PROMPT": "0"}) + for args in (("init", "-q", "-b", "main"), ("add", "-A"), + ("commit", "-q", "--no-gpg-sign", "-m", _COMMIT_MESSAGE)): + done = process.git(*args, cwd=work, env=env, runner=runner) + if done.returncode != 0: + raise RuntimeError(f"git {args[0]} failed staging the catalog: {process.git_failure(done)}") + return work + + +def probe_env(ws: AgentWorkspace, cfg: ProbeConfig, source: Optional[Dict[str, str]] = None) -> Dict[str, str]: + """The allowlisted env, scratch HOME/config; the bundled-skill seal only when not ``stock``.""" + env = agent_env(operator_source_env() if source is None else source) + if cfg.stock: + for k in _SEAL_ENV: + env.pop(k, None) + env["HOME"] = str(ws.root / "home") + env["CLAUDE_CONFIG_DIR"] = str(ws.root / "config") + env["TMPDIR"] = str(ws.root / "home") + return env + + +def probe_argv(query: str, cfg: ProbeConfig) -> List[str]: + """``claude`` argv: the query verbatim, stream-json out, and nothing that steers the model.""" + argv = [cfg.claude, "-p", query, "--output-format", "stream-json", "--verbose", + "--max-turns", str(cfg.max_turns), "--no-session-persistence"] + if cfg.model: + argv += ["--model", cfg.model] + if cfg.listing_budget_fraction is not None: + argv += ["--settings", json.dumps({"skillListingBudgetFraction": cfg.listing_budget_fraction})] + return argv + list(cfg.extra_args) + + +def _events(stream: str) -> Iterable[Dict[str, Any]]: + for line in stream.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + obj = json.loads(line) + except ValueError: + continue + if isinstance(obj, dict): + yield obj + + +def _skill_name(raw: Any) -> str: + # A plugin skill is invoked as `plugin:skill`; the catalog names the skill alone. + return str(raw or "").rsplit(":", 1)[-1] + + +def parse_stream(stream: str) -> Dict[str, Any]: + """Skills listed and invoked, per the transcript; ``first_turn_loaded`` is the choice.""" + listed: List[str] = [] + version = None + turns: List[List[str]] = [] + other_tools: List[str] = [] + for ev in _events(stream): + if ev.get("type") == "system" and ev.get("subtype") == "init": + listed = [str(s) for s in ev.get("skills") or []] + version = ev.get("claude_code_version") + if ev.get("type") != "assistant": + continue + content = (ev.get("message") or {}).get("content") or [] + uses = [c for c in content if isinstance(c, dict) and c.get("type") == "tool_use"] + skills = [_skill_name((c.get("input") or {}).get("skill")) for c in uses if c.get("name") == "Skill"] + other_tools += [str(c.get("name")) for c in uses if c.get("name") != "Skill"] + if skills: + turns.append(skills) + first = turns[0] if turns else [] + return { + "offered": True, + "listed": listed, + "selected": first[0] if first else None, + "first_turn_loaded": first, + "loaded": [s for t in turns for s in t], + "tools_before_choice": other_tools, + "claude_code_version": version, + } + + +def run_query(query: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Runner = process.run, + source_env: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + """One fresh harness run of *query*; the parsed skill block plus cost and any error.""" + ws = new_workspace() + try: + work = stage_workspace(catalog, ws, runner=runner) + argv = probe_argv(query, cfg) + done = runner(argv, cwd=work, timeout=cfg.timeout_sec, + env=process.Env.only((), probe_env(ws, cfg, source_env)), input="", + capture_limit=None) + stream = done.stdout + skill = parse_stream(stream) + result = result_event(stream) or {} + error = None + if done.timed_out: + error = "timeout" + elif not result: + error = f"no result event (exit {done.returncode})" + # Out of turns without a Skill call is an observation (nothing triggered), not a failure. + elif result.get("is_error") and result.get("subtype") != "error_max_turns" \ + and not skill["first_turn_loaded"]: + error = str(result.get("subtype") or "error") + return {"skill": skill, "cost_usd": result.get("total_cost_usd"), + "num_turns": result.get("num_turns"), "error": error} + finally: + ws.cleanup() + + +def probe_trigger_set(queries: Sequence[Any], catalog: Path, cfg: ProbeConfig, *, trials: int = 1, + arm_name: str = "claude_code", runner: process.Runner = process.run, + on_record: Optional[Callable[[Dict[str, Any]], None]] = None) -> List[Dict[str, Any]]: + """Run every labelled query *trials* times; answers-shaped records for the selection summary. + + *queries* are :class:`dokimi.external.trigger_set.TriggerQuery`. Each trial is its own + arm (``#``), so the summary reports trials side by side. + """ + out: List[Dict[str, Any]] = [] + for q in queries: + if not getattr(q, "labelled", True): + continue + arms = {} + for k in range(trials): + r = run_query(q.text, catalog, cfg, runner=runner) + arms[f"{arm_name}#{k + 1}"] = { + "answer": None, + "error": r["error"], + "tool_usage": {"skill": r["skill"]}, + "metrics": {"cost_usd": r["cost_usd"], "num_turns": r["num_turns"]}, + } + rec = {"question_id": q.id, "question_text": q.text, "expected_skill": q.expected_skill, "arms": arms} + out.append(rec) + if on_record: + on_record(rec) + return out diff --git a/dokimi/tests/test_payload_audit.py b/dokimi/tests/test_payload_audit.py new file mode 100644 index 00000000..74f85fcd --- /dev/null +++ b/dokimi/tests/test_payload_audit.py @@ -0,0 +1,117 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Leak audit: markers in what an agent is sent, applied to the skill-catalog and docker-claude paths.""" + +from __future__ import annotations + +from pathlib import Path + +import dokimi.eval.agent_runner as ar_mod +from dokimi.eval.agent_runner import run_agent_loop +from dokimi.harnesses import load_task +from dokimi.harnesses.docker_solver import DockerSolverHarness +from dokimi.harnesses.payload_audit import added_framing, audit_payload, markers_for, new_leaks +from dokimi.harnesses.skill_treatment import TREATMENT_NOTICE, Treatment +from dokimi.skills import load_skill +from dokimi.treatments.arms import SkillCatalogTreatment + + +def _skill(root: Path, name: str, description: str, body: str) -> Path: + d = root / name + d.mkdir(parents=True) + (d / "SKILL.md").write_text(f"---\nname: {name}\ndescription: {description}\n---\n{body}\n", + encoding="utf-8") + return d / "SKILL.md" + + +def test_planted_markers_are_found_with_their_path(): + payload = {"messages": [{"role": "user", "content": "Q-17: how do I sum?"}], + "meta": {"expected_skill": "x"}, "cwd": "/srv/dokimi-run-ab12/work"} + leaks = audit_payload(payload, markers_for(question_ids=["Q-17"])) + found = {(lk.marker, lk.path) for lk in leaks} + assert ("Q-17", "$.messages[0].content") in found + assert ("expected_skill", "$.meta.expected_skill#key") in found + # A hyphenated path still names the tool: `dokimi-run-` is a leak. + assert ("dokimi", "$.cwd") in found + + +def test_marker_inside_a_longer_word_is_not_a_leak(): + assert audit_payload({"t": "evaluate the baseline_x reevaluation"}, ["eval"]) == [] + assert [lk.marker for lk in audit_payload({"t": "see eval_queries.json"}, ["eval"])] == ["eval"] + + +def test_operator_content_is_blanked_but_dokimi_text_around_it_is_not(): + desc = "Compare against a baseline run before tuning." + payload = {"tools": [{"description": f"Load skill. {desc}"}], + "system": "You are in the treatment arm."} + leaks = audit_payload(payload, allowed=[desc]) + assert [(lk.marker, lk.path) for lk in leaks] == [("treatment", "$.system")] + # Without the allowance the skill's own word is reported too. + assert {lk.marker for lk in audit_payload(payload)} == {"baseline", "treatment"} + + +def test_new_leaks_ignore_what_the_stock_harness_already_says(): + reference = {"system": "Test the golden path and judge edge cases."} + measured = {"system": "Test the golden path and judge edge cases.", "user": "graded by the judge"} + assert [(lk.path, lk.marker) for lk in new_leaks(measured, reference)] == [("$.user", "judge")] + + +def test_skill_catalog_request_carries_no_dokimi_artifacts(tmp_path, monkeypatch): + # The request the skill-catalog arm actually sends (messages + tool schemas), captured + # at the LLM chokepoint, holds the query and the skills' own text and nothing else of ours. + paths = [_skill(tmp_path / "cat", "sum-fast", "Use when summing arrays; beats the baseline loop.", + "# Sum\nUse a reduction."), + _skill(tmp_path / "cat", "io-read", "Use when reading files.", "# IO\nRead in chunks.")] + skills = [load_skill(p) for p in paths] + arm = SkillCatalogTreatment(skills) + question = "How do I sum a big array quickly?" + cfg = arm.prepare(question, "lib") + sent = [] + + class _Msg: + content, tool_calls = "Use a reduction.", None + + class _Resp: + choices = [type("C", (), {"message": _Msg()})()] + usage = type("U", (), {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2})() + + def fake(messages, model, tools=None, **kw): + sent.append({"messages": [dict(m) for m in messages], "tools": tools}) + return _Resp() + + monkeypatch.setattr(ar_mod, "chat_completion", fake) + run_agent_loop(question, cfg.tools, model="m", system_prompt=cfg.system_prompt) + allowed = [question] + [s.description for s in skills] + [s.name for s in skills] + markers = markers_for(question_ids=["q-001"], arm_names=[arm.name], paths=[str(tmp_path)]) + assert sent and audit_payload(sent, markers, allowed) == [] + # The audit is live on this payload: the arm's own name, planted, is caught. + sent[0]["messages"].append({"role": "user", "content": f"arm {arm.name}"}) + assert {lk.marker for lk in audit_payload(sent, markers, allowed)} == {"skill_catalog", arm.name} + + +def _task(tmp_path: Path) -> Path: + task = tmp_path / "content" / "tasks" / "demo-task" + (task / "tests").mkdir(parents=True) + (task / "solution").mkdir() + (task / "environment").mkdir() + (task / "instruction.md").write_text("Make sum() faster.", encoding="utf-8") + (task / "task.toml").write_text("[environment]\n", encoding="utf-8") + return task + + +def test_docker_claude_discover_prompt_is_task_framing_only(tmp_path): + task = load_task(_task(tmp_path)) + treatment = Treatment(name="sum-fast", plugin="sum-fast") + out = tmp_path / "out" + out.mkdir() + prompts = {} + for notice in (True, False): + h = DockerSolverHarness(harness_id="docker-claude", mode="claude", skill_notice=notice) + argv = h._claude_args(task, treatment, "/plugins/sum-fast", "haiku", out) + prompt = argv[argv.index("-p") + 1] + prompts[notice] = added_framing(prompt, task.instruction) + assert audit_payload(prompt, markers_for(paths=[str(task.path)])) == [] + # Discover keeps dokimi's task framing but drops the notice that names the skill. + assert "sum-fast" not in prompts[False] and prompts[False].startswith("Solve this coding task.") + assert prompts[True].startswith(TREATMENT_NOTICE.format(skill_id=treatment.skill_id).strip()) diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py new file mode 100644 index 00000000..59015edf --- /dev/null +++ b/dokimi/tests/test_trigger_probe.py @@ -0,0 +1,131 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Real-harness trigger probe: staging, argv/env hygiene, transcript parsing, scoring.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from dokimi import process +from dokimi.eval.arm_summary import summarize_skill_selection +from dokimi.external.trigger_set import TriggerQuery +from dokimi.harnesses import trigger_probe as tp +from dokimi.harnesses.payload_audit import audit_payload, markers_for + + +def _catalog(tmp_path: Path) -> Path: + for name in ("sum-fast", "io-read"): + d = tmp_path / "cat" / name + d.mkdir(parents=True) + (d / "SKILL.md").write_text(f"---\nname: {name}\ndescription: Use for {name}.\n---\nbody\n", + encoding="utf-8") + return tmp_path / "cat" + + +def _stream(*skills_per_turn, other: str = "", version: str = "2.1.304") -> str: + lines = [{"type": "system", "subtype": "init", "skills": ["sum-fast", "io-read", "dataviz"], + "claude_code_version": version}] + if other: + lines.append({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": other, "input": {}}]}}) + for turn in skills_per_turn: + lines.append({"type": "assistant", "message": {"content": [ + {"type": "text", "text": "ok"}] + [ + {"type": "tool_use", "name": "Skill", "input": {"skill": s}} for s in turn]}}) + lines.append({"type": "result", "subtype": "success", "is_error": False, + "total_cost_usd": 0.01, "num_turns": 2}) + return "\n".join(json.dumps(x) for x in lines) + "\n" + + +def test_parse_stream_takes_the_first_skill_turn_as_the_choice(): + s = tp.parse_stream(_stream(["plug:sum-fast", "io-read"], ["io-read"], other="Bash")) + assert s["first_turn_loaded"] == ["sum-fast", "io-read"] + assert s["selected"] == "sum-fast" + assert s["loaded"] == ["sum-fast", "io-read", "io-read"] + assert s["tools_before_choice"] == ["Bash"] + assert s["listed"] == ["sum-fast", "io-read", "dataviz"] and s["claude_code_version"] == "2.1.304" + assert tp.parse_stream(_stream())["selected"] is None + + +def test_argv_is_the_query_verbatim_and_pins_only_what_is_asked(): + cfg = tp.ProbeConfig(model="haiku", max_turns=2) + argv = tp.probe_argv("How do I sum?", cfg) + assert argv[argv.index("-p") + 1] == "How do I sum?" + assert "--settings" not in argv and "--tools" not in argv and "--plugin-dir" not in argv + pinned = tp.probe_argv("q", tp.ProbeConfig(listing_budget_fraction=0.03)) + assert json.loads(pinned[pinned.index("--settings") + 1]) == {"skillListingBudgetFraction": 0.03} + + +def test_env_keeps_operator_config_out_and_bundled_skills_on_in_stock_mode(tmp_path): + ws = tp.new_workspace() + try: + source = {"PATH": "/bin", "AWS_REGION": "us-east-2", "CLAUDECODE": "1", + "CLAUDE_CONFIG_DIR": "/home/op/.claude", "GITHUB_TOKEN": "x"} + stock = tp.probe_env(ws, tp.ProbeConfig(), source) + sealed = tp.probe_env(ws, tp.ProbeConfig(stock=False), source) + assert stock["CLAUDE_CONFIG_DIR"] == str(ws.root / "config") and stock["HOME"] == str(ws.root / "home") + assert "GITHUB_TOKEN" not in stock and "CLAUDECODE" not in stock + assert "CLAUDE_CODE_DISABLE_BUNDLED_SKILLS" not in stock + assert sealed["CLAUDE_CODE_DISABLE_BUNDLED_SKILLS"] == "1" + finally: + ws.cleanup() + + +def test_run_query_stages_a_clean_repo_whose_path_names_no_experiment(tmp_path): + seen = {} + + def handler(call: process.Call): + if call.argv[0] == "git": + return None if call.argv[1] != "status" else process.reply("") + seen["cwd"] = call.cwd + seen["skills"] = sorted(p.name for p in (call.cwd / ".claude" / "skills").iterdir()) + seen["env"] = call.env + return process.reply(_stream(["io-read"])) + + fake = process.FakeRunner(handler) + # Real git for staging (it is the clean-repo claim under test), fake claude. + def runner(argv, **kw): + return process.run(argv, **kw) if argv[0] == "git" else fake(argv, **kw) + + r = tp.run_query("read a csv", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + assert r["skill"]["first_turn_loaded"] == ["io-read"] and r["error"] is None and r["cost_usd"] == 0.01 + assert seen["skills"] == ["io-read", "sum-fast"] + assert audit_payload(str(seen["cwd"]), markers_for()) == [] + assert not seen["cwd"].exists() # cleaned up + + +def test_out_of_turns_without_a_skill_is_no_trigger_not_a_failure(tmp_path): + stream = _stream().replace('"subtype": "success", "is_error": false', + '"subtype": "error_max_turns", "is_error": true') + + def runner(argv, **kw): + return process.run(argv, **kw) if argv[0] == "git" else process.reply(stream) + + r = tp.run_query("q", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, source_env={"PATH": "/usr/bin:/bin"}) + assert r["error"] is None and r["skill"]["first_turn_loaded"] == [] + broken = tp.run_query("q", _catalog(tmp_path / "b"), tp.ProbeConfig(), + runner=lambda a, **kw: process.run(a, **kw) if a[0] == "git" else process.reply("", returncode=1), + source_env={"PATH": "/usr/bin:/bin"}) + assert broken["error"] == "no result event (exit 1)" + + +def test_records_score_with_the_same_summary_as_the_emulated_arm(tmp_path): + answers = {"sum it": ["sum-fast"], "read it": ["sum-fast"], "hello": []} + + def runner(argv, **kw): + if argv[0] == "git": + return process.run(argv, **kw) + q = argv[list(argv).index("-p") + 1] + return process.reply(_stream(answers[q]) if answers[q] else _stream()) + + queries = [TriggerQuery("a", "sum it", "sum-fast"), TriggerQuery("b", "read it", "io-read"), + TriggerQuery("c", "hello", None), TriggerQuery("d", "unlabelled", None, labelled=False)] + recs = tp.probe_trigger_set(queries, _catalog(tmp_path), tp.ProbeConfig(), trials=2, runner=runner) + assert [r["question_id"] for r in recs] == ["a", "b", "c"] + s = summarize_skill_selection(recs) + assert set(s) == {"claude_code#1", "claude_code#2"} + assert s["claude_code#1"]["recall"] == 0.5 and s["claude_code#1"]["misroute_rate"] == 0.5 + assert s["claude_code#1"]["false_trigger_rate"] == 0.0 From 63ea270c60cb3c8d02d917fea412bd4a8d5a0ac0 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 24 Oct 2026 22:27:44 -0700 Subject: [PATCH 5/8] fix(catalog): group trigger-probe Skill calls by message.id Claude Code streams one assistant event per content block, so two Skill calls in one turn were scored as two turns and the strict first-turn rule, multi_load_rate and mean_first_turn_loads never fired. Group by message.id as agent_runner groups by iteration, and skip subagent events. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/harnesses/trigger_probe.py | 16 ++++++++++------ dokimi/tests/test_trigger_probe.py | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index bce1b005..01e845e9 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -132,20 +132,24 @@ def parse_stream(stream: str) -> Dict[str, Any]: """Skills listed and invoked, per the transcript; ``first_turn_loaded`` is the choice.""" listed: List[str] = [] version = None - turns: List[List[str]] = [] + # Claude Code emits one assistant event per content block; the blocks of one turn + # share message.id. Subagent events (parent_tool_use_id set) are not the choice. + by_turn: Dict[str, List[str]] = {} other_tools: List[str] = [] - for ev in _events(stream): + for n, ev in enumerate(_events(stream)): if ev.get("type") == "system" and ev.get("subtype") == "init": listed = [str(s) for s in ev.get("skills") or []] version = ev.get("claude_code_version") - if ev.get("type") != "assistant": + if ev.get("type") != "assistant" or ev.get("parent_tool_use_id"): continue - content = (ev.get("message") or {}).get("content") or [] - uses = [c for c in content if isinstance(c, dict) and c.get("type") == "tool_use"] + message = ev.get("message") or {} + uses = [c for c in message.get("content") or [] if isinstance(c, dict) and c.get("type") == "tool_use"] skills = [_skill_name((c.get("input") or {}).get("skill")) for c in uses if c.get("name") == "Skill"] other_tools += [str(c.get("name")) for c in uses if c.get("name") != "Skill"] if skills: - turns.append(skills) + turn = by_turn.setdefault(str(message.get("id") or f"#{n}"), []) + turn += [s for s in dict.fromkeys(skills) if s not in turn] + turns = list(by_turn.values()) first = turns[0] if turns else [] return { "offered": True, diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 59015edf..5dab7d42 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -49,6 +49,27 @@ def test_parse_stream_takes_the_first_skill_turn_as_the_choice(): assert tp.parse_stream(_stream())["selected"] is None +def test_parse_stream_groups_per_block_events_by_message_id(): + # Claude Code streams each content block as its own event; one turn shares message.id. + def block(mid, block, parent=None): + return {"type": "assistant", "parent_tool_use_id": parent, "message": {"id": mid, "content": [block]}} + + def skill(s): + return {"type": "tool_use", "name": "Skill", "input": {"skill": s}} + + events = [block("m1", {"type": "thinking", "thinking": "..."}), block("m1", skill("sum-fast")), + block("m1", skill("io-read")), {"type": "user", "message": {"content": []}}, + block("m2", skill("dataviz"), parent="toolu_1"), block("m3", skill("sum-fast")), + {"type": "result", "subtype": "success", "is_error": False}] + s = tp.parse_stream("\n".join(json.dumps(e) for e in events)) + assert s["first_turn_loaded"] == ["sum-fast", "io-read"] + assert s["loaded"] == ["sum-fast", "io-read", "sum-fast"] + rec = {"question_id": "a", "expected_skill": "sum-fast", + "arms": {"cc": {"answer": None, "error": None, "tool_usage": {"skill": s}}}} + summary = summarize_skill_selection([rec])["cc"] + assert summary["recall"] == 0.0 and summary["multi_load_rate"] == 1.0 + + def test_argv_is_the_query_verbatim_and_pins_only_what_is_asked(): cfg = tp.ProbeConfig(model="haiku", max_turns=2) argv = tp.probe_argv("How do I sum?", cfg) From d99619088c2584906863e030e348534e1f260a0c Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 24 Oct 2026 22:28:16 -0700 Subject: [PATCH 6/8] fix(catalog): score a trigger-probe choice made before a timeout run_query checked timed_out first, so a query whose Skill call was already in the transcript, or whose result event was read before the CLI hung on a backgrounded Bash call, was dropped as n_failed. A Skill call seen is now the choice, and a result event decides the rest. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/harnesses/trigger_probe.py | 11 ++++++----- dokimi/tests/test_trigger_probe.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 01e845e9..b4a0b7f0 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -176,13 +176,14 @@ def run_query(query: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Ru skill = parse_stream(stream) result = result_event(stream) or {} error = None - if done.timed_out: - error = "timeout" + # A Skill call seen is the choice, whatever came after it (a hang, a timeout, an + # error). A result event read decides the rest, even if the CLI then hung. + if skill["first_turn_loaded"]: + pass elif not result: - error = f"no result event (exit {done.returncode})" + error = "timeout" if done.timed_out else f"no result event (exit {done.returncode})" # Out of turns without a Skill call is an observation (nothing triggered), not a failure. - elif result.get("is_error") and result.get("subtype") != "error_max_turns" \ - and not skill["first_turn_loaded"]: + elif result.get("is_error") and result.get("subtype") != "error_max_turns": error = str(result.get("subtype") or "error") return {"skill": skill, "cost_usd": result.get("total_cost_usd"), "num_turns": result.get("num_turns"), "error": error} diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 5dab7d42..06e0933b 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -133,6 +133,22 @@ def runner(argv, **kw): assert broken["error"] == "no result event (exit 1)" +def test_a_hang_after_the_choice_is_scored_not_dropped(tmp_path): + # A stock-tool Bash call can outlive the turn; the CLI is then killed at --timeout. + with_result = _stream(["sum-fast"]) + no_result = with_result.rsplit('{"type": "result"', 1)[0] + cases = {"after-result": (with_result, None, ["sum-fast"]), "no-result": (no_result, None, ["sum-fast"]), + "no-skill-after-result": (_stream(), None, []), "nothing": ("", "timeout", [])} + for name, (stream, error, loaded) in cases.items(): + def runner(argv, stream=stream, **kw): + return process.run(argv, **kw) if argv[0] == "git" else process.reply(stream, timed_out=True) + + r = tp.run_query("q", _catalog(tmp_path / name), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + assert r["error"] == error, name + assert r["skill"]["first_turn_loaded"] == loaded, name + + def test_records_score_with_the_same_summary_as_the_emulated_arm(tmp_path): answers = {"sum it": ["sum-fast"], "read it": ["sum-fast"], "hello": []} From 25c992bd31afafd78b26d22cfa25d0081d834df8 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 24 Oct 2026 22:28:45 -0700 Subject: [PATCH 7/8] fix(catalog): apply the arms' plugin seal in trigger-probe --sealed --sealed set only the env seal, so the built-in plugin skills the docker-claude arms switch off via --settings stayed listed. Sealed argv now carries the same enabledPlugins, merged with --listing-budget-fraction into one --settings. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/harnesses/trigger_probe.py | 10 ++++++++-- dokimi/tests/test_trigger_probe.py | 9 +++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index b4a0b7f0..04b13473 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -34,7 +34,9 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence from dokimi import process -from dokimi.harnesses.agent_sandbox import _SEAL_ENV, AgentWorkspace, agent_env, operator_source_env +from dokimi.harnesses.agent_sandbox import ( + _SEAL_ENV, _SEAL_SETTINGS, AgentWorkspace, agent_env, operator_source_env, +) from dokimi.harnesses.claude_json import result_event #: Fixed identity and dates for the staging commit, so the git context the harness @@ -105,8 +107,12 @@ def probe_argv(query: str, cfg: ProbeConfig) -> List[str]: "--max-turns", str(cfg.max_turns), "--no-session-persistence"] if cfg.model: argv += ["--model", cfg.model] + # One --settings: sealed carries the arms' plugin seal, merged with any listing budget. + settings: Dict[str, Any] = {} if cfg.stock else json.loads(json.dumps(_SEAL_SETTINGS)) if cfg.listing_budget_fraction is not None: - argv += ["--settings", json.dumps({"skillListingBudgetFraction": cfg.listing_budget_fraction})] + settings["skillListingBudgetFraction"] = cfg.listing_budget_fraction + if settings: + argv += ["--settings", json.dumps(settings)] return argv + list(cfg.extra_args) diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 06e0933b..5dadfe75 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -12,6 +12,7 @@ from dokimi.eval.arm_summary import summarize_skill_selection from dokimi.external.trigger_set import TriggerQuery from dokimi.harnesses import trigger_probe as tp +from dokimi.harnesses.agent_sandbox import solver_cli_flags from dokimi.harnesses.payload_audit import audit_payload, markers_for @@ -79,6 +80,14 @@ def test_argv_is_the_query_verbatim_and_pins_only_what_is_asked(): assert json.loads(pinned[pinned.index("--settings") + 1]) == {"skillListingBudgetFraction": 0.03} +def test_sealed_argv_switches_off_the_builtin_plugins_as_the_arms_do(): + seal = json.loads(solver_cli_flags()[solver_cli_flags().index("--settings") + 1]) + for frac, extra in ((None, {}), (0.05, {"skillListingBudgetFraction": 0.05})): + argv = tp.probe_argv("q", tp.ProbeConfig(stock=False, listing_budget_fraction=frac)) + assert argv.count("--settings") == 1 + assert json.loads(argv[argv.index("--settings") + 1]) == {**seal, **extra} + + def test_env_keeps_operator_config_out_and_bundled_skills_on_in_stock_mode(tmp_path): ws = tp.new_workspace() try: From 0d834120b8d1665fd08fb1ab4918c18bb53e3cd0 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 24 Oct 2026 22:29:40 -0700 Subject: [PATCH 8/8] fix(catalog): keep trigger-probe results when one run fails A relative --claude passed the which() preflight but was spawned from the scratch work dir, and any OSError/RuntimeError in one run escaped pool.map and lost every paid result before --out was written. Resolve --claude to an absolute path and record a failed run as that trial's error. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/commands/catalog_probe.py | 7 +++-- dokimi/dokimi/harnesses/trigger_probe.py | 7 ++++- dokimi/tests/test_trigger_probe.py | 40 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index d55177e8..42b86d3c 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -7,6 +7,7 @@ import argparse import json +import os import shutil import sys from concurrent.futures import ThreadPoolExecutor @@ -21,7 +22,9 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: catalog = Path(args.catalog) if not catalog.is_dir(): raise SystemExit(f"--catalog is not a directory: {catalog}") - if shutil.which(args.claude) is None: + # Absolute: each run's cwd is a scratch dir, where a relative path names nothing. + claude = shutil.which(args.claude) + if claude is None: raise SystemExit(f"{args.claude!r} is not on PATH") try: data = json.loads(Path(args.trigger_set).read_text(encoding="utf-8")) @@ -31,7 +34,7 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: queries = [q for q in queries if q.labelled][: args.limit or None] cfg = tp.ProbeConfig(model=args.model, max_turns=args.max_turns, timeout_sec=args.timeout, stock=not args.sealed, listing_budget_fraction=args.listing_budget_fraction, - claude=args.claude) + claude=os.path.abspath(claude)) def one(q): rec = tp.probe_trigger_set([q], catalog, cfg, trials=args.trials)[0] diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 04b13473..f52e6c07 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -211,7 +211,12 @@ def probe_trigger_set(queries: Sequence[Any], catalog: Path, cfg: ProbeConfig, * continue arms = {} for k in range(trials): - r = run_query(q.text, catalog, cfg, runner=runner) + try: + r = run_query(q.text, catalog, cfg, runner=runner) + except (OSError, RuntimeError) as exc: + # One failed run (staging, spawn) is that trial's error, not the lost set. + r = {"skill": parse_stream(""), "cost_usd": None, "num_turns": None, + "error": f"{type(exc).__name__}: {exc}"} arms[f"{arm_name}#{k + 1}"] = { "answer": None, "error": r["error"], diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 5dadfe75..05f7afa1 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -6,9 +6,13 @@ from __future__ import annotations import json +import os from pathlib import Path +import pytest + from dokimi import process +from dokimi.cli import build_parser from dokimi.eval.arm_summary import summarize_skill_selection from dokimi.external.trigger_set import TriggerQuery from dokimi.harnesses import trigger_probe as tp @@ -175,3 +179,39 @@ def runner(argv, **kw): assert set(s) == {"claude_code#1", "claude_code#2"} assert s["claude_code#1"]["recall"] == 0.5 and s["claude_code#1"]["misroute_rate"] == 0.5 assert s["claude_code#1"]["false_trigger_rate"] == 0.0 + + +def test_one_failed_run_is_that_trials_error_not_the_lost_set(tmp_path): + def runner(argv, **kw): + if argv[0] == "git": + return process.run(argv, **kw) + if argv[list(argv).index("-p") + 1] == "read it": + raise FileNotFoundError(2, "No such file or directory", argv[0]) + return process.reply(_stream(["sum-fast"])) + + queries = [TriggerQuery("a", "sum it", "sum-fast"), TriggerQuery("b", "read it", "io-read")] + recs = tp.probe_trigger_set(queries, _catalog(tmp_path), tp.ProbeConfig(), runner=runner) + assert [r["question_id"] for r in recs] == ["a", "b"] + assert recs[1]["arms"]["claude_code#1"]["error"].startswith("FileNotFoundError") + s = summarize_skill_selection(recs)["claude_code#1"] + assert s["recall"] == 1.0 and s["n_failed"] == 1 + + +@pytest.mark.skipif(os.name == "nt", reason="a shell-script stand-in for claude") +def test_cli_runs_a_relative_claude_from_the_scratch_cwd(tmp_path, monkeypatch, capsys): + (tmp_path / "stream.jsonl").write_text(_stream(["sum-fast"]), encoding="utf-8") + fake = tmp_path / "bin" / "claude" + fake.parent.mkdir() + fake.write_text("#!/bin/sh\nexec cat \"$STREAM\"\n".replace("$STREAM", str(tmp_path / "stream.jsonl")), + encoding="utf-8") + fake.chmod(0o755) + (tmp_path / "ts.json").write_text(json.dumps({"questions": [ + {"id": "a", "question": "sum it", "expected_skill": "sum-fast"}]}), encoding="utf-8") + _catalog(tmp_path) + monkeypatch.chdir(tmp_path) + args = build_parser().parse_args(["catalog", "trigger-probe", "--catalog", "cat", "--trigger-set", + "ts.json", "--claude", "bin/claude", "--out", "probe.json"]) + args.func(args) + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + arm = out["answers"][0]["arms"]["claude_code#1"] + assert arm["error"] is None and arm["tool_usage"]["skill"]["first_turn_loaded"] == ["sum-fast"]