From f4ce883951644e8fa1d17e8a5c0bdce0b67dd5ce Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 22:18:57 -0700 Subject: [PATCH 1/7] feat(tasks): detect eval-task formats with `tasks inspect` A probe registry (dokimi.task_formats) names the format of a file, a task directory or a tree, and reports kind, item count, present/missing parts, lint problems and which dokimi backends consume it today. Probes: harbor and task-md (wrapping harnesses.tasks), terminal-bench v1, SWE-bench, Inspect AI samples and @task files (text scan, never imported), OpenAI evals, skill-creator evals.json and trigger sets, skill-reach QuerySet, promptfoo and dokimi questions. Unknown paths get a best-effort structure report; malformed files are lint problems. Detection only: nothing new runs. Signed-off-by: Nikolay Petrov --- ...1023_120000_napetrov_task-format-detect.md | 11 + dokimi/docs/architecture.md | 2 +- dokimi/docs/coding-harnesses.md | 29 ++ dokimi/dokimi/__init__.py | 3 +- dokimi/dokimi/commands/tasks.py | 45 ++ dokimi/dokimi/task_formats/__init__.py | 16 + dokimi/dokimi/task_formats/configs.py | 100 +++++ dokimi/dokimi/task_formats/datasets.py | 336 +++++++++++++++ dokimi/dokimi/task_formats/detect.py | 158 +++++++ dokimi/dokimi/task_formats/dirs.py | 155 +++++++ dokimi/dokimi/task_formats/model.py | 91 ++++ dokimi/dokimi/task_formats/source.py | 113 +++++ dokimi/tests/test_import_architecture.py | 2 +- dokimi/tests/test_task_format_detect.py | 387 ++++++++++++++++++ 14 files changed, 1445 insertions(+), 3 deletions(-) create mode 100644 dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md create mode 100644 dokimi/dokimi/task_formats/__init__.py create mode 100644 dokimi/dokimi/task_formats/configs.py create mode 100644 dokimi/dokimi/task_formats/datasets.py create mode 100644 dokimi/dokimi/task_formats/detect.py create mode 100644 dokimi/dokimi/task_formats/dirs.py create mode 100644 dokimi/dokimi/task_formats/model.py create mode 100644 dokimi/dokimi/task_formats/source.py create mode 100644 dokimi/tests/test_task_format_detect.py diff --git a/dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md b/dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md new file mode 100644 index 00000000..deafaf0d --- /dev/null +++ b/dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md @@ -0,0 +1,11 @@ +### Added + +- **`tasks inspect [--json]`** — detects the eval-task format of a file, + a task directory or a whole tree and reports its structure: kind + (executable, answer-only, trigger), item count, present and missing parts, + lint problems and which dokimi backends consume it today. Recognises Harbor, + BenchFlow `task.md`, terminal-bench v1, SWE-bench instances, Inspect AI + samples and `@task` files (scanned as text, never imported), OpenAI evals + JSONL, skill-creator `evals.json` and trigger sets, skill-reach query sets, + promptfoo configs and dokimi question sets; anything else is reported as + `unknown` with its files. Detection only: no new format runs yet. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 4a37c85d..64d8fcc9 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -70,7 +70,7 @@ points up: | Layer | Modules | | --- | --- | | front end | `cli`, `commands`, `__main__` (the argparse front end; no layer below imports it) | -| tracks | `runner`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, and the checkers `config_check`, `schema_check`, `schema_compat` | +| tracks | `runner`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, `task_formats`, and the checkers `config_check`, `schema_check`, `schema_compat` | | evaluation | `eval`, `personas`, `questions`, `treatments`, `plugins` | | components | `ingest`, `metrics`, `gate`, `mcp`, `skills`, `agent_profiles` | | foundation | `__init__` (the `dokimi` package itself), `_version`, `artifacts`, `records`, `llm`, `process`, `caching`, `config`, `registry`, `intents`, `defaults`, `console`, `utils`, `metrics/usage.py` | diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index 91cb3a87..a482d123 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -78,6 +78,35 @@ refused (a few hundred bytes of nested aliases expand without bound), and frontmatter nested too deeply to parse is a parse error. YAML and TOML dates and times become ISO strings, and YAML sets sorted lists. +### Other eval formats: `tasks inspect` + +`python cli.py tasks inspect [--json]` names the eval-task format of a +file, a task directory, or every item under a tree, and reports its structure: +kind, item count, which parts are present or missing (instruction, +environment, oracle, verifier, metadata), lint problems, and what in dokimi +can consume it today. It only reads: nothing is run or converted, a Python +file is scanned as text and never imported, and a file that does not parse is +a problem on its report, not an error. A path no format claims is reported as +`unknown` with its files (or keys) and likely prompt and verifier files; when +two formats claim a path about equally, the report lists the other as +`ambiguous` with its evidence. + +| Format | Recognised by | Kind | Consumed by | +|---|---|---|---| +| `harbor` | directory with `task.toml` | executable | Harbor-backed harnesses (unless Harbor would refuse it), `docker-*` | +| `task-md` | directory with a BenchFlow `task.md` | executable | `benchflow*`, `docker-*` | +| `terminal-bench-v1` | directory with `task.yaml` + `run-tests.sh` / `Dockerfile` / `solution.sh` | executable | none yet | +| `swe-bench` | JSON/JSONL rows with `instance_id`, `repo`, `base_commit`, `problem_statement` (`FAIL_TO_PASS`) | executable | none yet | +| `inspect-samples` | rows with `input` and `target` | answer-only | none yet | +| `inspect-task` | `.py` importing `inspect_ai` with a `@task` function | answer-only (executable with a `sandbox=`) | none yet | +| `openai-evals` | JSONL rows with `input` (chat messages) and `ideal` | answer-only | none yet | +| `skill-creator-evals` | `evals.json` with `skill_name` and `evals[].prompt` | answer-only | none yet | +| `skill-creator-trigger` | rows with `query` and `should_trigger` | trigger | none yet | +| `skill-reach` | `QuerySet` (`queries[]` with `query_id`/`id`, `text`, `expected_skill`) | trigger | none yet | +| `dokimi-questions` | questions.v1/v2 JSON | answer-only (trigger with `expected_skill`) | questions track | + +`tasks list` and `tasks run` still read only the first two. + The `docker-*` harnesses mount the oracle at `/solution` and `/oracle` and the verifier at `/tests` and `/verifier`, because a script written for one layout refers to its siblings by that layout's path. The directory's contents are diff --git a/dokimi/dokimi/__init__.py b/dokimi/dokimi/__init__.py index 81f93aa5..eabdfaaa 100644 --- a/dokimi/dokimi/__init__.py +++ b/dokimi/dokimi/__init__.py @@ -22,7 +22,8 @@ [`dashboard`][dokimi.dashboard] aggregates results across libraries and [`orchestrator`][dokimi.orchestrator] runs the one-command pipeline. - Executable tasks: [`harnesses`][dokimi.harnesses] runs coding agents on task - directories. + directories; [`task_formats`][dokimi.task_formats] names the format of other + eval-task files and directories (detection only). - [`context`][dokimi.context] reads what a run's context window held. - Content novelty: [`novelty`][dokimi.novelty]. Repositories as subjects: [`repos`][dokimi.repos]. Subject scorecards, one per run: diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py index 0564c8fc..79779562 100644 --- a/dokimi/dokimi/commands/tasks.py +++ b/dokimi/dokimi/commands/tasks.py @@ -47,6 +47,43 @@ def cmd_tasks_list(args: argparse.Namespace) -> None: print(f"{task.name}\t{difficulty}\t{','.join(tags)}") +def _inspect_line(report: dict) -> str: + n = report["items"] + items = "" if n is None else f"\t{n} item{'' if n == 1 else 's'}" + return f"{report['path']}\t{report['format']}\t{report['kind']}{items}\t-> {', '.join(report['consumers'])}" + + +def _print_inspect(reports: list[dict]) -> None: + for r in reports: + print(_inspect_line(r)) + if r["present"] or r["missing"]: + print(f" parts: {', '.join(r['present']) or '-'}; missing: {', '.join(r['missing']) or '-'}") + for alt in r["alternatives"]: + print(f" ambiguous: also {alt['format']} ({alt['confidence']:.2f}: {', '.join(alt['evidence'])})") + for problem in r["problems"]: + print(f" problem: {problem}") + by_format: dict[str, int] = {} + for r in reports: + by_format[r["format"]] = by_format.get(r["format"], 0) + 1 + print(f"{len(reports)} item{'' if len(reports) == 1 else 's'}: " + ", ".join(f"{f} {n}" for f, n in sorted(by_format.items()))) + + +def cmd_tasks_inspect(args: argparse.Namespace) -> None: + """Detect the eval-task format of a path (or of everything under it) and report its structure.""" + import json + + from dokimi.task_formats import inspect_path + + try: + reports = [r.as_dict() for r in inspect_path(Path(args.path))] + except FileNotFoundError as exc: + raise SystemExit(f"error: {exc}") from None + if args.json: + print(json.dumps({"root": str(args.path), "items": reports}, indent=2, default=str)) + else: + _print_inspect(reports) + + def cmd_tasks_harnesses(args: argparse.Namespace) -> None: from dokimi.harnesses import known_harnesses @@ -185,6 +222,14 @@ def register(sub, positive_int) -> None: list_p.add_argument("--tasks-root", default=None, help=_TASKS_ROOT_HELP) list_p.set_defaults(func=cmd_tasks_list) + inspect_p = tasks_sub.add_parser( + "inspect", + help="Detect the eval-task format of a file or directory tree and report its structure (no run)", + ) + inspect_p.add_argument("path", help="A task directory, a dataset file, or a tree holding either") + inspect_p.add_argument("--json", action="store_true", help="Print the reports as JSON") + inspect_p.set_defaults(func=cmd_tasks_inspect) + harnesses_p = tasks_sub.add_parser("harnesses", help="List known coding harness aliases") harnesses_p.set_defaults(func=cmd_tasks_harnesses) diff --git a/dokimi/dokimi/task_formats/__init__.py b/dokimi/dokimi/task_formats/__init__.py new file mode 100644 index 00000000..12f9ab34 --- /dev/null +++ b/dokimi/dokimi/task_formats/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Eval-task format detection: which format is a path, and what does it hold? + +Detection and a structure report only: nothing here runs or converts a task. +``dokimi tasks inspect `` is the command line; :func:`inspect_path` the API. +""" + +from dokimi.task_formats.detect import DIR_PROBES, FILE_PROBES, PROBES, detect, inspect_path, unknown +from dokimi.task_formats.model import Fingerprint, Probe, Report + +__all__ = [ + "DIR_PROBES", "FILE_PROBES", "PROBES", "Fingerprint", "Probe", "Report", "detect", + "inspect_path", "unknown", +] diff --git a/dokimi/dokimi/task_formats/configs.py b/dokimi/dokimi/task_formats/configs.py new file mode 100644 index 00000000..75ff306d --- /dev/null +++ b/dokimi/dokimi/task_formats/configs.py @@ -0,0 +1,100 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probes for eval definitions that are config or code: promptfoo, Inspect AI tasks. + +An Inspect ``@task`` file is Python: it is scanned as text with regular +expressions and never imported or run. +""" + +from __future__ import annotations + +import re +from typing import Any + +from dokimi.task_formats.model import KIND_ANSWER_ONLY, KIND_EXECUTABLE, Fingerprint, Report +from dokimi.task_formats.source import FileSource + +# --- promptfoo (promptfoo.dev/docs/configuration/reference) ------------------- + +_PROMPTFOO_NAMES = ("promptfooconfig.yaml", "promptfooconfig.yml", "promptfooconfig.json") + + +class PromptfooProbe: + name = "promptfoo" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix not in (".yaml", ".yml", ".json"): + return None + named = source.path.name in _PROMPTFOO_NAMES + data = source.data + if not isinstance(data, dict): + return Fingerprint(0.6, ("file name",)) if named else None + keys = [k for k in ("prompts", "providers", "targets", "tests", "scenarios") if k in data] + has_target = "providers" in data or "targets" in data + if not named and not ("prompts" in data and has_target and ("tests" in data or "scenarios" in data)): + return None + return Fingerprint(min(1.0, 0.5 + 0.1 * named + 0.1 * len(keys)), (("file name",) if named else ()) + tuple(keys)) + + def describe(self, source: FileSource) -> Report: + data: dict[str, Any] = source.data if isinstance(source.data, dict) else {} + tests = data.get("tests") + report = Report(source.path, self.name, KIND_ANSWER_ONLY, + items=len(tests) if isinstance(tests, list) else None) + report.problems.extend(source.problems) + if source.data is not None and not data: + report.problems.append("not a mapping") + asserts = isinstance(data.get("defaultTest"), dict) and "assert" in data["defaultTest"] + asserts = asserts or any(isinstance(t, dict) and t.get("assert") for t in tests or [] if isinstance(tests, list)) + report.set_parts({ + "instruction": bool(data.get("prompts")), + "environment": bool(data.get("providers") or data.get("targets")), + "verifier": asserts, + "metadata": bool(data.get("description")), + }) + if isinstance(tests, str): + report.details["tests_from"] = tests + elif data and tests is None and "scenarios" not in data: + report.problems.append("no tests") + if data and not data.get("prompts"): + report.problems.append("no prompts") + return report + + +# --- Inspect AI @task files ---------------------------------------------------- + +_INSPECT_IMPORT = re.compile(r"^\s*(?:from\s+inspect_ai(?:\.\w+)*\s+import|import\s+inspect_ai)\b", re.M) +_TASK_DECORATOR = re.compile(r"^\s*@(?:inspect_ai\.)?task\b", re.M) +_SANDBOX = re.compile(r"\bsandbox\s*=") +_SCORER = re.compile(r"\bscorer\s*=") +_DATASET = re.compile(r"\bdataset\s*=|\bSample\s*\(|\b\w+_dataset\s*\(") +_SOLVER = re.compile(r"\b(?:solver|plan)\s*=") + + +class InspectTaskProbe: + name = "inspect-task" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix != ".py" or not source.text: + return None + if not _INSPECT_IMPORT.search(source.text) or not _TASK_DECORATOR.search(source.text): + return None + return Fingerprint(0.9, ("imports inspect_ai", "@task")) + + def describe(self, source: FileSource) -> Report: + text = source.text or "" + sandboxed = bool(_SANDBOX.search(text)) + report = Report(source.path, self.name, KIND_EXECUTABLE if sandboxed else KIND_ANSWER_ONLY, + items=len(_TASK_DECORATOR.findall(text))) + report.problems.extend(source.problems) + report.set_parts({ + "instruction": bool(_DATASET.search(text)), + "environment": sandboxed, + "verifier": bool(_SCORER.search(text)), + "metadata": False, + }) + report.details["solver"] = bool(_SOLVER.search(text)) + report.details["scan"] = "static text scan; the file was not imported" + return report diff --git a/dokimi/dokimi/task_formats/datasets.py b/dokimi/dokimi/task_formats/datasets.py new file mode 100644 index 00000000..e433ff1d --- /dev/null +++ b/dokimi/dokimi/task_formats/datasets.py @@ -0,0 +1,336 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probes for dataset files: one file, many items. + +Field names follow each upstream's own definition: + +- SWE-bench: ``swebench/types.py`` ``SWEbenchInstance`` (SWE-bench/SWE-bench). +- Inspect AI: ``Sample`` / ``FieldSpec`` defaults in ``inspect_ai/dataset/_dataset.py``. +- OpenAI evals: ``docs/build-eval.md`` (``input`` as chat messages, ``ideal``). +- anthropics/skills skill-creator: ``references/schemas.md`` (``evals.json``) and + the trigger eval set of its ``SKILL.md`` (``[{query, should_trigger}]``). +- google/skill-reach: ``Query`` / ``QuerySet`` in ``src/reach/models.py`` and + ``src/reach/queries.py`` (``id`` is the legacy spelling of ``query_id``). +""" + +from __future__ import annotations + +import json +from collections import Counter +from typing import Any + +from dokimi.task_formats.model import ( + BY_QUESTIONS, KIND_ANSWER_ONLY, KIND_EXECUTABLE, KIND_TRIGGER, Fingerprint, Report, +) +from dokimi.task_formats.source import FileSource, first_rows, key_share + +_ROW_SUFFIXES = (".json", ".jsonl", ".yaml", ".yml") + + +def _records(source: FileSource, marker: str) -> list[dict[str, Any]] | None: + """The file's rows; a single object holding *marker* is a one-row file.""" + if source.suffix not in _ROW_SUFFIXES: + return None + if source.rows is not None: + return source.rows + data = source.data + return [data] if isinstance(data, dict) and marker in data else None + + +def _without(rows: list[dict[str, Any]], key: str) -> int: + return sum(1 for r in rows if not r.get(key)) + + +def _lint_missing(report: Report, rows: list[dict[str, Any]], keys: tuple[str, ...]) -> None: + for key in keys: + n = _without(rows, key) + if n: + report.problems.append(f"{n} of {len(rows)} items have no {key}") + + +def _lint_duplicate_ids(report: Report, rows: list[dict[str, Any]], *keys: str) -> None: + ids = Counter(str(next((r[k] for k in keys if k in r), "")) for r in rows) + dupes = sorted(i for i, n in ids.items() if i and n > 1) + if dupes: + report.problems.append(f"duplicate ids: {', '.join(dupes[:5])}") + + +def _base(source: FileSource, name: str, kind: str, rows: list[dict[str, Any]]) -> Report: + report = Report(source.path, name, kind, items=len(rows)) + report.problems.extend(source.problems) + return report + + +# --- SWE-bench --------------------------------------------------------------- + +_SWE_REQUIRED = ("instance_id", "repo", "base_commit", "problem_statement") + + +def _test_list(value: Any) -> list[Any] | None: + """FAIL_TO_PASS / PASS_TO_PASS: a list, or (as the HF datasets ship it) its JSON string.""" + if isinstance(value, list): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except ValueError: + return None + return parsed if isinstance(parsed, list) else None + return None + + +class SweBenchProbe: + name = "swe-bench" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + rows = first_rows(_records(source, "instance_id")) + share = key_share(rows, *_SWE_REQUIRED) + if share < 0.5: + return None + tests = key_share(rows, "FAIL_TO_PASS", "PASS_TO_PASS") + return Fingerprint(0.6 * share + 0.4 * tests, (*_SWE_REQUIRED, "FAIL_TO_PASS/PASS_TO_PASS" if tests else "")) + + def describe(self, source: FileSource) -> Report: + rows = _records(source, "instance_id") or [] + report = _base(source, self.name, KIND_EXECUTABLE, rows) + report.set_parts({ + "instruction": key_share(rows, "problem_statement") == 1, + "environment": key_share(rows, "repo", "base_commit") == 1, + "oracle": key_share(rows, "patch") == 1, + "verifier": key_share(rows, "FAIL_TO_PASS") == 1, + "metadata": key_share(rows, "version") == 1, + }) + _lint_missing(report, rows, (*_SWE_REQUIRED, "FAIL_TO_PASS")) + bad = sum(1 for r in rows if "FAIL_TO_PASS" in r and _test_list(r["FAIL_TO_PASS"]) is None) + if bad: + report.problems.append(f"{bad} items have a FAIL_TO_PASS that is not a list of test ids") + _lint_duplicate_ids(report, rows, "instance_id") + report.details["repos"] = sorted({str(r.get("repo")) for r in rows if r.get("repo")})[:20] + return report + + +# --- Inspect AI samples and OpenAI evals -------------------------------------- + + +def _is_chat(value: Any) -> bool: + return isinstance(value, list) and bool(value) and all( + isinstance(m, dict) and "role" in m and "content" in m for m in value) + + +class InspectSamplesProbe: + name = "inspect-samples" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + rows = first_rows(_records(source, "input")) + with_target = key_share(rows, "input", "target") + if with_target >= 0.5: + return Fingerprint(0.85 * with_target, ("input", "target")) + if key_share(rows, "input") >= 0.8 and not key_share(rows, "ideal"): + return Fingerprint(0.55, ("input (no target)",)) + return None + + def describe(self, source: FileSource) -> Report: + rows = _records(source, "input") or [] + report = _base(source, self.name, KIND_ANSWER_ONLY, rows) + report.set_parts({ + "instruction": key_share(rows, "input") == 1, + "environment": any(k in r for r in rows for k in ("sandbox", "files", "setup")), + "oracle": key_share(rows, "target") == 1, + # The scorer is code in the @task file, not in the samples. + "verifier": False, + "metadata": key_share(rows, "metadata") == 1, + }) + _lint_missing(report, rows, ("input",)) + bad = sum(1 for r in rows if "input" in r and not isinstance(r["input"], str) and not _is_chat(r["input"])) + if bad: + report.problems.append(f"{bad} items have an input that is neither text nor chat messages") + _lint_duplicate_ids(report, rows, "id") + report.details["with_choices"] = sum(1 for r in rows if "choices" in r) + return report + + +class OpenAIEvalsProbe: + name = "openai-evals" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix not in (".jsonl", ".json"): + return None + rows = first_rows(_records(source, "ideal")) + share = key_share(rows, "input", "ideal") + if share < 0.5: + return None + chat = sum(_is_chat(r.get("input")) for r in rows) / len(rows) + return Fingerprint(0.7 * share + 0.25 * chat, ("input", "ideal", "chat messages" if chat else "")) + + def describe(self, source: FileSource) -> Report: + rows = _records(source, "ideal") or [] + report = _base(source, self.name, KIND_ANSWER_ONLY, rows) + report.set_parts({ + "instruction": key_share(rows, "input") == 1, + "oracle": key_share(rows, "ideal") == 1, + }) + _lint_missing(report, rows, ("input", "ideal")) + chat = sum(1 for r in rows if _is_chat(r.get("input"))) + report.details["chat_inputs"] = chat + return report + + +# --- skill-creator ----------------------------------------------------------- + + +class SkillCreatorEvalsProbe: + name = "skill-creator-evals" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + data = source.data if source.suffix == ".json" else None + if not isinstance(data, dict) or not isinstance(data.get("evals"), list): + return None + rows = [r for r in data["evals"] if isinstance(r, dict)] + score = 0.5 + 0.2 * ("skill_name" in data) + 0.3 * key_share(first_rows(rows), "prompt") + return Fingerprint(score, ("evals[]", "skill_name" if "skill_name" in data else "", "prompt")) + + def describe(self, source: FileSource) -> Report: + data = source.data if isinstance(source.data, dict) else {} + rows = [r for r in data.get("evals", []) if isinstance(r, dict)] + report = _base(source, self.name, KIND_ANSWER_ONLY, rows) + report.set_parts({ + "instruction": key_share(rows, "prompt") == 1 and bool(rows), + "environment": any(r.get("files") for r in rows), + "oracle": key_share(rows, "expected_output") == 1 and bool(rows), + "verifier": any(r.get("expectations") for r in rows), + "metadata": bool(data.get("skill_name")), + }) + _lint_missing(report, rows, ("prompt",)) + bad = sum(1 for r in rows if not isinstance(r.get("expectations", []), list)) + if bad: + report.problems.append(f"{bad} items have expectations that are not a list") + _lint_duplicate_ids(report, rows, "id") + report.details["skill_name"] = data.get("skill_name") + return report + + +class SkillCreatorTriggerProbe: + name = "skill-creator-trigger" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + rows = first_rows(source.rows if source.suffix in (".json", ".jsonl") else None) + share = key_share(rows, "query", "should_trigger") + return Fingerprint(0.95 * share, ("query", "should_trigger")) if share >= 0.5 else None + + def describe(self, source: FileSource) -> Report: + rows = source.rows or [] + report = _base(source, self.name, KIND_TRIGGER, rows) + report.set_parts({"instruction": key_share(rows, "query") == 1, + "oracle": key_share(rows, "should_trigger") == 1}) + _lint_missing(report, rows, ("query",)) + bad = sum(1 for r in rows if not isinstance(r.get("should_trigger"), bool)) + if bad: + report.problems.append(f"{bad} items have a should_trigger that is not true/false") + report.details["should_trigger"] = sum(1 for r in rows if r.get("should_trigger") is True) + report.details["should_not_trigger"] = sum(1 for r in rows if r.get("should_trigger") is False) + return report + + +# --- skill-reach ------------------------------------------------------------- + +_REACH_QUERY_KEYS = frozenset({"query_id", "id", "text", "kind", "expected_skill", "acceptable_skills", "notes"}) +_REACH_SET_KEYS = frozenset({"catalog_id", "notes", "provenance", "queries"}) + + +def _reach_rows(source: FileSource) -> list[dict[str, Any]] | None: + data = source.data + if isinstance(data, dict) and isinstance(data.get("queries"), list): + return [q for q in data["queries"] if isinstance(q, dict)] + return source.rows + + +def _lint_reach_query(q: dict[str, Any]) -> str | None: + qid = q.get("query_id", q.get("id", "?")) + unknown = sorted(set(q) - _REACH_QUERY_KEYS) + if unknown: + return f"{qid}: keys skill-reach refuses: {', '.join(unknown)}" + if (q.get("kind") == "out_of_scope") != (q.get("expected_skill") is None): + return f"{qid}: kind={q.get('kind')!r} disagrees with expected_skill={q.get('expected_skill')!r}" + return None + + +class SkillReachProbe: + name = "skill-reach" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix not in _ROW_SUFFIXES: + return None + rows = first_rows(_reach_rows(source)) + texts = key_share(rows, "text", "expected_skill") + if texts < 0.5: + return None + ids = max(key_share(rows, "query_id"), key_share(rows, "id")) + wrapped = isinstance(source.data, dict) and "queries" in source.data + evidence = ("queries[]" if wrapped else "rows", "text", "expected_skill", "query_id/id" if ids else "") + return Fingerprint(0.5 * texts + 0.3 * ids + 0.2 * wrapped, evidence) + + def describe(self, source: FileSource) -> Report: + rows = _reach_rows(source) or [] + report = _base(source, self.name, KIND_TRIGGER, rows) + data = source.data if isinstance(source.data, dict) else {} + report.set_parts({"instruction": key_share(rows, "text") == 1, + "oracle": key_share(rows, "expected_skill") == 1, + "metadata": bool(data.get("catalog_id") or data.get("provenance"))}) + _lint_missing(report, rows, ("text",)) + extra = sorted(set(data) - _REACH_SET_KEYS) + if extra: + report.problems.append(f"QuerySet keys skill-reach refuses: {', '.join(extra)}") + report.problems.extend(p for p in map(_lint_reach_query, rows) if p) + _lint_duplicate_ids(report, rows, "query_id", "id") + report.details["out_of_scope"] = sum(1 for r in rows if r.get("expected_skill") is None) + return report + + +# --- dokimi questions -------------------------------------------------------- + +_QUESTIONS_STAMPS = ("questions.v1", "questions.v2") + + +class QuestionsProbe: + name = "dokimi-questions" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + data = source.data if source.suffix == ".json" else None + if isinstance(data, dict) and data.get("schema_version") in _QUESTIONS_STAMPS: + return Fingerprint(1.0, (f"schema_version {data['schema_version']}",)) + if not isinstance(data, dict) or not isinstance(data.get("questions"), list): + return None + rows = first_rows([q for q in data["questions"] if isinstance(q, dict)]) + share = max(key_share(rows, "question"), key_share(rows, "text")) + return Fingerprint(0.4 + 0.4 * share, ("questions[]", "question/text")) if share else None + + def describe(self, source: FileSource) -> Report: + from dokimi.questions.artifact import load_questions + + data = source.data if isinstance(source.data, dict) else {} + rows = [q for q in data.get("questions", []) if isinstance(q, dict)] + kind = KIND_TRIGGER if any("expected_skill" in q for q in rows) else KIND_ANSWER_ONLY + report = _base(source, self.name, kind, rows) + has_text = all(q.get("question") or q.get("text") for q in rows) and bool(rows) + report.set_parts({ + "instruction": has_text, + "oracle": any(q.get("ground_truth_chunk") or q.get("expected_skill") for q in rows), + "verifier": any(q.get("expected_topics") for q in rows), + "metadata": bool(data.get("product") or data.get("library")), + }) + try: + load_questions(source.path) + except Exception as exc: # noqa: BLE001 -- any refusal is a lint finding, the scan goes on + report.problems.append(" ".join(str(exc).split())[:300]) + else: + report.consumers.append(BY_QUESTIONS) + report.details["schema_version"] = data.get("schema_version") + return report diff --git a/dokimi/dokimi/task_formats/detect.py b/dokimi/dokimi/task_formats/detect.py new file mode 100644 index 00000000..f373d7d7 --- /dev/null +++ b/dokimi/dokimi/task_formats/detect.py @@ -0,0 +1,158 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The probe registry and the scan: which eval-task format is each path? + +A directory a directory probe claims (a task) is one item and is not descended +into; any other directory is walked, and each data file in it is offered to the +file probes. When two probes claim a path about equally, the best is reported +and the others are listed as alternatives with their evidence. A path nothing +claims is reported as ``unknown`` with what it holds, never raised. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator, Sequence +from pathlib import Path +from typing import Any + +from dokimi.task_formats.configs import InspectTaskProbe, PromptfooProbe +from dokimi.task_formats.datasets import ( + InspectSamplesProbe, OpenAIEvalsProbe, QuestionsProbe, SkillCreatorEvalsProbe, + SkillCreatorTriggerProbe, SkillReachProbe, SweBenchProbe, +) +from dokimi.task_formats.dirs import HarborProbe, TaskMdProbe, TerminalBenchV1Probe +from dokimi.task_formats.model import ( + AMBIGUITY_MARGIN, FORMAT_UNKNOWN, KIND_UNKNOWN, MATCH_THRESHOLD, Fingerprint, Probe, Report, +) +from dokimi.task_formats.source import FileSource + +DIR_PROBES: tuple[Probe, ...] = (HarborProbe(), TaskMdProbe(), TerminalBenchV1Probe()) +FILE_PROBES: tuple[Probe, ...] = ( + SweBenchProbe(), InspectSamplesProbe(), OpenAIEvalsProbe(), SkillCreatorEvalsProbe(), + SkillCreatorTriggerProbe(), SkillReachProbe(), QuestionsProbe(), PromptfooProbe(), + InspectTaskProbe(), +) +PROBES: tuple[Probe, ...] = DIR_PROBES + FILE_PROBES + +#: Directories a scan never enters. +SKIP_DIRS = frozenset({".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv", "venv", + ".tox", ".mypy_cache", ".pytest_cache", ".ruff_cache"}) +#: Suffixes a scan offers to the file probes. +SCAN_SUFFIXES = frozenset({".json", ".jsonl", ".yaml", ".yml", ".py"}) +#: A scan stops after this many files and says so. +MAX_SCAN_FILES = 20000 + + +def _claims(probes: Sequence[Probe], subject: Any) -> list[tuple[Fingerprint, Probe]]: + found = [] + for probe in probes: + fp = probe.fingerprint(subject) + if fp is not None and fp.confidence >= MATCH_THRESHOLD: + found.append((fp, probe)) + return sorted(found, key=lambda c: -c[0].confidence) + + +def _evidence(fp: Fingerprint) -> list[str]: + return [e for e in fp.evidence if e] + + +def detect(path: Path, probes: Sequence[Probe] | None = None) -> Report | None: + """The report of the best probe claiming *path*, or ``None`` if none does.""" + path = Path(path) + if probes is None: + probes = DIR_PROBES if path.is_dir() else FILE_PROBES + subject: Any = path if path.is_dir() else FileSource(path) + claims = _claims(probes, subject) + if not claims: + return None + (best_fp, best), rest = claims[0], claims[1:] + report = best.describe(subject) + report.confidence = best_fp.confidence + report.evidence = _evidence(best_fp) + report.alternatives = [ + {"format": p.name, "confidence": round(fp.confidence, 2), "evidence": _evidence(fp)} + for fp, p in rest if round(best_fp.confidence - fp.confidence, 6) <= AMBIGUITY_MARGIN + ] + return report + + +def inspect_path(path: Path) -> list[Report]: + """Every item under *path* (a file, a task directory, or a tree), or one ``unknown``.""" + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"no such file or directory: {path}") + if path.is_file(): + return [detect(path) or unknown(path)] + scan = _Scan() + reports = list(scan.walk(path)) + if not reports: + reports = [unknown(path)] + if scan.truncated: + reports[-1].problems.append(f"scan stopped after {MAX_SCAN_FILES} files") + return reports + + +class _Scan: + def __init__(self) -> None: + self.files = 0 + self.truncated = False + + def walk(self, directory: Path) -> Iterator[Report]: + task = detect(directory, DIR_PROBES) + if task is not None: + yield task + return + try: + children = sorted(directory.iterdir()) + except OSError: + return + for child in children: + if self.truncated: + return + yield from self._child(child) + + def _child(self, child: Path) -> Iterator[Report]: + if child.is_symlink(): + return + if child.is_dir(): + if child.name not in SKIP_DIRS and not child.name.startswith("."): + yield from self.walk(child) + return + if child.suffix.lower() not in SCAN_SUFFIXES: + return + self.files += 1 + if self.files > MAX_SCAN_FILES: + self.truncated = True + return + report = detect(child, FILE_PROBES) + if report is not None: + yield report + + +# --- unknown ----------------------------------------------------------------- + +_PROMPT_NAMES = re.compile(r"^(instruction|prompt|task|problem|question)s?[\w-]*\.(md|txt|ya?ml|json)$", re.I) +_VERIFIER_NAMES = re.compile(r"^(test|tests|verify|verifier|run-tests|check|grade|grader)[\w-]*(\.\w+)?$", re.I) +_LISTED = 30 + + +def unknown(path: Path) -> Report: + """Best-effort structure of a path no probe claims.""" + report = Report(path, FORMAT_UNKNOWN, KIND_UNKNOWN) + if path.is_dir(): + names = sorted(p.name + ("/" if p.is_dir() else "") for p in path.iterdir()) + report.details["files"] = names[:_LISTED] + report.details["prompt_candidates"] = [n for n in names if _PROMPT_NAMES.match(n)] + report.details["verifier_candidates"] = [n for n in names if _VERIFIER_NAMES.match(n.rstrip("/"))] + return report + source = FileSource(path) + data = source.data + report.problems.extend(source.problems) + if isinstance(data, dict): + report.details["keys"] = sorted(map(str, data))[:_LISTED] + elif source.rows: + report.items = len(source.rows) + report.details["row_keys"] = sorted(map(str, source.rows[0]))[:_LISTED] + return report diff --git a/dokimi/dokimi/task_formats/dirs.py b/dokimi/dokimi/task_formats/dirs.py new file mode 100644 index 00000000..0f2e2ff3 --- /dev/null +++ b/dokimi/dokimi/task_formats/dirs.py @@ -0,0 +1,155 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probes for task directories: Harbor, BenchFlow task.md, terminal-bench v1. + +Harbor and task-md recognition is :func:`dokimi.harnesses.tasks.task_format` and +their checks are :func:`~dokimi.harnesses.tasks.load_task` and +:func:`~dokimi.harnesses.tasks.harbor_task_problem`: the probe reports what the +runners would do, it does not decide again. +""" + +from __future__ import annotations + +import warnings +from pathlib import Path +from typing import Any + +from dokimi.harnesses import tasks as task_layouts +from dokimi.task_formats.model import ( + BY_BENCHFLOW, BY_DOCKER, BY_HARBOR, KIND_EXECUTABLE, Fingerprint, Report, +) +from dokimi.task_formats.source import FileSource + + +def _any_file(path: Path, *names: str) -> bool: + return any((path / n).is_file() for n in names) + + +def _any_dir(path: Path, *names: str) -> bool: + return any((path / n).is_dir() for n in names) + + +def _load(path: Path, report: Report) -> task_layouts.TaskSpec | None: + """load_task, with its exception (and any warning it raises) as lint problems.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + spec = task_layouts.load_task(path) + except (ValueError, OSError) as exc: + report.problems.append(str(exc)) + spec = None + report.problems.extend(str(w.message) for w in caught) + return spec + + +class HarborProbe: + name = task_layouts.FORMAT_HARBOR + target = "dir" + + def fingerprint(self, path: Path) -> Fingerprint | None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fmt = task_layouts.task_format(path) + if fmt != task_layouts.FORMAT_HARBOR: + return None + return Fingerprint(1.0, ("task.toml",)) + + def describe(self, path: Path) -> Report: + report = Report(path, self.name, KIND_EXECUTABLE) + spec = _load(path, report) + report.set_parts({ + "instruction": (path / "instruction.md").is_file(), + "environment": (path / "environment").is_dir(), + "oracle": _any_dir(path, "solution", "oracle"), + "verifier": _any_dir(path, "tests", "verifier"), + "metadata": bool(spec and spec.metadata), + }) + if spec is None: + return report + report.details["metadata"] = _summary(spec.metadata) + problem = task_layouts.harbor_task_problem(spec) + if problem: + report.problems.append(f"Harbor would refuse it: {problem}") + else: + report.consumers.append(BY_HARBOR) + if _any_dir(path, "tests", "verifier"): + report.consumers.append(BY_DOCKER) + return report + + +class TaskMdProbe: + name = task_layouts.FORMAT_TASK_MD + target = "dir" + + def fingerprint(self, path: Path) -> Fingerprint | None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fmt = task_layouts.task_format(path) + if fmt != task_layouts.FORMAT_TASK_MD: + return None + return Fingerprint(1.0, ("task.md with BenchFlow frontmatter",)) + + def describe(self, path: Path) -> Report: + report = Report(path, self.name, KIND_EXECUTABLE) + spec = _load(path, report) + report.set_parts({ + "instruction": bool(spec and spec.instruction.strip()), + "environment": (path / "environment").is_dir(), + "oracle": _any_dir(path, "oracle", "solution"), + "verifier": _any_dir(path, "verifier", "tests"), + "metadata": bool(spec and spec.metadata), + }) + if spec is None: + return report + report.details["metadata"] = _summary(spec.metadata) + report.details["schema_version"] = spec.schema_version + report.consumers.append(BY_BENCHFLOW) + if _any_dir(path, "verifier", "tests"): + report.consumers.append(BY_DOCKER) + return report + + +# terminal-bench 0.x (laude-institute/terminal-bench, original-tasks/): task.yaml +# holds the instruction and metadata; the container is Dockerfile + +# docker-compose.yaml, the oracle solution.sh (or solution.yaml), the verifier +# run-tests.sh running tests/. +_TB1_SIGNALS = ("run-tests.sh", "docker-compose.yaml", "Dockerfile", "solution.sh", "solution.yaml") + + +class TerminalBenchV1Probe: + name = "terminal-bench-v1" + target = "dir" + + def fingerprint(self, path: Path) -> Fingerprint | None: + if not (path / "task.yaml").is_file(): + return None + seen = tuple(n for n in _TB1_SIGNALS if (path / n).is_file()) + if not seen: + return None + return Fingerprint(min(1.0, 0.5 + 0.1 * len(seen)), ("task.yaml", *seen)) + + def describe(self, path: Path) -> Report: + report = Report(path, self.name, KIND_EXECUTABLE) + source = FileSource(path / "task.yaml") + data = source.data + report.problems.extend(f"task.yaml: {p}" for p in source.problems) + config: dict[str, Any] = data if isinstance(data, dict) else {} + if data is not None and not config: + report.problems.append("task.yaml: not a mapping") + instruction = config.get("instruction") or config.get("descriptions") + report.set_parts({ + "instruction": bool(instruction), + "environment": _any_file(path, "Dockerfile", "docker-compose.yaml"), + "oracle": _any_file(path, "solution.sh", "solution.yaml"), + "verifier": (path / "run-tests.sh").is_file(), + "metadata": any(k in config for k in ("difficulty", "category", "tags")), + }) + if config and not instruction: + report.problems.append("task.yaml: no instruction") + report.details["metadata"] = _summary(config, ("difficulty", "category", "tags", "parser_name")) + return report + + +def _summary(table: dict[str, Any], keys: tuple[str, ...] = ("difficulty", "category", "tags")) -> dict[str, Any]: + return {k: table[k] for k in keys if k in table} diff --git a/dokimi/dokimi/task_formats/model.py b/dokimi/dokimi/task_formats/model.py new file mode 100644 index 00000000..7843aa06 --- /dev/null +++ b/dokimi/dokimi/task_formats/model.py @@ -0,0 +1,91 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""What a format probe returns: a fingerprint and a structure report.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Protocol + +KIND_EXECUTABLE = "executable" # has an environment and a verifier +KIND_ANSWER_ONLY = "answer-only" # prompt/target samples +KIND_TRIGGER = "trigger" # skill-selection queries +KIND_UNKNOWN = "unknown" + +FORMAT_UNKNOWN = "unknown" + +#: The parts a report says are present or missing, in display order. +PARTS = ("instruction", "environment", "oracle", "verifier", "metadata") + +# Labels for what can consume a format today. +BY_HARBOR = "harbor" +BY_DOCKER = "docker-*" +BY_BENCHFLOW = "benchflow" +BY_QUESTIONS = "questions track" +NONE_YET = "none-yet" + +#: A probe at or above this confidence claims the path. +MATCH_THRESHOLD = 0.5 +#: Other claims within this margin of the best one are reported as ambiguous. +AMBIGUITY_MARGIN = 0.15 + + +@dataclass(frozen=True) +class Fingerprint: + """How sure a probe is that a path is its format, and why.""" + + confidence: float + evidence: tuple[str, ...] = () + + +@dataclass +class Report: + """Structure of one detected item (a task directory or a dataset file).""" + + path: Path + format: str + kind: str + confidence: float = 0.0 + evidence: list[str] = field(default_factory=list) + items: int | None = None + present: list[str] = field(default_factory=list) + missing: list[str] = field(default_factory=list) + problems: list[str] = field(default_factory=list) + consumers: list[str] = field(default_factory=list) + alternatives: list[dict[str, Any]] = field(default_factory=list) + details: dict[str, Any] = field(default_factory=dict) + + def set_parts(self, parts: dict[str, bool]) -> None: + """Record which of :data:`PARTS` are present; a part not named is missing.""" + self.present = [p for p in PARTS if parts.get(p)] + self.missing = [p for p in PARTS if not parts.get(p)] + + def as_dict(self) -> dict[str, Any]: + return { + "path": str(self.path), + "format": self.format, + "kind": self.kind, + "confidence": round(self.confidence, 2), + "evidence": list(self.evidence), + "items": self.items, + "present": list(self.present), + "missing": list(self.missing), + "problems": list(self.problems), + "consumers": list(self.consumers) or [NONE_YET], + "alternatives": list(self.alternatives), + "details": dict(self.details), + } + + +class Probe(Protocol): + """One format: recognise it (:meth:`fingerprint`) and describe it (:meth:`describe`).""" + + name: str + #: ``"dir"`` probes look at directories, ``"file"`` probes at files. + target: str + + def fingerprint(self, source: Any) -> Fingerprint | None: ... + + def describe(self, source: Any) -> Report: ... diff --git a/dokimi/dokimi/task_formats/source.py b/dokimi/dokimi/task_formats/source.py new file mode 100644 index 00000000..d239dfa6 --- /dev/null +++ b/dokimi/dokimi/task_formats/source.py @@ -0,0 +1,113 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A file read once for every probe: parsed data, rows, and what failed to parse. + +Files are data here, never code: JSON, JSONL, YAML and TOML are parsed with safe +loaders (YAML refuses aliases), and a ``.py`` file is only read as text. A parse +failure is a lint problem on the report, not an exception. +""" + +from __future__ import annotations + +import json +import tomllib +from functools import cached_property +from pathlib import Path +from typing import Any + +import yaml + +from dokimi.harnesses.tasks import _load_yaml + +#: Files larger than this are not parsed (their structure is reported as unread). +MAX_BYTES = 64 * 1024 * 1024 + + +class FileSource: + """One candidate file. Parsing is lazy and happens at most once.""" + + def __init__(self, path: Path) -> None: + self.path = Path(path) + self.problems: list[str] = [] + + @property + def suffix(self) -> str: + return self.path.suffix.lower() + + @cached_property + def text(self) -> str | None: + try: + if self.path.stat().st_size > MAX_BYTES: + self.problems.append(f"larger than {MAX_BYTES // (1024 * 1024)} MiB: not read") + return None + return self.path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + self.problems.append(f"cannot read: {exc}") + return None + + @cached_property + def data(self) -> Any: + """The parsed document (a JSONL file parses to its list of rows), or ``None``.""" + text = self.text + if text is None or self.suffix == ".py": + return None + try: + return _parse(self.suffix, text, self.problems) + except (ValueError, yaml.YAMLError, RecursionError) as exc: + self.problems.append(f"not valid {_LANG.get(self.suffix, 'data')}: {_one_line(exc)}") + return None + + @cached_property + def rows(self) -> list[dict[str, Any]] | None: + """The records of a list-shaped document (JSONL, a JSON/YAML list), else ``None``.""" + data = self.data + if not isinstance(data, list): + return None + return [r for r in data if isinstance(r, dict)] + + +_LANG = {".json": "JSON", ".jsonl": "JSONL", ".yaml": "YAML", ".yml": "YAML", ".toml": "TOML"} + + +def _one_line(exc: BaseException) -> str: + return " ".join(str(exc).split())[:200] + + +def _parse(suffix: str, text: str, problems: list[str]) -> Any: + if suffix == ".jsonl": + return _parse_jsonl(text, problems) + if suffix == ".json": + return json.loads(text) + if suffix == ".toml": + return tomllib.loads(text) + return _load_yaml(text) + + +def _parse_jsonl(text: str, problems: list[str]) -> list[Any]: + rows: list[Any] = [] + bad = 0 + for number, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except ValueError as exc: + bad += 1 + if bad <= 3: + problems.append(f"line {number}: not valid JSON: {_one_line(exc)}") + if bad > 3: + problems.append(f"{bad - 3} more lines are not valid JSON") + return rows + + +def first_rows(rows: list[dict[str, Any]] | None, n: int = 50) -> list[dict[str, Any]]: + """The rows a fingerprint looks at: the first *n*, so a large file costs no more.""" + return (rows or [])[:n] + + +def key_share(rows: list[dict[str, Any]], *keys: str) -> float: + """Share of *rows* that hold every one of *keys*.""" + if not rows: + return 0.0 + return sum(all(k in r for k in keys) for r in rows) / len(rows) diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index 5df0aac9..978567bb 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -52,7 +52,7 @@ "dokimi.datasets_export", "dokimi.explore", "dokimi.external", "dokimi.harnesses", "dokimi.novelty", "dokimi.orchestrator", "dokimi.report", "dokimi.repos", "dokimi.runner", "dokimi.schema_check", "dokimi.schema_compat", "dokimi.subjects", - "dokimi.tool_preference", + "dokimi.task_formats", "dokimi.tool_preference", )), # The LLM evaluation engine: answering, judging, panels, arms, questions, personas. ("evaluation", ( diff --git a/dokimi/tests/test_task_format_detect.py b/dokimi/tests/test_task_format_detect.py new file mode 100644 index 00000000..c5c3d77f --- /dev/null +++ b/dokimi/tests/test_task_format_detect.py @@ -0,0 +1,387 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for eval-task format detection (``dokimi.task_formats``, ``tasks inspect``). + +Each fixture is hand-written from the upstream shape it imitates: + +- terminal-bench v1: laude-institute/terminal-bench ``original-tasks/hello-world``. +- SWE-bench: ``SWEbenchInstance`` in SWE-bench/SWE-bench ``swebench/types.py`` + (``FAIL_TO_PASS`` as the JSON string the HF datasets ship). +- Inspect AI: ``Sample`` fields (``input``, ``target``, ``id``, ``sandbox``) and an + ``examples/hello_world.py``-style ``@task`` file. +- OpenAI evals: ``evals/registry/data/coqa/match.jsonl`` (chat ``input``, ``ideal``). +- skill-creator: ``references/schemas.md`` ``evals.json`` and the ``SKILL.md`` trigger set. +- skill-reach: ``QuerySet`` in google/skill-reach ``src/reach/queries.py``. +- promptfoo: ``examples/*/promptfooconfig.yaml``. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.external.trigger_set import TriggerQuery, to_reach, to_skill_creator +from dokimi.harnesses import discover_tasks +from dokimi.task_formats import detect, inspect_path + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _jsonl(path: Path, rows: list) -> Path: + return _write(path, "".join(json.dumps(r) + "\n" for r in rows)) + + +def _harbor(root: Path, name: str = "hb") -> Path: + task = root / name + _write(task / "task.toml", '[metadata]\ndifficulty = "easy"\ntags = ["x"]\n') + _write(task / "instruction.md", "Fix it.") + (task / "environment").mkdir() + _write(task / "solution" / "solve.sh", "#!/bin/bash\n") + _write(task / "tests" / "test.sh", "#!/bin/bash\n") + return task + + +def _task_md(root: Path, name: str = "tm") -> Path: + task = root / name + _write(task / "task.md", "---\nschema_version: '1.3'\nmetadata:\n difficulty: medium\n" + "verifier:\n timeout_sec: 60\n---\n\nDo the thing.\n") + _write(task / "oracle" / "solve.sh", "#!/bin/bash\n") + _write(task / "verifier" / "test.sh", "#!/bin/bash\n") + return task + + +def _tb1(root: Path, name: str = "hello-world") -> Path: + task = root / name + _write(task / "task.yaml", "instruction: |\n Create hello.txt with 'Hello, world!'.\n" + "author_name: a\ndifficulty: easy\ncategory: file-operations\n" + "tags:\n - file-operations\nparser_name: pytest\n") + _write(task / "Dockerfile", "FROM ubuntu:24.04\nWORKDIR /app\n") + _write(task / "docker-compose.yaml", "services: {}\n") + _write(task / "run-tests.sh", "#!/bin/bash\npytest $TEST_DIR/test_outputs.py\n") + _write(task / "solution.sh", "echo 'Hello, world!' > hello.txt\n") + _write(task / "tests" / "test_outputs.py", "def test_hello():\n pass\n") + return task + + +_SWE_ROW = { + "repo": "astropy/astropy", "instance_id": "astropy__astropy-12907", + "base_commit": "d16bfe05a744909de4b27f5875fe0d4ed41ce607", "patch": "diff --git a/x b/x\n", + "test_patch": "diff --git a/t b/t\n", "problem_statement": "separability_matrix is wrong", + "hints_text": "", "created_at": "2022-03-03T15:14:54Z", "version": "4.3", + "FAIL_TO_PASS": '["astropy/modeling/tests/test_separable.py::test_separable[compound_model6]"]', + "PASS_TO_PASS": "[]", "environment_setup_commit": "298ccb478e6bf092953bca67a3d29dc6c35f6752", +} +_CHAT = [{"role": "system", "content": "Answer briefly."}, {"role": "user", "content": "2+2?"}] + + +def _one(path: Path): + reports = inspect_path(path) + assert len(reports) == 1, [r.as_dict() for r in reports] + return reports[0] + + +# ── one probe per format ────────────────────────────────────────────────────── + + +def test_harbor_task_is_executable_and_consumable_by_harbor_and_docker(tmp_path): + r = _one(_harbor(tmp_path)) + assert (r.format, r.kind, r.consumers) == ("harbor", "executable", ["harbor", "docker-*"]) + assert r.missing == [] and r.problems == [] + + +def test_harbor_task_without_environment_reports_why_harbor_refuses_it(tmp_path): + task = _harbor(tmp_path) + (task / "environment").rmdir() + r = _one(task) + assert "environment" in r.missing + assert r.consumers == ["docker-*"] + assert any("Harbor would refuse it: no environment/" in p for p in r.problems) + + +def test_task_md_task_is_consumable_by_benchflow(tmp_path): + r = _one(_task_md(tmp_path)) + assert (r.format, r.kind) == ("task-md", "executable") + assert r.consumers == ["benchflow", "docker-*"] + assert r.details["schema_version"] == "1.3" + + +def test_terminal_bench_v1_task_has_every_part_and_no_consumer_yet(tmp_path): + r = _one(_tb1(tmp_path)) + assert (r.format, r.kind, r.missing) == ("terminal-bench-v1", "executable", []) + assert r.as_dict()["consumers"] == ["none-yet"] + assert r.details["metadata"]["difficulty"] == "easy" + + +def test_swe_bench_jsonl_counts_instances_and_parses_string_test_lists(tmp_path): + other = {**_SWE_ROW, "instance_id": "astropy__astropy-14182"} + r = _one(_jsonl(tmp_path / "lite.jsonl", [_SWE_ROW, other])) + assert (r.format, r.kind, r.items) == ("swe-bench", "executable", 2) + assert r.missing == [] and r.problems == [] + assert r.details["repos"] == ["astropy/astropy"] + + +def test_swe_bench_lints_missing_fields_and_duplicate_ids(tmp_path): + broken = {k: v for k, v in _SWE_ROW.items() if k != "FAIL_TO_PASS"} + r = _one(_jsonl(tmp_path / "bad.jsonl", [_SWE_ROW, broken, {**_SWE_ROW, "FAIL_TO_PASS": "nope"}])) + assert r.format == "swe-bench" + assert "1 of 3 items have no FAIL_TO_PASS" in r.problems + assert "1 items have a FAIL_TO_PASS that is not a list of test ids" in r.problems + assert "duplicate ids: astropy__astropy-12907" in r.problems + + +def test_inspect_samples_jsonl_is_answer_only(tmp_path): + rows = [{"id": 1, "input": "Capital of France?", "target": "Paris"}, + {"id": 2, "input": _CHAT, "target": "4", "sandbox": "docker"}] + r = _one(_jsonl(tmp_path / "samples.jsonl", rows)) + assert (r.format, r.kind, r.items) == ("inspect-samples", "answer-only", 2) + assert "environment" in r.present and "verifier" in r.missing + + +def test_inspect_task_file_is_scanned_not_imported(tmp_path): + marker = tmp_path / "imported" + src = ( + "from inspect_ai import Task, task\nfrom inspect_ai.dataset import Sample\n" + f"open({str(marker)!r}, 'w').write('ran')\n\n" + "@task\ndef hello():\n return Task(dataset=[Sample(input='hi', target='hi')],\n" + " solver=[generate()], scorer=exact(), sandbox='docker')\n" + ) + r = _one(_write(tmp_path / "hello.py", src)) + assert (r.format, r.kind, r.items) == ("inspect-task", "executable", 1) + assert {"instruction", "environment", "verifier"} <= set(r.present) + assert not marker.exists() + + +def test_python_without_inspect_import_is_not_an_inspect_task(tmp_path): + r = _one(_write(tmp_path / "x.py", "@task\ndef f():\n pass\n")) + assert r.format == "unknown" + + +def test_openai_evals_chat_input_with_ideal(tmp_path): + r = _one(_jsonl(tmp_path / "match.jsonl", [{"input": _CHAT, "ideal": "4"}] * 3)) + assert (r.format, r.kind, r.items) == ("openai-evals", "answer-only", 3) + assert r.details["chat_inputs"] == 3 + assert r.alternatives == [] + + +def test_skill_creator_evals_json(tmp_path): + data = {"skill_name": "pdf", "evals": [ + {"id": 1, "prompt": "Extract the tables", "expected_output": "a CSV", + "files": ["evals/files/a.pdf"], "expectations": ["The output includes X"]}, + {"id": 1, "prompt": "Split it", "expectations": "not a list"}, + ]} + r = _one(_write(tmp_path / "evals" / "evals.json", json.dumps(data))) + assert (r.format, r.kind, r.items) == ("skill-creator-evals", "answer-only", 2) + assert r.details["skill_name"] == "pdf" + assert "1 items have expectations that are not a list" in r.problems + assert "duplicate ids: 1" in r.problems + + +def _queries() -> list[TriggerQuery]: + return [TriggerQuery(id="q1", text="load my pdf", expected_skill="pdf"), + TriggerQuery(id="q2", text="what's the weather", expected_skill=None)] + + +def test_skill_creator_trigger_set_written_by_dokimi_is_detected(tmp_path): + path = _write(tmp_path / "trigger.json", json.dumps(to_skill_creator(_queries(), "pdf"))) + r = _one(path) + assert (r.format, r.kind, r.items) == ("skill-creator-trigger", "trigger", 2) + assert (r.details["should_trigger"], r.details["should_not_trigger"]) == (1, 1) + + +def test_skill_reach_queryset_written_by_dokimi_is_detected_and_clean(tmp_path): + path = _write(tmp_path / "queries.json", json.dumps(to_reach(_queries(), "cat"))) + r = _one(path) + assert (r.format, r.kind, r.items) == ("skill-reach", "trigger", 2) + assert r.problems == [] + assert r.details["out_of_scope"] == 1 + + +def test_skill_reach_lints_what_reach_would_refuse(tmp_path): + data = {"queries": [{"query_id": "a", "text": "t", "expected_skill": None}, + {"query_id": "a", "text": "u", "expected_skill": "s", "extra": 1}], + "owner": "me"} + r = _one(_write(tmp_path / "q.json", json.dumps(data))) + assert r.format == "skill-reach" + assert "QuerySet keys skill-reach refuses: owner" in r.problems + assert "a: kind=None disagrees with expected_skill=None" in r.problems + assert "a: keys skill-reach refuses: extra" in r.problems + assert "duplicate ids: a" in r.problems + + +def test_promptfoo_config(tmp_path): + cfg = ("description: suite\nprompts:\n - 'Answer: {{query}}'\nproviders:\n - openai:gpt-4o\n" + "tests:\n - vars: {query: hi}\n assert:\n - type: contains\n value: hi\n" + " - vars: {query: yo}\n") + r = _one(_write(tmp_path / "promptfooconfig.yaml", cfg)) + assert (r.format, r.kind, r.items) == ("promptfoo", "answer-only", 2) + assert r.present == ["instruction", "environment", "verifier", "metadata"] + + +def test_promptfoo_tests_from_a_file_reference(tmp_path): + cfg = "prompts: [p.txt]\nproviders: [echo]\ntests: file://tests.csv\n" + r = _one(_write(tmp_path / "eval.yaml", cfg)) + assert (r.format, r.items, r.details["tests_from"]) == ("promptfoo", None, "file://tests.csv") + + +def test_dokimi_questions_v2_is_consumable_by_the_questions_track(tmp_path): + data = {"schema_version": "questions.v2", "product": "p", + "questions": [{"id": "q1", "question": "How?", "expected_topics": ["a"]}]} + r = _one(_write(tmp_path / "golden.json", json.dumps(data))) + assert (r.format, r.kind, r.items, r.consumers) == ("dokimi-questions", "answer-only", 1, ["questions track"]) + assert r.problems == [] + + +def test_dokimi_questions_with_expected_skill_are_trigger_and_v1_loads(tmp_path): + data = {"questions": [{"id": "q1", "text": "How?", "expected_skill": "pdf"}]} + r = _one(_write(tmp_path / "v1.json", json.dumps(data))) + assert (r.format, r.kind, r.consumers) == ("dokimi-questions", "trigger", ["questions track"]) + + +def test_dokimi_questions_the_loader_refuses_is_a_problem_not_a_consumer(tmp_path): + data = {"schema_version": "questions.v2", "questions": [{"id": "q1", "question": ""}]} + r = _one(_write(tmp_path / "bad.json", json.dumps(data))) + assert r.format == "dokimi-questions" + assert r.consumers == [] and r.problems + + +# ── ambiguity, unknown, malformed ───────────────────────────────────────────── + + +def test_rows_with_target_and_ideal_list_the_other_format_as_ambiguous(tmp_path): + rows = [{"input": _CHAT, "target": "4", "ideal": "4"}] * 2 + r = _one(_jsonl(tmp_path / "both.jsonl", rows)) + assert r.format == "openai-evals" + assert [a["format"] for a in r.alternatives] == ["inspect-samples"] + assert r.alternatives[0]["evidence"] == ["input", "target"] + + +def test_directory_with_task_toml_and_task_yaml_lists_terminal_bench_as_alternative(tmp_path): + task = _harbor(tmp_path) + _write(task / "task.yaml", "instruction: hi\n") + _write(task / "run-tests.sh", "#!/bin/bash\n") + r = _one(task) + assert r.format == "harbor" + assert [a["format"] for a in r.alternatives] == [] # 0.6 is not within the margin of 1.0 + assert detect(task).format == "harbor" + + +def test_unknown_directory_lists_prompt_and_verifier_candidates(tmp_path): + _write(tmp_path / "prompt.md", "do x") + _write(tmp_path / "check.sh", "exit 0") + _write(tmp_path / "notes.txt", "n") + r = _one(tmp_path) + assert (r.format, r.kind) == ("unknown", "unknown") + assert r.details["prompt_candidates"] == ["prompt.md"] + assert r.details["verifier_candidates"] == ["check.sh"] + + +def test_unknown_jsonl_reports_row_keys(tmp_path): + r = _one(_jsonl(tmp_path / "qa.jsonl", [{"question": "q", "answer": "a"}])) + assert (r.format, r.items, r.details["row_keys"]) == ("unknown", 1, ["answer", "question"]) + + +@pytest.mark.parametrize("name, text, problem", [ + ("bad.json", "{not json", "not valid JSON"), + ("bad.yaml", "a: [1, 2\n", "not valid YAML"), + ("alias.yaml", "a: &x [1]\nb: *x\n", "not valid YAML"), + ("bad.jsonl", '{"input": "a", "target": "b"}\n{oops\n', "line 2: not valid JSON"), +]) +def test_malformed_data_file_is_a_lint_problem_not_a_crash(tmp_path, name, text, problem): + r = _one(_write(tmp_path / name, text)) + assert any(problem in p for p in r.problems), r.problems + + +def test_malformed_jsonl_keeps_its_good_rows(tmp_path): + r = _one(_write(tmp_path / "s.jsonl", '{"input": "a", "target": "b"}\n{oops\n')) + assert (r.format, r.items) == ("inspect-samples", 1) + + +def test_malformed_task_toml_is_a_problem(tmp_path): + task = _harbor(tmp_path) + _write(task / "task.toml", "[metadata\n") + r = _one(task) + assert r.format == "harbor" + assert any("not valid TOML" in p for p in r.problems) + + +def test_malformed_tb1_task_yaml_is_a_problem(tmp_path): + task = _tb1(tmp_path) + _write(task / "task.yaml", "instruction: [\n") + r = _one(task) + assert r.format == "terminal-bench-v1" + assert any(p.startswith("task.yaml: not valid YAML") for p in r.problems) + + +# ── scan ────────────────────────────────────────────────────────────────────── + + +def test_scan_of_a_mixed_tree_reports_each_item_once(tmp_path): + _harbor(tmp_path / "tasks", "a") + _task_md(tmp_path / "tasks", "b") + _tb1(tmp_path / "tb") + _jsonl(tmp_path / "data" / "swe.jsonl", [_SWE_ROW]) + _write(tmp_path / "data" / "package.json", '{"name": "x"}') # not an eval: skipped + _write(tmp_path / ".git" / "q.json", json.dumps(to_reach(_queries()))) # never entered + found = {(r.path.relative_to(tmp_path).as_posix(), r.format) for r in inspect_path(tmp_path)} + assert found == {("tasks/a", "harbor"), ("tasks/b", "task-md"), + ("tb/hello-world", "terminal-bench-v1"), ("data/swe.jsonl", "swe-bench")} + + +def test_inspect_missing_path_raises(tmp_path): + with pytest.raises(FileNotFoundError): + inspect_path(tmp_path / "nope") + + +def test_existing_discovery_is_unchanged_by_new_formats_in_the_root(tmp_path): + _harbor(tmp_path, "a") + _task_md(tmp_path, "b") + _tb1(tmp_path, "c") + assert [(t.name, t.format) for t in discover_tasks(tmp_path)] == [("a", "harbor"), ("b", "task-md")] + + +# ── CLI ─────────────────────────────────────────────────────────────────────── + + +def _run(argv: list[str]) -> None: + args = build_parser().parse_args(argv) + args.func(args) + + +def test_cli_inspect_prints_a_table_line_per_item(tmp_path, capsys): + _harbor(tmp_path) + _jsonl(tmp_path / "m.jsonl", [{"input": _CHAT, "ideal": "4"}]) + _run(["tasks", "inspect", str(tmp_path)]) + out = capsys.readouterr().out.splitlines() + assert out[0] == f"{tmp_path / 'hb'}\tharbor\texecutable\t-> harbor, docker-*" + assert f"{tmp_path / 'm.jsonl'}\topenai-evals\tanswer-only\t1 item\t-> none-yet" in out + assert out[-1] == "2 items: harbor 1, openai-evals 1" + + +def test_cli_inspect_json(tmp_path, capsys): + _tb1(tmp_path) + _run(["tasks", "inspect", str(tmp_path), "--json"]) + payload = json.loads(capsys.readouterr().out) + assert [i["format"] for i in payload["items"]] == ["terminal-bench-v1"] + assert payload["items"][0]["consumers"] == ["none-yet"] + + +def test_cli_inspect_shows_ambiguity_and_problems(tmp_path, capsys): + _write(tmp_path / "x.jsonl", '{"input": "a", "target": "b", "ideal": "b"}\n{bad\n') + _run(["tasks", "inspect", str(tmp_path / "x.jsonl")]) + out = capsys.readouterr().out + assert " ambiguous: also openai-evals" in out + assert " problem: line 2: not valid JSON" in out + + +def test_cli_inspect_missing_path_exits_with_a_message(tmp_path): + with pytest.raises(SystemExit) as excinfo: + _run(["tasks", "inspect", str(tmp_path / "nope")]) + assert "no such file or directory" in str(excinfo.value) From f7abf7c192b9fe51705c30b7c69187939a693a86 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 24 Oct 2026 09:54:55 -0700 Subject: [PATCH 2/7] feat(catalog): read trigger sets back and list skill package formats Readers for the three trigger-set formats dokimi already writes (skill-reach QuerySet, skill-creator / agentskills.io eval queries, questions with expected_skill) into TriggerQuery, held to the writers by round-trip tests. catalog trigger-set --trigger-set relints an imported set against the catalog and hands it to the skill-catalog: arm; arms run points external sets at it. catalog formats names the skill/rule packages of other ecosystems (Cursor rules, Copilot instructions, Gemini extensions, Claude plugins) next to agentskills.io SKILL.md, with spec lint and the selecting mechanism. Signed-off-by: Nikolay Petrov --- ...0261024_120000_napetrov_trigger_formats.md | 4 + dokimi/dokimi/commands/arms.py | 25 ++ dokimi/dokimi/commands/catalog.py | 111 ++++++++- dokimi/dokimi/external/trigger_set.py | 147 +++++++++++ dokimi/dokimi/skills/formats.py | 228 ++++++++++++++++++ dokimi/dokimi/task_formats/datasets.py | 6 +- dokimi/dokimi/task_formats/model.py | 2 + dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_skill_formats.py | 107 ++++++++ dokimi/tests/test_trigger_set_readers.py | 169 +++++++++++++ 10 files changed, 786 insertions(+), 15 deletions(-) create mode 100644 dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md create mode 100644 dokimi/dokimi/skills/formats.py create mode 100644 dokimi/tests/test_skill_formats.py create mode 100644 dokimi/tests/test_trigger_set_readers.py diff --git a/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md new file mode 100644 index 00000000..80294970 --- /dev/null +++ b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md @@ -0,0 +1,4 @@ +### Added + +- **Read trigger sets back** — `dokimi catalog trigger-set --trigger-set PATH[=TARGET]` reads a google/skill-reach QuerySet, an anthropics skill-creator / agentskills.io `[{query, should_trigger}]` set (TARGET names its skill), or a questions set with `expected_skill`, relints it against `--catalog`, and writes any of the three formats, so an imported set runs on the `skill-catalog:` arm. `arms run` given a skill-reach or skill-creator file names the conversion command; `tasks inspect` lists `catalog trigger-set` as the consumer of both formats. +- **Skill package formats** — `dokimi catalog formats --root DIR [--json]` lists Agent Skills `SKILL.md` packages (with the host read from the path, the agentskills.io name/description limits linted, and Codex `agents/openai.yaml` noted), Cursor `.mdc` rules, Copilot `*.instructions.md`, Gemini CLI `gemini-extension.json` and Claude Code `.claude-plugin/plugin.json` bundles, with how each is selected (description, globs, always, install). diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index e07da1f8..e540b827 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -450,6 +450,7 @@ def _load_questions(path: str) -> list[dict]: """ from dokimi.questions.artifact import load_questions + _refuse_external_trigger_set(path) questions = load_questions(path).questions ids = [question_id(q) for q in questions] for i, qid in enumerate(ids): @@ -461,6 +462,30 @@ def _load_questions(path: str) -> list[dict]: return questions +def _refuse_external_trigger_set(path: str) -> None: + """Point a skill-reach or skill-creator trigger set at the command that reads it. + + Those sets are read through ``catalog trigger-set --trigger-set``, which also + lints them against the catalog the arm will offer; a questions loader that + guessed at them would drop that lint and, for skill-creator, the target skill. + """ + import json + + from dokimi.external.trigger_set import FORMAT_QUESTIONS, detect_trigger_format + + try: + data = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return # the questions loader reports it + fmt = detect_trigger_format(data) + if fmt and fmt != FORMAT_QUESTIONS: + target = "=" if fmt == "skill-creator" else "" + raise ValueError( + f"{path} is a {fmt} trigger set, not a questions set; convert it with " + f"`dokimi catalog trigger-set --catalog --trigger-set {path}{target} " + f"--format questions --out q.json`, then run the skill-catalog: arm on q.json") + + def _refuse_duplicate_ids(ids: list, path: str) -> None: """Raise on a repeated id.""" counts = Counter(qid for qid in ids if qid is not None) diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index a365cb21..75fdcef4 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -67,7 +67,6 @@ def cmd_catalog_trigger_set(args: argparse.Namespace) -> None: from dokimi.external import trigger_set as ts from dokimi.skills import discover_skills - from dokimi.utils import read_json_input catalog_root = Path(args.catalog) if not catalog_root.is_dir(): @@ -80,16 +79,11 @@ def cmd_catalog_trigger_set(args: argparse.Namespace) -> None: if args.target not in {s.name for s in catalog}: raise SystemExit(f"--target {args.target!r} is not a skill in --catalog " f"(have: {', '.join(sorted(s.name for s in catalog))})") - sources = [] - for spec in args.questions: - path, _, rule = spec.rpartition("=") # labels never contain '=', paths may - if not path or not rule: - raise SystemExit(f"--questions needs PATH=LABEL (a skill name, 'none' or '@field'): {spec}") - data = read_json_input(path, flag="--questions") - questions = data.get("questions", data) if isinstance(data, dict) else data - if not isinstance(questions, list) or not all(isinstance(q, dict) for q in questions): - raise SystemExit(f"{path}: expected a list of question objects") - sources.append((questions, rule)) + sources = _question_sources(args.questions or []) + sources += _trigger_set_sources(args.trigger_set or [], catalog) + if not sources: + raise SystemExit("give --questions PATH=LABEL or --trigger-set PATH[=TARGET]") + specs = list(args.questions or []) + list(args.trigger_set or []) rewrite = None if args.rewrite: @@ -107,7 +101,7 @@ def rewrite(text: str) -> str: # An @field rule that resolves for no question in its source is a typo in the # path, not a set of negatives; refuse it rather than write "nothing loads". start = 0 - for (qs, rule), spec in zip(sources, args.questions, strict=True): + for (qs, rule), spec in zip(sources, specs, strict=True): chunk = queries[start:start + len(qs)] start += len(qs) if rule.startswith("@") and chunk and not any(q.labelled for q in chunk): @@ -143,6 +137,82 @@ def rewrite(text: str) -> str: print(f"wrote {args.out} ({args.format}, {n_out} queries, catalog of {len(catalog)})") +def cmd_catalog_formats(args: argparse.Namespace) -> None: + import json + + from dokimi.skills import formats + + root = Path(args.root) + if not root.is_dir(): + raise SystemExit(f"--root is not a directory: {root}") + packages = formats.detect_skill_packages(root) + summary = formats.summarize(packages) + if args.json: + print(json.dumps({"root": str(root), "summary": summary, + "packages": [p.to_dict(root) for p in packages]}, indent=2)) + return + for p in packages: + problems = "; ".join(p.problems) or "ok" + print(f"{p.format}\t{p.name or '-'}\t{p.selected_by}\t{p.host or '-'}\t" + f"{p.path.relative_to(root).as_posix()}\t{problems}") + print() + for fmt, row in summary.items(): + print(f"# {fmt}: {row['n']} ({row['loadable']} loadable as skills, " + f"{row['with_problems']} with problems); selected by {row['selected_by']}") + + +def _question_sources(specs: list) -> list: + """Each ``--questions PATH=LABEL`` as a (questions, rule) source.""" + from dokimi.utils import read_json_input + + sources = [] + for spec in specs: + path, _, rule = spec.rpartition("=") # labels never contain '=', paths may + if not path or not rule: + raise SystemExit(f"--questions needs PATH=LABEL (a skill name, 'none' or '@field'): {spec}") + data = read_json_input(path, flag="--questions") + questions = data.get("questions", data) if isinstance(data, dict) else data + if not isinstance(questions, list) or not all(isinstance(q, dict) for q in questions): + raise SystemExit(f"{path}: expected a list of question objects") + sources.append((questions, rule)) + return sources + + +def _trigger_set_sources(specs: list, catalog) -> list: + """Each ``--trigger-set PATH[=TARGET]`` read back as a (questions, rule) source. + + The set is read into the canonical trigger model and handed on as questions + labelled by their own ``expected_skill``, so it is linted (and rewritten) like + any other source. TARGET names the skill a skill-creator set was written for. + """ + from dokimi.external import trigger_set as ts + from dokimi.utils import read_json_input + + names = {s.name for s in catalog} + out = [] + for spec in specs: + path, target = spec, None + if not Path(spec).exists() and "=" in spec: + path, _, target = spec.rpartition("=") + data = read_json_input(path, flag="--trigger-set") + try: + fmt, queries = ts.read_trigger_set(data, target=target) + except ValueError as exc: + raise SystemExit(f"{path}: {exc}") from exc + if fmt == ts.FORMAT_SKILL_CREATOR: + if target not in names: + raise SystemExit(f"{spec}: skill-creator target {target!r} is not a skill in " + f"--catalog (have: {', '.join(sorted(names))})") + if len(names) > 1: + print(f"note: {path}: skill-creator negatives are read as 'no skill loads'; in a " + f"catalog of {len(names)} a rival load counts as a false trigger", file=sys.stderr) + elif target: + raise SystemExit(f"{spec}: =TARGET applies to skill-creator sets only, this is {fmt}") + print(f"read {path} as {fmt} ({len(queries)} queries)", file=sys.stderr) + out.append((ts.to_questions(queries)["questions"], "@expected_skill")) + return out + + def register(sub, positive_int) -> None: """Add the `catalog` subcommand group.""" @@ -167,16 +237,31 @@ def register(sub, positive_int) -> None: ) scan_p.set_defaults(func=cmd_catalog_scan) + fm_p = catalog_sub.add_parser( + "formats", + help="List the skill/rule packages under --root by ecosystem (Agent Skills, Cursor " + "rules, Copilot instructions, Gemini extensions, Claude plugins)", + ) + fm_p.add_argument("--root", required=True, help="Local checkout to scan") + fm_p.add_argument("--json", action="store_true", help="Print JSON instead of a table") + fm_p.set_defaults(func=cmd_catalog_formats) + ts_p = catalog_sub.add_parser( "trigger-set", help="Build a skill-selection query set (labels + leakage/framing lint) from question sets", ) ts_p.add_argument("--catalog", required=True, help="Directory of the skills the agent will see") ts_p.add_argument( - "--questions", action="append", required=True, metavar="PATH=LABEL", + "--questions", action="append", metavar="PATH=LABEL", help="Question file and its label: a skill name, 'none' (no skill should load), or " "'@field.path' read per question (e.g. @metadata.skill_under_test). Repeatable.", ) + ts_p.add_argument( + "--trigger-set", action="append", dest="trigger_set", metavar="PATH[=TARGET]", + help="An existing trigger set to read back: a skill-reach QuerySet, a skill-creator / " + "agentskills.io [{query, should_trigger}] set (=TARGET names its skill), or questions " + "with expected_skill. Repeatable; combines with --questions.", + ) ts_p.add_argument("--format", choices=["reach", "skill-creator", "questions"], default="reach", help="reach: google/skill-reach QuerySet; skill-creator: one target's " "eval set; questions: a questions artifact for the skill-catalog: arm") diff --git a/dokimi/dokimi/external/trigger_set.py b/dokimi/dokimi/external/trigger_set.py index 839fdfe7..441c6d19 100644 --- a/dokimi/dokimi/external/trigger_set.py +++ b/dokimi/dokimi/external/trigger_set.py @@ -24,6 +24,10 @@ (``QuerySet`` JSON), anthropics/skills skill-creator (``[{query, should_trigger}]`` for one target skill), and this repo's questions artifact (for the ``skill-catalog:`` arm; saved with :func:`~dokimi.questions.artifact.save_questions_artifact`). + +Readers take the same three formats back to :class:`TriggerQuery` (the +canonical model), so a set authored for one tool can be linted against a +catalog and run by the ``skill-catalog:`` arm, or re-written for another tool. """ from __future__ import annotations @@ -257,3 +261,146 @@ def to_questions(queries: Sequence[TriggerQuery]) -> Dict[str, Any]: row["source_text"] = q.source_text rows.append(row) return {"questions": rows} + + +# --- readers ----------------------------------------------------------------- + +FORMAT_REACH = "reach" +FORMAT_SKILL_CREATOR = "skill-creator" +FORMAT_QUESTIONS = "questions" +TRIGGER_FORMATS = (FORMAT_REACH, FORMAT_SKILL_CREATOR, FORMAT_QUESTIONS) + + +def detect_trigger_format(data: Any) -> Optional[str]: + """Which trigger-set format *data* (parsed JSON) is, or ``None``. + + By shape only: a ``{"queries": [...]}`` object is a skill-reach QuerySet, a list + of ``{query, should_trigger}`` rows is a skill-creator (and agentskills.io + ``eval_queries.json``) set, and ``{"questions": [...]}`` or a bare list of + ``question`` rows is a dokimi questions set. + """ + if isinstance(data, dict): + if isinstance(data.get("queries"), list): + return FORMAT_REACH + if isinstance(data.get("questions"), list): + return FORMAT_QUESTIONS + return None + if isinstance(data, list) and data and all(isinstance(r, dict) for r in data): + if all("query" in r and "should_trigger" in r for r in data): + return FORMAT_SKILL_CREATOR + if all("question" in r or "text" in r for r in data): + return FORMAT_QUESTIONS + return None + + +def _label(value: Any, where: str) -> ExpectedSkill: + if value is None: + return None + if isinstance(value, str) and value: + return value + if isinstance(value, list) and all(isinstance(v, str) and v for v in value): + return list(value) or None + raise ValueError(f"{where}: expected_skill must be a skill name, a list of names, or null; " + f"got {value!r}") + + +def _text(row: Dict[str, Any], keys: Sequence[str], where: str) -> str: + for key in keys: + value = row.get(key) + if isinstance(value, str) and value.strip(): + return value + raise ValueError(f"{where}: no query text (looked in {', '.join(keys)})") + + +def from_reach(data: Dict[str, Any]) -> List[TriggerQuery]: + """A google/skill-reach ``QuerySet``; ``acceptable_skills`` join ``expected_skill``. + + skill-reach credits only ``expected_skill`` and treats ``acceptable_skills`` as + neutral; dokimi's selection score credits any of them, so a query with + acceptable siblings scores at least as high here. An id-less query gets + skill-reach's own default, ``q-001`` by position. + """ + rows = data.get("queries") + if not isinstance(rows, list): + raise ValueError("skill-reach QuerySet has no queries list") + out = [] + for i, q in enumerate(rows, start=1): + if not isinstance(q, dict): + raise ValueError(f"queries[{i - 1}] is not an object") + qid = str(q.get("id") or q.get("query_id") or f"q-{i:03d}") + first = _label(q.get("expected_skill"), qid) + if isinstance(first, list): + raise ValueError(f"{qid}: skill-reach expected_skill is one name; use acceptable_skills") + extra = q.get("acceptable_skills") or [] + if not isinstance(extra, list) or not all(isinstance(v, str) and v for v in extra): + raise ValueError(f"{qid}: acceptable_skills must be a list of skill names") + if first is None and extra: + raise ValueError(f"{qid}: acceptable_skills without an expected_skill") + names = [first, *[v for v in extra if v != first]] if first else [] + label: ExpectedSkill = names if len(names) > 1 else (first or None) + out.append(TriggerQuery(id=qid, text=_text(q, ("text",), qid), expected_skill=label)) + return out + + +def from_skill_creator(rows: List[Dict[str, Any]], target: str) -> List[TriggerQuery]: + """An anthropics skill-creator / agentskills.io set ``[{query, should_trigger}]`` for *target*. + + The format has no ids, so they are positional (``sc-001``). A negative is + read as "no skill should load": exact for a catalog of the target alone, + stricter than the source in a larger one, where loading a rival was not + something the set's author ruled out. + """ + if not target: + raise ValueError("a skill-creator trigger set needs the target skill it was written for") + out = [] + for i, r in enumerate(rows, start=1): + qid = f"sc-{i:03d}" + if not isinstance(r, dict): + raise ValueError(f"{qid}: row is not an object") + flag = r.get("should_trigger") + if not isinstance(flag, bool): + raise ValueError(f"{qid}: should_trigger must be true or false, got {flag!r}") + out.append(TriggerQuery(id=qid, text=_text(r, ("query",), qid), + expected_skill=target if flag else None)) + return out + + +def from_questions(data: Any) -> List[TriggerQuery]: + """A dokimi questions set; a question without ``expected_skill`` stays unlabelled.""" + rows = data.get("questions") if isinstance(data, dict) else data + if not isinstance(rows, list): + raise ValueError("questions set has no questions list") + out = [] + for i, q in enumerate(rows): + if not isinstance(q, dict): + raise ValueError(f"questions[{i}] is not an object") + raw = question_id(q) + qid = str(f"q{i + 1}" if raw is None else raw) + text = _text(q, ("question", "text", "prompt", "question_text"), qid) + tq = TriggerQuery(id=qid, text=text, expected_skill=None, labelled="expected_skill" in q) + if tq.labelled: + tq.expected_skill = _label(q["expected_skill"], qid) + if isinstance(q.get("source_text"), str): + tq.source_text = q["source_text"] + if isinstance(q.get("trigger_lint"), dict): + tq.lint = dict(q["trigger_lint"]) + out.append(tq) + return out + + +def read_trigger_set(data: Any, target: Optional[str] = None, + fmt: Optional[str] = None) -> tuple: + """Read a parsed trigger set as ``(format, [TriggerQuery, ...])``. + + *fmt* overrides detection; *target* is required for a skill-creator set, + which names no skill of its own. + """ + fmt = fmt or detect_trigger_format(data) + if fmt == FORMAT_REACH: + return fmt, from_reach(data) + if fmt == FORMAT_SKILL_CREATOR: + return fmt, from_skill_creator(data, target or "") + if fmt == FORMAT_QUESTIONS: + return fmt, from_questions(data) + raise ValueError("not a trigger set this dokimi reads (skill-reach QuerySet, " + "skill-creator [{query, should_trigger}], or questions with expected_skill)") diff --git a/dokimi/dokimi/skills/formats.py b/dokimi/dokimi/skills/formats.py new file mode 100644 index 00000000..b1ee0344 --- /dev/null +++ b/dokimi/dokimi/skills/formats.py @@ -0,0 +1,228 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Detect the skill and rule package formats of other agent ecosystems. + +The loader reads one format, the Agent Skills ``SKILL.md`` package +(agentskills.io), which Claude Code, OpenAI Codex, Gemini CLI and GitHub Copilot +all read. Other ecosystems ship guidance in shapes the loader does not load; +this module names them so a catalog scan can say what a checkout holds and +which part of it dokimi can evaluate. Nothing here loads or runs a package. + +Formats, and what selects each one at run time: + +- ``agent-skill`` -- ``SKILL.md`` with ``name``/``description`` frontmatter; the + description triggers it. Linted against the agentskills.io field limits. The + host is read from the path (``.claude/skills``, ``.agents/skills``, ...); a + Codex skill may add ``agents/openai.yaml`` (display metadata only); +- ``cursor-rule`` -- ``.cursor/rules/*.mdc``; ``alwaysApply``, ``globs`` or (an + "agent requested" rule) its ``description`` selects it; +- ``copilot-instructions`` -- ``*.instructions.md`` with ``applyTo`` globs, and + the repo-wide ``.github/copilot-instructions.md``; file globs select them; +- ``gemini-extension`` -- ``gemini-extension.json``: a bundle (context file, MCP + servers, commands, and skills under ``skills/``), installed, not triggered; +- ``claude-plugin`` -- ``.claude-plugin/plugin.json``: a bundle whose + ``skills/`` hold agent skills; installed, not triggered. + +Only ``agent-skill`` is loadable (``discover_skills``); the other rows are +reported so a reader sees what a description-trigger eval does not cover. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + +from dokimi.utils import parse_frontmatter + +FORMAT_AGENT_SKILL = "agent-skill" +FORMAT_CURSOR_RULE = "cursor-rule" +FORMAT_COPILOT = "copilot-instructions" +FORMAT_GEMINI_EXTENSION = "gemini-extension" +FORMAT_CLAUDE_PLUGIN = "claude-plugin" + +# agentskills.io/specification: name 1-64 chars of [a-z0-9-], no leading, +# trailing or doubled hyphen, equal to the directory name; description 1-1024. +_SPEC_NAME = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +SPEC_NAME_MAX = 64 +SPEC_DESCRIPTION_MAX = 1024 + +# Path markers of the hosts that read SKILL.md, first match wins. +_HOSTS = ( + (".claude/skills", "claude-code"), + (".codex/skills", "codex"), + (".gemini/skills", "gemini-cli"), + (".github/skills", "copilot"), + (".agents/skills", "agents"), # the cross-tool alias Codex and Gemini CLI read + ("environment/skills", "task-bundled"), # a SkillsBench / Harbor task's own skills +) +_SKIP_DIRS = frozenset({".git", "node_modules", "__pycache__", ".venv"}) + + +@dataclass +class SkillPackage: + """One package found under a root.""" + + path: Path + format: str + name: str = "" + description: str = "" + selected_by: str = "" # description | globs | always | install + loadable: bool = False + host: Optional[str] = None + problems: List[str] = field(default_factory=list) + details: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self, root: Optional[Path] = None) -> Dict[str, Any]: + path = self.path.relative_to(root) if root else self.path + return {"path": path.as_posix(), "format": self.format, "name": self.name, + "selected_by": self.selected_by, "loadable": self.loadable, "host": self.host, + "description_chars": len(self.description), "problems": list(self.problems), + "details": dict(self.details)} + + +def _frontmatter(text: str) -> Dict[str, Any]: + """Frontmatter, or flat ``key: value`` lines where YAML refuses it. + + Cursor rules write ``globs: **/*``, which YAML reads as an alias and rejects. + """ + meta, _ = parse_frontmatter(text) + if meta or not text.startswith("---"): + return meta + flat: Dict[str, Any] = {} + for line in text.splitlines()[1:]: + if line.strip() == "---": + break + key, sep, value = line.partition(":") + if sep and key.strip() and not key.startswith(" "): + flat[key.strip()] = value.strip().strip("'\"") + return flat + + +def _host(path: Path) -> Optional[str]: + posix = path.as_posix() + for marker, host in _HOSTS: + if f"{marker}/" in posix: + return host + return None + + +def lint_agent_skill(meta: Dict[str, Any], skill_file: Path) -> List[str]: + """Where a SKILL.md breaks the agentskills.io field limits.""" + problems = [] + name = str(meta.get("name") or "") + desc = str(meta.get("description") or "") + if not name: + problems.append("no name") + elif len(name) > SPEC_NAME_MAX or not _SPEC_NAME.match(name): + problems.append(f"name {name!r} is not 1-{SPEC_NAME_MAX} chars of a-z, 0-9, single hyphens") + elif name != skill_file.parent.name: + problems.append(f"name {name!r} differs from its directory {skill_file.parent.name!r}") + if not desc.strip(): + problems.append("no description") + elif len(desc) > SPEC_DESCRIPTION_MAX: + problems.append(f"description is {len(desc)} chars, over {SPEC_DESCRIPTION_MAX}") + return problems + + +def _agent_skill(path: Path) -> SkillPackage: + meta = _frontmatter(path.read_text(encoding="utf-8", errors="replace")) + pkg = SkillPackage(path, FORMAT_AGENT_SKILL, str(meta.get("name") or path.parent.name), + str(meta.get("description") or ""), "description", + loadable=bool(str(meta.get("description") or "").strip()), host=_host(path)) + pkg.problems = lint_agent_skill(meta, path) + if (path.parent / "agents" / "openai.yaml").is_file(): + pkg.details["codex_metadata"] = "agents/openai.yaml" + return pkg + + +def _cursor_rule(path: Path) -> SkillPackage: + meta = _frontmatter(path.read_text(encoding="utf-8", errors="replace")) + always = str(meta.get("alwaysApply", "")).lower() == "true" + globs = meta.get("globs") + desc = str(meta.get("description") or "") + selected = "always" if always else "globs" if globs else "description" if desc else "manual" + pkg = SkillPackage(path, FORMAT_CURSOR_RULE, path.stem, desc, selected) + if globs: + pkg.details["globs"] = globs if isinstance(globs, (str, list)) else str(globs) + return pkg + + +def _copilot(path: Path) -> SkillPackage: + meta = _frontmatter(path.read_text(encoding="utf-8", errors="replace")) + name = path.name.removesuffix(".instructions.md").removesuffix(".md") + pkg = SkillPackage(path, FORMAT_COPILOT, name, str(meta.get("description") or ""), + "globs" if meta.get("applyTo") else "always") + if meta.get("applyTo"): + pkg.details["applyTo"] = str(meta["applyTo"]) + return pkg + + +def _manifest(path: Path, fmt: str) -> SkillPackage: + pkg = SkillPackage(path, fmt, selected_by="install") + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + pkg.problems.append(f"unreadable JSON: {exc}") + return pkg + if not isinstance(data, dict): + pkg.problems.append("manifest is not an object") + return pkg + pkg.name, pkg.description = str(data.get("name") or ""), str(data.get("description") or "") + if not pkg.name: + pkg.problems.append("no name") + bundle = path.parent if fmt == FORMAT_GEMINI_EXTENSION else path.parent.parent + pkg.details["skills"] = len(list((bundle / "skills").glob("*/SKILL.md"))) + if fmt == FORMAT_GEMINI_EXTENSION: + pkg.details["context_file"] = data.get("contextFileName") or ( + "GEMINI.md" if (bundle / "GEMINI.md").is_file() else None) + pkg.details["mcp_servers"] = len(data.get("mcpServers") or {}) + return pkg + + +def classify(path: Path) -> Optional[SkillPackage]: + """The package *path* is, or ``None``.""" + name = path.name + if name == "SKILL.md": + return _agent_skill(path) + if name.endswith(".mdc"): # Cursor's own extension + return _cursor_rule(path) + if name.endswith(".instructions.md") or ( + name == "copilot-instructions.md" and path.parent.name == ".github"): + return _copilot(path) + if name == "gemini-extension.json": + return _manifest(path, FORMAT_GEMINI_EXTENSION) + if name == "plugin.json" and path.parent.name == ".claude-plugin": + return _manifest(path, FORMAT_CLAUDE_PLUGIN) + return None + + +def detect_skill_packages(root) -> List[SkillPackage]: + """Every package under *root*, in path order (VCS and dependency dirs skipped).""" + root = Path(root) + found = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or _SKIP_DIRS.intersection(path.relative_to(root).parts): + continue + pkg = classify(path) + if pkg is not None: + found.append(pkg) + return found + + +def summarize(packages: List[SkillPackage]) -> Dict[str, Any]: + """Counts per format, how each is selected, and spec problems.""" + out: Dict[str, Any] = {} + for p in packages: + row = out.setdefault(p.format, {"n": 0, "loadable": 0, "with_problems": 0, + "selected_by": {}, "hosts": {}}) + row["n"] += 1 + row["loadable"] += p.loadable + row["with_problems"] += bool(p.problems) + row["selected_by"][p.selected_by] = row["selected_by"].get(p.selected_by, 0) + 1 + if p.host: + row["hosts"][p.host] = row["hosts"].get(p.host, 0) + 1 + return out diff --git a/dokimi/dokimi/task_formats/datasets.py b/dokimi/dokimi/task_formats/datasets.py index e433ff1d..951bf33b 100644 --- a/dokimi/dokimi/task_formats/datasets.py +++ b/dokimi/dokimi/task_formats/datasets.py @@ -21,7 +21,7 @@ from typing import Any from dokimi.task_formats.model import ( - BY_QUESTIONS, KIND_ANSWER_ONLY, KIND_EXECUTABLE, KIND_TRIGGER, Fingerprint, Report, + BY_QUESTIONS, BY_TRIGGER_SET, KIND_ANSWER_ONLY, KIND_EXECUTABLE, KIND_TRIGGER, Fingerprint, Report, ) from dokimi.task_formats.source import FileSource, first_rows, key_share @@ -234,6 +234,8 @@ def describe(self, source: FileSource) -> Report: report.problems.append(f"{bad} items have a should_trigger that is not true/false") report.details["should_trigger"] = sum(1 for r in rows if r.get("should_trigger") is True) report.details["should_not_trigger"] = sum(1 for r in rows if r.get("should_trigger") is False) + if not report.problems: + report.consumers.append(BY_TRIGGER_SET) return report @@ -290,6 +292,8 @@ def describe(self, source: FileSource) -> Report: report.problems.extend(p for p in map(_lint_reach_query, rows) if p) _lint_duplicate_ids(report, rows, "query_id", "id") report.details["out_of_scope"] = sum(1 for r in rows if r.get("expected_skill") is None) + if not report.problems: + report.consumers.append(BY_TRIGGER_SET) return report diff --git a/dokimi/dokimi/task_formats/model.py b/dokimi/dokimi/task_formats/model.py index 7843aa06..c02bec38 100644 --- a/dokimi/dokimi/task_formats/model.py +++ b/dokimi/dokimi/task_formats/model.py @@ -24,6 +24,8 @@ BY_DOCKER = "docker-*" BY_BENCHFLOW = "benchflow" BY_QUESTIONS = "questions track" +# Read back by `catalog trigger-set --trigger-set`, then run by the skill-catalog: arm. +BY_TRIGGER_SET = "catalog trigger-set" NONE_YET = "none-yet" #: A probe at or above this confidence claims the path. diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 13f91674..7023a3b7 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -21,7 +21,7 @@ "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 22}, "dokimi/commands/catalog.py::cmd_catalog_scan": {"complexity": 12}, - "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 18}, + "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 16}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"}, "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 12}, diff --git a/dokimi/tests/test_skill_formats.py b/dokimi/tests/test_skill_formats.py new file mode 100644 index 00000000..bbea2f36 --- /dev/null +++ b/dokimi/tests/test_skill_formats.py @@ -0,0 +1,107 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Skill/rule package detection across agent ecosystems (``dokimi.skills.formats``). + +The fixtures copy the shapes of public samples: openai/skills (``agents/openai.yaml``), +google-gemini/gemini-cli (``gemini-extension.json``, ``.gemini/skills``), +PatrickJS/awesome-cursorrules (``.mdc`` with ``globs: **/*``), github/awesome-copilot +(``*.instructions.md`` with ``applyTo``), anthropics/claude-plugins-official +(``.claude-plugin/plugin.json``) and benchflow-ai/skillsbench (``environment/skills``). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from dokimi.cli import build_parser +from dokimi.skills import discover_skills +from dokimi.skills import formats as sf + + +def _w(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _skill(d: Path, name: str, desc: str = "Use when profiling.") -> Path: + return _w(d / "SKILL.md", f"---\nname: {name}\ndescription: {desc}\n---\nbody\n") + + +def _tree(root: Path) -> Path: + _skill(root / ".codex/skills/imagegen", "imagegen") + _w(root / ".codex/skills/imagegen/agents/openai.yaml", "interface:\n display_name: Image Gen\n") + _skill(root / ".gemini/skills/code_explorer", "code_explorer") + _skill(root / "tasks/t1/environment/skills/xlsx", "xlsx", "x" * 1100) + _w(root / ".cursor/rules/ts.mdc", + "---\ndescription: \"TypeScript rules\"\nglobs: **/*.ts\nalwaysApply: false\n---\n# TS\n") + _w(root / ".cursor/rules/always.mdc", "---\nalwaysApply: true\n---\nbe terse\n") + _w(root / ".github/instructions/swift.instructions.md", + "---\ndescription: 'Swift MCP'\napplyTo: \"**/*.swift\"\n---\n# Swift\n") + _w(root / ".github/copilot-instructions.md", "Repo-wide rules\n") + ext = root / "ext/policy" + _w(ext / "gemini-extension.json", + json.dumps({"name": "policy", "version": "1", "mcpServers": {"s": {}}})) + _w(ext / "GEMINI.md", "context\n") + _skill(ext / "skills/audit", "audit") + plugin = root / "plugins/hookify" + _w(plugin / ".claude-plugin/plugin.json", json.dumps({"name": "hookify", "description": "d"})) + _skill(plugin / "skills/writing-rules", "writing-hookify-rules") + _w(root / "node_modules/x/SKILL.md", "---\nname: x\ndescription: d\n---\n") + return root + + +def test_each_ecosystem_is_named_with_how_it_is_selected(tmp_path): + root = _tree(tmp_path) + by_path = {p.path.relative_to(root).as_posix(): p for p in sf.detect_skill_packages(root)} + assert "node_modules/x/SKILL.md" not in by_path + codex = by_path[".codex/skills/imagegen/SKILL.md"] + assert (codex.format, codex.host, codex.loadable, codex.problems) == ( + "agent-skill", "codex", True, []) + assert codex.details == {"codex_metadata": "agents/openai.yaml"} + assert by_path[".gemini/skills/code_explorer/SKILL.md"].problems[0].startswith( + "name 'code_explorer' is not") + bundled = by_path["tasks/t1/environment/skills/xlsx/SKILL.md"] + assert bundled.host == "task-bundled" and "over 1024" in bundled.problems[0] + ts_rule = by_path[".cursor/rules/ts.mdc"] + assert (ts_rule.format, ts_rule.selected_by, ts_rule.description) == ( + "cursor-rule", "globs", "TypeScript rules") # YAML refuses `**/*.ts`; read anyway + assert by_path[".cursor/rules/always.mdc"].selected_by == "always" + assert by_path[".github/instructions/swift.instructions.md"].details == {"applyTo": "**/*.swift"} + assert by_path[".github/copilot-instructions.md"].selected_by == "always" + ext = by_path["ext/policy/gemini-extension.json"] + assert (ext.selected_by, ext.details) == ( + "install", {"skills": 1, "context_file": "GEMINI.md", "mcp_servers": 1}) + plugin = by_path["plugins/hookify/.claude-plugin/plugin.json"] + assert (plugin.format, plugin.details["skills"]) == ("claude-plugin", 1) + assert "differs from its directory" in by_path[ + "plugins/hookify/skills/writing-rules/SKILL.md"].problems[0] + + +def test_every_loadable_package_is_one_the_skill_loader_loads(tmp_path): + root = _tree(tmp_path) + loadable = {p.name for p in sf.detect_skill_packages(root) if p.loadable} + # The loader does not skip node_modules/ (a vendored copy loads as a skill); the scan does. + assert loadable == {s.name for s in discover_skills(root)} - {"x"} + + +def test_spec_lint_rules(): + f = Path("a-b/SKILL.md") + assert sf.lint_agent_skill({"name": "a-b", "description": "d"}, f) == [] + for name in ("A-b", "-ab", "a--b", "a" * 65): + assert "is not" in sf.lint_agent_skill({"name": name, "description": "d"}, Path(f"{name}/SKILL.md"))[0] + assert sf.lint_agent_skill({}, f) == ["no name", "no description"] + + +def test_catalog_formats_cli(tmp_path, capsys): + root = _tree(tmp_path / "r") + args = build_parser().parse_args(["catalog", "formats", "--root", str(root), "--json"]) + args.func(args) + data = json.loads(capsys.readouterr().out) + assert data["summary"]["agent-skill"]["n"] == 5 + assert data["summary"]["cursor-rule"]["selected_by"] == {"globs": 1, "always": 1} + args = build_parser().parse_args(["catalog", "formats", "--root", str(root)]) + args.func(args) + assert "# gemini-extension: 1" in capsys.readouterr().out diff --git a/dokimi/tests/test_trigger_set_readers.py b/dokimi/tests/test_trigger_set_readers.py new file mode 100644 index 00000000..6e70f6a2 --- /dev/null +++ b/dokimi/tests/test_trigger_set_readers.py @@ -0,0 +1,169 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Reading trigger sets back: skill-reach, skill-creator / agentskills.io, dokimi questions. + +Each reader is held to the writer of the same format in ``dokimi.external.trigger_set``: +what dokimi writes, it reads back to the same labels. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.external import trigger_set as ts +from dokimi.questions.artifact import load_questions +from dokimi.skills import discover_skills +from dokimi.task_formats import detect + + +def _write_skill(root: Path, name: str, description: str) -> None: + d = root / name + d.mkdir(parents=True) + (d / "SKILL.md").write_text(f"---\nname: {name}\ndescription: \"{description}\"\n---\n# {name}\n", + encoding="utf-8") + + +@pytest.fixture +def catalog(tmp_path: Path) -> Path: + root = tmp_path / "skills" + _write_skill(root, "linux-perf", "Profile Linux programs with perf: hotspots, IPC, cache misses.") + _write_skill(root, "dpnp-io", "Read and write dpnp arrays to files on Intel GPUs.") + _write_skill(root, "dpnp-memory", "Place dpnp arrays on devices and size memory.") + return root + + +@pytest.fixture +def queries(catalog): + return ts.build_trigger_set([ + ([{"id": "p", "text": "my program is slow, where is the time going"}], "linux-perf"), + ([{"id": "f", "text": "save this array to disk", "skills": ["dpnp-io", "dpnp-memory"]}], + "@skills"), + ([{"id": "n", "text": "write a haiku"}], "none"), + ([{"id": "u", "text": "unlabelled", "metadata": {}}], "@metadata.skill"), + ], discover_skills(catalog)) + + +def _labels(qs): + return [(q.id, q.text, q.expected_skill, q.labelled) for q in qs] + + +def test_reach_round_trip_keeps_ids_texts_and_acceptable_sets(queries): + back = ts.from_reach(ts.to_reach(queries, catalog_id="c")) + assert _labels(back) == _labels(ts.labelled(queries)) + assert ts.detect_trigger_format(ts.to_reach(queries)) == ts.FORMAT_REACH + + +def test_skill_creator_round_trip_is_identity_on_its_own_rows(queries): + rows = ts.to_skill_creator(queries, "linux-perf") + back = ts.from_skill_creator(rows, "linux-perf") + assert ts.to_skill_creator(back, "linux-perf") == rows + assert [q.expected_skill for q in back] == ["linux-perf", None, None] + assert [q.id for q in back] == ["sc-001", "sc-002", "sc-003"] + assert ts.detect_trigger_format(rows) == ts.FORMAT_SKILL_CREATOR + + +def test_questions_round_trip_keeps_unlabelled_and_lint(queries, tmp_path): + payload = ts.to_questions(queries) + back = ts.from_questions(payload) + assert _labels(back) == _labels(queries) + assert back[0].lint == queries[0].lint + # Through the stamped artifact the arms command loads, too. + from dokimi.questions.artifact import save_questions_artifact + + path = tmp_path / "q.json" + save_questions_artifact(payload, str(path)) + stamped = json.loads(path.read_text(encoding="utf-8")) + assert _labels(ts.from_questions(stamped)) == _labels(queries) + assert len(load_questions(path).questions) == 4 + + +def test_reach_reader_matches_skill_reach_defaults_and_refuses_bad_rows(): + qs = ts.from_reach({"queries": [ + {"text": "a", "expected_skill": "x", "acceptable_skills": ["y", "x"]}, + {"query_id": "k", "text": "b", "expected_skill": None, "kind": "out_of_scope"}, + ]}) + assert [(q.id, q.expected_skill) for q in qs] == [("q-001", ["x", "y"]), ("k", None)] + for bad, msg in [({"text": "a", "expected_skill": None, "acceptable_skills": ["y"]}, "without"), + ({"text": "", "expected_skill": "x"}, "no query text"), + ({"text": "a", "expected_skill": 3}, "skill name"), + ({"text": "a", "expected_skill": ["x"]}, "one name")]: + with pytest.raises(ValueError, match=msg): + ts.from_reach({"queries": [bad]}) + + +def test_skill_creator_reader_needs_a_target_and_booleans(): + with pytest.raises(ValueError, match="target"): + ts.read_trigger_set([{"query": "a", "should_trigger": True}]) + with pytest.raises(ValueError, match="true or false"): + ts.from_skill_creator([{"query": "a", "should_trigger": "yes"}], "x") + + +def test_detect_refuses_what_is_not_a_trigger_set(): + for data in ({"evals": []}, [], [{"prompt": "a"}], "text", {"queries": "x"}): + assert ts.detect_trigger_format(data) is None + with pytest.raises(ValueError, match="not a trigger set"): + ts.read_trigger_set({"evals": []}) + + +def _run(argv): + args = build_parser().parse_args(argv) + args.func(args) + + +def test_cli_reads_each_format_back_and_relints_against_the_catalog(tmp_path, catalog, queries, + capsys): + reach = tmp_path / "reach.json" + reach.write_text(json.dumps(ts.to_reach(queries)), encoding="utf-8") + sc = tmp_path / "sc.json" + sc.write_text(json.dumps([{"query": "use linux-perf on it", "should_trigger": True}]), + encoding="utf-8") + out = tmp_path / "q.json" + _run(["catalog", "trigger-set", "--catalog", str(catalog), "--format", "questions", + "--trigger-set", str(reach), "--trigger-set", f"{sc}=linux-perf", "--out", str(out)]) + rows = load_questions(out).questions + assert [r.get("expected_skill") for r in rows] == [ + "linux-perf", ["dpnp-io", "dpnp-memory"], None, "linux-perf"] + assert rows[3]["trigger_lint"]["names_skill"] == ["linux-perf"] # linted on import + err = capsys.readouterr().err + assert "as reach (3 queries)" in err and "rival load counts as a false trigger" in err + + +def test_cli_refuses_bad_trigger_set_specs(tmp_path, catalog): + sc = tmp_path / "sc.json" + sc.write_text(json.dumps([{"query": "a", "should_trigger": True}]), encoding="utf-8") + reach = tmp_path / "r.json" + reach.write_text(json.dumps({"queries": [{"id": "1", "text": "a", "expected_skill": None}]}), + encoding="utf-8") + base = ["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(tmp_path / "o.json")] + with pytest.raises(SystemExit, match="not a skill in --catalog"): + _run(base + ["--trigger-set", f"{sc}=nope"]) + with pytest.raises(SystemExit, match="needs the target"): + _run(base + ["--trigger-set", str(sc)]) + with pytest.raises(SystemExit, match="skill-creator sets only"): + _run(base + ["--trigger-set", f"{reach}=linux-perf"]) + with pytest.raises(SystemExit, match="--questions PATH=LABEL or --trigger-set"): + _run(base) + + +def test_arms_points_an_external_trigger_set_at_the_converter(tmp_path): + from dokimi.commands.arms import _load_questions + + sc = tmp_path / "sc.json" + sc.write_text(json.dumps([{"query": "a", "should_trigger": True}]), encoding="utf-8") + with pytest.raises(ValueError, match=r"skill-creator trigger set.*--trigger-set .*="): + _load_questions(str(sc)) + + +def test_task_formats_name_the_reader_as_consumer(tmp_path, queries): + reach = tmp_path / "reach.json" + reach.write_text(json.dumps(ts.to_reach(queries)), encoding="utf-8") + sc = tmp_path / "sc.json" + sc.write_text(json.dumps(ts.to_skill_creator(queries, "linux-perf")), encoding="utf-8") + for path in (reach, sc): + report = detect(path) + assert report.consumers == ["catalog trigger-set"], report From 19822bab06088e77f5070399651318f467f4ceef Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 24 Oct 2026 10:57:20 -0700 Subject: [PATCH 3/7] feat(arms): precision in skill_selection_summary Hits over the questions where the arm loaded any skill, strict on multi-load like recall; rendered in the arms report's Skill selection table. Replaces the ad-hoc calculation used for the #211 evidence table. Signed-off-by: Nikolay Petrov --- .../20261024_120000_napetrov_trigger_formats.md | 1 + dokimi/dokimi/eval/arm_summary.py | 5 ++++- dokimi/dokimi/report/arms_report.py | 6 +++--- dokimi/tests/test_skill_selection.py | 14 ++++++++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md index 80294970..1590aaa3 100644 --- a/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md +++ b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md @@ -2,3 +2,4 @@ - **Read trigger sets back** — `dokimi catalog trigger-set --trigger-set PATH[=TARGET]` reads a google/skill-reach QuerySet, an anthropics skill-creator / agentskills.io `[{query, should_trigger}]` set (TARGET names its skill), or a questions set with `expected_skill`, relints it against `--catalog`, and writes any of the three formats, so an imported set runs on the `skill-catalog:` arm. `arms run` given a skill-reach or skill-creator file names the conversion command; `tasks inspect` lists `catalog trigger-set` as the consumer of both formats. - **Skill package formats** — `dokimi catalog formats --root DIR [--json]` lists Agent Skills `SKILL.md` packages (with the host read from the path, the agentskills.io name/description limits linted, and Codex `agents/openai.yaml` noted), Cursor `.mdc` rules, Copilot `*.instructions.md`, Gemini CLI `gemini-extension.json` and Claude Code `.claude-plugin/plugin.json` bundles, with how each is selected (description, globs, always, install). +- **Skill selection precision** — `skill_selection_summary` and the arms report's *Skill selection* table carry `precision`: hits over the questions where the arm loaded any skill, so misroutes and false triggers count against it. diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index 968cbf2b..cf109ae5 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -130,7 +130,7 @@ def summarize_skill_selection(records: Iterable[RecordLike]) -> Dict[str, Any]: or ``None`` — no skill should load. Arms that offered no skill are skipped. The skills loaded in the first turn that loaded any are one choice. ``recall`` - is strict: a hit needs every one of them acceptable. ``recall_any`` (one + and ``precision`` (hits over questions where any skill loaded) are strict: a hit needs every one of them acceptable. ``recall_any`` (one acceptable skill among them), ``multi_load_rate`` and ``mean_first_turn_loads`` show how far the strict figure depends on multi-loading. """ @@ -192,6 +192,9 @@ def rate(a: int, b: int) -> Any: "n_failed": s["failed"], "selection_accuracy": rate(s["correct"], s["n"]), "recall": rate(s["hit"], s["pos"]), + # Of the questions where the arm loaded a skill, the share where every + # loaded skill was acceptable (misroutes and false triggers count against). + "precision": rate(s["hit"], s["loaded_n"]), # Lenient: the right skill was among those loaded in the first turn. "recall_any": rate(s["hit_any"], s["pos"]), "multi_load_rate": rate(s["multi"], s["loaded_n"]), diff --git a/dokimi/dokimi/report/arms_report.py b/dokimi/dokimi/report/arms_report.py index 702bf6de..da298f0e 100644 --- a/dokimi/dokimi/report/arms_report.py +++ b/dokimi/dokimi/report/arms_report.py @@ -265,13 +265,13 @@ def pct(v: Any) -> str: "harness does." ) lines.append("") - lines.append("| Arm | Accuracy | Recall | Recall (any) | Multi-load | Misroute | False trigger " + lines.append("| Arm | Accuracy | Recall | Precision | Recall (any) | Multi-load | Misroute | False trigger " "| +/− | n | Failed |") - lines.append("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") + lines.append("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") for arm_name, s in selection.items(): lines.append( f"| `{arm_name}` | {pct(s.get('selection_accuracy'))} | {pct(s.get('recall'))} | " - f"{pct(s.get('recall_any'))} | {pct(s.get('multi_load_rate'))} | " + f"{pct(s.get('precision'))} | {pct(s.get('recall_any'))} | {pct(s.get('multi_load_rate'))} | " f"{pct(s.get('misroute_rate'))} | {pct(s.get('false_trigger_rate'))} | " f"{s.get('n_positive', 0)}/{s.get('n_negative', 0)} | {s.get('n', 0)} | " f"{s.get('n_failed', 0)} |" diff --git a/dokimi/tests/test_skill_selection.py b/dokimi/tests/test_skill_selection.py index ae704b4a..eceed720 100644 --- a/dokimi/tests/test_skill_selection.py +++ b/dokimi/tests/test_skill_selection.py @@ -101,6 +101,8 @@ def test_selection_summary_scores_hits_misroutes_and_false_triggers(): assert s["recall"] == 0.5 assert s["misroute_rate"] == 0.25 assert s["false_trigger_rate"] == 0.5 + # Loaded on 4 of 6: two hits, one misroute, one false trigger. + assert s["precision"] == 0.5 assert s["selection_accuracy"] == round(3 / 6, 3) assert s["confusion"]["linux-perf"] == {"linux-perf": 1, "onetbb-quickstart": 1, "(none)": 1} assert s["confusion"]["(none)"] == {"(none)": 1, "linux-perf": 1} @@ -117,7 +119,7 @@ def test_report_renders_selection_table(): "skill_selection_summary": summarize_skill_selection([_rec("a", "a"), _rec(None, "a")]), }) assert "## Skill selection" in md - assert "| `cat` | 50% | 100% | 100% | 0% | 0% | 100% | 1/1 | 2 | 0 |" in md + assert "| `cat` | 50% | 100% | 50% | 100% | 0% | 0% | 100% | 1/1 | 2 | 0 |" in md # ── trigger set ────────────────────────────────────────────────────────── @@ -428,7 +430,7 @@ def test_combined_arm_rejects_duplicate_tools(catalog): def test_report_renders_dash_for_missing_rates(): md = render_arms_report({"arms": ["cat"], "answers": [], "skill_selection_summary": summarize_skill_selection([_rec(None, None)])}) - assert "| `cat` | 100% | — | — | — | — | 0% | 0/1 | 1 | 0 |" in md + assert "| `cat` | 100% | — | — | — | — | — | 0% | 0/1 | 1 | 0 |" in md def _qfile(tmp_path, name, rows): @@ -726,3 +728,11 @@ def test_loading_the_whole_catalog_in_one_turn_is_not_a_hit(catalog): assert (s["recall"], s["misroute_rate"], s["recall_any"]) == (0.0, 1.0, 1.0) assert s["mean_first_turn_loads"] == 3.0 assert s["confusion"] == {"linux-perf": {"dpnp-io+linux-perf+onetbb-quickstart": 1}} + + +def test_selection_precision_is_none_when_nothing_loaded_and_strict_on_multi_load(): + assert summarize_skill_selection([_rec("a", None), _rec(None, None)])["cat"]["precision"] is None + rec = _rec("a", "a") + rec["arms"]["cat"]["tool_usage"]["skill"]["first_turn_loaded"] = ["a", "b"] + s = summarize_skill_selection([rec, _rec("a", "a")])["cat"] + assert (s["precision"], s["recall_any"]) == (0.5, 1.0) From b4275386c9551c69338ba6a913dcb72043496013 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 24 Oct 2026 23:26:38 -0700 Subject: [PATCH 4/7] fix(catalog): keep source_text when a trigger set is read back build_trigger_set rebuilt each query from its text alone, so a questions set read back with --trigger-set (or --questions) lost its pre-rewrite source_text. Carry it, and keep it as the original on a second --rewrite. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/external/trigger_set.py | 4 +++- dokimi/tests/test_trigger_set_readers.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/external/trigger_set.py b/dokimi/dokimi/external/trigger_set.py index 441c6d19..33f875ef 100644 --- a/dokimi/dokimi/external/trigger_set.py +++ b/dokimi/dokimi/external/trigger_set.py @@ -196,8 +196,10 @@ def build_trigger_set( tq = TriggerQuery(id=qid, text=text, expected_skill=resolve_label(q, rule)) except LabelUnresolved: tq = TriggerQuery(id=qid, text=text, expected_skill=None, labelled=False) + prior = q.get("source_text") # a re-read set keeps its pre-rewrite original + tq.source_text = prior if isinstance(prior, str) else None if rewrite is not None: - tq.source_text, tq.text = text, rewrite(text).strip() + tq.source_text, tq.text = tq.source_text or text, rewrite(text).strip() if not tq.text: raise ValueError(f"rewrite returned empty text for question {qid}") tq.lint = lint_query(tq, catalog) diff --git a/dokimi/tests/test_trigger_set_readers.py b/dokimi/tests/test_trigger_set_readers.py index 6e70f6a2..02383af0 100644 --- a/dokimi/tests/test_trigger_set_readers.py +++ b/dokimi/tests/test_trigger_set_readers.py @@ -133,6 +133,21 @@ def test_cli_reads_each_format_back_and_relints_against_the_catalog(tmp_path, ca assert "as reach (3 queries)" in err and "rival load counts as a false trigger" in err +def test_cli_read_back_keeps_source_text(tmp_path, catalog): + src = tmp_path / "rw.json" + src.write_text(json.dumps({"questions": [{"id": "r1", "question": "rewritten text", + "source_text": "Scenario: original text", + "expected_skill": "linux-perf"}]}), encoding="utf-8") + out = tmp_path / "q.json" + _run(["catalog", "trigger-set", "--catalog", str(catalog), "--format", "questions", + "--trigger-set", str(src), "--out", str(out)]) + assert load_questions(out).questions[0]["source_text"] == "Scenario: original text" + queries = ts.build_trigger_set([(ts.to_questions(ts.from_questions(json.loads( + src.read_text(encoding="utf-8"))))["questions"], "@expected_skill")], + discover_skills(catalog), rewrite=lambda t: "again") + assert (queries[0].source_text, queries[0].text) == ("Scenario: original text", "again") + + def test_cli_refuses_bad_trigger_set_specs(tmp_path, catalog): sc = tmp_path / "sc.json" sc.write_text(json.dumps([{"query": "a", "should_trigger": True}]), encoding="utf-8") From e33173d07dcf5c7a367707a533a2311d35fa93e4 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 24 Oct 2026 23:27:04 -0700 Subject: [PATCH 5/7] fix(catalog): report a non-object mcpServers instead of crashing `catalog formats` aborted the whole scan with a TypeError when a gemini-extension.json had a number or boolean mcpServers, and counted a string by its characters. Record it as a package problem instead. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/skills/formats.py | 6 +++++- dokimi/tests/test_skill_formats.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/skills/formats.py b/dokimi/dokimi/skills/formats.py index b1ee0344..1272e643 100644 --- a/dokimi/dokimi/skills/formats.py +++ b/dokimi/dokimi/skills/formats.py @@ -179,7 +179,11 @@ def _manifest(path: Path, fmt: str) -> SkillPackage: if fmt == FORMAT_GEMINI_EXTENSION: pkg.details["context_file"] = data.get("contextFileName") or ( "GEMINI.md" if (bundle / "GEMINI.md").is_file() else None) - pkg.details["mcp_servers"] = len(data.get("mcpServers") or {}) + servers = data.get("mcpServers") or {} + if isinstance(servers, dict): + pkg.details["mcp_servers"] = len(servers) + else: + pkg.problems.append("mcpServers is not an object") return pkg diff --git a/dokimi/tests/test_skill_formats.py b/dokimi/tests/test_skill_formats.py index bbea2f36..273168de 100644 --- a/dokimi/tests/test_skill_formats.py +++ b/dokimi/tests/test_skill_formats.py @@ -87,6 +87,16 @@ def test_every_loadable_package_is_one_the_skill_loader_loads(tmp_path): assert loadable == {s.name for s in discover_skills(root)} - {"x"} +def test_a_malformed_manifest_is_a_problem_not_a_crash(tmp_path): + for value in (5, True, "abc"): + root = tmp_path / str(value) + _w(root / "ext/gemini-extension.json", json.dumps({"name": "e", "mcpServers": value})) + _skill(root / "s/ok", "ok") + by_format = {p.format: p for p in sf.detect_skill_packages(root)} + ext = by_format["gemini-extension"] + assert ext.problems == ["mcpServers is not an object"] and "mcp_servers" not in ext.details + assert by_format["agent-skill"].name == "ok" + def test_spec_lint_rules(): f = Path("a-b/SKILL.md") assert sf.lint_agent_skill({"name": "a-b", "description": "d"}, f) == [] From 98cb170d1a723b0c78d6a8a61a9d56e173db5b71 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sun, 25 Oct 2026 02:26:29 -0700 Subject: [PATCH 6/7] fix(questions): keep a reach entry's kind and notes when a set is read back Signed-off-by: Alexander Rybkin --- dokimi/dokimi/commands/catalog.py | 6 +++++- dokimi/dokimi/external/trigger_set.py | 15 ++++++++++++--- dokimi/tests/test_trigger_set_readers.py | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index 75fdcef4..3fe51df6 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -209,7 +209,11 @@ def _trigger_set_sources(specs: list, catalog) -> list: elif target: raise SystemExit(f"{spec}: =TARGET applies to skill-creator sets only, this is {fmt}") print(f"read {path} as {fmt} ({len(queries)} queries)", file=sys.stderr) - out.append((ts.to_questions(queries)["questions"], "@expected_skill")) + rows = ts.to_questions(queries)["questions"] + for row, q in zip(rows, queries, strict=True): + if q.reach: + row[ts.REACH_FIELDS] = q.reach + out.append((rows, "@expected_skill")) return out diff --git a/dokimi/dokimi/external/trigger_set.py b/dokimi/dokimi/external/trigger_set.py index 33f875ef..4c7990b8 100644 --- a/dokimi/dokimi/external/trigger_set.py +++ b/dokimi/dokimi/external/trigger_set.py @@ -89,6 +89,7 @@ class TriggerQuery: source_text: Optional[str] = None lint: Dict[str, Any] = field(default_factory=dict) labelled: bool = True + reach: Dict[str, Any] = field(default_factory=dict) # a read reach row's kind/notes @property def acceptable(self) -> List[str]: @@ -198,6 +199,8 @@ def build_trigger_set( tq = TriggerQuery(id=qid, text=text, expected_skill=None, labelled=False) prior = q.get("source_text") # a re-read set keeps its pre-rewrite original tq.source_text = prior if isinstance(prior, str) else None + extra = q.get(REACH_FIELDS) + tq.reach = dict(extra) if isinstance(extra, dict) else {} if rewrite is not None: tq.source_text, tq.text = tq.source_text or text, rewrite(text).strip() if not tq.text: @@ -236,8 +239,11 @@ def to_reach(queries: Sequence[TriggerQuery], catalog_id: str = "") -> Dict[str, for q in labelled(queries): acc = q.acceptable item: Dict[str, Any] = {"id": q.id, "text": q.text, "expected_skill": acc[0] if acc else None} - if not acc: - item["kind"] = "out_of_scope" + kind = q.reach.get("kind") if acc else "out_of_scope" + if kind and not (acc and kind == "out_of_scope"): + item["kind"] = kind + if q.reach.get("notes"): + item["notes"] = q.reach["notes"] if len(acc) > 1: item["acceptable_skills"] = acc[1:] items.append(item) @@ -271,6 +277,8 @@ def to_questions(queries: Sequence[TriggerQuery]) -> Dict[str, Any]: FORMAT_SKILL_CREATOR = "skill-creator" FORMAT_QUESTIONS = "questions" TRIGGER_FORMATS = (FORMAT_REACH, FORMAT_SKILL_CREATOR, FORMAT_QUESTIONS) +#: Key carrying a reach row's kind/notes through the in-memory questions rows; never written. +REACH_FIELDS = "_reach" def detect_trigger_format(data: Any) -> Optional[str]: @@ -340,7 +348,8 @@ def from_reach(data: Dict[str, Any]) -> List[TriggerQuery]: raise ValueError(f"{qid}: acceptable_skills without an expected_skill") names = [first, *[v for v in extra if v != first]] if first else [] label: ExpectedSkill = names if len(names) > 1 else (first or None) - out.append(TriggerQuery(id=qid, text=_text(q, ("text",), qid), expected_skill=label)) + out.append(TriggerQuery(id=qid, text=_text(q, ("text",), qid), expected_skill=label, + reach={k: q[k] for k in ("kind", "notes") if isinstance(q.get(k), str)})) return out diff --git a/dokimi/tests/test_trigger_set_readers.py b/dokimi/tests/test_trigger_set_readers.py index 02383af0..53ab065d 100644 --- a/dokimi/tests/test_trigger_set_readers.py +++ b/dokimi/tests/test_trigger_set_readers.py @@ -148,6 +148,20 @@ def test_cli_read_back_keeps_source_text(tmp_path, catalog): assert (queries[0].source_text, queries[0].text) == ("Scenario: original text", "again") +def test_cli_reach_read_back_keeps_kind_and_notes(tmp_path, catalog): + src = tmp_path / "nn.json" + src.write_text(json.dumps({"queries": [{"id": "nn", "text": "save a numpy array to csv", + "expected_skill": "linux-perf", "kind": "neighbor_negative", + "notes": "near dpnp-io"}]}), encoding="utf-8") + out, q_out = tmp_path / "r.json", tmp_path / "q.json" + base = ["catalog", "trigger-set", "--catalog", str(catalog), "--trigger-set", str(src)] + _run(base + ["--out", str(out)]) + row = json.loads(out.read_text(encoding="utf-8"))["queries"][0] + assert (row["kind"], row["notes"]) == ("neighbor_negative", "near dpnp-io") + _run(base + ["--format", "questions", "--out", str(q_out)]) + assert ts.REACH_FIELDS not in load_questions(q_out).questions[0] + + def test_cli_refuses_bad_trigger_set_specs(tmp_path, catalog): sc = tmp_path / "sc.json" sc.write_text(json.dumps([{"query": "a", "should_trigger": True}]), encoding="utf-8") From 21b643307a4dc796a2c1c6a16c033c48d29a80c4 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sun, 25 Oct 2026 02:39:31 -0700 Subject: [PATCH 7/7] fix(catalog): formats counts loadable skills as discover_skills does Signed-off-by: Alexander Rybkin --- dokimi/dokimi/skills/formats.py | 3 +++ dokimi/dokimi/skills/loader.py | 21 +++++++++++++-------- dokimi/tests/test_skill_formats.py | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/skills/formats.py b/dokimi/dokimi/skills/formats.py index 1272e643..26830c49 100644 --- a/dokimi/dokimi/skills/formats.py +++ b/dokimi/dokimi/skills/formats.py @@ -36,6 +36,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional +from dokimi.skills.loader import skill_files from dokimi.utils import parse_frontmatter FORMAT_AGENT_SKILL = "agent-skill" @@ -207,12 +208,14 @@ def classify(path: Path) -> Optional[SkillPackage]: def detect_skill_packages(root) -> List[SkillPackage]: """Every package under *root*, in path order (VCS and dependency dirs skipped).""" root = Path(root) + kept = set(skill_files(root)) # a template or a mirror is not loadable, as discover_skills found = [] for path in sorted(root.rglob("*")): if not path.is_file() or _SKIP_DIRS.intersection(path.relative_to(root).parts): continue pkg = classify(path) if pkg is not None: + pkg.loadable = pkg.loadable and path in kept found.append(pkg) return found diff --git a/dokimi/dokimi/skills/loader.py b/dokimi/dokimi/skills/loader.py index 52734027..c9d96ca8 100644 --- a/dokimi/dokimi/skills/loader.py +++ b/dokimi/dokimi/skills/loader.py @@ -175,20 +175,25 @@ def discover_skills(root, *, notes: Optional[List[str]] = None) -> List[Skill]: Returns: The skills, in path order. """ + return [load_skill(f) for f in skill_files(root, notes=notes)] + + +def skill_files(root, *, notes: Optional[List[str]] = None) -> List[Path]: + """The ``SKILL.md`` files :func:`discover_skills` loads, in path order, unloaded.""" root = Path(root) - skill_files: List[Path] = [] + found: List[Path] = [] for skill_file in sorted(root.rglob("SKILL.md")): if TEMPLATE_DIR_NAMES.intersection(skill_file.relative_to(root).parts[:-1]): if notes is not None: notes.append(f"left out {skill_file}: a template, not a skill") continue - skill_files.append(skill_file) + found.append(skill_file) key_of: Dict[Path, Tuple[str, str]] = {} copies_of: Dict[Tuple[str, str], List[Path]] = {} - for skill_file in skill_files: + for skill_file in found: data = skill_file.read_bytes() - meta, _ = parse_frontmatter(data.decode("utf-8")) + meta, _ = parse_frontmatter(data.decode("utf-8", errors="replace")) # the name only groups mirrors key = (_skill_name(meta, skill_file), hashlib.sha256(data).hexdigest()) key_of[skill_file] = key copies_of.setdefault(key, []).append(skill_file) @@ -197,12 +202,12 @@ def discover_skills(root, *, notes: Optional[List[str]] = None) -> List[Skill]: for group, copies in copies_of.items() } - skills: List[Skill] = [] - for skill_file in skill_files: + kept_files: List[Path] = [] + for skill_file in found: kept = keeper[key_of[skill_file]] if kept != skill_file: if notes is not None: notes.append(f"left out {skill_file}: identical to {kept}") continue - skills.append(load_skill(skill_file)) - return skills + kept_files.append(skill_file) + return kept_files diff --git a/dokimi/tests/test_skill_formats.py b/dokimi/tests/test_skill_formats.py index 273168de..f2ae8a0d 100644 --- a/dokimi/tests/test_skill_formats.py +++ b/dokimi/tests/test_skill_formats.py @@ -115,3 +115,21 @@ def test_catalog_formats_cli(tmp_path, capsys): args = build_parser().parse_args(["catalog", "formats", "--root", str(root)]) args.func(args) assert "# gemini-extension: 1" in capsys.readouterr().out + + +def test_catalog_formats_counts_loadable_skills_as_discover_skills(tmp_path, capsys): + _skill(tmp_path / "template", "template-skill") + for host in (".agents", ".claude"): + _skill(tmp_path / host / "skills/bar", "bar") + args = build_parser().parse_args(["catalog", "formats", "--root", str(tmp_path), "--json"]) + args.func(args) + row = json.loads(capsys.readouterr().out)["summary"]["agent-skill"] + assert (row["n"], row["loadable"]) == (3, len(discover_skills(tmp_path))) == (3, 1) + + +def test_catalog_formats_survives_a_skill_md_that_is_not_utf8(tmp_path, capsys): + (tmp_path / "bad").mkdir() + (tmp_path / "bad" / "SKILL.md").write_bytes(b"---\nname: bad\ndescription: caf\xe9\n---\nbody\n") + args = build_parser().parse_args(["catalog", "formats", "--root", str(tmp_path), "--json"]) + args.func(args) + assert json.loads(capsys.readouterr().out)["summary"]["agent-skill"]["n"] == 1