From 03890332dc337cbde0cf4f732bbfa3e646495912 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:28:49 -0700 Subject: [PATCH 1/7] feat(skills): draft dokimi-design-evals skill A bundled skill for designing evals: sources (user tasks, fixed failures, constraints, negatives; skill text secondary), a per-scenario template, outcome/process/cost layers, verifier self-test with repo validate, repo tasks check and docker-oracle/docker-probe, manual mutation controls, coverage report and draft -> reviewed -> holdout statuses. Bundled skill markdown now goes through test_docs_code_refs, and test_bundled_skills checks each skill loads as a treatment, its description fits the listing cap and its relative links resolve inside the skill. Part of #121 (item 6), J7 #97. Signed-off-by: Nikolay Petrov --- ...1021_120000_napetrov_skill-design-evals.md | 9 ++ .../data/skills/dokimi-design-evals/SKILL.md | 104 ++++++++++++++++++ .../references/coverage-report.md | 37 +++++++ .../references/example-scenario.md | 85 ++++++++++++++ .../references/scenario-template.md | 58 ++++++++++ .../references/verifier-self-test.md | 102 +++++++++++++++++ dokimi/tests/test_bundled_skills.py | 68 ++++++++++++ dokimi/tests/test_docs_code_refs.py | 2 + 8 files changed, 465 insertions(+) create mode 100644 dokimi/changelog.d/20261021_120000_napetrov_skill-design-evals.md create mode 100644 dokimi/data/skills/dokimi-design-evals/SKILL.md create mode 100644 dokimi/data/skills/dokimi-design-evals/references/coverage-report.md create mode 100644 dokimi/data/skills/dokimi-design-evals/references/example-scenario.md create mode 100644 dokimi/data/skills/dokimi-design-evals/references/scenario-template.md create mode 100644 dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md create mode 100644 dokimi/tests/test_bundled_skills.py diff --git a/dokimi/changelog.d/20261021_120000_napetrov_skill-design-evals.md b/dokimi/changelog.d/20261021_120000_napetrov_skill-design-evals.md new file mode 100644 index 00000000..e0378e04 --- /dev/null +++ b/dokimi/changelog.d/20261021_120000_napetrov_skill-design-evals.md @@ -0,0 +1,9 @@ +### Added + +- **`dokimi-design-evals` skill (draft)** — `data/skills/dokimi-design-evals/` + guides an agent through designing evals for a skill or repository: sources, + a per-scenario template, outcome/process/cost layers, verifier self-test + with the existing `repo validate`, `repo tasks check` and `docker-oracle` / + `docker-probe` commands, manual mutation controls, coverage report and + `draft → reviewed → holdout` statuses. Bundled skills are now covered by + the docs command check and a new frontmatter/link test. diff --git a/dokimi/data/skills/dokimi-design-evals/SKILL.md b/dokimi/data/skills/dokimi-design-evals/SKILL.md new file mode 100644 index 00000000..62f3f11c --- /dev/null +++ b/dokimi/data/skills/dokimi-design-evals/SKILL.md @@ -0,0 +1,104 @@ +--- +name: dokimi-design-evals +description: Design, review or repair evals (tasks, verifiers, negative controls, coverage report) for a skill or repository before a dokimi run. Use when asked to write evals, add tasks for a subject, check whether existing evals or graders can be trusted, or state what a suite does not cover. Not for running a benchmark or comparing arms. +--- +# Design evals that measure the user's work + +An eval set makes four separate claims. Keep them apart in everything you write: + +1. **Valid**: it parses against its schema. +2. **Correctly graded**: the verifier fails an untouched workspace, passes the + reference solution, and catches the mistakes it is meant to catch. +3. **Relevant**: its tasks are the work users actually do. +4. **Covering**: what it does not test is listed. + +A passing self-check proves claim 2 at most. Never report it as evidence of 3 or 4. + +## Workflow + +### 1. Collect sources (before writing any task) + +In this order of weight: + +- real user tasks and requests (traces, support threads, examples users wrote); +- issues and PRs, especially **fixed failures**: each one is a ready regression + scenario with a known base commit and fix; +- the supported workflows the docs promise, and the constraints they state + (platforms, versions, forbidden actions); +- negative cases: requests the subject should decline, or should not change. + +The skill or guidance text under test is a **secondary** source. Tasks written +only from it test whether the agent repeats the skill, not whether it helps. +Record the source of every scenario. A scenario you inferred from code or docs +is a **hypothesis** for the user to confirm, not confirmed user intent. + +`python cli.py repo inspect ` lists a tree's instruction files, build +and check entry points and risks without a model call; use it to find the +workflows a repository offers. + +### 2. Write each scenario + +Fill every field of [references/scenario-template.md](references/scenario-template.md): +initial state, request, expected outcome, allowed actions, forbidden +consequences, verifier, how to reproduce, source, status. A worked example is +in [references/example-scenario.md](references/example-scenario.md). + +### 3. Check three layers, in this priority + +- **Outcome**: is the end state correct? This is the primary verdict. +- **Process**: only steps that are truly mandatory (the user forbade a tool, + a test must be run before a claim, a file must not be touched). Do not + require a fixed tool sequence when it is only one valid way to solve the task. +- **Cost**: tokens, wall time, tool calls, retries. Report; gate only when the + user declared a budget up front. + +### 4. Choose the verifier + +Prefer deterministic checks: tests, file and diff assertions, exit codes, +parsed output. Use an LLM judge only for qualities no check can capture; then +use a written rubric, calibrate it +(`python cli.py eval judge-validity --answers --sequential`), +and keep a human review of a sample. +A judge that cannot discriminate degraded answers is not evidence. + +### 5. Self-test every verifier before any paid run + +Run the checks in [references/verifier-self-test.md](references/verifier-self-test.md): +schema validation, oracle passes, untouched workspace fails, isolation, then +the task-appropriate mutations (skipped step, false test claim, deleted tests, +weakened criterion, wrong tool). Pick only the mutations that fit the task; a +task where doing nothing is a correct answer must not be forced to fail on +inaction. Mutation controls have no command yet: they are a manual step, and +the reference says how. + +### 6. Multi-step scenarios + +Give each step a checkpoint verifier and keep a final-state check. Use +Harbor's multi-step `task.toml` (`steps`). The steps of one session are **one** +task in every count, pass rate and sample size; never report them as +independent tasks. + +### 7. Write the coverage report + +Use [references/coverage-report.md](references/coverage-report.md): happy path, +branches, recovery, ambiguity, constraints, negative tasks, neighbouring tasks, +each with the scenario ids that cover it or `none`. Put the gaps in the PR +that adds the suite. + +### 8. Set the status + +`draft` (written, self-test not passed or not reviewed) → `reviewed` (self-test +passed, a person confirmed relevance and the hypotheses) → `holdout` (kept out +of development; used only to confirm a change). Never tune a skill or prompt +against a `holdout` scenario; once it was used for tuning, it is `reviewed` +again. dokimi does not read this status yet; record it next to the task (see +the template). + +## Rules + +- Do not weaken a criterion to make a verifier pass. Fix the task or the oracle. +- Do not change a grader and the subject in one comparison. +- Keep answers out of the agent's reach: no solution, hidden test or expected + output inside the workspace or the image the agent sees. +- An egress or host requirement the run cannot meet is `unsupported`, not a pass. +- Report what you did not check, by name. diff --git a/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md b/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md new file mode 100644 index 00000000..ff1e2fcc --- /dev/null +++ b/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md @@ -0,0 +1,37 @@ +# Coverage report + +dokimi has no command that writes this report (`python -m dokimi.coverage` +prints a product x awareness/work task count, not scenario coverage). Write +it by hand, in the suite's README and in the PR that adds or changes the +suite. + +## Template + +```markdown +## Coverage: at , suite + +| Area | Scenarios | Status | Notes | +|---|---|---|---| +| Happy path | fix-null-deref | reviewed | | +| Branches (each supported variant, platform, option) | none | | GPU path untested: no device in CI | +| Recovery (agent hits an error and must recover) | build-fails-first | draft | | +| Ambiguity (request needs a question or a stated assumption) | none | | | +| Constraints (forbidden tools, files, versions) | no-network-install | reviewed | | +| Negative tasks (subject should decline or not apply) | unrelated-css-fix | reviewed | valid inaction | +| Neighbouring tasks (close to the subject, should not trigger it) | none | | | + +Hypotheses not yet confirmed by a user: . +Not covered, by decision: . +Self-test: . A pass shows the verifiers are sane on +null, oracle and the listed mutations; it does not show relevance or coverage. +``` + +## Rules + +- Every row has scenario ids or `none`. `none` with no note is a gap nobody + looked at; `none` with a note is a decision. +- Count a multi-step scenario once, however many checkpoints it has. +- List `draft` scenarios separately from `reviewed` and `holdout` ones; a + draft does not cover anything yet. +- Name each hypothesis (a scenario inferred from code, docs or issues and not + confirmed by a user) with its source. diff --git a/dokimi/data/skills/dokimi-design-evals/references/example-scenario.md b/dokimi/data/skills/dokimi-design-evals/references/example-scenario.md new file mode 100644 index 00000000..f6245fdd --- /dev/null +++ b/dokimi/data/skills/dokimi-design-evals/references/example-scenario.md @@ -0,0 +1,85 @@ +# Worked example: a fixed failure as a Harbor task + +Source: an issue reported that `parse_range("5-")` raised `IndexError` +instead of `ValueError`; the fix PR added a check and a test. That is a real +failure with a known fix, so it becomes a regression scenario. + +## Scenario + +```yaml +id: rangeparse-open-end +status: draft +source: + kind: fixed-failure + ref: , fixed by + confirmed_by: none +initial_state: > + /app holds the library at the parent of the fix commit: rangeparse.py + without the check, and tests/test_rangeparse.py without the new test. +request: > + parse_range("5-") crashes with IndexError. It should reject an open-ended + range with a clear ValueError. Fix it and keep the existing tests passing. +expected_outcome: > + parse_range("5-") raises ValueError; parse_range("1-3") still returns + [1, 2, 3]; the existing tests in /app/tests pass unchanged. +allowed_actions: > + Any edit to rangeparse.py; adding tests. Any way of detecting the open end. +forbidden_consequences: > + Existing tests deleted, skipped or edited; a bare `except Exception` + that turns every error into ValueError. +verifier: + kind: deterministic + checks: + - outcome: hidden test - "5-" and "-" raise ValueError, "1-3" -> [1, 2, 3], "x-2" still raises ValueError + - outcome: the existing tests/test_rangeparse.py passes + - process: the workspace's tests/test_rangeparse.py is byte-identical to the original +mutations: [deleted-tests, weakened-criterion] +valid_inaction: false +steps: none +reproduce: > + python cli.py tasks run --tasks-root --tasks rangeparse-open-end --harnesses docker-oracle --out-json results/oracle.json +``` + +Mutations not chosen: skipped step (one-step fix), false test claim (the +request does not ask for a report), wrong tool (no tool is forbidden). + +## Verifier (`test_outputs.py` in the task's tests directory, run by its `test.sh`) + +```python +import hashlib, importlib, subprocess, sys +from pathlib import Path + +import pytest + +sys.path.insert(0, "/app") +rp = importlib.import_module("rangeparse") +ORIGINAL_TEST_SHA256 = "" + +@pytest.mark.parametrize("bad", ["5-", "-", "x-2"]) +def test_bad_ranges_raise_value_error(bad): + with pytest.raises(ValueError): + rp.parse_range(bad) + +def test_closed_range_still_works(): + assert rp.parse_range("1-3") == [1, 2, 3] + +def test_existing_tests_untouched_and_passing(): + path = Path("/app/tests/test_rangeparse.py") + assert hashlib.sha256(path.read_bytes()).hexdigest() == ORIGINAL_TEST_SHA256 + assert subprocess.run([sys.executable, "-m", "pytest", "-q", str(path)], cwd="/app").returncode == 0 +``` + +## Self-test, first round (illustrative, not a recorded run) + +| Run | Expected | Observed | +|---|---|---| +| oracle (`docker-oracle`) | pass | pass | +| untouched (manual `docker run`) | fail | fail: `"5-"` raises `IndexError` | +| mutation: deleted tests | fail | fail: hash check | +| mutation: weakened criterion (`except Exception: raise ValueError`) | fail | **pass** | + +The last row is why mutation runs are worth doing: a blanket `except` raises +`ValueError` for every bad input, so the checks above pass it. The verifier +needs a case that tells a targeted fix from a blanket one, for example +`parse_range(None)` must still raise `TypeError`. Add it, rerun the oracle and +the mutation, then move the scenario to `reviewed` after a person confirms it. diff --git a/dokimi/data/skills/dokimi-design-evals/references/scenario-template.md b/dokimi/data/skills/dokimi-design-evals/references/scenario-template.md new file mode 100644 index 00000000..d1ae8e1e --- /dev/null +++ b/dokimi/data/skills/dokimi-design-evals/references/scenario-template.md @@ -0,0 +1,58 @@ +# Scenario template + +One scenario per task directory. Fill every field; write `none` rather than +leaving one out, so a reviewer can tell "considered, empty" from "forgotten". + +```yaml +id: +status: draft # draft | reviewed | holdout +source: # where this scenario comes from; hypotheses say so + kind: issue # user-task | issue | pr | fixed-failure | workflow | constraint | negative | inferred + ref: + confirmed_by: none # a person who confirmed it is real user work; `none` = hypothesis +initial_state: > + The repository at , plus any setup patch (injected bug, missing + file). What exists, what is broken, what the agent can and cannot see. +request: > + The exact prompt the agent gets, worded as a user would word it. No hint of + the grader, the expected diff or the skill under test. +expected_outcome: > + The end state that counts as success, stated so a check can decide it. +allowed_actions: > + What the agent may do. Name the alternatives when more than one valid way + exists, so none of them is graded as a failure. +forbidden_consequences: > + End states that fail the task however the outcome looks: tests deleted or + skipped, a criterion weakened, files outside scope changed, a claim of + "tests pass" with no run, a forbidden tool used. +verifier: + kind: deterministic # deterministic | rubric (calibrated judge + human review) + checks: # one line per check, each tagged with its layer + - outcome: + - process: + - cost: +mutations: # verifier self-test controls that fit this task (verifier-self-test.md) + - skipped-step +valid_inaction: false # true when declining or doing nothing is a correct answer +steps: none # multi-step: list checkpoints, each with its own verifier +reproduce: > + The commands that build the image and run oracle and null, with the SHA. +``` + +## Where the fields go in each task format + +| Field | Harbor task (`tasks run`) | Repository task (`repo run`) | +|---|---|---| +| request | `instruction.md` | `task.json` `prompt` | +| initial state | `environment/Dockerfile` + seed files | `task.json` `base`, `setup_patches` | +| expected outcome, forbidden consequences | `tests/test.sh` + `tests/test_*.py` | the repository's `grade` entry point, `task.json` `gate` | +| reference solution | `solution/solve.sh` | the `oracle` entry point | +| multi-step checkpoints | `task.toml` `steps` (Harbor runs them) | not supported | +| status, source, allowed actions, mutations | not read by dokimi | extra `task.json` keys are passed to the grader unread | + +`status`, `source` and `mutations` have no field dokimi reads today. Keep them +in the task's `task.toml` `[metadata]` table, or as extra `task.json` keys, +and in the suite's README, so the record travels with the task. + +Format references, in the dokimi project: `docs/contributing-terminal-bench-task.md` +(Harbor) and `docs/repo-evaluation-contract.md` §3 (`task.json`). diff --git a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md new file mode 100644 index 00000000..8c7e1b1c --- /dev/null +++ b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md @@ -0,0 +1,102 @@ +# Verifier self-test + +Run these before any paid agent run. None of them calls a model, except the +judge check. Commands run from the dokimi project directory. + +## 1. Schema and paths + +Repository tasks (`repo-eval.yaml` + `task.json`): + +```bash +python cli.py repo validate --manifest --fail-on-warning +``` + +Harbor tasks: discovery is the check. A task missing from the list, or listed +with an error, is not runnable. + +```bash +python cli.py tasks list --tasks-root +``` + +## 2. Oracle passes, untouched workspace fails, isolation + +Repository tasks: one command does all three (null, oracle, the agent cannot +read tasks or source; `docs/repo-evaluation-contract.md` §4.3). It exits 1 on +any broken or unsupported task. + +```bash +python cli.py repo tasks check --manifest --task +``` + +Harbor tasks: oracle and isolation have harnesses; expect `pass_rate: 1.0` +from the oracle and a clean probe. + +```bash +python cli.py tasks run --tasks-root --tasks --harnesses docker-oracle --out-json results/oracle.json +python cli.py tasks run --tasks-root --tasks --harnesses docker-probe --out-json results/probe.json +``` + +**Manual for Harbor:** there is no null harness. Run the verifier on the +untouched image by hand and expect reward 0, as in +`docs/contributing-terminal-bench-task.md` §8: + +```bash +docker run --rm --network none -v "$TASK:/task" \ + bash -c "bash /task/tests/test.sh; cat /logs/verifier/reward.txt" +``` + +A task that passes the oracle but also passes untouched does not discriminate. +Fix the assertion, not the oracle. + +## 3. Mutation controls (manual: no command yet) + +dokimi has no mutation-control command. Do it by hand with the commands above, +changing only the reference solution so it produces the mutated end state: + +- Harbor: copy the task directory to a scratch tasks root outside the suite, + edit the copy's `solution/solve.sh`, run `docker-oracle` on the copy. Expect + `pass_rate: 0.0`. +- Repository: write a scratch copy of `repo-eval.yaml` whose `oracle` command + runs the real oracle and then the mutation, and pass it with + `--manifest-file` to `repo tasks check`. Expect the oracle check to fail. + +The verifier must **fail** every mutated solution. Delete the copies; never +commit them. + +Pick only the mutations that fit the task. Write the chosen list in the +scenario's `mutations` field, and why the others do not apply. + +| Mutation | Mutated solution | Fits when | +|---|---|---| +| skipped step | the oracle minus one mandatory step (no rebuild, no migration, config not updated) | the task has a step the outcome depends on | +| false test claim | the change without the fix, plus a summary or file claiming tests pass | the task asks for tests to be run or reported | +| deleted tests | the fix plus deleting or skipping the failing test | the workspace has tests the agent could touch | +| weakened criterion | the fix replaced by loosening an assertion, tolerance or check | the success criterion lives in files the agent can edit | +| wrong tool | the right end state reached through a tool or path the user forbade | the request forbids a tool or names a required one | + +**Valid inaction.** When declining, asking, or leaving the tree unchanged is +a correct answer (a negative task, an out-of-scope request), set +`valid_inaction: true`: the null workspace is then expected to **pass**, and +the "untouched fails" check does not apply. Use a mutation that does act +wrongly instead (an unrequested edit). Do not reword the task so that +inaction fails just to satisfy the template. Note that `repo tasks check` +always requires the null to fail; a valid-inaction task needs a grader that +fails on the unwanted action and its own manual null check. + +## 4. Judged verifiers + +A rubric judge must tell good answers from degraded ones before its scores +count. It costs model calls; the sequential mode stops early. + +```bash +python cli.py eval judge-validity --answers --sequential +``` + +Then have a person review a sample of judged cells against the rubric and +record the agreement. A run that hits its sample cap without a decision is +not a pass. + +## What a passing self-test does not show + +It shows the verifier is sane on the cases tried. It does not show the tasks +are relevant, or that the suite covers the subject; say so in the report. diff --git a/dokimi/tests/test_bundled_skills.py b/dokimi/tests/test_bundled_skills.py new file mode 100644 index 00000000..d7d45db2 --- /dev/null +++ b/dokimi/tests/test_bundled_skills.py @@ -0,0 +1,68 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The skills bundled under ``data/skills/`` load as treatments and their links resolve. + +Commands, flags and paths these skills name are checked by ``test_docs_code_refs``; this +file checks the bundle itself: the frontmatter a skill arm reads, a listing entry the +harness shows in full, and relative links to the skill's own reference files. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from dokimi.harnesses.skill_treatment import check_skill +from dokimi.repos.budget import LISTING_TEXT_CAP +from dokimi.utils import parse_frontmatter + +SKILLS = Path(__file__).resolve().parents[1] / "data" / "skills" +BUNDLED = sorted(d for d in SKILLS.iterdir() if (d / "SKILL.md").is_file()) +LINK = re.compile(r"\]\(([^)#\s]+)(?:#[^)]*)?\)") + + +def _broken_links(skill_dir: Path, root: Path = SKILLS) -> list[str]: + broken = [] + for doc in sorted(skill_dir.rglob("*.md")): + for target in LINK.findall(doc.read_text(encoding="utf-8")): + if re.match(r"[a-z]+:", target): + continue + resolved = (doc.parent / target).resolve() + if not resolved.exists() or not resolved.is_relative_to(skill_dir.resolve()): + broken.append(f"{doc.relative_to(root)} -> {target}") + return broken + + +def test_the_bundle_is_not_empty(): + assert {d.name for d in BUNDLED} >= {"fastcontext", "dokimi-design-evals"} + + +@pytest.mark.parametrize("skill_dir", BUNDLED, ids=lambda d: d.name) +def test_bundled_skill_loads_as_a_treatment_named_after_its_directory(skill_dir): + assert check_skill(skill_dir) == skill_dir.name + + +@pytest.mark.parametrize("skill_dir", BUNDLED, ids=lambda d: d.name) +def test_bundled_skill_description_fits_the_listing(skill_dir): + meta, _ = parse_frontmatter((skill_dir / "SKILL.md").read_text(encoding="utf-8")) + description = str(meta.get("description", "")).strip() + assert 40 <= len(description) <= LISTING_TEXT_CAP, description + + +@pytest.mark.parametrize("skill_dir", BUNDLED, ids=lambda d: d.name) +def test_bundled_skill_links_stay_inside_and_resolve(skill_dir): + assert _broken_links(skill_dir) == [] + + +def test_the_link_check_catches_a_dangling_and_an_escaping_link(tmp_path): + skill = tmp_path / "s" + (skill / "references").mkdir(parents=True) + (skill / "references" / "ok.md").write_text("x") + (tmp_path / "outside.md").write_text("x") + (skill / "SKILL.md").write_text( + "[a](references/ok.md) [b](references/gone.md) [c](../outside.md) [d](https://x.org)\n") + assert _broken_links(skill, tmp_path) == [ + "s/SKILL.md -> references/gone.md", "s/SKILL.md -> ../outside.md"] diff --git a/dokimi/tests/test_docs_code_refs.py b/dokimi/tests/test_docs_code_refs.py index 890abbe4..a9f5f14c 100644 --- a/dokimi/tests/test_docs_code_refs.py +++ b/dokimi/tests/test_docs_code_refs.py @@ -28,6 +28,8 @@ DOCS = [ *sorted((PROJECT / "docs").glob("*.md")), *sorted((PROJECT / "docs" / "user-journeys").glob("*.md")), + # Bundled skills tell an agent which commands to run: a renamed one breaks the skill. + *sorted((PROJECT / "data" / "skills").rglob("*.md")), PROJECT / "README.md", PROJECT / "CONTRIBUTING.md", PROJECT.parent / "README.md", From e2aae98c330ebb2f939c32ba94798ca103494892 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 13:18:14 +0000 Subject: [PATCH 2/7] Add --strict to the sequential judge-validity example Signed-off-by: Anton Volkov --- .../skills/dokimi-design-evals/references/verifier-self-test.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md index 8c7e1b1c..8ce468f3 100644 --- a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md +++ b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md @@ -89,7 +89,7 @@ A rubric judge must tell good answers from degraded ones before its scores count. It costs model calls; the sequential mode stops early. ```bash -python cli.py eval judge-validity --answers --sequential +python cli.py eval judge-validity --answers --sequential --strict ``` Then have a person review a sample of judged cells against the rubric and From 4a0f048b520aa68fff27580aaff8bb9e883da6e0 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 13:18:36 +0000 Subject: [PATCH 3/7] Let the skill link check see titled and angle-bracket links Signed-off-by: Anton Volkov --- dokimi/tests/test_bundled_skills.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dokimi/tests/test_bundled_skills.py b/dokimi/tests/test_bundled_skills.py index d7d45db2..135a8edf 100644 --- a/dokimi/tests/test_bundled_skills.py +++ b/dokimi/tests/test_bundled_skills.py @@ -21,7 +21,7 @@ SKILLS = Path(__file__).resolve().parents[1] / "data" / "skills" BUNDLED = sorted(d for d in SKILLS.iterdir() if (d / "SKILL.md").is_file()) -LINK = re.compile(r"\]\(([^)#\s]+)(?:#[^)]*)?\)") +LINK = re.compile(r"\]\(\s][^)#<>]*?)>?(?:#[^)\s]*)?(?:\s+\"[^\"]*\")?\)") def _broken_links(skill_dir: Path, root: Path = SKILLS) -> list[str]: @@ -63,6 +63,8 @@ def test_the_link_check_catches_a_dangling_and_an_escaping_link(tmp_path): (skill / "references" / "ok.md").write_text("x") (tmp_path / "outside.md").write_text("x") (skill / "SKILL.md").write_text( - "[a](references/ok.md) [b](references/gone.md) [c](../outside.md) [d](https://x.org)\n") + "[a](references/ok.md) [b](references/gone.md) [c](../outside.md) [d](https://x.org)\n" + '[e](references/gone.md "title") [f]() [g](references/ok.md#top "t")\n') assert _broken_links(skill, tmp_path) == [ - "s/SKILL.md -> references/gone.md", "s/SKILL.md -> ../outside.md"] + "s/SKILL.md -> references/gone.md", "s/SKILL.md -> ../outside.md", + "s/SKILL.md -> references/gone.md", "s/SKILL.md -> references/a b.md"] From 294bf2e401a961fa5d1118b847e07a96878efe53 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 08:13:03 -0700 Subject: [PATCH 4/7] docs(skills): say tasks list is not a runnability check; allow the dokimi-design-evals skill under data/ tasks list never prints an error row and lists a task with only task.toml and instruction.md, so the first runnability check is the docker-oracle run. AGENTS.md and data/README.md now cover the second bundled skill. Signed-off-by: Nikolay Petrov --- AGENTS.md | 6 ++++-- dokimi/data/README.md | 3 ++- .../dokimi-design-evals/references/verifier-self-test.md | 8 ++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 74248429..16510a2d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,8 +93,10 @@ Registries, bundled in the package (`dokimi/dokimi/`) so they ship in the wheel: `baselines/`. Logs, coverage files and tarballs stay local too. - `dokimi/data/` holds only the fixtures this repo's own tests and examples read: `data/skills/fastcontext/`, `data/agent_profiles/`, - `data/explore_tasks/`. Question sets, product skills and tasks belong in - their content repository. + `data/explore_tasks/`. The one other skill there, `data/skills/dokimi-design-evals/`, + is guidance for using dokimi itself, not a product skill; its links and + commands are checked by `tests/test_bundled_skills.py`. Question sets, + product skills and tasks belong in their content repository. - A file the package reads at run time goes inside `dokimi/dokimi/` and into `[tool.setuptools.package-data]`, or the installed wheel goes without it. - Design rationale, status and plans go in the PR description or an issue; diff --git a/dokimi/data/README.md b/dokimi/data/README.md index 2c5a8912..afd960b3 100644 --- a/dokimi/data/README.md +++ b/dokimi/data/README.md @@ -19,7 +19,8 @@ write to the git-ignored `results/`, `reports/`, and the default scratch dirs | Path | What it is | Produced by / consumed by | | --- | --- | --- | -| `skills/fastcontext/SKILL.md` | The one committed skill: a product-neutral "explore before you act" skill, used by the explore track and as the skill fixture in examples | `skill:`/`skill-agent:` treatment arms | +| `skills/fastcontext/SKILL.md` | A product-neutral "explore before you act" skill, used by the explore track and as the skill fixture in examples | `skill:`/`skill-agent:` treatment arms | +| `skills/dokimi-design-evals/` | The one committed skill about dokimi itself: how to design and self-test evals with its commands (`SKILL.md` plus `references/`). It is not product content. | `skill:` arms; `tests/test_bundled_skills.py` checks its loading, links and the commands it names | | `agent_profiles/concise_expert.md` | An answer-style persona (terse, code-first) | `profile:` treatment arm | | `agent_profiles/generic-expert.md` | Negative control: an expert framing that names no product, to pair with a product persona | `profile:` treatment arm | | `explore_tasks/.yaml` | Localization tasks over this repository's own code, with known-good references | `cli.py explore run` | diff --git a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md index 8ce468f3..b4334e0c 100644 --- a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md +++ b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md @@ -11,8 +11,12 @@ Repository tasks (`repo-eval.yaml` + `task.json`): python cli.py repo validate --manifest --fail-on-warning ``` -Harbor tasks: discovery is the check. A task missing from the list, or listed -with an error, is not runnable. +Harbor tasks: there is no validate command. `tasks list` only shows that the +task was discovered (it reads `task.toml` and `instruction.md`); it never prints +an error row, and it lists a task that has no `tests/test.sh`, `solution/` or +`environment/`. A task without `instruction.md` aborts the listing with a +traceback. So a listed task is not known to be runnable: the first check that +the files Harbor needs exist is the `docker-oracle` run in step 2. ```bash python cli.py tasks list --tasks-root From d45d4b2ef8833ba4abb8e49966a23ac8cab642ae Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 00:37:58 -0700 Subject: [PATCH 5/7] docs(skills): run the Harbor null check through docker-oracle The manual null command mounted the task at /task and skipped mkdir -p /logs/verifier, so a test.sh written to the /tests contract never produced reward 1 and every verifier read as "untouched fails". A no-op solve.sh on a scratch copy uses the harness's own mounts. Signed-off-by: Alexander Rybkin --- .../references/verifier-self-test.md | 11 ++++++----- dokimi/tests/test_bundled_skills.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md index b4334e0c..b577c8c6 100644 --- a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md +++ b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md @@ -40,13 +40,14 @@ python cli.py tasks run --tasks-root --tasks --harnesses docke python cli.py tasks run --tasks-root --tasks --harnesses docker-probe --out-json results/probe.json ``` -**Manual for Harbor:** there is no null harness. Run the verifier on the -untouched image by hand and expect reward 0, as in -`docs/contributing-terminal-bench-task.md` §8: +**Manual for Harbor:** there is no null harness. Copy the task directory to a +scratch tasks root outside the suite, make the copy's `solution/solve.sh` do +nothing (`#!/bin/sh` and `true`), and run `docker-oracle` on the copy, so the +verifier runs on the untouched workspace with the harness's own mounts. Expect +`pass_rate: 0.0`. ```bash -docker run --rm --network none -v "$TASK:/task" \ - bash -c "bash /task/tests/test.sh; cat /logs/verifier/reward.txt" +python cli.py tasks run --tasks-root --tasks --harnesses docker-oracle --out-json results/null.json ``` A task that passes the oracle but also passes untouched does not discriminate. diff --git a/dokimi/tests/test_bundled_skills.py b/dokimi/tests/test_bundled_skills.py index 135a8edf..b69d048b 100644 --- a/dokimi/tests/test_bundled_skills.py +++ b/dokimi/tests/test_bundled_skills.py @@ -22,6 +22,7 @@ SKILLS = Path(__file__).resolve().parents[1] / "data" / "skills" BUNDLED = sorted(d for d in SKILLS.iterdir() if (d / "SKILL.md").is_file()) LINK = re.compile(r"\]\(\s][^)#<>]*?)>?(?:#[^)\s]*)?(?:\s+\"[^\"]*\")?\)") +SHELL_BLOCK = re.compile(r"^```(?:bash|sh|shell)\s*\n(.*?)^```", re.M | re.S) def _broken_links(skill_dir: Path, root: Path = SKILLS) -> list[str]: @@ -57,6 +58,17 @@ def test_bundled_skill_links_stay_inside_and_resolve(skill_dir): assert _broken_links(skill_dir) == [] +@pytest.mark.parametrize("skill_dir", BUNDLED, ids=lambda d: d.name) +def test_bundled_skill_shell_blocks_run_only_cli_commands(skill_dir): + # test_docs_code_refs resolves `python cli.py` lines; a raw `docker run` drifts from a harness. + other = [] + for doc in sorted(skill_dir.rglob("*.md")): + for block in SHELL_BLOCK.findall(doc.read_text(encoding="utf-8")): + lines = [ln.strip() for ln in re.sub(r"\\\r?\n", " ", block).splitlines()] + other += [ln for ln in lines if ln and not ln.startswith(("#", "python cli.py "))] + assert other == [] + + def test_the_link_check_catches_a_dangling_and_an_escaping_link(tmp_path): skill = tmp_path / "s" (skill / "references").mkdir(parents=True) From e1a836c261859c050ce789ed0183799308c4e7b4 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 00:38:21 -0700 Subject: [PATCH 6/7] docs(skills): pass --strict to judge-validity in SKILL.md e2aae98 added --strict to the reference file only. The command in SKILL.md, the file an agent always loads, exits 0 on a run that stops at its sample cap without a decision, which the skill says is no pass. Signed-off-by: Alexander Rybkin --- dokimi/data/skills/dokimi-design-evals/SKILL.md | 2 +- dokimi/tests/test_bundled_skills.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dokimi/data/skills/dokimi-design-evals/SKILL.md b/dokimi/data/skills/dokimi-design-evals/SKILL.md index 62f3f11c..9c55f11c 100644 --- a/dokimi/data/skills/dokimi-design-evals/SKILL.md +++ b/dokimi/data/skills/dokimi-design-evals/SKILL.md @@ -57,7 +57,7 @@ in [references/example-scenario.md](references/example-scenario.md). Prefer deterministic checks: tests, file and diff assertions, exit codes, parsed output. Use an LLM judge only for qualities no check can capture; then use a written rubric, calibrate it -(`python cli.py eval judge-validity --answers --sequential`), +(`python cli.py eval judge-validity --answers --sequential --strict`), and keep a human review of a sample. A judge that cannot discriminate degraded answers is not evidence. diff --git a/dokimi/tests/test_bundled_skills.py b/dokimi/tests/test_bundled_skills.py index b69d048b..b630ce65 100644 --- a/dokimi/tests/test_bundled_skills.py +++ b/dokimi/tests/test_bundled_skills.py @@ -11,10 +11,12 @@ from __future__ import annotations import re +import shlex from pathlib import Path import pytest +from dokimi.cli import build_parser from dokimi.harnesses.skill_treatment import check_skill from dokimi.repos.budget import LISTING_TEXT_CAP from dokimi.utils import parse_frontmatter @@ -22,6 +24,7 @@ SKILLS = Path(__file__).resolve().parents[1] / "data" / "skills" BUNDLED = sorted(d for d in SKILLS.iterdir() if (d / "SKILL.md").is_file()) LINK = re.compile(r"\]\(\s][^)#<>]*?)>?(?:#[^)\s]*)?(?:\s+\"[^\"]*\")?\)") +JUDGE_VALIDITY = re.compile(r"python cli\.py eval judge-validity[^`\n]*") SHELL_BLOCK = re.compile(r"^```(?:bash|sh|shell)\s*\n(.*?)^```", re.M | re.S) @@ -69,6 +72,15 @@ def test_bundled_skill_shell_blocks_run_only_cli_commands(skill_dir): assert other == [] +@pytest.mark.parametrize("skill_dir", BUNDLED, ids=lambda d: d.name) +def test_bundled_skill_judge_validity_commands_fail_on_an_inconclusive_run(skill_dir): + # Without --strict an inconclusive judge-validity run exits 0 (test_judge_validity). + for doc in sorted(skill_dir.rglob("*.md")): + for command in JUDGE_VALIDITY.findall(doc.read_text(encoding="utf-8")): + args = build_parser().parse_args(shlex.split(command)[2:]) + assert args.strict, f"{doc.relative_to(SKILLS)}: {command}" + + def test_the_link_check_catches_a_dangling_and_an_escaping_link(tmp_path): skill = tmp_path / "s" (skill / "references").mkdir(parents=True) From c4dd52628c904ed85a6a353e6c8cebfe49bf5dca Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 12:46:31 -0700 Subject: [PATCH 7/7] docs(skills): design-evals self-test notes (sh -c oracle, manual step checks, judge scope, layout); test harness names Signed-off-by: Nikolay Petrov --- .../data/skills/dokimi-design-evals/SKILL.md | 8 ++++-- .../references/scenario-template.md | 2 +- .../references/verifier-self-test.md | 25 +++++++++++++++++++ dokimi/tests/test_bundled_skills.py | 24 ++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/dokimi/data/skills/dokimi-design-evals/SKILL.md b/dokimi/data/skills/dokimi-design-evals/SKILL.md index 9c55f11c..42295f4d 100644 --- a/dokimi/data/skills/dokimi-design-evals/SKILL.md +++ b/dokimi/data/skills/dokimi-design-evals/SKILL.md @@ -58,7 +58,9 @@ Prefer deterministic checks: tests, file and diff assertions, exit codes, parsed output. Use an LLM judge only for qualities no check can capture; then use a written rubric, calibrate it (`python cli.py eval judge-validity --answers --sequential --strict`), -and keep a human review of a sample. +and keep a human review of a sample. The command calibrates only dokimi's own +answer judge (rows with `question_id`), not a task's rubric judge; check that one +by hand. A judge that cannot discriminate degraded answers is not evidence. ### 5. Self-test every verifier before any paid run @@ -74,7 +76,9 @@ the reference says how. ### 6. Multi-step scenarios Give each step a checkpoint verifier and keep a final-state check. Use -Harbor's multi-step `task.toml` (`steps`). The steps of one session are **one** +Harbor's multi-step `task.toml` (`steps`) and a top-level `instruction.md`. +`docker-oracle` does not run the checkpoint verifiers, so self-test each step's verifier +by hand ([references/verifier-self-test.md](references/verifier-self-test.md) §5). The steps of one session are **one** task in every count, pass rate and sample size; never report them as independent tasks. diff --git a/dokimi/data/skills/dokimi-design-evals/references/scenario-template.md b/dokimi/data/skills/dokimi-design-evals/references/scenario-template.md index d1ae8e1e..2cac0529 100644 --- a/dokimi/data/skills/dokimi-design-evals/references/scenario-template.md +++ b/dokimi/data/skills/dokimi-design-evals/references/scenario-template.md @@ -47,7 +47,7 @@ reproduce: > | initial state | `environment/Dockerfile` + seed files | `task.json` `base`, `setup_patches` | | expected outcome, forbidden consequences | `tests/test.sh` + `tests/test_*.py` | the repository's `grade` entry point, `task.json` `gate` | | reference solution | `solution/solve.sh` | the `oracle` entry point | -| multi-step checkpoints | `task.toml` `steps` (Harbor runs them) | not supported | +| multi-step checkpoints | `task.toml` `steps` (Harbor runs them; the per-step self-test is manual) | not supported | | status, source, allowed actions, mutations | not read by dokimi | extra `task.json` keys are passed to the grader unread | `status`, `source` and `mutations` have no field dokimi reads today. Keep them diff --git a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md index b577c8c6..03815eed 100644 --- a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md +++ b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md @@ -18,6 +18,12 @@ an error row, and it lists a task that has no `tests/test.sh`, `solution/` or traceback. So a listed task is not known to be runnable: the first check that the files Harbor needs exist is the `docker-oracle` run in step 2. +Use Harbor's layout: `tests/test.sh`, `solution/solve.sh`, `environment/` and +`instruction.md` at the top of the task. `docker-oracle` also passes a task laid +out as `verifier/` + `oracle/`, but the Harbor-backed arms (`codex`, +`claude-code`, `terminal-bench:`) refuse it, so a pass there does not mean +the task will run in the paid arms. + ```bash python cli.py tasks list --tasks-root ``` @@ -64,6 +70,10 @@ changing only the reference solution so it produces the mutated end state: - Repository: write a scratch copy of `repo-eval.yaml` whose `oracle` command runs the real oracle and then the mutation, and pass it with `--manifest-file` to `repo tasks check`. Expect the oracle check to fail. + The oracle command is split into argv and run **without a shell**, so `&&`, + pipes and redirects are passed to the program as arguments and the mutation + never runs (the check then passes, which looks like a verifier that survived). + Write it as `sh -c ' && '`. The verifier must **fail** every mutated solution. Delete the copies; never commit them. @@ -93,6 +103,12 @@ fails on the unwanted action and its own manual null check. A rubric judge must tell good answers from degraded ones before its scores count. It costs model calls; the sequential mode stops early. +`eval judge-validity` calibrates only dokimi's own answer judge: its answers +file needs a `question_id` and the question text per row. It stops with +`missing question_id` (exit 2) on a task-rubric answers file, so a task's own +rubric judge is not covered by this command. Check that judge by hand: run it on +a known-good and a degraded answer you wrote, and have a person read the scores. + ```bash python cli.py eval judge-validity --answers --sequential --strict ``` @@ -101,6 +117,15 @@ Then have a person review a sample of judged cells against the rubric and record the agreement. A run that hits its sample cap without a decision is not a pass. +## 5. Multi-step tasks (manual) + +`docker-oracle` does not run the per-step checkpoint verifiers of a task with +`steps` in `task.toml`. A checkpoint verifier of a single step that always fails still gives +`pass_rate: 1.0`, so the per-step self-test is manual: for each step, run its +verifier on that step's reference state and on the state before it, and expect +pass then fail. The task needs a top-level `instruction.md`; without it the task +does not load. + ## What a passing self-test does not show It shows the verifier is sane on the cases tried. It does not show the tasks diff --git a/dokimi/tests/test_bundled_skills.py b/dokimi/tests/test_bundled_skills.py index b630ce65..bd808a92 100644 --- a/dokimi/tests/test_bundled_skills.py +++ b/dokimi/tests/test_bundled_skills.py @@ -17,6 +17,7 @@ import pytest from dokimi.cli import build_parser +from dokimi.harnesses.registry import known_harnesses from dokimi.harnesses.skill_treatment import check_skill from dokimi.repos.budget import LISTING_TEXT_CAP from dokimi.utils import parse_frontmatter @@ -92,3 +93,26 @@ def test_the_link_check_catches_a_dangling_and_an_escaping_link(tmp_path): assert _broken_links(skill, tmp_path) == [ "s/SKILL.md -> references/gone.md", "s/SKILL.md -> ../outside.md", "s/SKILL.md -> references/gone.md", "s/SKILL.md -> references/a b.md"] + + +HARNESSES_FLAG = re.compile(r"--harnesses\s+(\S+)") + + +def _harness_names(skill_dir: Path) -> list[str]: + names = [] + for doc in sorted(skill_dir.rglob("*.md")): + for value in HARNESSES_FLAG.findall(doc.read_text(encoding="utf-8")): + names += value.strip("`'\"").split(",") + return names + + +def test_the_harness_name_check_reads_the_skills_commands(): + assert "docker-oracle" in _harness_names(SKILLS / "dokimi-design-evals") + + +@pytest.mark.parametrize("skill_dir", BUNDLED, ids=lambda d: d.name) +def test_bundled_skill_harness_names_are_registered(skill_dir): + # test_docs_code_refs checks that a flag exists, not its value: a typo would pass. + registered = {key.split(":")[0] for key in known_harnesses()} + unknown = [n for n in _harness_names(skill_dir) if n.split(":")[0] not in registered] + assert unknown == []