diff --git a/AGENTS.md b/AGENTS.md index 90097d09..74248429 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,7 +98,9 @@ Registries, bundled in the package (`dokimi/dokimi/`) so they ship in the wheel: - A file the package reads at run time goes inside `dokimi/dokimi/` and into `[tool.setuptools.package-data]`, or the installed wheel goes without it. - Design rationale, status and plans go in the PR description or an issue; - the tree has no ADRs or status reports. + the tree has no ADRs or status reports. The one exception is the research + register, `dokimi/research/records/`: one record per idea, experiment, + finding, decision or implementation link, negative results included (`dokimi/docs/research-register.md`). - A new file starts with the SPDX header (`Copyright (C) Intel Corporation` / `SPDX-License-Identifier: Apache-2.0`) in its comment syntax. CI's license header step (static job) checks it and `make ci-local` does diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index 61a4436a..6b0158ee 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -76,11 +76,11 @@ future: # on the include/exclude globs, not a target: min_docs catches a glob that # matched almost nothing, max_docs catches one that escaped into the tree (8 # markdown files live outside docs/). Measured selection at the time of -# writing: 30 files. Adding a doc past the ceiling is meant to be a deliberate +# writing: 31 files. Adding a doc past the ceiling is meant to be a deliberate # edit here, so raise max_docs in the same commit that adds the doc. golden_manifest: min_docs: 10 - max_docs: 30 + max_docs: 31 include: - docs/**/*.md exclude: diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index 9dc1a984..32cfc1c0 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -321,3 +321,5 @@ code. metric extracts and runs fenced code examples in isolation. - [ingestion-docling.md](ingestion-docling.md) — ingesting non-Markdown documents (PDF, HTML, …) via Docling, installed with the optional `ocr` extra. +- [research-register.md](research-register.md) — the register of Dokimi's own + ideas, experiments (negative ones included), findings and decisions. diff --git a/dokimi/docs/research-register.md b/dokimi/docs/research-register.md new file mode 100644 index 00000000..1a96da0c --- /dev/null +++ b/dokimi/docs/research-register.md @@ -0,0 +1,117 @@ +# Research register + +Dokimi's own research is itself a set of experiments, and many of them come back negative or +inconclusive. The register in `research/records/` keeps every one of them findable, so the +next person who wants to try an idea can see what was already tried, what came out of it and +why a direction was kept or dropped. + +The register is plain files in this repository: no server, history from git, review through +pull requests, and any agent with file access can read and write it. + +## Record types + +Each record is one Markdown file, `research/records//.md`. The YAML front matter +holds the typed fields that `research/record.v1.json` defines; the body is free Markdown for +the setup, result, rationale and anything else a reader needs, after the +license header every file in the repository carries. + +| Type | Id | What it holds | Points at | +|---|---|---|---| +| `idea` | `I001` | A question worth answering, its map theme (`P`…`W`) and `open`/`parked`/`closed` status | — | +| `experiment` | `X001` | Hypothesis, what was run, and the `outcome`: `planned`, `running`, `supported`, `refuted`, `inconclusive` or `aborted` | `idea` | +| `finding` | `F001` | What was learned, with `low`/`med`/`high` confidence | `experiments` | +| `decision` | `D001` | `accepted`, `rejected`, `deferred` or `superseded`, and why | `findings`, `ideas`, `superseded_by` | +| `implref` | `R001` | The issue or pull request that carries a decision out, with its own `status` | `decision` | + +Three states stay separate on purpose: an experiment's `outcome` (what the evidence showed), +a decision's `state` (what the team chose) and an implementation reference's `status` (what +was built). A closed issue does not tell which of the three happened. + +An experiment that turns out wrong is not deleted: set `retracted: true` and say in the body +what was wrong with it. `retracted` means the whole experiment is invalid, so `check` fails +any finding that still cites it; query, export and render keep it, marked. When only part of a +result is withdrawn, keep the outcome, leave `retracted` unset and list the withdrawn claims +under a `## Corrections` heading. Every record has a `date` (`YYYY-MM-DD`) and optional `links` (required on +`implref`), which must be public `github.com/intel/` or `github.com/napetrov/` URLs. The register is public: +leave out internal hosts, paths, names and raw traces, and describe local evidence in words +("local run, 96 runs"). + +## Working with it + +Run from the `dokimi/` project directory: + +```bash +python scripts/research_register.py check +python scripts/research_register.py next experiment +python scripts/research_register.py query --type experiment --ref I001 --where outcome=refuted,inconclusive +python scripts/research_register.py query --type decision --where state=rejected +python scripts/research_register.py export > register.json +python scripts/research_register.py render > map.md +python scripts/research_register.py render --html --live > map.html +python scripts/research_register.py gaps --live +``` + +- `check` validates every record against its schema, its path and id prefix, and every + reference. `tests/test_research_register.py` runs it in CI, so a broken record fails the PR. +- `next` prints the next free id for a type. Two PRs can claim the same id; both add the same + file, so the second PR to merge gets an add/add conflict (`check` never sees a duplicate) + and renumbers then. To renumber: take the new id + from `next`, rename the file and change its `id:`, then rewrite every reference to the old + id in that PR's own records (`idea`, `experiments`, `findings`, `ideas`, `superseded_by`, + `decision`). Find them with `query --ref OLDID`; `check` must pass afterwards. +- `query` lists the records matching every filter: `--where FIELD=V1,V2` matches any of the + values, `--ref ID` keeps the records that point at `ID`. An unknown field, a clause + without `=` or a missing `ID` exits 2. Booleans are spelled as in the records: + `--where retracted=true`. +- `export` writes all records, the reference edges and the ideas grouped by map theme as one + JSON document, stamped with the commit, for the plans-and-research map and other views. + `dirty` is true when `research/` has uncommitted changes, and `render` says so next to the + commit. +- `render` writes the plans-and-research map from the same tree: a Mermaid mindmap (theme, + idea, experiments marked by outcome) and a table, as one Markdown file. It is generated on + demand and not committed; open it in any Mermaid-aware viewer, such as GitHub or an editor + preview. +- `render --html` writes the same register as one self-contained page to open in a browser, + with no network and no library: a collapsible tree (theme, idea, experiment, finding, + decision, implref), a search box (`/` focuses it; every word must match the id, a field or + the body), filters for type, theme, outcome or state and implementation, a sortable table, + and a panel with the clicked record and the records that cite it. `#X003` in the URL opens + that record. +- `gaps` lists accepted decisions with nothing merged and implrefs still `open` or + `proposed`. An implref's `status` is written by hand and goes stale when its pull request + merges, so `--live` (here and on `render --html`) asks GitHub through `gh` for the state of + every link, reports implrefs whose recorded status differs, and lists the open + `intel/dokimi` issues that no record links to. It only reads: fix a stale status in a PR. + `check` stays offline. +- An open issue that is engineering work with no research behind it (a dependency + dashboard, a build chore, a bug) goes in `research/untracked.yaml` with its reason + instead of a record; `gaps --live` skips those and reports an entry whose issue has closed. + An implref may also point at the file that carries the change when it landed before the + public history and has no pull request. + +## Adding a record + +1. `python scripts/research_register.py next ` for the id. +2. Write `research/records//.md`: + + ```markdown + --- + id: X012 + type: experiment + title: Compressed skill keeps step ordering + date: 2026-10-14 + idea: I004 + hypothesis: A skill cut to 40% of its tokens keeps the ordering constraints. + outcome: refuted + --- + + ## Setup + ... + ## Result + ... + ``` + +3. `python scripts/research_register.py check`, then open the PR. diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index 14844ef7..8c8775be 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -135,5 +135,6 @@ nav: - difficulty-rubric.md - example-execution.md - ingestion-docling.md + - research-register.md - CLI reference: reference/cli/ - API reference: reference/api/ diff --git a/dokimi/research/record.v1.json b/dokimi/research/record.v1.json new file mode 100644 index 00000000..5851151d --- /dev/null +++ b/dokimi/research/record.v1.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/research/record.v1.json", + "title": "Research register record (front matter of research/records//.md)", + "description": "One schema per record type under $defs; scripts/research_register.py picks it by `type`. References are ids; the register checks they resolve to a record of the right type.", + "$defs": { + "id": {"type": "string", "pattern": "^[A-Z]+[0-9]{3}$"}, + "ids": {"type": "array", "items": {"$ref": "#/$defs/id"}, "uniqueItems": true}, + "text": {"type": "string", "minLength": 1}, + "date": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"}, + "url": {"type": "string", "pattern": "^https://github\\.com/(intel|napetrov)/(?!(.*/)?\\.\\.(/|$))[A-Za-z0-9_./#?=&~+:@,-]+$"}, + "theme": { + "description": "Branch of the plans-and-research map.", + "enum": ["P", "E", "V", "K", "G", "O", "C", "D", "W"] + }, + "common": { + "type": "object", + "properties": { + "id": {"$ref": "#/$defs/id"}, + "type": {"type": "string"}, + "title": {"$ref": "#/$defs/text"}, + "date": {"$ref": "#/$defs/date"}, + "links": {"type": "array", "items": {"$ref": "#/$defs/url"}, "uniqueItems": true} + }, + "required": ["id", "type", "title", "date"] + }, + "idea": { + "$ref": "#/$defs/common", + "properties": { + "type": {"const": "idea"}, + "theme": {"$ref": "#/$defs/theme"}, + "question": {"$ref": "#/$defs/text"}, + "status": {"enum": ["open", "parked", "closed"]} + }, + "required": ["theme", "question", "status"], + "unevaluatedProperties": false + }, + "experiment": { + "$ref": "#/$defs/common", + "properties": { + "type": {"const": "experiment"}, + "idea": {"$ref": "#/$defs/id"}, + "hypothesis": {"$ref": "#/$defs/text"}, + "outcome": { + "description": "What the evidence showed. planned/running: no result yet. Negative and inconclusive outcomes stay in the register.", + "enum": ["planned", "running", "supported", "refuted", "inconclusive", "aborted"] + }, + "retracted": {"type": "boolean"} + }, + "required": ["idea", "hypothesis", "outcome"], + "unevaluatedProperties": false + }, + "finding": { + "$ref": "#/$defs/common", + "properties": { + "type": {"const": "finding"}, + "experiments": {"$ref": "#/$defs/ids", "minItems": 1}, + "confidence": {"enum": ["low", "med", "high"]} + }, + "required": ["experiments", "confidence"], + "unevaluatedProperties": false + }, + "decision": { + "$ref": "#/$defs/common", + "properties": { + "type": {"const": "decision"}, + "findings": {"$ref": "#/$defs/ids"}, + "ideas": {"$ref": "#/$defs/ids"}, + "state": {"enum": ["accepted", "rejected", "deferred", "superseded"]}, + "superseded_by": {"$ref": "#/$defs/id"} + }, + "required": ["state"], + "unevaluatedProperties": false + }, + "implref": { + "$ref": "#/$defs/common", + "properties": { + "type": {"const": "implref"}, + "decision": {"$ref": "#/$defs/id"}, + "status": { + "description": "Implementation state, kept apart from the decision state.", + "enum": ["proposed", "open", "merged", "closed"] + } + }, + "required": ["decision", "status", "links"], + "unevaluatedProperties": false + } + } +} diff --git a/dokimi/research/records/decision/D001.md b/dokimi/research/records/decision/D001.md new file mode 100644 index 00000000..a88f667b --- /dev/null +++ b/dokimi/research/records/decision/D001.md @@ -0,0 +1,16 @@ +--- +id: D001 +type: decision +title: Do not ship the local defect battery as a judge replacement +date: '2026-10-19' +findings: +- F001 +- F004 +state: rejected +--- + +## Rationale +Corrected baseline: the hosted judge was not saturated (0.93 pairwise) so the battery only tied it at higher cost. diff --git a/dokimi/research/records/decision/D002.md b/dokimi/research/records/decision/D002.md new file mode 100644 index 00000000..45a156df --- /dev/null +++ b/dokimi/research/records/decision/D002.md @@ -0,0 +1,15 @@ +--- +id: D002 +type: decision +title: Recompute aggregate in code; weight correctness x2; zero-weight specificity +date: '2026-10-19' +findings: +- F002 +state: accepted +--- + +## Rationale +Judged from pre-public history, not from a record here: the five dimensions appeared to move together, and averaging diluted correctness. F002 is cited only for the judge-prompt context (enumeration beats description); it does not support the weights themselves. diff --git a/dokimi/research/records/decision/D003.md b/dokimi/research/records/decision/D003.md new file mode 100644 index 00000000..5a90706e --- /dev/null +++ b/dokimi/research/records/decision/D003.md @@ -0,0 +1,15 @@ +--- +id: D003 +type: decision +title: Persist judge material by digest and validate the judge in the arms setting +date: '2026-10-19' +findings: +- F005 +state: accepted +--- + +## Rationale +Issue #48; material kept in artifact (#69), gate --context stored (#73, open). diff --git a/dokimi/research/records/decision/D004.md b/dokimi/research/records/decision/D004.md new file mode 100644 index 00000000..d5647eb5 --- /dev/null +++ b/dokimi/research/records/decision/D004.md @@ -0,0 +1,15 @@ +--- +id: D004 +type: decision +title: Do not build stratified probe sampler, SliCK split, LLMLingua shortener +date: '2026-10-19' +findings: +- F009 +state: rejected +--- + +## Rationale +Gains were negligible or refuted. The LLMLingua shortener was refuted (X009). F009 reports the grader spread, and stratified probe sampling is judged not to pay for its probe budget. The SliCK split was dropped for the same reason. The sampling and SliCK figures come from local runs not recorded here. \ No newline at end of file diff --git a/dokimi/research/records/decision/D005.md b/dokimi/research/records/decision/D005.md new file mode 100644 index 00000000..74231c23 --- /dev/null +++ b/dokimi/research/records/decision/D005.md @@ -0,0 +1,15 @@ +--- +id: D005 +type: decision +title: Infer/measure mechanism; [utility] descriptor is an override only +date: '2026-10-19' +findings: +- F010 +state: accepted +--- + +## Rationale +No skill standard declares mechanisms; few public skills carry a type field. diff --git a/dokimi/research/records/decision/D006.md b/dokimi/research/records/decision/D006.md new file mode 100644 index 00000000..9bc64027 --- /dev/null +++ b/dokimi/research/records/decision/D006.md @@ -0,0 +1,15 @@ +--- +id: D006 +type: decision +title: Move dokimi guidance to AGENTS.md; CLAUDE.md removed +date: '2026-10-19' +findings: +- F012 +state: accepted +--- + +## Rationale +Measured process gains; merged. diff --git a/dokimi/research/records/decision/D007.md b/dokimi/research/records/decision/D007.md new file mode 100644 index 00000000..01255506 --- /dev/null +++ b/dokimi/research/records/decision/D007.md @@ -0,0 +1,15 @@ +--- +id: D007 +type: decision +title: Writing-for-agents prose rewrite rejected; value is the lint gate +date: '2026-10-19' +findings: +- F013 +state: rejected +--- + +## Rationale +Rewrite dropped facts; gate-only kept them. diff --git a/dokimi/research/records/decision/D008.md b/dokimi/research/records/decision/D008.md new file mode 100644 index 00000000..822631cd --- /dev/null +++ b/dokimi/research/records/decision/D008.md @@ -0,0 +1,16 @@ +--- +id: D008 +type: decision +title: Ship LLM-free context tools (trace, budget) first +date: '2026-10-19' +findings: +- F015 +- F016 +state: accepted +--- + +## Rationale +Measurement tools are deterministic; behavioural fixes stay experimental. diff --git a/dokimi/research/records/decision/D009.md b/dokimi/research/records/decision/D009.md new file mode 100644 index 00000000..87f63c7a --- /dev/null +++ b/dokimi/research/records/decision/D009.md @@ -0,0 +1,15 @@ +--- +id: D009 +type: decision +title: Product repo keeps evals only; mechanics move to dokimi; policy rules dropped +date: '2026-10-19' +findings: +- F018 +state: accepted +--- + +## Rationale +Reviewer objections to generic harness in the product repo; tracking #41. diff --git a/dokimi/research/records/decision/D010.md b/dokimi/research/records/decision/D010.md new file mode 100644 index 00000000..adb071e6 --- /dev/null +++ b/dokimi/research/records/decision/D010.md @@ -0,0 +1,13 @@ +--- +id: D010 +type: decision +title: Number changes never go in a refactor; stored values carry a computation stamp +date: '2026-10-19' +state: accepted +--- + +## Rationale +Keeps stored results comparable across refactors. diff --git a/dokimi/research/records/decision/D011.md b/dokimi/research/records/decision/D011.md new file mode 100644 index 00000000..d5010f16 --- /dev/null +++ b/dokimi/research/records/decision/D011.md @@ -0,0 +1,13 @@ +--- +id: D011 +type: decision +title: Outcome contracts as pytest plugin in-repo (no extra tooling/runtime) +date: '2026-10-19' +state: accepted +--- + +## Rationale +Simpler adoption; audit found --questions-root did not pin the golden set and code was fixed. diff --git a/dokimi/research/records/decision/D012.md b/dokimi/research/records/decision/D012.md new file mode 100644 index 00000000..4adfa0cb --- /dev/null +++ b/dokimi/research/records/decision/D012.md @@ -0,0 +1,15 @@ +--- +id: D012 +type: decision +title: git+markdown canonical store with validator; optional UI pilot; GitHub for discussion +date: '2026-10-19' +findings: +- F019 +state: accepted +--- + +## Rationale +Lowest cost, history via git, existing MCP access. diff --git a/dokimi/research/records/decision/D013.md b/dokimi/research/records/decision/D013.md new file mode 100644 index 00000000..5a82fc92 --- /dev/null +++ b/dokimi/research/records/decision/D013.md @@ -0,0 +1,15 @@ +--- +id: D013 +type: decision +title: Reject GitLab CE, OpenProject CE as register +date: '2026-10-19' +findings: +- F019 +state: rejected +--- + +## Rationale +Schema features paid or not via API; high RAM. diff --git a/dokimi/research/records/decision/D014.md b/dokimi/research/records/decision/D014.md new file mode 100644 index 00000000..1babe372 --- /dev/null +++ b/dokimi/research/records/decision/D014.md @@ -0,0 +1,15 @@ +--- +id: D014 +type: decision +date: '2026-10-21' +title: Catalogs run dokimi in their own CI and promote skills on measured lift +ideas: +- I008 +state: accepted +--- + +## Rationale +A catalog entry should move from published to validated on per-skill evidence, not on review alone. The catalog consumes the generic engine (plan, result, compare, gate policy) planned in #121, so the catalog work lists only what a catalog adds on top. diff --git a/dokimi/research/records/decision/D015.md b/dokimi/research/records/decision/D015.md new file mode 100644 index 00000000..63b638ab --- /dev/null +++ b/dokimi/research/records/decision/D015.md @@ -0,0 +1,15 @@ +--- +id: D015 +type: decision +date: '2026-10-21' +title: CLI restructure into task-oriented groups waits for the user loop +ideas: +- I020 +state: deferred +--- + +## Rationale +The proposal regroups the 64 existing operations under task-oriented roots. The groups should follow the plan/validate/compare/gate/record loop (R030), so the restructure waits until that loop is settled. diff --git a/dokimi/research/records/decision/D016.md b/dokimi/research/records/decision/D016.md new file mode 100644 index 00000000..2e05e9bf --- /dev/null +++ b/dokimi/research/records/decision/D016.md @@ -0,0 +1,15 @@ +--- +id: D016 +type: decision +date: '2026-10-21' +title: A run records only the conditions it enforced +ideas: +- I018 +state: accepted +--- + +## Rationale +A cell record that names a condition the run did not enforce overstates what was measured. Example: `--delegation none` swaps tool arguments, but a skill with `context: fork` still starts a subagent. Either enforce the condition or report the cell as not meeting it. diff --git a/dokimi/research/records/experiment/X001.md b/dokimi/research/records/experiment/X001.md new file mode 100644 index 00000000..779ae55b --- /dev/null +++ b/dokimi/research/records/experiment/X001.md @@ -0,0 +1,21 @@ +--- +id: X001 +type: experiment +title: Question polarity vs ceiling saturation (local small model) +date: '2026-10-19' +idea: I001 +hypothesis: Typed one-token decisions fix saturation. +outcome: refuted +--- + +## Setup +Local 4B/14B model, deterministic degradations of perf golden answers; positive vs negative wording of the same decision. + +## Result +Positive 'covers point?' gave P(yes)=1.000 for intact and degraded copies; negative 'is it absent?' separated them (0.825 vs 0.250). Negative wording then saturated at the floor on a less acquiescent 14B model. + +## Evidence +- local run, Qwen3 4B/14B, 20 bases diff --git a/dokimi/research/records/experiment/X002.md b/dokimi/research/records/experiment/X002.md new file mode 100644 index 00000000..cfe3d0b6 --- /dev/null +++ b/dokimi/research/records/experiment/X002.md @@ -0,0 +1,21 @@ +--- +id: X002 +type: experiment +title: Length-only baseline for judge validity +date: '2026-10-19' +idea: I001 +hypothesis: Judge validity accuracy reflects reading content. +outcome: inconclusive +--- + +## Setup +Compare judge pairwise accuracy on degradations to a predictor using answer length only. + +## Result +Defect battery 0.94 vs length-only 0.81 on 5 variants; length beat the battery on strip_code and truncate. Shipped hosted judge 9-variant: 0.920 vs length 0.663; 4 of 9 variants <= length. + +## Evidence +- local run, Haiku judge, 20 bases, 176 pairs diff --git a/dokimi/research/records/experiment/X003.md b/dokimi/research/records/experiment/X003.md new file mode 100644 index 00000000..8257a6a2 --- /dev/null +++ b/dokimi/research/records/experiment/X003.md @@ -0,0 +1,21 @@ +--- +id: X003 +type: experiment +title: 'Wrong-magnitude detection: criterion wording only' +date: '2026-10-19' +idea: I001 +hypothesis: Describing a numeric-claims criterion lets the judge catch wrong numbers. +outcome: refuted +--- + +## Setup +Haiku judge, 4 seeds; criterion only vs the baseline prompt. The enumeration arm is X028. + +## Result +Criterion only +0.045, McNemar p=0.43: within run-to-run noise. + +## Evidence +- local run, Haiku, 156 pairs, 4 seeds diff --git a/dokimi/research/records/experiment/X004.md b/dokimi/research/records/experiment/X004.md new file mode 100644 index 00000000..67dba7fd --- /dev/null +++ b/dokimi/research/records/experiment/X004.md @@ -0,0 +1,21 @@ +--- +id: X004 +type: experiment +title: Context-blind judge on skill arm +date: '2026-10-19' +idea: I002 +hypothesis: The skill arm wins on the perf golden set. +outcome: inconclusive +--- + +## Setup +81 paired questions, hosted judge, skill answers re-judged with and without the skill manifest in context. + +## Result +Without context skill lost 29/81 (judge called existing skill file paths 'fabricated', 17/81 flagged). With context 60/81. Symmetric control: baseline alone gained ~+4 from context; 36/81, p=0.087. + +## Evidence +- local run, n=81 paired diff --git a/dokimi/research/records/experiment/X005.md b/dokimi/research/records/experiment/X005.md new file mode 100644 index 00000000..28893f91 --- /dev/null +++ b/dokimi/research/records/experiment/X005.md @@ -0,0 +1,21 @@ +--- +id: X005 +type: experiment +title: Judge-validity gate with stored context +date: '2026-10-19' +idea: I002 +hypothesis: A no-context validity gate reflects the arms setting. +outcome: supported +links: +- https://github.com/intel/dokimi/issues/48 +- https://github.com/intel/dokimi/pull/73 +--- + +## Setup +Haiku judge, 24 bases, snippet vs full material. + +## Result +With full material intact mean 85.9->89.9 while overall accuracy fell 0.869->0.808; wrong_tools 0.891->0.674. diff --git a/dokimi/research/records/experiment/X006.md b/dokimi/research/records/experiment/X006.md new file mode 100644 index 00000000..9d6a0ef7 --- /dev/null +++ b/dokimi/research/records/experiment/X006.md @@ -0,0 +1,23 @@ +--- +id: X006 +type: experiment +title: Judge ladder and cross-judge agreement +date: '2026-10-19' +idea: I003 +hypothesis: Skill-vs-baseline verdict is judge-independent. +outcome: refuted +--- + +## Setup +Same answers judged by Haiku, gpt-oss, Sonnet, Opus; absolute vs pairwise; sequential (SPRT) gate. + +## Result +Verdict sign flipped with judge (Haiku -3.4 p=.024; Opus +1.2 p=.005). Per-question direction agreement 46-76%. Only one absolute judge passed the sequential gate; pairwise gpt-oss passed at 0.962 for ~$0.5. +## Corrections +A pair-keying audit retracted three later claims: that every absolute judge, Opus included, failed the gate; that margin cascades never pass strip_code; and that Haiku pairwise prefers the contradicting answer 84% of the time. The sign flip above was not retracted. + +## Evidence +- local run, ~11k judge calls diff --git a/dokimi/research/records/experiment/X007.md b/dokimi/research/records/experiment/X007.md new file mode 100644 index 00000000..820ec7f0 --- /dev/null +++ b/dokimi/research/records/experiment/X007.md @@ -0,0 +1,21 @@ +--- +id: X007 +type: experiment +title: Automatic prompt optimisation of the judge rubric (GEPA) +date: '2026-10-19' +idea: I003 +hypothesis: Reflective prompt optimisation improves the judge rubric. +outcome: refuted +--- + +## Setup +GEPA with default and 'general' reflection templates; question-disjoint split, held-out test of 57 questions. + +## Result +Default template leaked the test generator, zero accepted improvement in 1408 metric calls; validation gains were winner's curse; no rubric beat the hand-written prompt on held-out test. + +## Evidence +- local run, held-out 57 questions diff --git a/dokimi/research/records/experiment/X008.md b/dokimi/research/records/experiment/X008.md new file mode 100644 index 00000000..d70325e8 --- /dev/null +++ b/dokimi/research/records/experiment/X008.md @@ -0,0 +1,21 @@ +--- +id: X008 +type: experiment +title: FormatSpread wording sensitivity +date: '2026-10-19' +idea: I003 +hypothesis: Meaning-preserving rewrites don't change judge accuracy. +outcome: inconclusive +--- + +## Setup +6 rewrites vs 3 identical-prompt redraws. + +## Result +Pooled accuracy within redraw noise, but wrong_numbers 0.47-0.72 and padding 0.37-0.67 were wording-fragile; 10.6% of verdicts flip between identical draws. + +## Evidence +- local run diff --git a/dokimi/research/records/experiment/X009.md b/dokimi/research/records/experiment/X009.md new file mode 100644 index 00000000..38779bdc --- /dev/null +++ b/dokimi/research/records/experiment/X009.md @@ -0,0 +1,21 @@ +--- +id: X009 +type: experiment +title: 'Novelty toolscan: cheap matchers and compressors' +date: '2026-10-19' +idea: I005 +hypothesis: Cheap NLI matchers / LLMLingua can replace LLM adjudication. +outcome: refuted +--- + +## Setup +Matcher bake-off on hard band; LLMLingua-2 force_tokens; judge-free items. + +## Result +Small NLI at chance (AUC 0.576); HHEM-2.1 AUC 0.804 but no matcher removed LLM adjudication; LLMLingua force_tokens did not protect atoms (refuted); every cheap instrument failed on numbers/versions/negation. + +## Evidence +- local run, 100 paired items diff --git a/dokimi/research/records/experiment/X010.md b/dokimi/research/records/experiment/X010.md new file mode 100644 index 00000000..81208267 --- /dev/null +++ b/dokimi/research/records/experiment/X010.md @@ -0,0 +1,21 @@ +--- +id: X010 +type: experiment +title: Planted contradictions (value adoption) +date: '2026-10-19' +idea: I005 +hypothesis: Absurd planted values are resisted by the reader model. +outcome: refuted +--- + +## Setup +20 atoms, 8 artifacts, 3 models, 240 calls. + +## Result +planted_near adopted 0.942, planted_far 0.846, no-skill true value 0.115; dose response nearly flat (p=0.062); only independently anchored values resisted. + +## Evidence +- local run, 52 valid cells diff --git a/dokimi/research/records/experiment/X011.md b/dokimi/research/records/experiment/X011.md new file mode 100644 index 00000000..e1b9cbc5 --- /dev/null +++ b/dokimi/research/records/experiment/X011.md @@ -0,0 +1,21 @@ +--- +id: X011 +type: experiment +title: Infer skill mechanism and predict lift +date: '2026-10-19' +idea: I006 +hypothesis: A declared or inferred mechanism (or novelty) predicts task delta. +outcome: refuted +--- + +## Setup +3 LLM classifiers on 33 public skills; static regex; 13 skills with task deltas. + +## Result +Classifier kappa 0.41-0.54; static rules usable only for automation/contract/guardrail; |rho|<=0.19 vs task delta for every mechanism and for novelty. + +## Evidence +- local run, 33 skills, 24 cells diff --git a/dokimi/research/records/experiment/X012.md b/dokimi/research/records/experiment/X012.md new file mode 100644 index 00000000..7980e49e --- /dev/null +++ b/dokimi/research/records/experiment/X012.md @@ -0,0 +1,24 @@ +--- +id: X012 +type: experiment +title: Skill ablation and checklist control +date: '2026-10-19' +idea: I006 +hypothesis: Full skill outperforms ablations and a short checklist. +outcome: inconclusive +--- + +## Setup +onetbb quickstart, 7 tasks x 3 reps, Haiku; arms none/full/instr_only/facts_only/checklist; later irrelevant same-length control. + +## Result +none 8/21, full 15/21, checklist 16/21; no variant distinguishable from full (McNemar p=1.0); lift is ~one knowledge atom checked by a marker verifier; skills raised cost. Same-length irrelevant control ~= none, so lift is content. + +## Corrections +An earlier claim, 'full guardrail worse than none', was retracted: 3 reps are too few. + +## Evidence +- local run, 7 tasks x 3 reps diff --git a/dokimi/research/records/experiment/X013.md b/dokimi/research/records/experiment/X013.md new file mode 100644 index 00000000..77bf9070 --- /dev/null +++ b/dokimi/research/records/experiment/X013.md @@ -0,0 +1,20 @@ +--- +id: X013 +type: experiment +title: Skill discover arm smoke +date: '2026-10-19' +idea: I007 +hypothesis: Skill helps without notice. +outcome: refuted +links: +- https://github.com/intel/dokimi/pull/110 +--- + +## Setup +onetbb transpose task, Haiku, 3 reps: no-skill / skill / discover. + +## Result +no-skill 0/3, skill 3/3, discover 0/3 with 0 Skill calls. diff --git a/dokimi/research/records/experiment/X014.md b/dokimi/research/records/experiment/X014.md new file mode 100644 index 00000000..cb49c5df --- /dev/null +++ b/dokimi/research/records/experiment/X014.md @@ -0,0 +1,21 @@ +--- +id: X014 +type: experiment +title: Invocation drives skill effect +date: '2026-10-19' +idea: I007 +hypothesis: Skill effect is independent of whether it is invoked. +outcome: refuted +--- + +## Setup +Determinism suite, 96+ runs, Haiku/Sonnet. + +## Result +When invoked done right 11/11, not invoked 0/25; trigger-heavy description raised invocation 2/12 -> 7/12 (p=0.087); a Stop hook removed the dependency. + +## Evidence +- local run, n=96 diff --git a/dokimi/research/records/experiment/X015.md b/dokimi/research/records/experiment/X015.md new file mode 100644 index 00000000..aefa4ea6 --- /dev/null +++ b/dokimi/research/records/experiment/X015.md @@ -0,0 +1,21 @@ +--- +id: X015 +type: experiment +title: AGENTS.md vs CLAUDE.md vs none on dokimi's own tasks +date: '2026-10-19' +idea: I009 +hypothesis: Moving guidance to AGENTS.md does not change outcomes. +outcome: refuted +links: +- https://github.com/intel/dokimi/pull/56 +- https://github.com/intel/dokimi/pull/57 +--- + +## Setup +10 repo tasks, Haiku/Sonnet/Opus, n=3, 270 cells. + +## Result +Schema-required-field: AGENTS 9/9 vs CLAUDE 4/9 (Fisher p=.015); commands killed by timeout 0/90 vs 42/90; first 'legacy keys' wording regressed (2/6) and was fixed before merge. diff --git a/dokimi/research/records/experiment/X016.md b/dokimi/research/records/experiment/X016.md new file mode 100644 index 00000000..e8cbab39 --- /dev/null +++ b/dokimi/research/records/experiment/X016.md @@ -0,0 +1,21 @@ +--- +id: X016 +type: experiment +title: Pointer and prohibition wording +date: '2026-10-19' +idea: I009 +hypothesis: Descriptive index lines and prohibitions guide agents. +outcome: refuted +--- + +## Setup +Synthetic fixtures; 96 + 192 runs, Haiku/Sonnet. + +## Result +Conditional 'Before X, read Y' 10/12 vs bare/descriptive 3/12 (Haiku); prohibition 0/48 violations but preferred target only 28/48 vs 48/48 for positive. + +## Evidence +- local run, n=288 diff --git a/dokimi/research/records/experiment/X017.md b/dokimi/research/records/experiment/X017.md new file mode 100644 index 00000000..ea9d4d20 --- /dev/null +++ b/dokimi/research/records/experiment/X017.md @@ -0,0 +1,21 @@ +--- +id: X017 +type: experiment +title: Rewriting guidance with a writing-for-agents skill +date: '2026-10-19' +idea: I009 +hypothesis: A rewrite skill improves guidance. +outcome: refuted +--- + +## Setup +3 repos x 4 rewrite modes x 3 models x 2. + +## Result +Skill rewrite shrank text and dropped facts (57% kept in one case); lint-only gate kept 93-100% of facts. + +## Evidence +- local run, 72 runs diff --git a/dokimi/research/records/experiment/X018.md b/dokimi/research/records/experiment/X018.md new file mode 100644 index 00000000..4019928c --- /dev/null +++ b/dokimi/research/records/experiment/X018.md @@ -0,0 +1,21 @@ +--- +id: X018 +type: experiment +title: Router vs flat skill listing +date: '2026-10-19' +idea: I009 +hypothesis: Grouping skills in a router improves selection. +outcome: refuted +--- + +## Setup +Public skills catalog, 4 groups, Haiku 5.5. + +## Result +Router 0.20 vs flat 0.38. + +## Evidence +- local run diff --git a/dokimi/research/records/experiment/X019.md b/dokimi/research/records/experiment/X019.md new file mode 100644 index 00000000..babd64d2 --- /dev/null +++ b/dokimi/research/records/experiment/X019.md @@ -0,0 +1,21 @@ +--- +id: X019 +type: experiment +title: 'Knowledge quiz: does a CLAUDE.md add knowledge' +date: '2026-10-19' +idea: I009 +hypothesis: Guidance file raises open-book accuracy. +outcome: refuted +--- + +## Setup +17-question quiz, closed/open book, n=2. + +## Result +Open-book accuracy same with or without file; value was cost (tokens -25..-45%). + +## Evidence +- local run, n=2 diff --git a/dokimi/research/records/experiment/X020.md b/dokimi/research/records/experiment/X020.md new file mode 100644 index 00000000..c0760031 --- /dev/null +++ b/dokimi/research/records/experiment/X020.md @@ -0,0 +1,21 @@ +--- +id: X020 +type: experiment +title: Cold-agent onboarding probe +date: '2026-10-19' +idea: I010 +hypothesis: README suffices to review a skill or repo. +outcome: supported +links: +- https://github.com/intel/dokimi/pull/59 +- https://github.com/intel/dokimi/pull/60 +--- + +## Setup +4 cold agents (2 models x 2 tasks), only the dokimi tree. + +## Result +All found the right commands in 2-5 turns; gaps: no USD cap on novelty (one run spent $14.50 vs $6 budget), manifests required for guidance-only scans, fabricated competitors for routing. diff --git a/dokimi/research/records/experiment/X021.md b/dokimi/research/records/experiment/X021.md new file mode 100644 index 00000000..cb41db1d --- /dev/null +++ b/dokimi/research/records/experiment/X021.md @@ -0,0 +1,21 @@ +--- +id: X021 +type: experiment +title: Context failure-mode suite +date: '2026-10-19' +idea: I013 +hypothesis: Verify prompts and compaction mitigate context failures. +outcome: inconclusive +links: +- https://github.com/intel/dokimi/pull/68 +- https://github.com/intel/dokimi/pull/71 +--- + +## Setup +421 runs, Haiku 4.5 / Sonnet 5.5, n=3-5 per cell (directional). + +## Result +Stale facts adopted near-totally; 'verify' prompt helped only Sonnet (4/5); 150k irrelevant history kept accuracy but cost 16-31x; guidance file beat code and skill on clash 10/10; prune ~10-20x cheaper than compaction at equal accuracy. Follow-up on compaction: it dropped ~7% of tokens (107,547 -> 99,842) with 5/6 accuracy (Haiku miscounted once); pruning reads peaked at 9.6k/5.8k tokens with 6/6. diff --git a/dokimi/research/records/experiment/X022.md b/dokimi/research/records/experiment/X022.md new file mode 100644 index 00000000..a657b4a0 --- /dev/null +++ b/dokimi/research/records/experiment/X022.md @@ -0,0 +1,21 @@ +--- +id: X022 +type: experiment +title: Near-duplicate MCP tools under deferred loading +date: '2026-10-19' +idea: I013 +hypothesis: Tool descriptions disambiguate tools. +outcome: refuted +--- + +## Setup +k near-duplicate tools, deferred vs flat loading. + +## Result +Deferred: 1/40 right at k>=10 (Haiku)/k>=50 (Sonnet); flat: 56/56 right at +9k-22k tokens per call. + +## Evidence +- local run diff --git a/dokimi/research/records/experiment/X023.md b/dokimi/research/records/experiment/X023.md new file mode 100644 index 00000000..d171875b --- /dev/null +++ b/dokimi/research/records/experiment/X023.md @@ -0,0 +1,21 @@ +--- +id: X023 +type: experiment +title: Determinism profile on a procedure task +date: '2026-10-19' +idea: I015 +hypothesis: Harness success and path stability indicate correctness. +outcome: refuted +--- + +## Setup +96 runs, $5.27; arms none/prose/script/hook; repeat/paraphrase/fault series. + +## Result +Harness success 100% everywhere (uninformative); stable failure common; hook vs script p=0.0006; 'script arm has most stable path' did not hold; after a transient failure 0/24 answers mentioned it. + +## Evidence +- local run, n=96 diff --git a/dokimi/research/records/experiment/X024.md b/dokimi/research/records/experiment/X024.md new file mode 100644 index 00000000..f53c8d18 --- /dev/null +++ b/dokimi/research/records/experiment/X024.md @@ -0,0 +1,20 @@ +--- +id: X024 +type: experiment +title: Built-in review graders equivalence +date: '2026-10-19' +idea: I017 +hypothesis: Built-in graders reproduce the product repo's graders. +outcome: supported +links: +- https://github.com/intel/dokimi/pull/120 +--- + +## Setup +Regrade 1414 answers over 9 review tasks. + +## Result +0 mismatches. diff --git a/dokimi/research/records/experiment/X025.md b/dokimi/research/records/experiment/X025.md new file mode 100644 index 00000000..6de87ddc --- /dev/null +++ b/dokimi/research/records/experiment/X025.md @@ -0,0 +1,23 @@ +--- +id: X025 +type: experiment +title: L0 static-check parity with seeded defects +date: '2026-10-19' +idea: I017 +hypothesis: Generic L0 rules match the product's static check. +outcome: inconclusive +links: +- https://github.com/intel/dokimi/pull/115 +--- + +## Setup +7 seeded stale defects. + +## Result +Product check 6/7; dokimi 1/7 before, 5/7 after. + +## Corrections +One product 'real failure' was retracted as the product check's own false positive. diff --git a/dokimi/research/records/experiment/X026.md b/dokimi/research/records/experiment/X026.md new file mode 100644 index 00000000..d500c002 --- /dev/null +++ b/dokimi/research/records/experiment/X026.md @@ -0,0 +1,20 @@ +--- +id: X026 +type: experiment +title: Test-quality audit of dokimi tests +date: '2026-10-19' +idea: I018 +hypothesis: Some tests cannot fail for wrong behaviour. +outcome: supported +links: +- https://github.com/intel/dokimi/pull/103 +--- + +## Setup +Static AST audit of the test suite. + +## Result +31 of 3340 tests flagged in pilot; 30 findings on main, 27 fixed. diff --git a/dokimi/research/records/experiment/X027.md b/dokimi/research/records/experiment/X027.md new file mode 100644 index 00000000..738ff1d5 --- /dev/null +++ b/dokimi/research/records/experiment/X027.md @@ -0,0 +1,21 @@ +--- +id: X027 +type: experiment +title: Research workspace tool trial +date: '2026-10-19' +idea: I021 +hypothesis: A self-hosted tool can enforce the record schema. +outcome: refuted +--- + +## Setup +5 tools in local containers, same protocol and seed records. + +## Result +No server tool enforced typed refs/enums on write; git+markdown passed all tests with no server. + +## Evidence +- local trial, 5 tools diff --git a/dokimi/research/records/experiment/X028.md b/dokimi/research/records/experiment/X028.md new file mode 100644 index 00000000..ed7cde2f --- /dev/null +++ b/dokimi/research/records/experiment/X028.md @@ -0,0 +1,21 @@ +--- +id: X028 +type: experiment +title: 'Wrong-magnitude detection: criterion plus enumeration' +date: '2026-10-19' +idea: I001 +hypothesis: Telling the judge to enumerate every figure lets it catch wrong numbers. +outcome: supported +--- + +## Setup +Haiku judge, 4 seeds; criterion plus an instruction to enumerate every figure, against the baseline prompt. + +## Result +Pooled 0.599 -> 0.712, p=0.0144 over 156 pairs. Still under the 0.80 gate, so the judge is better but not reliable on numbers (see F003). + +## Evidence +- local run, Haiku, 156 pairs, 4 seeds diff --git a/dokimi/research/records/experiment/X029.md b/dokimi/research/records/experiment/X029.md new file mode 100644 index 00000000..4a500fd7 --- /dev/null +++ b/dokimi/research/records/experiment/X029.md @@ -0,0 +1,21 @@ +--- +id: X029 +type: experiment +title: Grader agreement on the novelty judge band +date: '2026-10-19' +idea: I005 +hypothesis: Two graders agree closely enough to read small 'added' differences. +outcome: refuted +--- + +## Setup +25 probes graded by two graders on the novelty judge band. + +## Result +Cohen's kappa 0.576, fine-grained agreement 0.664; grader-to-grader spread of 8-12 percentage points. + +## Evidence +- local run, 25 dual-graded probes diff --git a/dokimi/research/records/finding/F001.md b/dokimi/research/records/finding/F001.md new file mode 100644 index 00000000..d7d0536f --- /dev/null +++ b/dokimi/research/records/finding/F001.md @@ -0,0 +1,18 @@ +--- +id: F001 +type: finding +title: Judges must be checked against a length-only baseline +date: '2026-10-19' +experiments: +- X002 +confidence: high +--- + +## Statement +A degradation variant whose judge accuracy does not exceed length-only accuracy proves nothing about content reading. + +## Applies to +Any LLM-judge validity gate diff --git a/dokimi/research/records/finding/F002.md b/dokimi/research/records/finding/F002.md new file mode 100644 index 00000000..0f58a64e --- /dev/null +++ b/dokimi/research/records/finding/F002.md @@ -0,0 +1,19 @@ +--- +id: F002 +type: finding +title: Instructing the judge to enumerate beats describing the criterion +date: '2026-10-19' +experiments: +- X003 +- X028 +confidence: med +--- + +## Statement +Enumeration of figures moved wrong-number detection; wording alone stayed within run-to-run noise (~0.04). + +## Applies to +LLM-judge prompt design diff --git a/dokimi/research/records/finding/F003.md b/dokimi/research/records/finding/F003.md new file mode 100644 index 00000000..52c48f69 --- /dev/null +++ b/dokimi/research/records/finding/F003.md @@ -0,0 +1,19 @@ +--- +id: F003 +type: finding +title: No measured judge sees wrong magnitudes reliably +date: '2026-10-19' +experiments: +- X008 +- X028 +confidence: med +--- + +## Statement +wrong_numbers detection measured 0.47-0.72 across prompt wordings (X008) and 0.599, raised to 0.712 by the X028 change, which is still under the 0.80 gate: no measured judge catches wrong values reliably. + +## Applies to +Judging technical answers with numbers diff --git a/dokimi/research/records/finding/F004.md b/dokimi/research/records/finding/F004.md new file mode 100644 index 00000000..a41f97e9 --- /dev/null +++ b/dokimi/research/records/finding/F004.md @@ -0,0 +1,18 @@ +--- +id: F004 +type: finding +title: Polarity must be matched to the model +date: '2026-10-19' +experiments: +- X001 +confidence: med +--- + +## Statement +Negative-polarity questions fix ceiling saturation on an acquiescent model but saturate at the floor on a non-acquiescent one. + +## Applies to +Typed judge batteries diff --git a/dokimi/research/records/finding/F005.md b/dokimi/research/records/finding/F005.md new file mode 100644 index 00000000..81b17d62 --- /dev/null +++ b/dokimi/research/records/finding/F005.md @@ -0,0 +1,19 @@ +--- +id: F005 +type: finding +title: Judge context is a second treatment +date: '2026-10-19' +experiments: +- X004 +- X005 +confidence: high +--- + +## Statement +Per-arm judge context shifts scores by itself; no arm comparison is trustworthy if a treatment cites material the judge cannot see, and the skill effect itself was not established. + +## Applies to +Arms runs with skill/docs treatments diff --git a/dokimi/research/records/finding/F006.md b/dokimi/research/records/finding/F006.md new file mode 100644 index 00000000..5f19ed88 --- /dev/null +++ b/dokimi/research/records/finding/F006.md @@ -0,0 +1,20 @@ +--- +id: F006 +type: finding +title: Judge choice can flip arm verdicts; per-question tables are noise +date: '2026-10-19' +experiments: +- X006 +confidence: med +--- + +## Statement +Record judge model and gate the judge before trusting a delta; per-question agreement between judges is 46-76%. + +One skill/baseline pair only, so confidence stays med. + +## Applies to +Arms and model-compare reports diff --git a/dokimi/research/records/finding/F007.md b/dokimi/research/records/finding/F007.md new file mode 100644 index 00000000..da0c0a4b --- /dev/null +++ b/dokimi/research/records/finding/F007.md @@ -0,0 +1,18 @@ +--- +id: F007 +type: finding +title: Prompt optimisation overfits the validity generator +date: '2026-10-19' +experiments: +- X007 +confidence: med +--- + +## Statement +Rubric optimisation against synthetic degradations learns the degradations, not judging. + +## Applies to +Judge tuning diff --git a/dokimi/research/records/finding/F008.md b/dokimi/research/records/finding/F008.md new file mode 100644 index 00000000..090e12af --- /dev/null +++ b/dokimi/research/records/finding/F008.md @@ -0,0 +1,18 @@ +--- +id: F008 +type: finding +title: A reader model adopts document values ~85-94% of the time +date: '2026-10-19' +experiments: +- X010 +confidence: med +--- + +## Statement +Correctness of a value cannot be delegated to the reader; ship decision values with an independently checkable correlate. + +## Applies to +Skill and doc authoring diff --git a/dokimi/research/records/finding/F009.md b/dokimi/research/records/finding/F009.md new file mode 100644 index 00000000..31d3301b --- /dev/null +++ b/dokimi/research/records/finding/F009.md @@ -0,0 +1,19 @@ +--- +id: F009 +type: finding +title: Quote no 'added' difference below 0.15 +date: '2026-10-19' +experiments: +- X009 +- X029 +confidence: med +--- + +## Statement +Grader agreement kappa 0.576 and ±8-12pp grader spread make smaller differences unreadable. + +## Applies to +Novelty reports diff --git a/dokimi/research/records/finding/F010.md b/dokimi/research/records/finding/F010.md new file mode 100644 index 00000000..2b04d074 --- /dev/null +++ b/dokimi/research/records/finding/F010.md @@ -0,0 +1,18 @@ +--- +id: F010 +type: finding +title: Per-check (process/contract) grading is what discriminates skill utility +date: '2026-10-19' +experiments: +- X012 +confidence: med +--- + +## Statement +Marker-based verifiers measure one atom; tasks need verifiers that check process or contract. Use >=6 reps before quoting per-check defects. + +## Applies to +Skill eval task design diff --git a/dokimi/research/records/finding/F011.md b/dokimi/research/records/finding/F011.md new file mode 100644 index 00000000..3aee469b --- /dev/null +++ b/dokimi/research/records/finding/F011.md @@ -0,0 +1,19 @@ +--- +id: F011 +type: finding +title: For weaker models the skill effect is entirely invocation +date: '2026-10-19' +experiments: +- X013 +- X014 +confidence: med +--- + +## Statement +Measure invocation separately; hooks or descriptions that trigger invocation matter more than skill body. + +## Applies to +Skill eval reports diff --git a/dokimi/research/records/finding/F012.md b/dokimi/research/records/finding/F012.md new file mode 100644 index 00000000..12a95b23 --- /dev/null +++ b/dokimi/research/records/finding/F012.md @@ -0,0 +1,18 @@ +--- +id: F012 +type: finding +title: A guidance line can regress behaviour; evaluate wording changes +date: '2026-10-19' +experiments: +- X015 +confidence: high +--- + +## Statement +Measured regressions occurred from a plausible wording; eval caught it before merge. + +## Applies to +Any repo guidance change diff --git a/dokimi/research/records/finding/F013.md b/dokimi/research/records/finding/F013.md new file mode 100644 index 00000000..48bfb8b9 --- /dev/null +++ b/dokimi/research/records/finding/F013.md @@ -0,0 +1,18 @@ +--- +id: F013 +type: finding +title: Conditional pointers work; descriptive index lines do not +date: '2026-10-19' +experiments: +- X016 +confidence: med +--- + +## Statement +Tie a doc pointer to the triggering action; give a positive target with prohibitions. + +## Applies to +Guidance writing, synthetic only (real-repo transfer untested) diff --git a/dokimi/research/records/finding/F014.md b/dokimi/research/records/finding/F014.md new file mode 100644 index 00000000..4bc58f12 --- /dev/null +++ b/dokimi/research/records/finding/F014.md @@ -0,0 +1,18 @@ +--- +id: F014 +type: finding +title: Guidance value is often cost, not accuracy +date: '2026-10-19' +experiments: +- X019 +confidence: low +--- + +## Statement +When rules are discoverable, guidance saves tokens rather than changing answers. + +## Applies to +Repo guidance evaluation diff --git a/dokimi/research/records/finding/F015.md b/dokimi/research/records/finding/F015.md new file mode 100644 index 00000000..ab37a914 --- /dev/null +++ b/dokimi/research/records/finding/F015.md @@ -0,0 +1,18 @@ +--- +id: F015 +type: finding +title: Under deferred MCP loading the distinction must be in the tool name +date: '2026-10-19' +experiments: +- X022 +confidence: med +--- + +## Statement +Descriptions are not seen at selection time. + +## Applies to +MCP server design diff --git a/dokimi/research/records/finding/F016.md b/dokimi/research/records/finding/F016.md new file mode 100644 index 00000000..6c686925 --- /dev/null +++ b/dokimi/research/records/finding/F016.md @@ -0,0 +1,18 @@ +--- +id: F016 +type: finding +title: Prune/curate beats compaction +date: '2026-10-19' +experiments: +- X021 +confidence: low +--- + +## Statement +Pruning reads cut peak tokens ~10x with same accuracy. + +## Applies to +Context management diff --git a/dokimi/research/records/finding/F017.md b/dokimi/research/records/finding/F017.md new file mode 100644 index 00000000..fb1ae8eb --- /dev/null +++ b/dokimi/research/records/finding/F017.md @@ -0,0 +1,18 @@ +--- +id: F017 +type: finding +title: Harness success carries no information; stable paths can be stably wrong +date: '2026-10-19' +experiments: +- X023 +confidence: med +--- + +## Statement +Grade outcome and process separately; tool-call grain must split shell commands. + +## Applies to +Trace-based eval diff --git a/dokimi/research/records/finding/F018.md b/dokimi/research/records/finding/F018.md new file mode 100644 index 00000000..5bb15741 --- /dev/null +++ b/dokimi/research/records/finding/F018.md @@ -0,0 +1,20 @@ +--- +id: F018 +type: finding +title: Zero delta at ceiling is not 'no effect' +date: '2026-10-19' +experiments: +- X024 +confidence: low +--- + +## Statement +When both arms pass every paired task, compare must flag no headroom. + +Only source is the regrade in X024, which reports 0 mismatches against the product repo's graders; it does not itself test the headroom flag, so this is a design claim with low confidence. + +## Applies to +repo compare diff --git a/dokimi/research/records/finding/F019.md b/dokimi/research/records/finding/F019.md new file mode 100644 index 00000000..a76907b1 --- /dev/null +++ b/dokimi/research/records/finding/F019.md @@ -0,0 +1,18 @@ +--- +id: F019 +type: finding +title: Validation must live in our tooling +date: '2026-10-19' +experiments: +- X027 +confidence: med +--- + +## Statement +No trialled tool enforced typed references or enums on API write. + +## Applies to +Research register diff --git a/dokimi/research/records/idea/I001.md b/dokimi/research/records/idea/I001.md new file mode 100644 index 00000000..717970c2 --- /dev/null +++ b/dokimi/research/records/idea/I001.md @@ -0,0 +1,18 @@ +--- +id: I001 +type: idea +title: Absolute single-answer judge scoring +date: '2026-10-19' +theme: C +question: Can a judge score one answer on an absolute scale without saturating or rewarding length? +status: open +--- + +## Benefit +Arm comparisons need a trustworthy absolute score, not only pairwise preference. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I002.md b/dokimi/research/records/idea/I002.md new file mode 100644 index 00000000..efed2432 --- /dev/null +++ b/dokimi/research/records/idea/I002.md @@ -0,0 +1,18 @@ +--- +id: I002 +type: idea +title: Judge context in arm comparisons +date: '2026-10-19' +theme: C +question: Does what the judge sees per arm bias skill-vs-baseline verdicts? +status: open +--- + +## Benefit +Prevents false wins/losses for treatments that cite their own material. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I003.md b/dokimi/research/records/idea/I003.md new file mode 100644 index 00000000..e1cfe0f8 --- /dev/null +++ b/dokimi/research/records/idea/I003.md @@ -0,0 +1,18 @@ +--- +id: I003 +type: idea +title: 'Judge reliability: cross-judge, pairwise, sequential gates' +date: '2026-10-19' +theme: C +question: Which judge mode/model passes a validity gate and is stable across judges? +status: open +--- + +## Benefit +Cheaper and more reliable verdicts; know when a verdict is judge-dependent. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I004.md b/dokimi/research/records/idea/I004.md new file mode 100644 index 00000000..b508a50f --- /dev/null +++ b/dokimi/research/records/idea/I004.md @@ -0,0 +1,18 @@ +--- +id: I004 +type: idea +title: Malformed model rows must never read as results +date: '2026-10-19' +theme: C +question: Do parse failures in novelty/judge stages silently count as verdicts? +status: closed +--- + +## Benefit +Silent-failure removal across the pipeline. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I005.md b/dokimi/research/records/idea/I005.md new file mode 100644 index 00000000..75ebde9b --- /dev/null +++ b/dokimi/research/records/idea/I005.md @@ -0,0 +1,18 @@ +--- +id: I005 +type: idea +title: 'Novelty: what knowledge does a skill add' +date: '2026-10-19' +theme: K +question: How much of a skill's content is new to a model, and can it be measured cheaply? +status: open +--- + +## Benefit +Tells skill authors what is worth shipping. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I006.md b/dokimi/research/records/idea/I006.md new file mode 100644 index 00000000..d5986cf7 --- /dev/null +++ b/dokimi/research/records/idea/I006.md @@ -0,0 +1,18 @@ +--- +id: I006 +type: idea +title: Skill utility without novelty +date: '2026-10-19' +theme: K +question: How to evaluate skills whose value is workflow, automation or guardrails rather than new facts? +status: open +--- + +## Benefit +Avoids discarding useful skills that add no knowledge. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I007.md b/dokimi/research/records/idea/I007.md new file mode 100644 index 00000000..2d316b28 --- /dev/null +++ b/dokimi/research/records/idea/I007.md @@ -0,0 +1,18 @@ +--- +id: I007 +type: idea +title: Skill discovery and invocation +date: '2026-10-19' +theme: K +question: Does an installed skill help when the agent is not told it exists? +status: open +--- + +## Benefit +Measures real-world discovery, not treatment-notice effect. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I008.md b/dokimi/research/records/idea/I008.md new file mode 100644 index 00000000..3b9cd096 --- /dev/null +++ b/dokimi/research/records/idea/I008.md @@ -0,0 +1,18 @@ +--- +id: I008 +type: idea +title: Skill catalog integration +date: '2026-10-19' +theme: K +question: Run per-skill evals in a catalog's CI and promote skills to validated on evidence. +status: open +--- + +## Benefit +Evidence-backed skill catalogs. + +## Projects +dokimi, intel/skills, skill_search diff --git a/dokimi/research/records/idea/I009.md b/dokimi/research/records/idea/I009.md new file mode 100644 index 00000000..d5405000 --- /dev/null +++ b/dokimi/research/records/idea/I009.md @@ -0,0 +1,18 @@ +--- +id: I009 +type: idea +title: Agent guidance wording and placement +date: '2026-10-19' +theme: D +question: Which guidance file and wording actually change agent behaviour? +status: open +--- + +## Benefit +Evidence-based AGENTS.md / skill writing rules. + +## Projects +dokimi, oneDAL, sklearnex diff --git a/dokimi/research/records/idea/I010.md b/dokimi/research/records/idea/I010.md new file mode 100644 index 00000000..b78fe070 --- /dev/null +++ b/dokimi/research/records/idea/I010.md @@ -0,0 +1,18 @@ +--- +id: I010 +type: idea +title: Cold-agent onboarding +date: '2026-10-19' +theme: D +question: Can a fresh agent use dokimi from the README alone? +status: open +--- + +## Benefit +Lower adoption friction. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I011.md b/dokimi/research/records/idea/I011.md new file mode 100644 index 00000000..c70cb53a --- /dev/null +++ b/dokimi/research/records/idea/I011.md @@ -0,0 +1,18 @@ +--- +id: I011 +type: idea +title: 'Docs structure: Learn vs Use tracks' +date: '2026-10-19' +theme: D +question: Should docs split education and reference? +status: parked +--- + +## Benefit +Clearer adoption path. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I012.md b/dokimi/research/records/idea/I012.md new file mode 100644 index 00000000..06556f18 --- /dev/null +++ b/dokimi/research/records/idea/I012.md @@ -0,0 +1,18 @@ +--- +id: I012 +type: idea +title: Preparing a repository for coding agents +date: '2026-10-19' +theme: D +question: What does a repository offer an agent, statically? +status: open +--- + +## Benefit +Readiness diagnostics without a score. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I013.md b/dokimi/research/records/idea/I013.md new file mode 100644 index 00000000..e023f879 --- /dev/null +++ b/dokimi/research/records/idea/I013.md @@ -0,0 +1,18 @@ +--- +id: I013 +type: idea +title: 'Context failure modes: poisoning, distraction, confusion, clash' +date: '2026-10-19' +theme: G +question: How do agents fail under bad context, and which fixes work? +status: open +--- + +## Benefit +Operational context tooling. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I014.md b/dokimi/research/records/idea/I014.md new file mode 100644 index 00000000..12b35bad --- /dev/null +++ b/dokimi/research/records/idea/I014.md @@ -0,0 +1,18 @@ +--- +id: I014 +type: idea +title: Tool preference factors +date: '2026-10-19' +theme: G +question: What moves a model's choice between two equal tools? +status: open +--- + +## Benefit +MCP/tool authors learn which description changes matter. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I015.md b/dokimi/research/records/idea/I015.md new file mode 100644 index 00000000..9af3aafd --- /dev/null +++ b/dokimi/research/records/idea/I015.md @@ -0,0 +1,18 @@ +--- +id: I015 +type: idea +title: Result vs path stability +date: '2026-10-19' +theme: O +question: Is a stable agent path a correct one; how to measure process conformance from traces? +status: open +--- + +## Benefit +Separates result stability from path stability in evals. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I016.md b/dokimi/research/records/idea/I016.md new file mode 100644 index 00000000..af48d644 --- /dev/null +++ b/dokimi/research/records/idea/I016.md @@ -0,0 +1,18 @@ +--- +id: I016 +type: idea +title: Cost-aware eval and tokenomics +date: '2026-10-19' +theme: O +question: How should eval cost be capped and reported? +status: open +--- + +## Benefit +Predictable spend. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I017.md b/dokimi/research/records/idea/I017.md new file mode 100644 index 00000000..bd0396d4 --- /dev/null +++ b/dokimi/research/records/idea/I017.md @@ -0,0 +1,19 @@ +--- +id: I017 +type: idea +title: Repository eval mechanics in dokimi +date: '2026-10-19' +theme: V +question: Move generic repo-eval mechanics (compare, graders, host requirements) out of a product repo + into dokimi. +status: open +--- + +## Benefit +One harness, evals stay with each repo. + +## Projects +dokimi, oneDAL diff --git a/dokimi/research/records/idea/I018.md b/dokimi/research/records/idea/I018.md new file mode 100644 index 00000000..242785bb --- /dev/null +++ b/dokimi/research/records/idea/I018.md @@ -0,0 +1,18 @@ +--- +id: I018 +type: idea +title: Repo quality gates and measurement integrity +date: '2026-10-19' +theme: E +question: Which CI gates catch real defects in an eval framework? +status: open +--- + +## Benefit +Trustworthy numbers from the tool itself. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I019.md b/dokimi/research/records/idea/I019.md new file mode 100644 index 00000000..ab686e7b --- /dev/null +++ b/dokimi/research/records/idea/I019.md @@ -0,0 +1,18 @@ +--- +id: I019 +type: idea +title: Closed user loop and journeys +date: '2026-10-19' +theme: P +question: Can dokimi take a user from task to evidence to regression check with contracts per journey? +status: open +--- + +## Benefit +Product promise: make a skill or repo more useful and prove it. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I020.md b/dokimi/research/records/idea/I020.md new file mode 100644 index 00000000..503cc6cf --- /dev/null +++ b/dokimi/research/records/idea/I020.md @@ -0,0 +1,18 @@ +--- +id: I020 +type: idea +title: CLI restructure around tasks +date: '2026-10-19' +theme: P +question: Should 64 operations be regrouped into task-oriented groups? +status: parked +--- + +## Benefit +Discoverability. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I021.md b/dokimi/research/records/idea/I021.md new file mode 100644 index 00000000..84f0023a --- /dev/null +++ b/dokimi/research/records/idea/I021.md @@ -0,0 +1,18 @@ +--- +id: I021 +type: idea +title: Tool-independent research record register +date: '2026-10-19' +theme: W +question: Where should idea/experiment/finding/decision records live? +status: open +--- + +## Benefit +Negative results and decisions stay traceable; export drives the mindmap. + +## Projects +dokimi diff --git a/dokimi/research/records/implref/R001.md b/dokimi/research/records/implref/R001.md new file mode 100644 index 00000000..705339ea --- /dev/null +++ b/dokimi/research/records/implref/R001.md @@ -0,0 +1,17 @@ +--- +id: R001 +type: implref +title: Keep judge treatment material in the artifact +date: '2026-10-19' +decision: D003 +status: merged +links: +- https://github.com/intel/dokimi/issues/48 +- https://github.com/intel/dokimi/pull/69 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R002.md b/dokimi/research/records/implref/R002.md new file mode 100644 index 00000000..00831082 --- /dev/null +++ b/dokimi/research/records/implref/R002.md @@ -0,0 +1,17 @@ +--- +id: R002 +type: implref +title: judge-validity --context stored +date: '2026-10-19' +decision: D003 +status: open +links: +- https://github.com/intel/dokimi/issues/48 +- https://github.com/intel/dokimi/pull/73 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R003.md b/dokimi/research/records/implref/R003.md new file mode 100644 index 00000000..51bb3653 --- /dev/null +++ b/dokimi/research/records/implref/R003.md @@ -0,0 +1,16 @@ +--- +id: R003 +type: implref +title: Stamp judge_context from what the judge saw +date: '2026-10-19' +decision: D003 +status: merged +links: +- https://github.com/intel/dokimi/pull/13 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R006.md b/dokimi/research/records/implref/R006.md new file mode 100644 index 00000000..34a3c18b --- /dev/null +++ b/dokimi/research/records/implref/R006.md @@ -0,0 +1,16 @@ +--- +id: R006 +type: implref +title: Novelty coverage basis stamped; unmeasured share over readable words +date: '2026-10-19' +decision: D004 +status: merged +links: +- https://github.com/intel/dokimi/pull/101 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R010.md b/dokimi/research/records/implref/R010.md new file mode 100644 index 00000000..cbb59426 --- /dev/null +++ b/dokimi/research/records/implref/R010.md @@ -0,0 +1,16 @@ +--- +id: R010 +type: implref +title: AGENTS.md guidance +date: '2026-10-19' +decision: D006 +status: merged +links: +- https://github.com/intel/dokimi/pull/56 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R011.md b/dokimi/research/records/implref/R011.md new file mode 100644 index 00000000..f7bcf08b --- /dev/null +++ b/dokimi/research/records/implref/R011.md @@ -0,0 +1,16 @@ +--- +id: R011 +type: implref +title: dokimi's own repository evals in .agent-eval +date: '2026-10-19' +decision: D006 +status: merged +links: +- https://github.com/intel/dokimi/pull/57 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R016.md b/dokimi/research/records/implref/R016.md new file mode 100644 index 00000000..b32f03d5 --- /dev/null +++ b/dokimi/research/records/implref/R016.md @@ -0,0 +1,16 @@ +--- +id: R016 +type: implref +title: context trace +date: '2026-10-19' +decision: D008 +status: merged +links: +- https://github.com/intel/dokimi/pull/68 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R017.md b/dokimi/research/records/implref/R017.md new file mode 100644 index 00000000..35dcfc8b --- /dev/null +++ b/dokimi/research/records/implref/R017.md @@ -0,0 +1,16 @@ +--- +id: R017 +type: implref +title: repo budget +date: '2026-10-19' +decision: D008 +status: merged +links: +- https://github.com/intel/dokimi/pull/71 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R021.md b/dokimi/research/records/implref/R021.md new file mode 100644 index 00000000..e9180e03 --- /dev/null +++ b/dokimi/research/records/implref/R021.md @@ -0,0 +1,17 @@ +--- +id: R021 +type: implref +title: repo compare (D1) +date: '2026-10-19' +decision: D009 +status: merged +links: +- https://github.com/intel/dokimi/issues/41 +- https://github.com/intel/dokimi/pull/42 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R022.md b/dokimi/research/records/implref/R022.md new file mode 100644 index 00000000..6894ac75 --- /dev/null +++ b/dokimi/research/records/implref/R022.md @@ -0,0 +1,17 @@ +--- +id: R022 +type: implref +title: compare flags no headroom +date: '2026-10-19' +decision: D009 +status: open +links: +- https://github.com/intel/dokimi/issues/41 +- https://github.com/intel/dokimi/pull/114 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R023.md b/dokimi/research/records/implref/R023.md new file mode 100644 index 00000000..816a9e70 --- /dev/null +++ b/dokimi/research/records/implref/R023.md @@ -0,0 +1,17 @@ +--- +id: R023 +type: implref +title: task host requirements (D3) +date: '2026-10-19' +decision: D009 +status: merged +links: +- https://github.com/intel/dokimi/issues/41 +- https://github.com/intel/dokimi/pull/112 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R024.md b/dokimi/research/records/implref/R024.md new file mode 100644 index 00000000..c85bbc28 --- /dev/null +++ b/dokimi/research/records/implref/R024.md @@ -0,0 +1,17 @@ +--- +id: R024 +type: implref +title: reusable repo-eval workflow (D8) +date: '2026-10-19' +decision: D009 +status: merged +links: +- https://github.com/intel/dokimi/issues/41 +- https://github.com/intel/dokimi/pull/113 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R025.md b/dokimi/research/records/implref/R025.md new file mode 100644 index 00000000..d4e66144 --- /dev/null +++ b/dokimi/research/records/implref/R025.md @@ -0,0 +1,16 @@ +--- +id: R025 +type: implref +title: no mypy exemptions +date: '2026-10-19' +decision: D010 +status: merged +links: +- https://github.com/intel/dokimi/pull/108 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R026.md b/dokimi/research/records/implref/R026.md new file mode 100644 index 00000000..7de6f547 --- /dev/null +++ b/dokimi/research/records/implref/R026.md @@ -0,0 +1,16 @@ +--- +id: R026 +type: implref +title: gate tests that cannot fail +date: '2026-10-19' +decision: D010 +status: open +links: +- https://github.com/intel/dokimi/pull/103 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R027.md b/dokimi/research/records/implref/R027.md new file mode 100644 index 00000000..eba7509d --- /dev/null +++ b/dokimi/research/records/implref/R027.md @@ -0,0 +1,16 @@ +--- +id: R027 +type: implref +title: Architecture cleanup tracking +date: '2026-10-19' +decision: D010 +status: open +links: +- https://github.com/intel/dokimi/issues/4 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R028.md b/dokimi/research/records/implref/R028.md new file mode 100644 index 00000000..6149e04e --- /dev/null +++ b/dokimi/research/records/implref/R028.md @@ -0,0 +1,16 @@ +--- +id: R028 +type: implref +title: Outcome contracts in tests +date: '2026-10-19' +decision: D011 +status: merged +links: +- https://github.com/intel/dokimi/pull/74 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R029.md b/dokimi/research/records/implref/R029.md new file mode 100644 index 00000000..acad6c0b --- /dev/null +++ b/dokimi/research/records/implref/R029.md @@ -0,0 +1,17 @@ +--- +id: R029 +type: implref +title: User-journey catalog J1-J15 +date: '2026-10-19' +decision: D011 +status: merged +links: +- https://github.com/intel/dokimi/issues/95 +- https://github.com/intel/dokimi/pull/102 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R030.md b/dokimi/research/records/implref/R030.md new file mode 100644 index 00000000..22bbaf60 --- /dev/null +++ b/dokimi/research/records/implref/R030.md @@ -0,0 +1,16 @@ +--- +id: R030 +type: implref +title: 'User loop: plan/validate/compare/gate/record' +date: '2026-10-19' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/121 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R031.md b/dokimi/research/records/implref/R031.md new file mode 100644 index 00000000..52dbf000 --- /dev/null +++ b/dokimi/research/records/implref/R031.md @@ -0,0 +1,16 @@ +--- +id: R031 +type: implref +date: '2026-10-21' +title: 'Journeys: bind available but untraced criteria to tests' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/100 +--- + +## What +AC-J03-002, AC-J03-005, AC-J12-001, AC-J14-005 already hold; bind them to test ids in the journeys catalog. diff --git a/dokimi/research/records/implref/R032.md b/dokimi/research/records/implref/R032.md new file mode 100644 index 00000000..dcef840e --- /dev/null +++ b/dokimi/research/records/implref/R032.md @@ -0,0 +1,16 @@ +--- +id: R032 +type: implref +date: '2026-10-21' +title: 'J11/J12: distinct inconclusive verdict and blocking policy' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/99 +--- + +## What +A sample cap or budget stop is reported as inconclusive, never as pass, and a policy decides whether it blocks. diff --git a/dokimi/research/records/implref/R033.md b/dokimi/research/records/implref/R033.md new file mode 100644 index 00000000..3e103e12 --- /dev/null +++ b/dokimi/research/records/implref/R033.md @@ -0,0 +1,16 @@ +--- +id: R033 +type: implref +date: '2026-10-21' +title: 'J2: join the novelty diagnosis to a task-level effect' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/98 +--- + +## What +Report novelty next to a measured task effect; re-measure an edited skill against its original in one step. diff --git a/dokimi/research/records/implref/R034.md b/dokimi/research/records/implref/R034.md new file mode 100644 index 00000000..e52a24f1 --- /dev/null +++ b/dokimi/research/records/implref/R034.md @@ -0,0 +1,16 @@ +--- +id: R034 +type: implref +date: '2026-10-21' +title: 'J7: draft tasks, negative controls and coverage gaps' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/97 +--- + +## What +Draft tasks as hypotheses with their source; list the areas an eval set has no task for. diff --git a/dokimi/research/records/implref/R035.md b/dokimi/research/records/implref/R035.md new file mode 100644 index 00000000..16d0ea79 --- /dev/null +++ b/dokimi/research/records/implref/R035.md @@ -0,0 +1,16 @@ +--- +id: R035 +type: implref +date: '2026-10-21' +title: 'J4: adherence and task-impact evidence for instructions' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/96 +--- + +## What +Measure adherence to specific instructions apart from reach; report task impact of guidance arms. diff --git a/dokimi/research/records/implref/R036.md b/dokimi/research/records/implref/R036.md new file mode 100644 index 00000000..0ba8621b --- /dev/null +++ b/dokimi/research/records/implref/R036.md @@ -0,0 +1,16 @@ +--- +id: R036 +type: implref +date: '2026-10-21' +title: 'Skill catalog integration (intel/skills)' +decision: D014 +status: open +links: +- https://github.com/intel/dokimi/issues/138 +--- + +## What +Per-skill evals in catalog CI, published results, promotion to validated. diff --git a/dokimi/research/records/implref/R037.md b/dokimi/research/records/implref/R037.md new file mode 100644 index 00000000..d23165b6 --- /dev/null +++ b/dokimi/research/records/implref/R037.md @@ -0,0 +1,16 @@ +--- +id: R037 +type: implref +date: '2026-10-21' +title: 'CLI restructure: nine task-oriented groups' +decision: D015 +status: proposed +links: +- https://github.com/intel/dokimi/issues/94 +--- + +## What +Proposal and migration plan; no code yet. diff --git a/dokimi/research/records/implref/R038.md b/dokimi/research/records/implref/R038.md new file mode 100644 index 00000000..6198167b --- /dev/null +++ b/dokimi/research/records/implref/R038.md @@ -0,0 +1,16 @@ +--- +id: R038 +type: implref +date: '2026-10-21' +title: 'repo run: block fork-context skills under --delegation none' +decision: D016 +status: open +links: +- https://github.com/intel/dokimi/issues/148 +--- + +## What +Refuse or flag `context: fork` skills in a `--delegation none` cell. diff --git a/dokimi/research/records/implref/R039.md b/dokimi/research/records/implref/R039.md new file mode 100644 index 00000000..67e666d3 --- /dev/null +++ b/dokimi/research/records/implref/R039.md @@ -0,0 +1,16 @@ +--- +id: R039 +type: implref +title: 'Research register: records, validator, map' +date: '2026-10-21' +decision: D012 +status: open +links: +- https://github.com/intel/dokimi/pull/139 +--- + +## What +This register: schema, records, `check`/`query`/`export`/`render`/`gaps`, and the HTML map. diff --git a/dokimi/research/records/implref/R040.md b/dokimi/research/records/implref/R040.md new file mode 100644 index 00000000..abaf2c30 --- /dev/null +++ b/dokimi/research/records/implref/R040.md @@ -0,0 +1,16 @@ +--- +id: R040 +type: implref +title: 'Judge aggregate recomputed in code with correctness weighted x2' +date: '2026-10-21' +decision: D002 +status: merged +links: +- https://github.com/intel/dokimi/blob/main/dokimi/dokimi/eval/judge.py +--- + +## What +The aggregate is computed from the dimension scores in `eval/judge.py`, correctness at weight 2.0 and specificity outside the single-judge aggregate. Landed before the public history, so there is no pull request to link. diff --git a/dokimi/research/untracked.yaml b/dokimi/research/untracked.yaml new file mode 100644 index 00000000..065a1d1f --- /dev/null +++ b/dokimi/research/untracked.yaml @@ -0,0 +1,15 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# Open intel/dokimi issues that are engineering work, not research: no idea, +# experiment or decision behind them. `research_register.py gaps --live` does +# not list them as missing from the register, and reports an entry here whose +# issue has closed so the list does not go stale. +- issue: 19 + reason: docs build hook rewrites links on the element tree; docs tooling +- issue: 30 + reason: Renovate dependency dashboard +- issue: 61 + reason: hash the dependency lock; supply-chain hardening +- issue: 136 + reason: eval cross-judge drops the provider from the judge spec; a bug diff --git a/dokimi/scripts/research_map.html b/dokimi/scripts/research_map.html new file mode 100644 index 00000000..19014c21 --- /dev/null +++ b/dokimi/scripts/research_map.html @@ -0,0 +1,214 @@ + + + + + +Dokimi plans and research map + + + +
+

Plans and research map

+ + + + + + + + +
+
+
+ +
+
+ + + + diff --git a/dokimi/scripts/research_register.py b/dokimi/scripts/research_register.py new file mode 100755 index 00000000..d8f1faf7 --- /dev/null +++ b/dokimi/scripts/research_register.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Check, query and export the research register in ``research/records/``. + +Each record is one Markdown file, ``research/records//.md``: YAML +front matter with the typed fields of ``research/record.v1.json`` and a free +Markdown body (setup, result, rationale). Ideas, experiments, findings, +decisions and implementation references point at each other by id, so a +refuted experiment stays findable from the idea it tested. + + python scripts/research_register.py check + python scripts/research_register.py query --type experiment --where outcome=refuted,inconclusive + python scripts/research_register.py query --type experiment --ref I001 + python scripts/research_register.py next experiment + python scripts/research_register.py export > register.json + python scripts/research_register.py render > map.md + +``check`` exits 1 when a record breaks its schema, sits under the wrong path, +points at a missing record or one of the wrong type, or cites a retracted +experiment. ``export`` writes every record plus the reference edges and an idea +tree grouped by map theme. ``render`` turns that tree into the plans-and-research +map: a Mermaid mindmap and a table, as Markdown. +""" + +from __future__ import annotations + +import argparse +import io +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import jsonschema +import yaml + +PROJECT = Path(__file__).resolve().parents[1] +RESEARCH = PROJECT / "research" +SCHEMA = RESEARCH / "record.v1.json" +# The license header every file carries, as an HTML comment after the front matter. +_HEADER = re.compile(r"\A\s*\s*", re.DOTALL) + +PREFIX = {"idea": "I", "experiment": "X", "finding": "F", "decision": "D", "implref": "R"} +# field -> record type it must point at, per record type. +REFS: dict[str, dict[str, str]] = { + "experiment": {"idea": "idea"}, + "finding": {"experiments": "experiment"}, + "decision": {"findings": "finding", "ideas": "idea", "superseded_by": "decision"}, + "implref": {"decision": "decision"}, +} +THEMES = { + "P": "User workflows and integrations", + "E": "Engineering foundations", + "V": "Eval design, quality and infrastructure", + "K": "Skill scope, knowledge and utility", + "G": "Graphs, context and MCP", + "O": "Traces, optimization and visualization", + "C": "Judging and empirical validation", + "D": "Documentation and adoption", + "W": "Research workspace and records", +} + + +@dataclass +class Record: + fields: dict[str, Any] + body: str + path: Path + + @property + def id(self) -> str: + return str(self.fields.get("id", self.path.stem)) + + @property + def type(self) -> str: + return str(self.fields.get("type", "")) + + +def parse(path: Path) -> Record: + text = path.read_text(encoding="utf-8") + if not text.startswith("---\n") or "\n---\n" not in text[4:]: + raise ValueError(f"{path}: no YAML front matter") + front, body = text[4:].split("\n---\n", 1) + fields = yaml.safe_load(front) + if not isinstance(fields, dict): + raise ValueError(f"{path}: front matter is not a mapping") + # YAML reads an unquoted 2026-10-14 as a date; the schema wants the string. + fields = {k: (v.isoformat() if hasattr(v, "isoformat") else v) for k, v in fields.items()} + return Record(fields, _HEADER.sub("", body).strip(), path) + + +def load(root: Path = RESEARCH) -> tuple[dict[str, Record], list[str]]: + records: dict[str, Record] = {} + errors: list[str] = [] + base = root / "records" + for path in sorted(base.rglob("*")): + if path.is_dir() or path.name.startswith("."): + continue + if path.suffix != ".md" or len(path.relative_to(base).parts) != 2: + errors.append(f"{path}: not a record, records live at records//.md") + continue + try: + record = parse(path) + except (ValueError, yaml.YAMLError) as exc: + errors.append(str(exc)) + continue + if record.id in records: + errors.append(f"{record.id}: duplicate id in {path} and {records[record.id].path}") + continue + records[record.id] = record + return records, errors + + +def ref_values(record: Record, field: str) -> list[str]: + value = record.fields.get(field) + if value is None: + return [] + return list(value) if isinstance(value, list) else [value] + + +def check(records: dict[str, Record], schema: dict[str, Any]) -> list[str]: + errors: list[str] = [] + for rid, record in sorted(records.items()): + kind = record.type + if kind not in PREFIX: + errors.append(f"{rid}: unknown type {kind!r}") + continue + sub = {"$schema": schema["$schema"], "$defs": schema["$defs"], "$ref": f"#/$defs/{kind}"} + for err in jsonschema.Draft202012Validator(sub).iter_errors(record.fields): + errors.append(f"{rid}: {err.json_path}: {err.message}") + if record.path.parent.name != kind or record.path.stem != rid: + errors.append(f"{rid}: expected at records/{kind}/{rid}.md, found {record.path.name}") + if not rid.startswith(PREFIX[kind]): + errors.append(f"{rid}: a {kind} id starts with {PREFIX[kind]}") + for field, want in REFS.get(kind, {}).items(): + for target in ref_values(record, field): + if not isinstance(target, str): + continue # the schema already reports a non-id value + if target not in records: + errors.append(f"{rid}.{field}: no record {target}") + elif records[target].type != want: + errors.append(f"{rid}.{field}: {target} is {records[target].type}, " + f"want {want}") + state = record.fields.get("state") + if state == "superseded" and "superseded_by" not in record.fields: + errors.append(f"{rid}: a superseded decision names its superseded_by") + if "superseded_by" in record.fields and state != "superseded": + errors.append(f"{rid}: superseded_by is set but state is {state!r}, not 'superseded'") + if record.fields.get("superseded_by") == rid: + errors.append(f"{rid}: superseded_by names the record itself") + else: + seen = {rid} + step = record.fields.get("superseded_by") + while isinstance(step, str) and step in records and step not in seen: + seen.add(step) + step = records[step].fields.get("superseded_by") + if step == rid: + errors.append(f"{rid}: superseded_by chain loops back to {rid}") + for target in ref_values(record, "experiments"): + if not isinstance(target, str): + errors.append(f"{rid}.experiments: {target!r} is not a record id") + continue + if target in records and records[target].fields.get("retracted"): + errors.append(f"{rid}.experiments: {target} is retracted") + return errors + + +def _text(value: object) -> str: + # Spell booleans as the records do (`retracted: true`), not as Python's str() does. + return str(value).lower() if isinstance(value, bool) else str(value) + + +def matches(record: Record, where: list[str], ref: str | None, kind: str | None) -> bool: + if kind and record.type != kind: + return False + for clause in where: + field, _, wanted = clause.partition("=") + value = record.fields.get(field) + if value is None: + return False # a missing field matches no value, not the string "None" + values = [_text(v) for v in value] if isinstance(value, list) else [_text(value)] + if not set(values) & set(wanted.split(",")): + return False + if ref: + return any(ref in ref_values(record, f) for f in REFS.get(record.type, {})) + return True + + +def next_id(records: dict[str, Record], kind: str) -> str: + prefix = PREFIX[kind] + used = [int(r[len(prefix):]) for r in records if r.startswith(prefix) and r[len(prefix):].isdigit()] + return f"{prefix}{max(used, default=0) + 1:03d}" + + +def _git(root: Path, *args: str) -> str: + try: + return subprocess.run(["git", "-C", str(root), *args], + capture_output=True, text=True, check=False).stdout.strip() + except OSError: + return "" # git is not installed + + +_GITHUB = re.compile(r"^https://github\.com/([^/]+/[^/]+)/(issues|pull)/([0-9]+)") + + +def github_state(url: str) -> str | None: + """The live state of an issue or pull request: open, closed or merged; None if unknown.""" + match = _GITHUB.match(url) + if not match: + return None + repo, _, number = match.groups() + try: + out = subprocess.run(["gh", "api", f"repos/{repo}/issues/{number}", + "--jq", "[.state, .pull_request.merged_at] | @tsv"], + capture_output=True, text=True, check=False, timeout=20) + except (OSError, subprocess.TimeoutExpired): + return None # gh is not installed or the network is down + state, _, merged = out.stdout.strip().partition("\t") + if out.returncode or state not in ("open", "closed"): + return None + return "merged" if merged else state + + +def open_issues(repo: str) -> list[dict[str, Any]]: + """Open issues (not pull requests) of a GitHub repository, as number and title.""" + try: + out = subprocess.run(["gh", "api", "--paginate", f"repos/{repo}/issues?state=open&per_page=100", + "--jq", ".[] | select(.pull_request == null) | [.number, .title] | @tsv"], + capture_output=True, text=True, check=False, timeout=120) + except (OSError, subprocess.TimeoutExpired): + return [] + rows = [line.split("\t", 1) for line in out.stdout.splitlines() if "\t" in line] + return [{"number": int(n), "title": title} for n, title in rows] + + +def implementation(records: dict[str, Record], live: bool = False) -> dict[str, dict[str, Any]]: + """Per decision: its implrefs with recorded and (with live) GitHub state, and a summary. + + The summary is merged when any implref is merged, open when any is open or + proposed, closed when all are closed, and none without an implref. + """ + refs: dict[str, list[dict[str, Any]]] = {} + cache: dict[str, str | None] = {} + asked = 0 + for record in sorted(records.values(), key=lambda r: r.id): + if record.type != "implref": + continue + for url in record.fields["links"] if live else []: + if url not in cache: + cache[url] = github_state(url) + asked += 1 + states = [cache[u] for u in record.fields["links"]] if live else [] + known = [s for s in states if s] + # Several links: the most advanced one says how far the work got. + live_state = next((s for s in ("merged", "open", "closed") if s in known), None) + refs.setdefault(record.fields["decision"], []).append( + {"id": record.id, "title": record.fields["title"], "status": record.fields["status"], + "live": live_state, "links": record.fields["links"]}) + if live and asked and not any(cache.values()): + print("warning: --live got no state from GitHub (is gh installed and logged in?); " + "the recorded status is shown instead", file=sys.stderr) + result = {} + for rid, record in sorted(records.items()): + if record.type != "decision": + continue + mine = refs.get(rid, []) + effective = {(ref["live"] or ref["status"]) for ref in mine} + summary = ("merged" if "merged" in effective else + "open" if effective & {"open", "proposed"} else + "closed" if effective else "none") + result[rid] = {"state": record.fields["state"], "summary": summary, "implrefs": mine} + return result + + +def untracked(root: Path = RESEARCH) -> dict[int, str]: + """Issue number -> reason, from research/untracked.yaml: open issues that are not research.""" + path = root / "untracked.yaml" + if not path.exists(): + return {} + entries = yaml.safe_load(path.read_text(encoding="utf-8")) or [] + if not isinstance(entries, list) or not all( + isinstance(e, dict) and isinstance(e.get("issue"), int) and isinstance(e.get("reason"), str) + for e in entries): + raise ValueError(f"{path}: want a list of {{issue: , reason: }}") + return {e["issue"]: e["reason"] for e in entries} + + +def gaps(records: dict[str, Record], live: bool = False, repo: str = "intel/dokimi", + root: Path = RESEARCH) -> str: + skip = untracked(root) + impl = implementation(records, live) + lines = [] + todo = [(d, i) for d, i in impl.items() if i["state"] == "accepted" and i["summary"] != "merged"] + lines.append(f"Accepted decisions with nothing merged: {len(todo)}") + for did, info in todo: + refs = ", ".join(f"{r['id']} {r['live'] or r['status']}" for r in info["implrefs"]) or "no implref" + lines.append(f" {did}\t{info['summary']}\t{records[did].fields['title']}\t({refs})") + pending = [r for i in impl.values() for r in i["implrefs"] + if (r["live"] or r["status"]) in ("open", "proposed")] + lines.append(f"Implrefs not done: {len(pending)}") + lines += [f" {r['id']}\t{r['live'] or r['status']}\t{r['title']}" for r in pending] + if not live: + lines.append("(recorded status only; --live asks GitHub for drift and open issues)") + return "\n".join(lines) + "\n" + # "proposed" is how an open issue with no code yet is recorded, not drift. + drift = [r for i in impl.values() for r in i["implrefs"] if r["live"] and r["live"] != r["status"] + and (r["live"], r["status"]) != ("open", "proposed")] + lines.append(f"Implrefs whose recorded status differs from GitHub: {len(drift)}") + lines += [f" {r['id']}\trecorded {r['status']}, GitHub {r['live']}\t{r['title']}" for r in drift] + cited = {int(m.group(3)) for record in records.values() + for url in record.fields.get("links", []) + if (m := _GITHUB.match(url)) and m.group(1) == repo} + issues = open_issues(repo) + loose = [i for i in issues if i["number"] not in cited and i["number"] not in skip] + lines.append(f"Open {repo} issues no record links to: {len(loose)}") + lines += [f" #{i['number']}\t{i['title']}" for i in loose] + if issues: # an empty list means gh failed, not that every issue closed + gone = sorted(set(skip) - {i["number"] for i in issues}) + lines.append(f"research/untracked.yaml entries no longer open: {len(gone)}") + lines += [f" #{n}\t{skip[n]}" for n in gone] + return "\n".join(lines) + "\n" + + +def export(records: dict[str, Record], root: Path = RESEARCH) -> dict[str, Any]: + edges = [{"from": rid, "to": target, "rel": field} + for rid, record in sorted(records.items()) + for field in REFS.get(record.type, {}) + for target in ref_values(record, field)] + pointing: dict[str, list[str]] = {} + for edge in edges: + pointing.setdefault(edge["to"], []).append(edge["from"]) + themes = [] + for key, title in THEMES.items(): + ideas = [r for r in records.values() if r.type == "idea" and r.fields["theme"] == key] + themes.append({"id": key, "title": title, "ideas": [ + {"id": r.id, "title": r.fields["title"], "status": r.fields["status"], + "experiments": [{"id": x, "outcome": records[x].fields["outcome"], + "retracted": bool(records[x].fields.get("retracted"))} + for x in sorted(pointing.get(r.id, [])) + if records[x].type == "experiment"]} + for r in sorted(ideas, key=lambda r: r.id)]}) + commit = _git(root, "rev-parse", "HEAD") + # Records edited after that commit make the stamp a lie unless it says so. + dirty = bool(commit and _git(root, "status", "--porcelain", "--", ".")) + return {"commit": commit or None, "dirty": dirty, + "records": [{**r.fields, "body": r.body} for _, r in sorted(records.items())], + "edges": edges, "themes": themes} + + +OUTCOME_MARK = {"supported": "+", "refuted": "-", "inconclusive": "?", "aborted": "x", + "planned": "...", "running": "..."} + + +def _label(text: str) -> str: + # Mermaid mindmap labels break on brackets, parentheses and line breaks. + return " ".join(re.sub(r"[()\[\]{}]", "", text).split()) + + +def render(data: dict[str, Any]) -> str: + titles = {r["id"]: r["title"] for r in data["records"]} + stamp = f"`{data['commit'] or 'unknown'}`" + if data.get("dirty"): + stamp += " (with uncommitted changes)" + lines = ["# Plans and research map", "", + f"Generated by `scripts/research_register.py render` from commit " + f"{stamp}. Outcome marks: + supported, - refuted, " + "? inconclusive, x aborted, ... planned or running.", "", + "```mermaid", "mindmap", " root((Dokimi research))"] + rows = [] + for theme in data["themes"]: + if not theme["ideas"]: + continue + lines.append(f" {theme['id']}[{_label(theme['title'])}]") + for idea in theme["ideas"]: + lines.append(f" {idea['id']}[{idea['id']} {_label(idea['title'])} - {idea['status']}]") + for exp in idea["experiments"]: + mark = "retracted" if exp["retracted"] else OUTCOME_MARK[exp["outcome"]] + lines.append(f" {exp['id']}[{exp['id']} {mark} {_label(titles[exp['id']])}]") + outcome = exp["outcome"] + (", retracted" if exp["retracted"] else "") + rows.append(f"| {theme['id']} | {idea['id']} | {exp['id']} | {outcome} | " + f"{' '.join(titles[exp['id']].split()).replace('|', '/')} |") + if not idea["experiments"]: + rows.append(f"| {theme['id']} | {idea['id']} | | | " + f"{' '.join(idea['title'].split()).replace('|', '/')} |") + lines += ["```", "", "| Theme | Idea | Experiment | Outcome | Title |", + "|---|---|---|---|---|", *rows] + return "\n".join(lines) + "\n" + + +HTML_TEMPLATE = Path(__file__).with_name("research_map.html") + + +def render_html(data: dict[str, Any], impl: dict[str, dict[str, Any]]) -> str: + """One self-contained page: collapsible map, search, filters, table. No network needed.""" + payload = json.dumps({**data, "implementation": impl}, ensure_ascii=False) + # A record body cannot close the script element that carries it, nor open a comment + # (\n## Result\nworse") + _write(tmp_path, "experiment", "X002", + "title: Run 2\nidea: I001\nhypothesis: H\noutcome: supported\n") + _write(tmp_path, "finding", "F001", "title: F\nexperiments: [X001]\nconfidence: med\n") + _write(tmp_path, "decision", "D001", "title: D\nfindings: [F001]\nstate: rejected\n") + return tmp_path + + +def test_repository_register_is_valid() -> None: + assert rr.main(["check"]) == 0 + + +def test_valid_register_passes(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "check"]) == 0 + assert "OK: 5 records" in capsys.readouterr().out + + +@pytest.mark.parametrize(("kind", "rid", "front", "message"), [ + ("experiment", "X003", "title: T\nidea: I009\nhypothesis: H\noutcome: refuted\n", + "no record I009"), + ("experiment", "X003", "title: T\nidea: F001\nhypothesis: H\noutcome: refuted\n", + "F001 is finding, want idea"), + ("experiment", "X003", "title: T\nidea: I001\nhypothesis: H\noutcome: maybe\n", + "'maybe' is not one of"), + ("experiment", "X003", "title: T\nidea: I001\nhypothesis: H\noutcome: refuted\nextra: 1\n", + "Unevaluated properties"), + ("decision", "D002", "title: T\nstate: superseded\n", "names its superseded_by"), + ("finding", "X009", "title: T\nexperiments: [X001]\nconfidence: low\n", + "a finding id starts with F"), + ("idea", "I002", "title: T\ntheme: O\nquestion: Q\nstatus: open\nlinks: [https://example.com/x]\n", + "does not match"), + ("experiment", "X003", "title: T\nidea: [[I001]]\nhypothesis: H\noutcome: refuted\n", + "is not of type 'string'"), + ("decision", "D002", "title: T\nstate: accepted\nsuperseded_by: D001\n", + "superseded_by is set but state is 'accepted'"), + ("decision", "D002", "title: T\nstate: superseded\nsuperseded_by: D002\n", + "names the record itself"), + ("idea", "I002", "title: T\ntheme: O\nquestion: Q\nstatus: open\nextra: 1\n", + "Unevaluated properties"), + ("finding", "F002", "title: T\nexperiments: [X001]\nconfidence: low\nextra: 1\n", + "Unevaluated properties"), + ("decision", "D002", "title: T\nstate: accepted\nextra: 1\n", "Unevaluated properties"), + ("implref", "R001", "title: T\ndecision: D001\nstatus: open\nlinks: []\nextra: 1\n", + "Unevaluated properties"), + ("finding", "F002", "title: T\nexperiments: [X009]\nconfidence: low\n", "no record X009"), + ("finding", "F002", "title: T\nexperiments: [I001]\nconfidence: low\n", + "I001 is idea, want experiment"), + ("finding", "F002", "title: T\nexperiments: [[X001]]\nconfidence: low\n", + "F002.experiments: ['X001'] is not a record id"), + ("finding", "F002", "title: T\nexperiments: [{a: 1}]\nconfidence: low\n", + "F002.experiments: {'a': 1} is not a record id"), + ("decision", "D002", "title: T\nfindings: [F009]\nstate: accepted\n", "no record F009"), + ("decision", "D002", "title: T\nfindings: [X001]\nstate: accepted\n", + "X001 is experiment, want finding"), + ("decision", "D002", "title: T\nideas: [I009]\nstate: accepted\n", "no record I009"), + ("decision", "D002", "title: T\nideas: [F001]\nstate: accepted\n", + "F001 is finding, want idea"), + ("decision", "D002", "title: T\nstate: superseded\nsuperseded_by: D009\n", + "no record D009"), + ("decision", "D002", "title: T\nstate: superseded\nsuperseded_by: F001\n", + "F001 is finding, want decision"), + ("implref", "R001", "title: T\ndecision: D009\nstatus: open\nlinks: []\n", "no record D009"), + ("implref", "R001", "title: T\ndecision: F001\nstatus: open\nlinks: []\n", + "F001 is finding, want decision"), + ("implref", "R001", "title: T\ndecision: [[D001]]\nstatus: open\nlinks: []\n", + "is not of type 'string'"), +]) +def test_bad_record_fails(root: Path, capsys: pytest.CaptureFixture[str], + kind: str, rid: str, front: str, message: str) -> None: + _write(root, kind, rid, front) + assert rr.main(["--root", str(root), "check"]) == 1 + assert message in capsys.readouterr().out + + +def test_supersession_cycle_fails(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "decision", "D002", "title: T\nstate: superseded\nsuperseded_by: D003\n") + _write(root, "decision", "D003", "title: T\nstate: superseded\nsuperseded_by: D002\n") + assert rr.main(["--root", str(root), "check"]) == 1 + out = capsys.readouterr().out + assert "D002: superseded_by chain loops back to D002" in out + assert "D003: superseded_by chain loops back to D003" in out + + +@pytest.mark.parametrize("link", [ + "https://github.com/intel/x https://example.com/secret", + "https://github.com/intel/../../other/repo", + "https://github.com/intel/x/..", + "https://github.com/intel/%2e%2e/other/repo", + "https://github.com/intel/..\\\\other/repo", + "https://github.com/intel/x\\nhttps://example.com/secret", +]) +def test_link_outside_public_repos_fails(root: Path, capsys: pytest.CaptureFixture[str], + link: str) -> None: + _write(root, "idea", "I002", + f"title: T\ntheme: O\nquestion: Q\nstatus: open\nlinks: [\"{link}\"]\n") + assert rr.main(["--root", str(root), "check"]) == 1 + assert "I002: $.links[0]: " in capsys.readouterr().out + + +def test_public_repo_links_pass(root: Path) -> None: + _write(root, "idea", "I002", "title: T\ntheme: O\nquestion: Q\nstatus: open\nlinks: [" + "\"https://github.com/intel/dokimi/compare/main...napetrov:dokimi:research-records\", " + "\"https://github.com/intel/dokimi/pulls?q=is:pr+label:research#top\", " + "\"https://github.com/napetrov/dokimi/blob/main/docs/a_b-c.md#L1-L2\"]\n") + assert rr.main(["--root", str(root), "check"]) == 0 + + +def test_duplicate_id_fails(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + (root / "records" / "idea" / "I001.md").rename(root / "records" / "idea" / "copy.md") + _write(root, "idea", "I001", "title: Again\ntheme: O\nquestion: Q?\nstatus: open\n") + assert rr.main(["--root", str(root), "check"]) == 1 + assert "duplicate id" in capsys.readouterr().out + + +def test_query_negative_outcomes_for_idea(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "query", "--type", "experiment", "--ref", "I001", + "--where", "outcome=refuted,inconclusive"]) == 0 + out = capsys.readouterr().out.splitlines() + assert [line.split("\t")[0] for line in out] == ["X001"] + + +def test_query_ref_and_type_both_filter(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "idea", "I002", "title: Other\ntheme: O\nquestion: Q?\nstatus: open\n") + _write(root, "experiment", "X003", "title: Run 3\nidea: I002\nhypothesis: H\noutcome: refuted\n") + + def listed(*argv: str) -> list[str]: + capsys.readouterr() + assert rr.main(["--root", str(root), "query", *argv]) == 0 + return [line.split("\t")[0] for line in capsys.readouterr().out.splitlines()] + + assert listed("--ref", "I001") == ["X001", "X002"] + assert listed("--ref", "I002") == ["X003"] + assert listed("--type", "idea") == ["I001", "I002"] + assert listed("--type", "experiment", "--ref", "I002") == ["X003"] + assert listed("--type", "finding", "--ref", "I001") == [] + + +def test_misplaced_record_fails(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "idea", "I002", "title: T\ntheme: O\nquestion: Q\nstatus: open\n") + (root / "records" / "idea" / "I002.md").rename(root / "records" / "finding" / "I002.md") + assert rr.main(["--root", str(root), "check"]) == 1 + assert "expected at records/idea/I002.md, found I002.md" in capsys.readouterr().out + + +@pytest.mark.parametrize("where", ["X003.md", "experiment/sub/X003.md", "experiment/X003.markdown"]) +def test_file_outside_type_dir_fails(root: Path, capsys: pytest.CaptureFixture[str], + where: str) -> None: + path = root / "records" / where + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("---\nid: X003\n---\n") + assert rr.main(["--root", str(root), "check"]) == 1 + assert "not a record, records live at records//.md" in capsys.readouterr().out + + +def test_wrong_filename_fails(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "idea", "I002", "title: T\ntheme: O\nquestion: Q\nstatus: open\n") + (root / "records" / "idea" / "I002.md").rename(root / "records" / "idea" / "I009.md") + assert rr.main(["--root", str(root), "check"]) == 1 + assert "expected at records/idea/I002.md, found I009.md" in capsys.readouterr().out + + +def test_next_id_skips_gaps(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "experiment", "X005", "title: Run 5\nidea: I001\nhypothesis: H\noutcome: planned\n") + assert rr.main(["--root", str(root), "next", "experiment"]) == 0 + assert capsys.readouterr().out.strip() == "X006" + + +def test_next_id(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "next", "experiment"]) == 0 + assert capsys.readouterr().out.strip() == "X003" + + +@pytest.mark.parametrize("argv", [["query"], ["next", "experiment"], ["export"]]) +def test_commands_refuse_broken_register(root: Path, capsys: pytest.CaptureFixture[str], + argv: list[str]) -> None: + _write(root, "idea", "I002", "title: No theme\nquestion: Q\nstatus: open\n") + assert rr.main(["--root", str(root), *argv]) == 1 + captured = capsys.readouterr() + assert "I002: $: 'theme' is a required property" in captured.err + assert captured.out == "" + + +def test_export_refuses_record_without_theme_status(root: Path, + capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "idea", "I002", "title: No status\ntheme: O\nquestion: Q\n") + assert rr.main(["--root", str(root), "export"]) == 1 + assert "'status' is a required property" in capsys.readouterr().err + + +def test_export_tree(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "export"]) == 0 + data = json.loads(capsys.readouterr().out) + assert {"from": "X001", "to": "I001", "rel": "idea"} in data["edges"] + theme = next(t for t in data["themes"] if t["id"] == "O") + assert theme["ideas"][0]["experiments"] == [ + {"id": "X001", "outcome": "refuted", "retracted": False}, + {"id": "X002", "outcome": "supported", "retracted": False}] + assert next(r for r in data["records"] if r["id"] == "X001")["body"] == "## Result\nworse" + + +def test_export_ignores_decision_pointing_at_idea(root: Path, + capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "decision", "D002", "title: Uses idea\nfindings: [F001]\nideas: [I001]\n" + "state: accepted\n") + assert rr.main(["--root", str(root), "export"]) == 0 + data = json.loads(capsys.readouterr().out) + theme = next(t for t in data["themes"] if t["id"] == "O") + assert theme["ideas"][0]["experiments"] == [ + {"id": "X001", "outcome": "refuted", "retracted": False}, + {"id": "X002", "outcome": "supported", "retracted": False}] + + +@pytest.mark.parametrize("clause", ["outcom=refuted", "noequals", "bogus=None"]) +def test_query_rejects_bad_where_clause(root: Path, capsys: pytest.CaptureFixture[str], + clause: str) -> None: + assert rr.main(["--root", str(root), "query", "--where", clause]) == 2 + captured = capsys.readouterr() + assert clause in captured.err + assert captured.out == "" + + +def test_query_missing_field_matches_no_value(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "query", "--type", "experiment", + "--where", "retracted=None"]) == 0 + assert capsys.readouterr().out == "" + + +def test_query_finds_and_marks_retracted(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "experiment", "X002", + "title: Run 2\nidea: I001\nhypothesis: H\noutcome: supported\nretracted: true\n") + assert rr.main(["--root", str(root), "query", "--where", "retracted=true"]) == 0 + assert capsys.readouterr().out == "X002\texperiment\tsupported, retracted\tRun 2\n" + assert rr.main(["--root", str(root), "query", "--where", "outcome=supported"]) == 0 + assert capsys.readouterr().out == "X002\texperiment\tsupported, retracted\tRun 2\n" + + +def test_query_rejects_unknown_ref(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "query", "--ref", "I009"]) == 2 + assert "no record I009" in capsys.readouterr().err + + +def test_citing_retracted_experiment_fails(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "experiment", "X001", + "title: Run 1\nidea: I001\nhypothesis: H\noutcome: refuted\nretracted: true\n") + assert rr.main(["--root", str(root), "check"]) == 1 + assert "F001.experiments: X001 is retracted" in capsys.readouterr().out + + +def test_export_marks_uncommitted_records(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + def git(*args: str) -> None: + subprocess.run(["git", "-C", str(root), "-c", "user.name=t", "-c", "user.email=t@example.com", + "-c", "commit.gpgsign=false", *args], check=True, capture_output=True) + + git("init", "-q") + git("add", "-A") + git("commit", "-q", "-m", "records") + assert rr.main(["--root", str(root), "export"]) == 0 + clean = json.loads(capsys.readouterr().out) + assert clean["commit"] and clean["dirty"] is False + _write(root, "experiment", "X002", + "title: Run 2\nidea: I001\nhypothesis: H\noutcome: refuted\n") + assert rr.main(["--root", str(root), "export"]) == 0 + edited = json.loads(capsys.readouterr().out) + assert (edited["commit"], edited["dirty"]) == (clean["commit"], True) + assert rr.main(["--root", str(root), "render"]) == 0 + assert f"`{clean['commit']}` (with uncommitted changes)." in capsys.readouterr().out + + +def test_export_without_git_installed(root: Path, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch) -> None: + def missing(*args: object, **kwargs: object) -> None: + raise FileNotFoundError("git") + + monkeypatch.setattr(rr.subprocess, "run", missing) + assert rr.main(["--root", str(root), "export"]) == 0 + assert json.loads(capsys.readouterr().out)["commit"] is None + + +def test_export_and_render_mark_retracted(root: Path, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(rr.subprocess, "run", lambda *a, **k: rr.subprocess.CompletedProcess(a, 0, "")) + _write(root, "experiment", "X003", + "title: Run [3] a|b\nidea: I001\nhypothesis: H\noutcome: supported\nretracted: true\n") + _write(root, "idea", "I002", "title: Bare (idea)\ntheme: O\nquestion: Q?\nstatus: open\n") + assert rr.main(["--root", str(root), "export"]) == 0 + theme = next(t for t in json.loads(capsys.readouterr().out)["themes"] if t["id"] == "O") + assert theme["ideas"][0]["experiments"][2] == {"id": "X003", "outcome": "supported", + "retracted": True} + assert rr.main(["--root", str(root), "render"]) == 0 + out = capsys.readouterr().out + assert " X003[X003 retracted Run 3 a|b]\n I002[I002 Bare idea - open]\n" in out + assert "| O | I001 | X003 | supported, retracted | Run [3] a/b |" in out + assert "| O | I002 | | | Bare (idea) |" in out + + +@pytest.mark.parametrize("command", ["export", "query", "render"]) +def test_output_is_utf8_under_a_legacy_code_page(root: Path, command: str) -> None: + (root / "records" / "experiment" / "X002.md").write_text( + "---\nid: X002\ntype: experiment\ndate: 2026-10-14\ntitle: Run 2 → ±\nidea: I001\n" + "hypothesis: H\noutcome: supported\n---\n", encoding="utf-8") + script = PROJECT / "scripts" / "research_register.py" + result = subprocess.run([sys.executable, str(script), "--root", str(root), command], + capture_output=True, check=False, + env={**os.environ, "PYTHONIOENCODING": "cp1252"}) + assert result.returncode == 0, result.stderr + out = result.stdout.decode("utf-8") + if command == "export": + assert next(r for r in json.loads(out)["records"] if r["id"] == "X002")["title"] == \ + "Run 2 → ±" + else: + assert "Run 2 → ±" in out + + +def test_render_map(root: Path, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(rr.subprocess, "run", lambda *a, **k: rr.subprocess.CompletedProcess(a, 0, "")) + assert rr.main(["--root", str(root), "render"]) == 0 + out = capsys.readouterr().out + assert "```mermaid\nmindmap\n root((Dokimi research))\n" \ + " O[Traces, optimization and visualization]\n" \ + " I001[I001 Compress - open]\n" \ + " X001[X001 - Run 1]\n" \ + " X002[X002 + Run 2]\n```" in out + assert "| O | I001 | X001 | refuted | Run 1 |" in out + assert "from commit `unknown`" in out + + +def test_render_flattens_multiline_title(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "idea", "I002", + "title: >\n Folded title\n over lines\ntheme: O\nquestion: Q?\nstatus: open\n") + _write(root, "experiment", "X003", + "title: >\n Run\n split\nidea: I002\nhypothesis: H\noutcome: refuted\n") + assert rr.main(["--root", str(root), "render"]) == 0 + out = capsys.readouterr().out + assert "I002[I002 Folded title over lines - open]" in out + assert "X003[X003 - Run split]" in out + assert all(line.count("[") == line.count("]") for line in out.splitlines()) + assert "| O | I002 | X003 | refuted | Run split |" in out + + +def _implref(root: Path, rid: str, decision: str, status: str, number: int) -> None: + _write(root, "implref", rid, f"title: Build {rid}\ndecision: {decision}\nstatus: {status}\n" + f"links: [https://github.com/intel/dokimi/pull/{number}]\n") + + +@pytest.fixture +def built(root: Path) -> Path: + _write(root, "decision", "D002", "title: Do it\nfindings: [F001]\nstate: accepted\n") + _write(root, "decision", "D003", "title: Done\nfindings: [F001]\nstate: accepted\n") + _implref(root, "R001", "D002", "open", 1) + _implref(root, "R002", "D003", "merged", 2) + return root + + +def test_implementation_summary(built: Path) -> None: + records, _ = rr.load(built) + impl = rr.implementation(records) + assert {d: i["summary"] for d, i in impl.items()} == { + "D001": "none", "D002": "open", "D003": "merged"} + assert impl["D002"]["implrefs"][0]["live"] is None + + +def test_gaps_offline(built: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(built), "gaps"]) == 0 + out = capsys.readouterr().out + assert "Accepted decisions with nothing merged: 1\n D002\topen\tDo it\t(R001 open)" in out + assert "Implrefs not done: 1\n R001\topen\tBuild R001" in out + assert "--live asks GitHub" in out + + +def test_gaps_live_reports_drift_and_unlinked_issues(built: Path, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(rr, "github_state", lambda url: "merged") + monkeypatch.setattr(rr, "open_issues", lambda repo: [{"number": 1, "title": "linked"}, + {"number": 7, "title": "Loose end"}]) + assert rr.main(["--root", str(built), "gaps", "--live"]) == 0 + out = capsys.readouterr().out + assert "Accepted decisions with nothing merged: 0" in out + assert "R001\trecorded open, GitHub merged\tBuild R001" in out + assert "Open intel/dokimi issues no record links to: 1\n #7\tLoose end" in out + + +def test_gaps_live_skips_untracked_and_reports_stale_entries( + built: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch) -> None: + (built / "untracked.yaml").write_text("- issue: 7\n reason: chore\n- issue: 9\n reason: done\n") + monkeypatch.setattr(rr, "github_state", lambda url: None) + monkeypatch.setattr(rr, "open_issues", lambda repo: [{"number": 7, "title": "Loose end"}]) + assert rr.main(["--root", str(built), "gaps", "--live"]) == 0 + out = capsys.readouterr().out + assert "no record links to: 0" in out + assert "untracked.yaml entries no longer open: 1\n #9\tdone" in out + + +def test_gaps_rejects_malformed_untracked(built: Path, capsys: pytest.CaptureFixture[str]) -> None: + (built / "untracked.yaml").write_text("- issue: seven\n") + assert rr.main(["--root", str(built), "gaps"]) == 1 + assert "want a list of" in capsys.readouterr().err + + +def test_repository_untracked_list_is_valid() -> None: + assert rr.untracked() + + +def test_github_state_without_gh(monkeypatch: pytest.MonkeyPatch) -> None: + def missing(*a: object, **k: object) -> None: + raise FileNotFoundError("gh") + monkeypatch.setattr(rr.subprocess, "run", missing) + assert rr.github_state("https://github.com/intel/dokimi/pull/1") is None + assert rr.github_state("https://github.com/intel/dokimi/blob/main/x") is None + assert rr.open_issues("intel/dokimi") == [] + + +def test_render_html_embeds_data_safely(built: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(built, "experiment", "X009", "title: Run 9\nidea: I001\nhypothesis: H\noutcome: planned\n", + "") + assert rr.main(["--root", str(built), "render", "--html"]) == 0 + page = capsys.readouterr().out + assert page.count("") == 2 # the data block and the code block, nothing from a body + data = page.split('type="application/json">', 1)[1].split("", 1)[0] + loaded = json.loads(data) + assert loaded["implementation"]["D002"]["summary"] == "open" + assert next(r for r in loaded["records"] if r["id"] == "X009")["body"].startswith("") + assert "/*DATA*/" not in page + + +def test_render_html_escapes_comment_opener(built: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(built, "experiment", "X009", "title: Run 9\nidea: I001\nhypothesis: H\noutcome: planned\n", + "