From 5dbb1127dd7a144fea624354c8ca6b5a2a87096b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 19 Oct 2026 23:14:13 -0700 Subject: [PATCH 01/21] Add a research register for Dokimi's ideas, experiments and decisions Records live as Markdown with typed YAML front matter under dokimi/research/records//.md, validated by research/record.v1.json. scripts/research_register.py checks schemas, paths and cross-references, queries by field or reference, hands out the next id, and exports records, edges and a per-theme idea tree for the plans-and-research map. Negative and inconclusive outcomes are first-class; experiment outcome, decision state and implementation status are separate fields. Seeds 98 records from the project's public PRs/issues and local research notes; AGENTS.md names the register as the one exception to the no-status-reports rule. Signed-off-by: Nikolay Petrov --- AGENTS.md | 4 +- dokimi/docs/README.md | 2 + dokimi/docs/research-register.md | 82 ++++++++ dokimi/mkdocs.yml | 1 + dokimi/research/record.v1.json | 89 ++++++++ dokimi/research/records/decision/D001.md | 16 ++ dokimi/research/records/decision/D002.md | 15 ++ dokimi/research/records/decision/D003.md | 15 ++ dokimi/research/records/decision/D004.md | 15 ++ dokimi/research/records/decision/D005.md | 15 ++ dokimi/research/records/decision/D006.md | 15 ++ dokimi/research/records/decision/D007.md | 15 ++ dokimi/research/records/decision/D008.md | 16 ++ dokimi/research/records/decision/D009.md | 15 ++ dokimi/research/records/decision/D010.md | 13 ++ dokimi/research/records/decision/D011.md | 13 ++ dokimi/research/records/decision/D012.md | 15 ++ dokimi/research/records/decision/D013.md | 15 ++ dokimi/research/records/experiment/X001.md | 21 ++ dokimi/research/records/experiment/X002.md | 21 ++ dokimi/research/records/experiment/X003.md | 21 ++ dokimi/research/records/experiment/X004.md | 21 ++ dokimi/research/records/experiment/X005.md | 21 ++ dokimi/research/records/experiment/X006.md | 22 ++ dokimi/research/records/experiment/X007.md | 21 ++ dokimi/research/records/experiment/X008.md | 21 ++ dokimi/research/records/experiment/X009.md | 21 ++ dokimi/research/records/experiment/X010.md | 21 ++ dokimi/research/records/experiment/X011.md | 21 ++ dokimi/research/records/experiment/X012.md | 22 ++ dokimi/research/records/experiment/X013.md | 20 ++ dokimi/research/records/experiment/X014.md | 21 ++ dokimi/research/records/experiment/X015.md | 21 ++ dokimi/research/records/experiment/X016.md | 21 ++ dokimi/research/records/experiment/X017.md | 21 ++ dokimi/research/records/experiment/X018.md | 21 ++ dokimi/research/records/experiment/X019.md | 21 ++ dokimi/research/records/experiment/X020.md | 21 ++ dokimi/research/records/experiment/X021.md | 21 ++ dokimi/research/records/experiment/X022.md | 21 ++ dokimi/research/records/experiment/X023.md | 21 ++ dokimi/research/records/experiment/X024.md | 20 ++ dokimi/research/records/experiment/X025.md | 21 ++ dokimi/research/records/experiment/X026.md | 20 ++ dokimi/research/records/experiment/X027.md | 21 ++ dokimi/research/records/finding/F001.md | 18 ++ dokimi/research/records/finding/F002.md | 18 ++ dokimi/research/records/finding/F003.md | 19 ++ dokimi/research/records/finding/F004.md | 18 ++ dokimi/research/records/finding/F005.md | 19 ++ dokimi/research/records/finding/F006.md | 18 ++ dokimi/research/records/finding/F007.md | 18 ++ dokimi/research/records/finding/F008.md | 18 ++ dokimi/research/records/finding/F009.md | 18 ++ dokimi/research/records/finding/F010.md | 18 ++ dokimi/research/records/finding/F011.md | 19 ++ dokimi/research/records/finding/F012.md | 18 ++ dokimi/research/records/finding/F013.md | 18 ++ dokimi/research/records/finding/F014.md | 18 ++ dokimi/research/records/finding/F015.md | 18 ++ dokimi/research/records/finding/F016.md | 18 ++ dokimi/research/records/finding/F017.md | 18 ++ dokimi/research/records/finding/F018.md | 18 ++ dokimi/research/records/finding/F019.md | 18 ++ dokimi/research/records/idea/I001.md | 18 ++ dokimi/research/records/idea/I002.md | 18 ++ dokimi/research/records/idea/I003.md | 18 ++ dokimi/research/records/idea/I004.md | 18 ++ dokimi/research/records/idea/I005.md | 18 ++ dokimi/research/records/idea/I006.md | 18 ++ dokimi/research/records/idea/I007.md | 18 ++ dokimi/research/records/idea/I008.md | 18 ++ dokimi/research/records/idea/I009.md | 18 ++ dokimi/research/records/idea/I010.md | 18 ++ dokimi/research/records/idea/I011.md | 18 ++ dokimi/research/records/idea/I012.md | 18 ++ dokimi/research/records/idea/I013.md | 18 ++ dokimi/research/records/idea/I014.md | 18 ++ dokimi/research/records/idea/I015.md | 18 ++ dokimi/research/records/idea/I016.md | 18 ++ dokimi/research/records/idea/I017.md | 19 ++ dokimi/research/records/idea/I018.md | 18 ++ dokimi/research/records/idea/I019.md | 18 ++ dokimi/research/records/idea/I020.md | 18 ++ dokimi/research/records/idea/I021.md | 18 ++ dokimi/research/records/implref/R001.md | 17 ++ dokimi/research/records/implref/R002.md | 17 ++ dokimi/research/records/implref/R003.md | 16 ++ dokimi/research/records/implref/R006.md | 16 ++ dokimi/research/records/implref/R010.md | 16 ++ dokimi/research/records/implref/R011.md | 16 ++ dokimi/research/records/implref/R016.md | 16 ++ dokimi/research/records/implref/R017.md | 16 ++ dokimi/research/records/implref/R021.md | 17 ++ dokimi/research/records/implref/R022.md | 17 ++ dokimi/research/records/implref/R023.md | 17 ++ dokimi/research/records/implref/R024.md | 17 ++ dokimi/research/records/implref/R025.md | 16 ++ dokimi/research/records/implref/R026.md | 16 ++ dokimi/research/records/implref/R027.md | 16 ++ dokimi/research/records/implref/R028.md | 16 ++ dokimi/research/records/implref/R029.md | 17 ++ dokimi/research/records/implref/R030.md | 16 ++ dokimi/scripts/research_register.py | 225 +++++++++++++++++++++ dokimi/tests/test_research_register.py | 103 ++++++++++ 105 files changed, 2283 insertions(+), 1 deletion(-) create mode 100644 dokimi/docs/research-register.md create mode 100644 dokimi/research/record.v1.json create mode 100644 dokimi/research/records/decision/D001.md create mode 100644 dokimi/research/records/decision/D002.md create mode 100644 dokimi/research/records/decision/D003.md create mode 100644 dokimi/research/records/decision/D004.md create mode 100644 dokimi/research/records/decision/D005.md create mode 100644 dokimi/research/records/decision/D006.md create mode 100644 dokimi/research/records/decision/D007.md create mode 100644 dokimi/research/records/decision/D008.md create mode 100644 dokimi/research/records/decision/D009.md create mode 100644 dokimi/research/records/decision/D010.md create mode 100644 dokimi/research/records/decision/D011.md create mode 100644 dokimi/research/records/decision/D012.md create mode 100644 dokimi/research/records/decision/D013.md create mode 100644 dokimi/research/records/experiment/X001.md create mode 100644 dokimi/research/records/experiment/X002.md create mode 100644 dokimi/research/records/experiment/X003.md create mode 100644 dokimi/research/records/experiment/X004.md create mode 100644 dokimi/research/records/experiment/X005.md create mode 100644 dokimi/research/records/experiment/X006.md create mode 100644 dokimi/research/records/experiment/X007.md create mode 100644 dokimi/research/records/experiment/X008.md create mode 100644 dokimi/research/records/experiment/X009.md create mode 100644 dokimi/research/records/experiment/X010.md create mode 100644 dokimi/research/records/experiment/X011.md create mode 100644 dokimi/research/records/experiment/X012.md create mode 100644 dokimi/research/records/experiment/X013.md create mode 100644 dokimi/research/records/experiment/X014.md create mode 100644 dokimi/research/records/experiment/X015.md create mode 100644 dokimi/research/records/experiment/X016.md create mode 100644 dokimi/research/records/experiment/X017.md create mode 100644 dokimi/research/records/experiment/X018.md create mode 100644 dokimi/research/records/experiment/X019.md create mode 100644 dokimi/research/records/experiment/X020.md create mode 100644 dokimi/research/records/experiment/X021.md create mode 100644 dokimi/research/records/experiment/X022.md create mode 100644 dokimi/research/records/experiment/X023.md create mode 100644 dokimi/research/records/experiment/X024.md create mode 100644 dokimi/research/records/experiment/X025.md create mode 100644 dokimi/research/records/experiment/X026.md create mode 100644 dokimi/research/records/experiment/X027.md create mode 100644 dokimi/research/records/finding/F001.md create mode 100644 dokimi/research/records/finding/F002.md create mode 100644 dokimi/research/records/finding/F003.md create mode 100644 dokimi/research/records/finding/F004.md create mode 100644 dokimi/research/records/finding/F005.md create mode 100644 dokimi/research/records/finding/F006.md create mode 100644 dokimi/research/records/finding/F007.md create mode 100644 dokimi/research/records/finding/F008.md create mode 100644 dokimi/research/records/finding/F009.md create mode 100644 dokimi/research/records/finding/F010.md create mode 100644 dokimi/research/records/finding/F011.md create mode 100644 dokimi/research/records/finding/F012.md create mode 100644 dokimi/research/records/finding/F013.md create mode 100644 dokimi/research/records/finding/F014.md create mode 100644 dokimi/research/records/finding/F015.md create mode 100644 dokimi/research/records/finding/F016.md create mode 100644 dokimi/research/records/finding/F017.md create mode 100644 dokimi/research/records/finding/F018.md create mode 100644 dokimi/research/records/finding/F019.md create mode 100644 dokimi/research/records/idea/I001.md create mode 100644 dokimi/research/records/idea/I002.md create mode 100644 dokimi/research/records/idea/I003.md create mode 100644 dokimi/research/records/idea/I004.md create mode 100644 dokimi/research/records/idea/I005.md create mode 100644 dokimi/research/records/idea/I006.md create mode 100644 dokimi/research/records/idea/I007.md create mode 100644 dokimi/research/records/idea/I008.md create mode 100644 dokimi/research/records/idea/I009.md create mode 100644 dokimi/research/records/idea/I010.md create mode 100644 dokimi/research/records/idea/I011.md create mode 100644 dokimi/research/records/idea/I012.md create mode 100644 dokimi/research/records/idea/I013.md create mode 100644 dokimi/research/records/idea/I014.md create mode 100644 dokimi/research/records/idea/I015.md create mode 100644 dokimi/research/records/idea/I016.md create mode 100644 dokimi/research/records/idea/I017.md create mode 100644 dokimi/research/records/idea/I018.md create mode 100644 dokimi/research/records/idea/I019.md create mode 100644 dokimi/research/records/idea/I020.md create mode 100644 dokimi/research/records/idea/I021.md create mode 100644 dokimi/research/records/implref/R001.md create mode 100644 dokimi/research/records/implref/R002.md create mode 100644 dokimi/research/records/implref/R003.md create mode 100644 dokimi/research/records/implref/R006.md create mode 100644 dokimi/research/records/implref/R010.md create mode 100644 dokimi/research/records/implref/R011.md create mode 100644 dokimi/research/records/implref/R016.md create mode 100644 dokimi/research/records/implref/R017.md create mode 100644 dokimi/research/records/implref/R021.md create mode 100644 dokimi/research/records/implref/R022.md create mode 100644 dokimi/research/records/implref/R023.md create mode 100644 dokimi/research/records/implref/R024.md create mode 100644 dokimi/research/records/implref/R025.md create mode 100644 dokimi/research/records/implref/R026.md create mode 100644 dokimi/research/records/implref/R027.md create mode 100644 dokimi/research/records/implref/R028.md create mode 100644 dokimi/research/records/implref/R029.md create mode 100644 dokimi/research/records/implref/R030.md create mode 100755 dokimi/scripts/research_register.py create mode 100644 dokimi/tests/test_research_register.py diff --git a/AGENTS.md b/AGENTS.md index ea54a31e..b879d0de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,7 +87,9 @@ Registries, bundled in the package (`dokimi/dokimi/`) so they ship in the wheel: - A file the package reads at run time goes inside `dokimi/dokimi/` and into `[tool.setuptools.package-data]`, or the installed wheel goes without it. - Design rationale, status and plans go in the PR description or an issue; - the tree has no ADRs or status reports. + the tree has no ADRs or status reports. The one exception is the research + register, `dokimi/research/records/`: one record per idea, experiment, + finding or decision, negative results included (`dokimi/docs/research-register.md`). - A new file starts with the SPDX header (`Copyright (C) Intel Corporation` / `SPDX-License-Identifier: Apache-2.0`) in its comment syntax. CI's license header step (static job) checks it and `make ci-local` does diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index dcbb0bbd..57625fd5 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -300,3 +300,5 @@ code. metric extracts and runs fenced code examples in isolation. - [ingestion-docling.md](ingestion-docling.md) — ingesting non-Markdown documents (PDF, HTML, …) via Docling, installed with the optional `ocr` extra. +- [research-register.md](research-register.md) — the register of Dokimi's own + ideas, experiments (negative ones included), findings and decisions. diff --git a/dokimi/docs/research-register.md b/dokimi/docs/research-register.md new file mode 100644 index 00000000..2e3e6d5e --- /dev/null +++ b/dokimi/docs/research-register.md @@ -0,0 +1,82 @@ +# Research register + +Dokimi's own research is itself a set of experiments, and many of them come back negative or +inconclusive. The register in `research/records/` keeps every one of them findable, so the +next person who wants to try an idea can see what was already tried, what came out of it and +why a direction was kept or dropped. + +The register is plain files in this repository: no server, history from git, review through +pull requests, and any agent with file access can read and write it. + +## Record types + +Each record is one Markdown file, `research/records//.md`. The YAML front matter +holds the typed fields that `research/record.v1.json` defines; the body is free Markdown for +the setup, result, rationale and anything else a reader needs, after the +license header every file in the repository carries. + +| Type | Id | What it holds | Points at | +|---|---|---|---| +| `idea` | `I001` | A question worth answering, its map theme (`P`…`W`) and `open`/`parked`/`closed` status | — | +| `experiment` | `X001` | Hypothesis, what was run, and the `outcome`: `planned`, `running`, `supported`, `refuted`, `inconclusive` or `aborted` | `idea` | +| `finding` | `F001` | What was learned, with `low`/`med`/`high` confidence | `experiments` | +| `decision` | `D001` | `accepted`, `rejected`, `deferred` or `superseded`, and why | `findings`, `ideas`, `superseded_by` | +| `implref` | `R001` | The issue or pull request that carries a decision out, with its own `status` | `decision` | + +Three states stay separate on purpose: an experiment's `outcome` (what the evidence showed), +a decision's `state` (what the team chose) and an implementation reference's `status` (what +was built). A closed issue does not tell which of the three happened. + +An experiment that turns out wrong is not deleted: set `retracted: true` and say in the body +what was wrong with it. Every record has a `date` (`YYYY-MM-DD`) and optional `links`, which +must be public `github.com/intel/` or `github.com/napetrov/` URLs. The register is public: +leave out internal hosts, paths, names and raw traces, and describe local evidence in words +("local run, 96 runs"). + +## Working with it + +Run from the `dokimi/` project directory: + +```bash +python scripts/research_register.py check +python scripts/research_register.py next experiment +python scripts/research_register.py query --type experiment --ref I001 --where outcome=refuted,inconclusive +python scripts/research_register.py query --type decision --where state=rejected +python scripts/research_register.py export > register.json +``` + +- `check` validates every record against its schema, its path and id prefix, and every + reference. `tests/test_research_register.py` runs it in CI, so a broken record fails the PR. +- `next` prints the next free id for a type. Two PRs can claim the same id; `check` reports + the duplicate on whichever merges second, which renumbers. +- `query` lists the records matching every filter: `--where FIELD=V1,V2` matches any of the + values, `--ref ID` keeps the records that point at `ID`. +- `export` writes all records, the reference edges and the ideas grouped by map theme as one + JSON document, stamped with the commit, for the plans-and-research map and other views. + +## Adding a record + +1. `python scripts/research_register.py next ` for the id. +2. Write `research/records//.md`: + + ```markdown + --- + id: X012 + type: experiment + title: Compressed skill keeps step ordering + date: 2026-10-14 + idea: I004 + hypothesis: A skill cut to 40% of its tokens keeps the ordering constraints. + outcome: refuted + --- + + ## Setup + ... + ## Result + ... + ``` + +3. `python scripts/research_register.py check`, then open the PR. diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index 83a30887..6ce46b21 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -128,5 +128,6 @@ nav: - difficulty-rubric.md - example-execution.md - ingestion-docling.md + - research-register.md - CLI reference: reference/cli/ - API reference: reference/api/ diff --git a/dokimi/research/record.v1.json b/dokimi/research/record.v1.json new file mode 100644 index 00000000..275d0a46 --- /dev/null +++ b/dokimi/research/record.v1.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/research/record.v1.json", + "title": "Research register record (front matter of research/records//.md)", + "description": "One schema per record type under $defs; scripts/research_register.py picks it by `type`. References are ids; the register checks they resolve to a record of the right type.", + "$defs": { + "id": {"type": "string", "pattern": "^[A-Z]+[0-9]{3}$"}, + "ids": {"type": "array", "items": {"$ref": "#/$defs/id"}, "uniqueItems": true}, + "text": {"type": "string", "minLength": 1}, + "date": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"}, + "url": {"type": "string", "pattern": "^https://github\\.com/(intel|napetrov)/"}, + "theme": { + "description": "Branch of the plans-and-research map.", + "enum": ["P", "E", "V", "K", "G", "O", "C", "D", "W"] + }, + "common": { + "type": "object", + "properties": { + "id": {"$ref": "#/$defs/id"}, + "type": {"type": "string"}, + "title": {"$ref": "#/$defs/text"}, + "date": {"$ref": "#/$defs/date"}, + "links": {"type": "array", "items": {"$ref": "#/$defs/url"}, "uniqueItems": true} + }, + "required": ["id", "type", "title", "date"] + }, + "idea": { + "$ref": "#/$defs/common", + "properties": { + "type": {"const": "idea"}, + "theme": {"$ref": "#/$defs/theme"}, + "question": {"$ref": "#/$defs/text"}, + "status": {"enum": ["open", "parked", "closed"]} + }, + "required": ["theme", "question", "status"], + "unevaluatedProperties": false + }, + "experiment": { + "$ref": "#/$defs/common", + "properties": { + "type": {"const": "experiment"}, + "idea": {"$ref": "#/$defs/id"}, + "hypothesis": {"$ref": "#/$defs/text"}, + "outcome": { + "description": "What the evidence showed. planned/running: no result yet. Negative and inconclusive outcomes stay in the register.", + "enum": ["planned", "running", "supported", "refuted", "inconclusive", "aborted"] + }, + "retracted": {"type": "boolean"} + }, + "required": ["idea", "hypothesis", "outcome"], + "unevaluatedProperties": false + }, + "finding": { + "$ref": "#/$defs/common", + "properties": { + "type": {"const": "finding"}, + "experiments": {"$ref": "#/$defs/ids", "minItems": 1}, + "confidence": {"enum": ["low", "med", "high"]} + }, + "required": ["experiments", "confidence"], + "unevaluatedProperties": false + }, + "decision": { + "$ref": "#/$defs/common", + "properties": { + "type": {"const": "decision"}, + "findings": {"$ref": "#/$defs/ids"}, + "ideas": {"$ref": "#/$defs/ids"}, + "state": {"enum": ["accepted", "rejected", "deferred", "superseded"]}, + "superseded_by": {"$ref": "#/$defs/id"} + }, + "required": ["state"], + "unevaluatedProperties": false + }, + "implref": { + "$ref": "#/$defs/common", + "properties": { + "type": {"const": "implref"}, + "decision": {"$ref": "#/$defs/id"}, + "status": { + "description": "Implementation state, kept apart from the decision state.", + "enum": ["proposed", "open", "merged", "closed"] + } + }, + "required": ["decision", "status", "links"], + "unevaluatedProperties": false + } + } +} diff --git a/dokimi/research/records/decision/D001.md b/dokimi/research/records/decision/D001.md new file mode 100644 index 00000000..d69a6ff2 --- /dev/null +++ b/dokimi/research/records/decision/D001.md @@ -0,0 +1,16 @@ +--- +id: D001 +type: decision +title: Do not ship the local defect battery as a judge replacement +date: '2026-10-19' +findings: +- F001 +- F004 +state: accepted +--- + +## Rationale +Corrected baseline: the hosted judge was not saturated (0.93 pairwise) so the battery only tied it at higher cost. diff --git a/dokimi/research/records/decision/D002.md b/dokimi/research/records/decision/D002.md new file mode 100644 index 00000000..d178b5df --- /dev/null +++ b/dokimi/research/records/decision/D002.md @@ -0,0 +1,15 @@ +--- +id: D002 +type: decision +title: Recompute aggregate in code; weight correctness x2; zero-weight specificity +date: '2026-10-19' +findings: +- F002 +state: accepted +--- + +## Rationale +The five dimensions behaved as one factor; averaging diluted correctness. Done in pre-public history. diff --git a/dokimi/research/records/decision/D003.md b/dokimi/research/records/decision/D003.md new file mode 100644 index 00000000..5a90706e --- /dev/null +++ b/dokimi/research/records/decision/D003.md @@ -0,0 +1,15 @@ +--- +id: D003 +type: decision +title: Persist judge material by digest and validate the judge in the arms setting +date: '2026-10-19' +findings: +- F005 +state: accepted +--- + +## Rationale +Issue #48; material kept in artifact (#69), gate --context stored (#73, open). diff --git a/dokimi/research/records/decision/D004.md b/dokimi/research/records/decision/D004.md new file mode 100644 index 00000000..5a43b9b8 --- /dev/null +++ b/dokimi/research/records/decision/D004.md @@ -0,0 +1,15 @@ +--- +id: D004 +type: decision +title: Do not build stratified probe sampler, SliCK split, LLMLingua shortener +date: '2026-10-19' +findings: +- F009 +state: rejected +--- + +## Rationale +Measured gains negligible (+0.007) or refuted. diff --git a/dokimi/research/records/decision/D005.md b/dokimi/research/records/decision/D005.md new file mode 100644 index 00000000..74231c23 --- /dev/null +++ b/dokimi/research/records/decision/D005.md @@ -0,0 +1,15 @@ +--- +id: D005 +type: decision +title: Infer/measure mechanism; [utility] descriptor is an override only +date: '2026-10-19' +findings: +- F010 +state: accepted +--- + +## Rationale +No skill standard declares mechanisms; few public skills carry a type field. diff --git a/dokimi/research/records/decision/D006.md b/dokimi/research/records/decision/D006.md new file mode 100644 index 00000000..9bc64027 --- /dev/null +++ b/dokimi/research/records/decision/D006.md @@ -0,0 +1,15 @@ +--- +id: D006 +type: decision +title: Move dokimi guidance to AGENTS.md; CLAUDE.md removed +date: '2026-10-19' +findings: +- F012 +state: accepted +--- + +## Rationale +Measured process gains; merged. diff --git a/dokimi/research/records/decision/D007.md b/dokimi/research/records/decision/D007.md new file mode 100644 index 00000000..01255506 --- /dev/null +++ b/dokimi/research/records/decision/D007.md @@ -0,0 +1,15 @@ +--- +id: D007 +type: decision +title: Writing-for-agents prose rewrite rejected; value is the lint gate +date: '2026-10-19' +findings: +- F013 +state: rejected +--- + +## Rationale +Rewrite dropped facts; gate-only kept them. diff --git a/dokimi/research/records/decision/D008.md b/dokimi/research/records/decision/D008.md new file mode 100644 index 00000000..822631cd --- /dev/null +++ b/dokimi/research/records/decision/D008.md @@ -0,0 +1,16 @@ +--- +id: D008 +type: decision +title: Ship LLM-free context tools (trace, budget) first +date: '2026-10-19' +findings: +- F015 +- F016 +state: accepted +--- + +## Rationale +Measurement tools are deterministic; behavioural fixes stay experimental. diff --git a/dokimi/research/records/decision/D009.md b/dokimi/research/records/decision/D009.md new file mode 100644 index 00000000..87f63c7a --- /dev/null +++ b/dokimi/research/records/decision/D009.md @@ -0,0 +1,15 @@ +--- +id: D009 +type: decision +title: Product repo keeps evals only; mechanics move to dokimi; policy rules dropped +date: '2026-10-19' +findings: +- F018 +state: accepted +--- + +## Rationale +Reviewer objections to generic harness in the product repo; tracking #41. diff --git a/dokimi/research/records/decision/D010.md b/dokimi/research/records/decision/D010.md new file mode 100644 index 00000000..adb071e6 --- /dev/null +++ b/dokimi/research/records/decision/D010.md @@ -0,0 +1,13 @@ +--- +id: D010 +type: decision +title: Number changes never go in a refactor; stored values carry a computation stamp +date: '2026-10-19' +state: accepted +--- + +## Rationale +Keeps stored results comparable across refactors. diff --git a/dokimi/research/records/decision/D011.md b/dokimi/research/records/decision/D011.md new file mode 100644 index 00000000..d5010f16 --- /dev/null +++ b/dokimi/research/records/decision/D011.md @@ -0,0 +1,13 @@ +--- +id: D011 +type: decision +title: Outcome contracts as pytest plugin in-repo (no extra tooling/runtime) +date: '2026-10-19' +state: accepted +--- + +## Rationale +Simpler adoption; audit found --questions-root did not pin the golden set and code was fixed. diff --git a/dokimi/research/records/decision/D012.md b/dokimi/research/records/decision/D012.md new file mode 100644 index 00000000..4adfa0cb --- /dev/null +++ b/dokimi/research/records/decision/D012.md @@ -0,0 +1,15 @@ +--- +id: D012 +type: decision +title: git+markdown canonical store with validator; optional UI pilot; GitHub for discussion +date: '2026-10-19' +findings: +- F019 +state: accepted +--- + +## Rationale +Lowest cost, history via git, existing MCP access. diff --git a/dokimi/research/records/decision/D013.md b/dokimi/research/records/decision/D013.md new file mode 100644 index 00000000..5a82fc92 --- /dev/null +++ b/dokimi/research/records/decision/D013.md @@ -0,0 +1,15 @@ +--- +id: D013 +type: decision +title: Reject GitLab CE, OpenProject CE as register +date: '2026-10-19' +findings: +- F019 +state: rejected +--- + +## Rationale +Schema features paid or not via API; high RAM. diff --git a/dokimi/research/records/experiment/X001.md b/dokimi/research/records/experiment/X001.md new file mode 100644 index 00000000..779ae55b --- /dev/null +++ b/dokimi/research/records/experiment/X001.md @@ -0,0 +1,21 @@ +--- +id: X001 +type: experiment +title: Question polarity vs ceiling saturation (local small model) +date: '2026-10-19' +idea: I001 +hypothesis: Typed one-token decisions fix saturation. +outcome: refuted +--- + +## Setup +Local 4B/14B model, deterministic degradations of perf golden answers; positive vs negative wording of the same decision. + +## Result +Positive 'covers point?' gave P(yes)=1.000 for intact and degraded copies; negative 'is it absent?' separated them (0.825 vs 0.250). Negative wording then saturated at the floor on a less acquiescent 14B model. + +## Evidence +- local run, Qwen3 4B/14B, 20 bases diff --git a/dokimi/research/records/experiment/X002.md b/dokimi/research/records/experiment/X002.md new file mode 100644 index 00000000..934e6086 --- /dev/null +++ b/dokimi/research/records/experiment/X002.md @@ -0,0 +1,21 @@ +--- +id: X002 +type: experiment +title: Length-only baseline for judge validity +date: '2026-10-19' +idea: I001 +hypothesis: Judge validity accuracy reflects reading content. +outcome: supported +--- + +## Setup +Compare judge pairwise accuracy on degradations to a predictor using answer length only. + +## Result +Defect battery 0.94 vs length-only 0.81 on 5 variants; length beat the battery on strip_code and truncate. Shipped hosted judge 9-variant: 0.920 vs length 0.663; 4 of 9 variants <= length. + +## Evidence +- local run, Haiku judge, 20 bases, 176 pairs diff --git a/dokimi/research/records/experiment/X003.md b/dokimi/research/records/experiment/X003.md new file mode 100644 index 00000000..20c48c01 --- /dev/null +++ b/dokimi/research/records/experiment/X003.md @@ -0,0 +1,21 @@ +--- +id: X003 +type: experiment +title: 'Wrong-magnitude detection: criterion vs enumeration' +date: '2026-10-19' +idea: I001 +hypothesis: Describing a numeric-claims criterion lets the judge catch wrong numbers. +outcome: supported +--- + +## Setup +Haiku judge, 4 seeds; criterion only vs criterion + instruction to enumerate every figure. + +## Result +Criterion only +0.045, McNemar p=0.43 (refuted). Criterion + enumeration pooled 0.599 -> 0.712, p=0.0144 over 156 pairs; still under a 0.80 gate. + +## Evidence +- local run, Haiku, 156 pairs, 4 seeds diff --git a/dokimi/research/records/experiment/X004.md b/dokimi/research/records/experiment/X004.md new file mode 100644 index 00000000..67dba7fd --- /dev/null +++ b/dokimi/research/records/experiment/X004.md @@ -0,0 +1,21 @@ +--- +id: X004 +type: experiment +title: Context-blind judge on skill arm +date: '2026-10-19' +idea: I002 +hypothesis: The skill arm wins on the perf golden set. +outcome: inconclusive +--- + +## Setup +81 paired questions, hosted judge, skill answers re-judged with and without the skill manifest in context. + +## Result +Without context skill lost 29/81 (judge called existing skill file paths 'fabricated', 17/81 flagged). With context 60/81. Symmetric control: baseline alone gained ~+4 from context; 36/81, p=0.087. + +## Evidence +- local run, n=81 paired diff --git a/dokimi/research/records/experiment/X005.md b/dokimi/research/records/experiment/X005.md new file mode 100644 index 00000000..28893f91 --- /dev/null +++ b/dokimi/research/records/experiment/X005.md @@ -0,0 +1,21 @@ +--- +id: X005 +type: experiment +title: Judge-validity gate with stored context +date: '2026-10-19' +idea: I002 +hypothesis: A no-context validity gate reflects the arms setting. +outcome: supported +links: +- https://github.com/intel/dokimi/issues/48 +- https://github.com/intel/dokimi/pull/73 +--- + +## Setup +Haiku judge, 24 bases, snippet vs full material. + +## Result +With full material intact mean 85.9->89.9 while overall accuracy fell 0.869->0.808; wrong_tools 0.891->0.674. diff --git a/dokimi/research/records/experiment/X006.md b/dokimi/research/records/experiment/X006.md new file mode 100644 index 00000000..ea2b2d55 --- /dev/null +++ b/dokimi/research/records/experiment/X006.md @@ -0,0 +1,22 @@ +--- +id: X006 +type: experiment +title: Judge ladder and cross-judge agreement +date: '2026-10-19' +idea: I003 +hypothesis: Skill-vs-baseline verdict is judge-independent. +outcome: supported +retracted: true +--- + +## Setup +Same answers judged by Haiku, gpt-oss, Sonnet, Opus; absolute vs pairwise; sequential (SPRT) gate. + +## Result +Verdict sign flipped with judge (Haiku -3.4 p=.024; Opus +1.2 p=.005). Per-question direction agreement 46-76%. Only one absolute judge passed the sequential gate; pairwise gpt-oss passed at 0.962 for ~$0.5. Several earlier claims retracted after a pair-keying audit. + +## Evidence +- local run, ~11k judge calls diff --git a/dokimi/research/records/experiment/X007.md b/dokimi/research/records/experiment/X007.md new file mode 100644 index 00000000..820ec7f0 --- /dev/null +++ b/dokimi/research/records/experiment/X007.md @@ -0,0 +1,21 @@ +--- +id: X007 +type: experiment +title: Automatic prompt optimisation of the judge rubric (GEPA) +date: '2026-10-19' +idea: I003 +hypothesis: Reflective prompt optimisation improves the judge rubric. +outcome: refuted +--- + +## Setup +GEPA with default and 'general' reflection templates; question-disjoint split, held-out test of 57 questions. + +## Result +Default template leaked the test generator, zero accepted improvement in 1408 metric calls; validation gains were winner's curse; no rubric beat the hand-written prompt on held-out test. + +## Evidence +- local run, held-out 57 questions diff --git a/dokimi/research/records/experiment/X008.md b/dokimi/research/records/experiment/X008.md new file mode 100644 index 00000000..0049c9f4 --- /dev/null +++ b/dokimi/research/records/experiment/X008.md @@ -0,0 +1,21 @@ +--- +id: X008 +type: experiment +title: FormatSpread wording sensitivity +date: '2026-10-19' +idea: I003 +hypothesis: Meaning-preserving rewrites don't change judge accuracy. +outcome: supported +--- + +## Setup +6 rewrites vs 3 identical-prompt redraws. + +## Result +Pooled accuracy within redraw noise, but wrong_numbers 0.47-0.72 and padding 0.37-0.67 were wording-fragile; 10.6% of verdicts flip between identical draws. + +## Evidence +- local run diff --git a/dokimi/research/records/experiment/X009.md b/dokimi/research/records/experiment/X009.md new file mode 100644 index 00000000..38779bdc --- /dev/null +++ b/dokimi/research/records/experiment/X009.md @@ -0,0 +1,21 @@ +--- +id: X009 +type: experiment +title: 'Novelty toolscan: cheap matchers and compressors' +date: '2026-10-19' +idea: I005 +hypothesis: Cheap NLI matchers / LLMLingua can replace LLM adjudication. +outcome: refuted +--- + +## Setup +Matcher bake-off on hard band; LLMLingua-2 force_tokens; judge-free items. + +## Result +Small NLI at chance (AUC 0.576); HHEM-2.1 AUC 0.804 but no matcher removed LLM adjudication; LLMLingua force_tokens did not protect atoms (refuted); every cheap instrument failed on numbers/versions/negation. + +## Evidence +- local run, 100 paired items diff --git a/dokimi/research/records/experiment/X010.md b/dokimi/research/records/experiment/X010.md new file mode 100644 index 00000000..81208267 --- /dev/null +++ b/dokimi/research/records/experiment/X010.md @@ -0,0 +1,21 @@ +--- +id: X010 +type: experiment +title: Planted contradictions (value adoption) +date: '2026-10-19' +idea: I005 +hypothesis: Absurd planted values are resisted by the reader model. +outcome: refuted +--- + +## Setup +20 atoms, 8 artifacts, 3 models, 240 calls. + +## Result +planted_near adopted 0.942, planted_far 0.846, no-skill true value 0.115; dose response nearly flat (p=0.062); only independently anchored values resisted. + +## Evidence +- local run, 52 valid cells diff --git a/dokimi/research/records/experiment/X011.md b/dokimi/research/records/experiment/X011.md new file mode 100644 index 00000000..e1b9cbc5 --- /dev/null +++ b/dokimi/research/records/experiment/X011.md @@ -0,0 +1,21 @@ +--- +id: X011 +type: experiment +title: Infer skill mechanism and predict lift +date: '2026-10-19' +idea: I006 +hypothesis: A declared or inferred mechanism (or novelty) predicts task delta. +outcome: refuted +--- + +## Setup +3 LLM classifiers on 33 public skills; static regex; 13 skills with task deltas. + +## Result +Classifier kappa 0.41-0.54; static rules usable only for automation/contract/guardrail; |rho|<=0.19 vs task delta for every mechanism and for novelty. + +## Evidence +- local run, 33 skills, 24 cells diff --git a/dokimi/research/records/experiment/X012.md b/dokimi/research/records/experiment/X012.md new file mode 100644 index 00000000..1e24ba57 --- /dev/null +++ b/dokimi/research/records/experiment/X012.md @@ -0,0 +1,22 @@ +--- +id: X012 +type: experiment +title: Skill ablation and checklist control +date: '2026-10-19' +idea: I006 +hypothesis: Full skill outperforms ablations and a short checklist. +outcome: inconclusive +retracted: true +--- + +## Setup +onetbb quickstart, 7 tasks x 3 reps, Haiku; arms none/full/instr_only/facts_only/checklist; later irrelevant same-length control. + +## Result +none 8/21, full 15/21, checklist 16/21; no variant distinguishable from full (McNemar p=1.0); lift is ~one knowledge atom checked by a marker verifier; skills raised cost. Same-length irrelevant control ~= none, so lift is content. Earlier 'full guardrail worse than none' retracted (3 reps too few). + +## Evidence +- local run, 7 tasks x 3 reps diff --git a/dokimi/research/records/experiment/X013.md b/dokimi/research/records/experiment/X013.md new file mode 100644 index 00000000..1e4274dd --- /dev/null +++ b/dokimi/research/records/experiment/X013.md @@ -0,0 +1,20 @@ +--- +id: X013 +type: experiment +title: Skill discover arm smoke +date: '2026-10-19' +idea: I007 +hypothesis: Skill helps without notice. +outcome: inconclusive +links: +- https://github.com/intel/dokimi/pull/110 +--- + +## Setup +onetbb transpose task, Haiku, 3 reps: no-skill / skill / discover. + +## Result +no-skill 0/3, skill 3/3, discover 0/3 with 0 Skill calls. diff --git a/dokimi/research/records/experiment/X014.md b/dokimi/research/records/experiment/X014.md new file mode 100644 index 00000000..cb49c5df --- /dev/null +++ b/dokimi/research/records/experiment/X014.md @@ -0,0 +1,21 @@ +--- +id: X014 +type: experiment +title: Invocation drives skill effect +date: '2026-10-19' +idea: I007 +hypothesis: Skill effect is independent of whether it is invoked. +outcome: refuted +--- + +## Setup +Determinism suite, 96+ runs, Haiku/Sonnet. + +## Result +When invoked done right 11/11, not invoked 0/25; trigger-heavy description raised invocation 2/12 -> 7/12 (p=0.087); a Stop hook removed the dependency. + +## Evidence +- local run, n=96 diff --git a/dokimi/research/records/experiment/X015.md b/dokimi/research/records/experiment/X015.md new file mode 100644 index 00000000..aefa4ea6 --- /dev/null +++ b/dokimi/research/records/experiment/X015.md @@ -0,0 +1,21 @@ +--- +id: X015 +type: experiment +title: AGENTS.md vs CLAUDE.md vs none on dokimi's own tasks +date: '2026-10-19' +idea: I009 +hypothesis: Moving guidance to AGENTS.md does not change outcomes. +outcome: refuted +links: +- https://github.com/intel/dokimi/pull/56 +- https://github.com/intel/dokimi/pull/57 +--- + +## Setup +10 repo tasks, Haiku/Sonnet/Opus, n=3, 270 cells. + +## Result +Schema-required-field: AGENTS 9/9 vs CLAUDE 4/9 (Fisher p=.015); commands killed by timeout 0/90 vs 42/90; first 'legacy keys' wording regressed (2/6) and was fixed before merge. diff --git a/dokimi/research/records/experiment/X016.md b/dokimi/research/records/experiment/X016.md new file mode 100644 index 00000000..34bcf9ce --- /dev/null +++ b/dokimi/research/records/experiment/X016.md @@ -0,0 +1,21 @@ +--- +id: X016 +type: experiment +title: Pointer and prohibition wording +date: '2026-10-19' +idea: I009 +hypothesis: Descriptive index lines and prohibitions guide agents. +outcome: supported +--- + +## Setup +Synthetic fixtures; 96 + 192 runs, Haiku/Sonnet. + +## Result +Conditional 'Before X, read Y' 10/12 vs bare/descriptive 3/12 (Haiku); prohibition 0/48 violations but preferred target only 28/48 vs 48/48 for positive. + +## Evidence +- local run, n=288 diff --git a/dokimi/research/records/experiment/X017.md b/dokimi/research/records/experiment/X017.md new file mode 100644 index 00000000..ea9d4d20 --- /dev/null +++ b/dokimi/research/records/experiment/X017.md @@ -0,0 +1,21 @@ +--- +id: X017 +type: experiment +title: Rewriting guidance with a writing-for-agents skill +date: '2026-10-19' +idea: I009 +hypothesis: A rewrite skill improves guidance. +outcome: refuted +--- + +## Setup +3 repos x 4 rewrite modes x 3 models x 2. + +## Result +Skill rewrite shrank text and dropped facts (57% kept in one case); lint-only gate kept 93-100% of facts. + +## Evidence +- local run, 72 runs diff --git a/dokimi/research/records/experiment/X018.md b/dokimi/research/records/experiment/X018.md new file mode 100644 index 00000000..4019928c --- /dev/null +++ b/dokimi/research/records/experiment/X018.md @@ -0,0 +1,21 @@ +--- +id: X018 +type: experiment +title: Router vs flat skill listing +date: '2026-10-19' +idea: I009 +hypothesis: Grouping skills in a router improves selection. +outcome: refuted +--- + +## Setup +Public skills catalog, 4 groups, Haiku 5.5. + +## Result +Router 0.20 vs flat 0.38. + +## Evidence +- local run diff --git a/dokimi/research/records/experiment/X019.md b/dokimi/research/records/experiment/X019.md new file mode 100644 index 00000000..babd64d2 --- /dev/null +++ b/dokimi/research/records/experiment/X019.md @@ -0,0 +1,21 @@ +--- +id: X019 +type: experiment +title: 'Knowledge quiz: does a CLAUDE.md add knowledge' +date: '2026-10-19' +idea: I009 +hypothesis: Guidance file raises open-book accuracy. +outcome: refuted +--- + +## Setup +17-question quiz, closed/open book, n=2. + +## Result +Open-book accuracy same with or without file; value was cost (tokens -25..-45%). + +## Evidence +- local run, n=2 diff --git a/dokimi/research/records/experiment/X020.md b/dokimi/research/records/experiment/X020.md new file mode 100644 index 00000000..c0760031 --- /dev/null +++ b/dokimi/research/records/experiment/X020.md @@ -0,0 +1,21 @@ +--- +id: X020 +type: experiment +title: Cold-agent onboarding probe +date: '2026-10-19' +idea: I010 +hypothesis: README suffices to review a skill or repo. +outcome: supported +links: +- https://github.com/intel/dokimi/pull/59 +- https://github.com/intel/dokimi/pull/60 +--- + +## Setup +4 cold agents (2 models x 2 tasks), only the dokimi tree. + +## Result +All found the right commands in 2-5 turns; gaps: no USD cap on novelty (one run spent $14.50 vs $6 budget), manifests required for guidance-only scans, fabricated competitors for routing. diff --git a/dokimi/research/records/experiment/X021.md b/dokimi/research/records/experiment/X021.md new file mode 100644 index 00000000..60e23fe5 --- /dev/null +++ b/dokimi/research/records/experiment/X021.md @@ -0,0 +1,21 @@ +--- +id: X021 +type: experiment +title: Context failure-mode suite +date: '2026-10-19' +idea: I013 +hypothesis: Verify prompts and compaction mitigate context failures. +outcome: supported +links: +- https://github.com/intel/dokimi/pull/68 +- https://github.com/intel/dokimi/pull/71 +--- + +## Setup +421 runs, Haiku 4.5 / Sonnet 5.5, n=3-5 per cell (directional). + +## Result +Stale facts adopted near-totally; 'verify' prompt helped only Sonnet (4/5); 150k irrelevant history kept accuracy but cost 16-31x; guidance file beat code and skill on clash 10/10; prune ~10-20x cheaper than compaction at equal accuracy. diff --git a/dokimi/research/records/experiment/X022.md b/dokimi/research/records/experiment/X022.md new file mode 100644 index 00000000..a657b4a0 --- /dev/null +++ b/dokimi/research/records/experiment/X022.md @@ -0,0 +1,21 @@ +--- +id: X022 +type: experiment +title: Near-duplicate MCP tools under deferred loading +date: '2026-10-19' +idea: I013 +hypothesis: Tool descriptions disambiguate tools. +outcome: refuted +--- + +## Setup +k near-duplicate tools, deferred vs flat loading. + +## Result +Deferred: 1/40 right at k>=10 (Haiku)/k>=50 (Sonnet); flat: 56/56 right at +9k-22k tokens per call. + +## Evidence +- local run diff --git a/dokimi/research/records/experiment/X023.md b/dokimi/research/records/experiment/X023.md new file mode 100644 index 00000000..d171875b --- /dev/null +++ b/dokimi/research/records/experiment/X023.md @@ -0,0 +1,21 @@ +--- +id: X023 +type: experiment +title: Determinism profile on a procedure task +date: '2026-10-19' +idea: I015 +hypothesis: Harness success and path stability indicate correctness. +outcome: refuted +--- + +## Setup +96 runs, $5.27; arms none/prose/script/hook; repeat/paraphrase/fault series. + +## Result +Harness success 100% everywhere (uninformative); stable failure common; hook vs script p=0.0006; 'script arm has most stable path' did not hold; after a transient failure 0/24 answers mentioned it. + +## Evidence +- local run, n=96 diff --git a/dokimi/research/records/experiment/X024.md b/dokimi/research/records/experiment/X024.md new file mode 100644 index 00000000..f53c8d18 --- /dev/null +++ b/dokimi/research/records/experiment/X024.md @@ -0,0 +1,20 @@ +--- +id: X024 +type: experiment +title: Built-in review graders equivalence +date: '2026-10-19' +idea: I017 +hypothesis: Built-in graders reproduce the product repo's graders. +outcome: supported +links: +- https://github.com/intel/dokimi/pull/120 +--- + +## Setup +Regrade 1414 answers over 9 review tasks. + +## Result +0 mismatches. diff --git a/dokimi/research/records/experiment/X025.md b/dokimi/research/records/experiment/X025.md new file mode 100644 index 00000000..6d3da85a --- /dev/null +++ b/dokimi/research/records/experiment/X025.md @@ -0,0 +1,21 @@ +--- +id: X025 +type: experiment +title: L0 static-check parity with seeded defects +date: '2026-10-19' +idea: I017 +hypothesis: Generic L0 rules match the product's static check. +outcome: supported +retracted: true +links: +- https://github.com/intel/dokimi/pull/115 +--- + +## Setup +7 seeded stale defects. + +## Result +Product check 6/7; dokimi 1/7 before, 5/7 after; one product 'real failure' retracted as its own false positive. diff --git a/dokimi/research/records/experiment/X026.md b/dokimi/research/records/experiment/X026.md new file mode 100644 index 00000000..d500c002 --- /dev/null +++ b/dokimi/research/records/experiment/X026.md @@ -0,0 +1,20 @@ +--- +id: X026 +type: experiment +title: Test-quality audit of dokimi tests +date: '2026-10-19' +idea: I018 +hypothesis: Some tests cannot fail for wrong behaviour. +outcome: supported +links: +- https://github.com/intel/dokimi/pull/103 +--- + +## Setup +Static AST audit of the test suite. + +## Result +31 of 3340 tests flagged in pilot; 30 findings on main, 27 fixed. diff --git a/dokimi/research/records/experiment/X027.md b/dokimi/research/records/experiment/X027.md new file mode 100644 index 00000000..5634f5d9 --- /dev/null +++ b/dokimi/research/records/experiment/X027.md @@ -0,0 +1,21 @@ +--- +id: X027 +type: experiment +title: Research workspace tool trial +date: '2026-10-19' +idea: I021 +hypothesis: A self-hosted tool can enforce the record schema. +outcome: supported +--- + +## Setup +5 tools in local containers, same protocol and seed records. + +## Result +No server tool enforced typed refs/enums on write; git+markdown passed all tests with no server. + +## Evidence +- local trial, 5 tools diff --git a/dokimi/research/records/finding/F001.md b/dokimi/research/records/finding/F001.md new file mode 100644 index 00000000..d7d0536f --- /dev/null +++ b/dokimi/research/records/finding/F001.md @@ -0,0 +1,18 @@ +--- +id: F001 +type: finding +title: Judges must be checked against a length-only baseline +date: '2026-10-19' +experiments: +- X002 +confidence: high +--- + +## Statement +A degradation variant whose judge accuracy does not exceed length-only accuracy proves nothing about content reading. + +## Applies to +Any LLM-judge validity gate diff --git a/dokimi/research/records/finding/F002.md b/dokimi/research/records/finding/F002.md new file mode 100644 index 00000000..a6ec5ea9 --- /dev/null +++ b/dokimi/research/records/finding/F002.md @@ -0,0 +1,18 @@ +--- +id: F002 +type: finding +title: Instructing the judge to enumerate beats describing the criterion +date: '2026-10-19' +experiments: +- X003 +confidence: med +--- + +## Statement +Enumeration of figures moved wrong-number detection; wording alone stayed within run-to-run noise (~0.04). + +## Applies to +LLM-judge prompt design diff --git a/dokimi/research/records/finding/F003.md b/dokimi/research/records/finding/F003.md new file mode 100644 index 00000000..151e2526 --- /dev/null +++ b/dokimi/research/records/finding/F003.md @@ -0,0 +1,19 @@ +--- +id: F003 +type: finding +title: No measured judge sees wrong magnitudes reliably +date: '2026-10-19' +experiments: +- X002 +- X003 +confidence: med +--- + +## Statement +wrong_numbers stayed 0.67-0.82 across every judge configuration measured; wrong names are caught, wrong values mostly not. + +## Applies to +Judging technical answers with numbers diff --git a/dokimi/research/records/finding/F004.md b/dokimi/research/records/finding/F004.md new file mode 100644 index 00000000..a41f97e9 --- /dev/null +++ b/dokimi/research/records/finding/F004.md @@ -0,0 +1,18 @@ +--- +id: F004 +type: finding +title: Polarity must be matched to the model +date: '2026-10-19' +experiments: +- X001 +confidence: med +--- + +## Statement +Negative-polarity questions fix ceiling saturation on an acquiescent model but saturate at the floor on a non-acquiescent one. + +## Applies to +Typed judge batteries diff --git a/dokimi/research/records/finding/F005.md b/dokimi/research/records/finding/F005.md new file mode 100644 index 00000000..81b17d62 --- /dev/null +++ b/dokimi/research/records/finding/F005.md @@ -0,0 +1,19 @@ +--- +id: F005 +type: finding +title: Judge context is a second treatment +date: '2026-10-19' +experiments: +- X004 +- X005 +confidence: high +--- + +## Statement +Per-arm judge context shifts scores by itself; no arm comparison is trustworthy if a treatment cites material the judge cannot see, and the skill effect itself was not established. + +## Applies to +Arms runs with skill/docs treatments diff --git a/dokimi/research/records/finding/F006.md b/dokimi/research/records/finding/F006.md new file mode 100644 index 00000000..66f8389b --- /dev/null +++ b/dokimi/research/records/finding/F006.md @@ -0,0 +1,18 @@ +--- +id: F006 +type: finding +title: Judge choice can flip arm verdicts; per-question tables are noise +date: '2026-10-19' +experiments: +- X006 +confidence: high +--- + +## Statement +Record judge model and gate the judge before trusting a delta; per-question agreement between judges is 46-76%. + +## Applies to +Arms and model-compare reports diff --git a/dokimi/research/records/finding/F007.md b/dokimi/research/records/finding/F007.md new file mode 100644 index 00000000..da0c0a4b --- /dev/null +++ b/dokimi/research/records/finding/F007.md @@ -0,0 +1,18 @@ +--- +id: F007 +type: finding +title: Prompt optimisation overfits the validity generator +date: '2026-10-19' +experiments: +- X007 +confidence: med +--- + +## Statement +Rubric optimisation against synthetic degradations learns the degradations, not judging. + +## Applies to +Judge tuning diff --git a/dokimi/research/records/finding/F008.md b/dokimi/research/records/finding/F008.md new file mode 100644 index 00000000..090e12af --- /dev/null +++ b/dokimi/research/records/finding/F008.md @@ -0,0 +1,18 @@ +--- +id: F008 +type: finding +title: A reader model adopts document values ~85-94% of the time +date: '2026-10-19' +experiments: +- X010 +confidence: med +--- + +## Statement +Correctness of a value cannot be delegated to the reader; ship decision values with an independently checkable correlate. + +## Applies to +Skill and doc authoring diff --git a/dokimi/research/records/finding/F009.md b/dokimi/research/records/finding/F009.md new file mode 100644 index 00000000..20daf006 --- /dev/null +++ b/dokimi/research/records/finding/F009.md @@ -0,0 +1,18 @@ +--- +id: F009 +type: finding +title: Quote no 'added' difference below 0.15 +date: '2026-10-19' +experiments: +- X009 +confidence: med +--- + +## Statement +Grader agreement kappa 0.576 and ±8-12pp grader spread make smaller differences unreadable. + +## Applies to +Novelty reports diff --git a/dokimi/research/records/finding/F010.md b/dokimi/research/records/finding/F010.md new file mode 100644 index 00000000..2b04d074 --- /dev/null +++ b/dokimi/research/records/finding/F010.md @@ -0,0 +1,18 @@ +--- +id: F010 +type: finding +title: Per-check (process/contract) grading is what discriminates skill utility +date: '2026-10-19' +experiments: +- X012 +confidence: med +--- + +## Statement +Marker-based verifiers measure one atom; tasks need verifiers that check process or contract. Use >=6 reps before quoting per-check defects. + +## Applies to +Skill eval task design diff --git a/dokimi/research/records/finding/F011.md b/dokimi/research/records/finding/F011.md new file mode 100644 index 00000000..3aee469b --- /dev/null +++ b/dokimi/research/records/finding/F011.md @@ -0,0 +1,19 @@ +--- +id: F011 +type: finding +title: For weaker models the skill effect is entirely invocation +date: '2026-10-19' +experiments: +- X013 +- X014 +confidence: med +--- + +## Statement +Measure invocation separately; hooks or descriptions that trigger invocation matter more than skill body. + +## Applies to +Skill eval reports diff --git a/dokimi/research/records/finding/F012.md b/dokimi/research/records/finding/F012.md new file mode 100644 index 00000000..12a95b23 --- /dev/null +++ b/dokimi/research/records/finding/F012.md @@ -0,0 +1,18 @@ +--- +id: F012 +type: finding +title: A guidance line can regress behaviour; evaluate wording changes +date: '2026-10-19' +experiments: +- X015 +confidence: high +--- + +## Statement +Measured regressions occurred from a plausible wording; eval caught it before merge. + +## Applies to +Any repo guidance change diff --git a/dokimi/research/records/finding/F013.md b/dokimi/research/records/finding/F013.md new file mode 100644 index 00000000..48bfb8b9 --- /dev/null +++ b/dokimi/research/records/finding/F013.md @@ -0,0 +1,18 @@ +--- +id: F013 +type: finding +title: Conditional pointers work; descriptive index lines do not +date: '2026-10-19' +experiments: +- X016 +confidence: med +--- + +## Statement +Tie a doc pointer to the triggering action; give a positive target with prohibitions. + +## Applies to +Guidance writing, synthetic only (real-repo transfer untested) diff --git a/dokimi/research/records/finding/F014.md b/dokimi/research/records/finding/F014.md new file mode 100644 index 00000000..4bc58f12 --- /dev/null +++ b/dokimi/research/records/finding/F014.md @@ -0,0 +1,18 @@ +--- +id: F014 +type: finding +title: Guidance value is often cost, not accuracy +date: '2026-10-19' +experiments: +- X019 +confidence: low +--- + +## Statement +When rules are discoverable, guidance saves tokens rather than changing answers. + +## Applies to +Repo guidance evaluation diff --git a/dokimi/research/records/finding/F015.md b/dokimi/research/records/finding/F015.md new file mode 100644 index 00000000..ab37a914 --- /dev/null +++ b/dokimi/research/records/finding/F015.md @@ -0,0 +1,18 @@ +--- +id: F015 +type: finding +title: Under deferred MCP loading the distinction must be in the tool name +date: '2026-10-19' +experiments: +- X022 +confidence: med +--- + +## Statement +Descriptions are not seen at selection time. + +## Applies to +MCP server design diff --git a/dokimi/research/records/finding/F016.md b/dokimi/research/records/finding/F016.md new file mode 100644 index 00000000..c19c348e --- /dev/null +++ b/dokimi/research/records/finding/F016.md @@ -0,0 +1,18 @@ +--- +id: F016 +type: finding +title: Prune/curate beats compaction +date: '2026-10-19' +experiments: +- X021 +confidence: low +--- + +## Statement +Pruning reads cut peak tokens ~10x with same accuracy; forced compaction dropped ~7% and lost accuracy once. + +## Applies to +Context management diff --git a/dokimi/research/records/finding/F017.md b/dokimi/research/records/finding/F017.md new file mode 100644 index 00000000..fb1ae8eb --- /dev/null +++ b/dokimi/research/records/finding/F017.md @@ -0,0 +1,18 @@ +--- +id: F017 +type: finding +title: Harness success carries no information; stable paths can be stably wrong +date: '2026-10-19' +experiments: +- X023 +confidence: med +--- + +## Statement +Grade outcome and process separately; tool-call grain must split shell commands. + +## Applies to +Trace-based eval diff --git a/dokimi/research/records/finding/F018.md b/dokimi/research/records/finding/F018.md new file mode 100644 index 00000000..22f9a9a8 --- /dev/null +++ b/dokimi/research/records/finding/F018.md @@ -0,0 +1,18 @@ +--- +id: F018 +type: finding +title: Zero delta at ceiling is not 'no effect' +date: '2026-10-19' +experiments: +- X024 +confidence: high +--- + +## Statement +When both arms pass every paired task, compare must flag no headroom. + +## Applies to +repo compare diff --git a/dokimi/research/records/finding/F019.md b/dokimi/research/records/finding/F019.md new file mode 100644 index 00000000..a76907b1 --- /dev/null +++ b/dokimi/research/records/finding/F019.md @@ -0,0 +1,18 @@ +--- +id: F019 +type: finding +title: Validation must live in our tooling +date: '2026-10-19' +experiments: +- X027 +confidence: med +--- + +## Statement +No trialled tool enforced typed references or enums on API write. + +## Applies to +Research register diff --git a/dokimi/research/records/idea/I001.md b/dokimi/research/records/idea/I001.md new file mode 100644 index 00000000..717970c2 --- /dev/null +++ b/dokimi/research/records/idea/I001.md @@ -0,0 +1,18 @@ +--- +id: I001 +type: idea +title: Absolute single-answer judge scoring +date: '2026-10-19' +theme: C +question: Can a judge score one answer on an absolute scale without saturating or rewarding length? +status: open +--- + +## Benefit +Arm comparisons need a trustworthy absolute score, not only pairwise preference. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I002.md b/dokimi/research/records/idea/I002.md new file mode 100644 index 00000000..efed2432 --- /dev/null +++ b/dokimi/research/records/idea/I002.md @@ -0,0 +1,18 @@ +--- +id: I002 +type: idea +title: Judge context in arm comparisons +date: '2026-10-19' +theme: C +question: Does what the judge sees per arm bias skill-vs-baseline verdicts? +status: open +--- + +## Benefit +Prevents false wins/losses for treatments that cite their own material. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I003.md b/dokimi/research/records/idea/I003.md new file mode 100644 index 00000000..e1cfe0f8 --- /dev/null +++ b/dokimi/research/records/idea/I003.md @@ -0,0 +1,18 @@ +--- +id: I003 +type: idea +title: 'Judge reliability: cross-judge, pairwise, sequential gates' +date: '2026-10-19' +theme: C +question: Which judge mode/model passes a validity gate and is stable across judges? +status: open +--- + +## Benefit +Cheaper and more reliable verdicts; know when a verdict is judge-dependent. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I004.md b/dokimi/research/records/idea/I004.md new file mode 100644 index 00000000..b508a50f --- /dev/null +++ b/dokimi/research/records/idea/I004.md @@ -0,0 +1,18 @@ +--- +id: I004 +type: idea +title: Malformed model rows must never read as results +date: '2026-10-19' +theme: C +question: Do parse failures in novelty/judge stages silently count as verdicts? +status: closed +--- + +## Benefit +Silent-failure removal across the pipeline. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I005.md b/dokimi/research/records/idea/I005.md new file mode 100644 index 00000000..75ebde9b --- /dev/null +++ b/dokimi/research/records/idea/I005.md @@ -0,0 +1,18 @@ +--- +id: I005 +type: idea +title: 'Novelty: what knowledge does a skill add' +date: '2026-10-19' +theme: K +question: How much of a skill's content is new to a model, and can it be measured cheaply? +status: open +--- + +## Benefit +Tells skill authors what is worth shipping. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I006.md b/dokimi/research/records/idea/I006.md new file mode 100644 index 00000000..d5986cf7 --- /dev/null +++ b/dokimi/research/records/idea/I006.md @@ -0,0 +1,18 @@ +--- +id: I006 +type: idea +title: Skill utility without novelty +date: '2026-10-19' +theme: K +question: How to evaluate skills whose value is workflow, automation or guardrails rather than new facts? +status: open +--- + +## Benefit +Avoids discarding useful skills that add no knowledge. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I007.md b/dokimi/research/records/idea/I007.md new file mode 100644 index 00000000..2d316b28 --- /dev/null +++ b/dokimi/research/records/idea/I007.md @@ -0,0 +1,18 @@ +--- +id: I007 +type: idea +title: Skill discovery and invocation +date: '2026-10-19' +theme: K +question: Does an installed skill help when the agent is not told it exists? +status: open +--- + +## Benefit +Measures real-world discovery, not treatment-notice effect. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I008.md b/dokimi/research/records/idea/I008.md new file mode 100644 index 00000000..3b9cd096 --- /dev/null +++ b/dokimi/research/records/idea/I008.md @@ -0,0 +1,18 @@ +--- +id: I008 +type: idea +title: Skill catalog integration +date: '2026-10-19' +theme: K +question: Run per-skill evals in a catalog's CI and promote skills to validated on evidence. +status: open +--- + +## Benefit +Evidence-backed skill catalogs. + +## Projects +dokimi, intel/skills, skill_search diff --git a/dokimi/research/records/idea/I009.md b/dokimi/research/records/idea/I009.md new file mode 100644 index 00000000..d5405000 --- /dev/null +++ b/dokimi/research/records/idea/I009.md @@ -0,0 +1,18 @@ +--- +id: I009 +type: idea +title: Agent guidance wording and placement +date: '2026-10-19' +theme: D +question: Which guidance file and wording actually change agent behaviour? +status: open +--- + +## Benefit +Evidence-based AGENTS.md / skill writing rules. + +## Projects +dokimi, oneDAL, sklearnex diff --git a/dokimi/research/records/idea/I010.md b/dokimi/research/records/idea/I010.md new file mode 100644 index 00000000..b78fe070 --- /dev/null +++ b/dokimi/research/records/idea/I010.md @@ -0,0 +1,18 @@ +--- +id: I010 +type: idea +title: Cold-agent onboarding +date: '2026-10-19' +theme: D +question: Can a fresh agent use dokimi from the README alone? +status: open +--- + +## Benefit +Lower adoption friction. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I011.md b/dokimi/research/records/idea/I011.md new file mode 100644 index 00000000..c70cb53a --- /dev/null +++ b/dokimi/research/records/idea/I011.md @@ -0,0 +1,18 @@ +--- +id: I011 +type: idea +title: 'Docs structure: Learn vs Use tracks' +date: '2026-10-19' +theme: D +question: Should docs split education and reference? +status: parked +--- + +## Benefit +Clearer adoption path. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I012.md b/dokimi/research/records/idea/I012.md new file mode 100644 index 00000000..06556f18 --- /dev/null +++ b/dokimi/research/records/idea/I012.md @@ -0,0 +1,18 @@ +--- +id: I012 +type: idea +title: Preparing a repository for coding agents +date: '2026-10-19' +theme: D +question: What does a repository offer an agent, statically? +status: open +--- + +## Benefit +Readiness diagnostics without a score. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I013.md b/dokimi/research/records/idea/I013.md new file mode 100644 index 00000000..e023f879 --- /dev/null +++ b/dokimi/research/records/idea/I013.md @@ -0,0 +1,18 @@ +--- +id: I013 +type: idea +title: 'Context failure modes: poisoning, distraction, confusion, clash' +date: '2026-10-19' +theme: G +question: How do agents fail under bad context, and which fixes work? +status: open +--- + +## Benefit +Operational context tooling. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I014.md b/dokimi/research/records/idea/I014.md new file mode 100644 index 00000000..12b35bad --- /dev/null +++ b/dokimi/research/records/idea/I014.md @@ -0,0 +1,18 @@ +--- +id: I014 +type: idea +title: Tool preference factors +date: '2026-10-19' +theme: G +question: What moves a model's choice between two equal tools? +status: open +--- + +## Benefit +MCP/tool authors learn which description changes matter. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I015.md b/dokimi/research/records/idea/I015.md new file mode 100644 index 00000000..9af3aafd --- /dev/null +++ b/dokimi/research/records/idea/I015.md @@ -0,0 +1,18 @@ +--- +id: I015 +type: idea +title: Result vs path stability +date: '2026-10-19' +theme: O +question: Is a stable agent path a correct one; how to measure process conformance from traces? +status: open +--- + +## Benefit +Separates result stability from path stability in evals. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I016.md b/dokimi/research/records/idea/I016.md new file mode 100644 index 00000000..af48d644 --- /dev/null +++ b/dokimi/research/records/idea/I016.md @@ -0,0 +1,18 @@ +--- +id: I016 +type: idea +title: Cost-aware eval and tokenomics +date: '2026-10-19' +theme: O +question: How should eval cost be capped and reported? +status: open +--- + +## Benefit +Predictable spend. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I017.md b/dokimi/research/records/idea/I017.md new file mode 100644 index 00000000..bd0396d4 --- /dev/null +++ b/dokimi/research/records/idea/I017.md @@ -0,0 +1,19 @@ +--- +id: I017 +type: idea +title: Repository eval mechanics in dokimi +date: '2026-10-19' +theme: V +question: Move generic repo-eval mechanics (compare, graders, host requirements) out of a product repo + into dokimi. +status: open +--- + +## Benefit +One harness, evals stay with each repo. + +## Projects +dokimi, oneDAL diff --git a/dokimi/research/records/idea/I018.md b/dokimi/research/records/idea/I018.md new file mode 100644 index 00000000..242785bb --- /dev/null +++ b/dokimi/research/records/idea/I018.md @@ -0,0 +1,18 @@ +--- +id: I018 +type: idea +title: Repo quality gates and measurement integrity +date: '2026-10-19' +theme: E +question: Which CI gates catch real defects in an eval framework? +status: open +--- + +## Benefit +Trustworthy numbers from the tool itself. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I019.md b/dokimi/research/records/idea/I019.md new file mode 100644 index 00000000..ab686e7b --- /dev/null +++ b/dokimi/research/records/idea/I019.md @@ -0,0 +1,18 @@ +--- +id: I019 +type: idea +title: Closed user loop and journeys +date: '2026-10-19' +theme: P +question: Can dokimi take a user from task to evidence to regression check with contracts per journey? +status: open +--- + +## Benefit +Product promise: make a skill or repo more useful and prove it. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I020.md b/dokimi/research/records/idea/I020.md new file mode 100644 index 00000000..503cc6cf --- /dev/null +++ b/dokimi/research/records/idea/I020.md @@ -0,0 +1,18 @@ +--- +id: I020 +type: idea +title: CLI restructure around tasks +date: '2026-10-19' +theme: P +question: Should 64 operations be regrouped into task-oriented groups? +status: parked +--- + +## Benefit +Discoverability. + +## Projects +dokimi diff --git a/dokimi/research/records/idea/I021.md b/dokimi/research/records/idea/I021.md new file mode 100644 index 00000000..84f0023a --- /dev/null +++ b/dokimi/research/records/idea/I021.md @@ -0,0 +1,18 @@ +--- +id: I021 +type: idea +title: Tool-independent research record register +date: '2026-10-19' +theme: W +question: Where should idea/experiment/finding/decision records live? +status: open +--- + +## Benefit +Negative results and decisions stay traceable; export drives the mindmap. + +## Projects +dokimi diff --git a/dokimi/research/records/implref/R001.md b/dokimi/research/records/implref/R001.md new file mode 100644 index 00000000..705339ea --- /dev/null +++ b/dokimi/research/records/implref/R001.md @@ -0,0 +1,17 @@ +--- +id: R001 +type: implref +title: Keep judge treatment material in the artifact +date: '2026-10-19' +decision: D003 +status: merged +links: +- https://github.com/intel/dokimi/issues/48 +- https://github.com/intel/dokimi/pull/69 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R002.md b/dokimi/research/records/implref/R002.md new file mode 100644 index 00000000..00831082 --- /dev/null +++ b/dokimi/research/records/implref/R002.md @@ -0,0 +1,17 @@ +--- +id: R002 +type: implref +title: judge-validity --context stored +date: '2026-10-19' +decision: D003 +status: open +links: +- https://github.com/intel/dokimi/issues/48 +- https://github.com/intel/dokimi/pull/73 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R003.md b/dokimi/research/records/implref/R003.md new file mode 100644 index 00000000..51bb3653 --- /dev/null +++ b/dokimi/research/records/implref/R003.md @@ -0,0 +1,16 @@ +--- +id: R003 +type: implref +title: Stamp judge_context from what the judge saw +date: '2026-10-19' +decision: D003 +status: merged +links: +- https://github.com/intel/dokimi/pull/13 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R006.md b/dokimi/research/records/implref/R006.md new file mode 100644 index 00000000..34a3c18b --- /dev/null +++ b/dokimi/research/records/implref/R006.md @@ -0,0 +1,16 @@ +--- +id: R006 +type: implref +title: Novelty coverage basis stamped; unmeasured share over readable words +date: '2026-10-19' +decision: D004 +status: merged +links: +- https://github.com/intel/dokimi/pull/101 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R010.md b/dokimi/research/records/implref/R010.md new file mode 100644 index 00000000..cbb59426 --- /dev/null +++ b/dokimi/research/records/implref/R010.md @@ -0,0 +1,16 @@ +--- +id: R010 +type: implref +title: AGENTS.md guidance +date: '2026-10-19' +decision: D006 +status: merged +links: +- https://github.com/intel/dokimi/pull/56 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R011.md b/dokimi/research/records/implref/R011.md new file mode 100644 index 00000000..f7bcf08b --- /dev/null +++ b/dokimi/research/records/implref/R011.md @@ -0,0 +1,16 @@ +--- +id: R011 +type: implref +title: dokimi's own repository evals in .agent-eval +date: '2026-10-19' +decision: D006 +status: merged +links: +- https://github.com/intel/dokimi/pull/57 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R016.md b/dokimi/research/records/implref/R016.md new file mode 100644 index 00000000..b32f03d5 --- /dev/null +++ b/dokimi/research/records/implref/R016.md @@ -0,0 +1,16 @@ +--- +id: R016 +type: implref +title: context trace +date: '2026-10-19' +decision: D008 +status: merged +links: +- https://github.com/intel/dokimi/pull/68 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R017.md b/dokimi/research/records/implref/R017.md new file mode 100644 index 00000000..f00cd0f5 --- /dev/null +++ b/dokimi/research/records/implref/R017.md @@ -0,0 +1,16 @@ +--- +id: R017 +type: implref +title: repo budget +date: '2026-10-19' +decision: D008 +status: open +links: +- https://github.com/intel/dokimi/pull/71 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R021.md b/dokimi/research/records/implref/R021.md new file mode 100644 index 00000000..e9180e03 --- /dev/null +++ b/dokimi/research/records/implref/R021.md @@ -0,0 +1,17 @@ +--- +id: R021 +type: implref +title: repo compare (D1) +date: '2026-10-19' +decision: D009 +status: merged +links: +- https://github.com/intel/dokimi/issues/41 +- https://github.com/intel/dokimi/pull/42 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R022.md b/dokimi/research/records/implref/R022.md new file mode 100644 index 00000000..6894ac75 --- /dev/null +++ b/dokimi/research/records/implref/R022.md @@ -0,0 +1,17 @@ +--- +id: R022 +type: implref +title: compare flags no headroom +date: '2026-10-19' +decision: D009 +status: open +links: +- https://github.com/intel/dokimi/issues/41 +- https://github.com/intel/dokimi/pull/114 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R023.md b/dokimi/research/records/implref/R023.md new file mode 100644 index 00000000..84755926 --- /dev/null +++ b/dokimi/research/records/implref/R023.md @@ -0,0 +1,17 @@ +--- +id: R023 +type: implref +title: task host requirements (D3) +date: '2026-10-19' +decision: D009 +status: open +links: +- https://github.com/intel/dokimi/issues/41 +- https://github.com/intel/dokimi/pull/112 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R024.md b/dokimi/research/records/implref/R024.md new file mode 100644 index 00000000..fafb3dda --- /dev/null +++ b/dokimi/research/records/implref/R024.md @@ -0,0 +1,17 @@ +--- +id: R024 +type: implref +title: reusable repo-eval workflow (D8) +date: '2026-10-19' +decision: D009 +status: open +links: +- https://github.com/intel/dokimi/issues/41 +- https://github.com/intel/dokimi/pull/113 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R025.md b/dokimi/research/records/implref/R025.md new file mode 100644 index 00000000..d4e66144 --- /dev/null +++ b/dokimi/research/records/implref/R025.md @@ -0,0 +1,16 @@ +--- +id: R025 +type: implref +title: no mypy exemptions +date: '2026-10-19' +decision: D010 +status: merged +links: +- https://github.com/intel/dokimi/pull/108 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R026.md b/dokimi/research/records/implref/R026.md new file mode 100644 index 00000000..7de6f547 --- /dev/null +++ b/dokimi/research/records/implref/R026.md @@ -0,0 +1,16 @@ +--- +id: R026 +type: implref +title: gate tests that cannot fail +date: '2026-10-19' +decision: D010 +status: open +links: +- https://github.com/intel/dokimi/pull/103 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R027.md b/dokimi/research/records/implref/R027.md new file mode 100644 index 00000000..eba7509d --- /dev/null +++ b/dokimi/research/records/implref/R027.md @@ -0,0 +1,16 @@ +--- +id: R027 +type: implref +title: Architecture cleanup tracking +date: '2026-10-19' +decision: D010 +status: open +links: +- https://github.com/intel/dokimi/issues/4 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R028.md b/dokimi/research/records/implref/R028.md new file mode 100644 index 00000000..6149e04e --- /dev/null +++ b/dokimi/research/records/implref/R028.md @@ -0,0 +1,16 @@ +--- +id: R028 +type: implref +title: Outcome contracts in tests +date: '2026-10-19' +decision: D011 +status: merged +links: +- https://github.com/intel/dokimi/pull/74 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R029.md b/dokimi/research/records/implref/R029.md new file mode 100644 index 00000000..bc291bb7 --- /dev/null +++ b/dokimi/research/records/implref/R029.md @@ -0,0 +1,17 @@ +--- +id: R029 +type: implref +title: User-journey catalog J1-J15 +date: '2026-10-19' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/95 +- https://github.com/intel/dokimi/pull/102 +--- + +## Repository +intel/dokimi diff --git a/dokimi/research/records/implref/R030.md b/dokimi/research/records/implref/R030.md new file mode 100644 index 00000000..22bbaf60 --- /dev/null +++ b/dokimi/research/records/implref/R030.md @@ -0,0 +1,16 @@ +--- +id: R030 +type: implref +title: 'User loop: plan/validate/compare/gate/record' +date: '2026-10-19' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/121 +--- + +## Repository +intel/dokimi diff --git a/dokimi/scripts/research_register.py b/dokimi/scripts/research_register.py new file mode 100755 index 00000000..38448899 --- /dev/null +++ b/dokimi/scripts/research_register.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Check, query and export the research register in ``research/records/``. + +Each record is one Markdown file, ``research/records//.md``: YAML +front matter with the typed fields of ``research/record.v1.json`` and a free +Markdown body (setup, result, rationale). Ideas, experiments, findings, +decisions and implementation references point at each other by id, so a +refuted experiment stays findable from the idea it tested. + + python scripts/research_register.py check + python scripts/research_register.py query --type experiment --where outcome=refuted,inconclusive + python scripts/research_register.py query --type experiment --ref I001 + python scripts/research_register.py next experiment + python scripts/research_register.py export > register.json + +``check`` exits 1 when a record breaks its schema, sits under the wrong path, +or points at a missing record or one of the wrong type. ``export`` writes every +record plus the reference edges and an idea tree grouped by map theme, the input +for the plans-and-research map. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import jsonschema +import yaml + +PROJECT = Path(__file__).resolve().parents[1] +RESEARCH = PROJECT / "research" +SCHEMA = RESEARCH / "record.v1.json" +# The license header every file carries, as an HTML comment after the front matter. +_HEADER = re.compile(r"\A\s*\s*", re.DOTALL) + +PREFIX = {"idea": "I", "experiment": "X", "finding": "F", "decision": "D", "implref": "R"} +# field -> record type it must point at, per record type. +REFS: dict[str, dict[str, str]] = { + "experiment": {"idea": "idea"}, + "finding": {"experiments": "experiment"}, + "decision": {"findings": "finding", "ideas": "idea", "superseded_by": "decision"}, + "implref": {"decision": "decision"}, +} +THEMES = { + "P": "User workflows and integrations", + "E": "Engineering foundations", + "V": "Eval design, quality and infrastructure", + "K": "Skill scope, knowledge and utility", + "G": "Graphs, context and MCP", + "O": "Traces, optimization and visualization", + "C": "Judging and empirical validation", + "D": "Documentation and adoption", + "W": "Research workspace and records", +} + + +@dataclass +class Record: + fields: dict[str, Any] + body: str + path: Path + + @property + def id(self) -> str: + return str(self.fields.get("id", self.path.stem)) + + @property + def type(self) -> str: + return str(self.fields.get("type", "")) + + +def parse(path: Path) -> Record: + text = path.read_text(encoding="utf-8") + if not text.startswith("---\n") or "\n---\n" not in text[4:]: + raise ValueError(f"{path}: no YAML front matter") + front, body = text[4:].split("\n---\n", 1) + fields = yaml.safe_load(front) + if not isinstance(fields, dict): + raise ValueError(f"{path}: front matter is not a mapping") + # YAML reads an unquoted 2026-10-14 as a date; the schema wants the string. + fields = {k: (v.isoformat() if hasattr(v, "isoformat") else v) for k, v in fields.items()} + return Record(fields, _HEADER.sub("", body).strip(), path) + + +def load(root: Path = RESEARCH) -> tuple[dict[str, Record], list[str]]: + records: dict[str, Record] = {} + errors: list[str] = [] + for path in sorted((root / "records").glob("*/*.md")): + try: + record = parse(path) + except (ValueError, yaml.YAMLError) as exc: + errors.append(str(exc)) + continue + if record.id in records: + errors.append(f"{record.id}: duplicate id in {path} and {records[record.id].path}") + continue + records[record.id] = record + return records, errors + + +def ref_values(record: Record, field: str) -> list[str]: + value = record.fields.get(field) + if value is None: + return [] + return list(value) if isinstance(value, list) else [value] + + +def check(records: dict[str, Record], schema: dict[str, Any]) -> list[str]: + errors: list[str] = [] + for rid, record in sorted(records.items()): + kind = record.type + if kind not in PREFIX: + errors.append(f"{rid}: unknown type {kind!r}") + continue + sub = {"$schema": schema["$schema"], "$defs": schema["$defs"], "$ref": f"#/$defs/{kind}"} + for err in jsonschema.Draft202012Validator(sub).iter_errors(record.fields): + errors.append(f"{rid}: {err.json_path}: {err.message}") + if record.path.parent.name != kind or record.path.stem != rid: + errors.append(f"{rid}: expected at records/{kind}/{rid}.md, found {record.path.name}") + if not rid.startswith(PREFIX[kind]): + errors.append(f"{rid}: a {kind} id starts with {PREFIX[kind]}") + for field, want in REFS.get(kind, {}).items(): + for target in ref_values(record, field): + if target not in records: + errors.append(f"{rid}.{field}: no record {target}") + elif records[target].type != want: + errors.append(f"{rid}.{field}: {target} is {records[target].type}, " + f"want {want}") + if record.fields.get("state") == "superseded" and "superseded_by" not in record.fields: + errors.append(f"{rid}: a superseded decision names its superseded_by") + return errors + + +def matches(record: Record, where: list[str], ref: str | None, kind: str | None) -> bool: + if kind and record.type != kind: + return False + for clause in where: + field, _, wanted = clause.partition("=") + value = record.fields.get(field) + values = [str(v) for v in value] if isinstance(value, list) else [str(value)] + if not set(values) & set(wanted.split(",")): + return False + if ref: + return any(ref in ref_values(record, f) for f in REFS.get(record.type, {})) + return True + + +def next_id(records: dict[str, Record], kind: str) -> str: + prefix = PREFIX[kind] + used = [int(r[len(prefix):]) for r in records if r.startswith(prefix) and r[len(prefix):].isdigit()] + return f"{prefix}{max(used, default=0) + 1:03d}" + + +def export(records: dict[str, Record]) -> dict[str, Any]: + edges = [{"from": rid, "to": target, "rel": field} + for rid, record in sorted(records.items()) + for field in REFS.get(record.type, {}) + for target in ref_values(record, field)] + pointing: dict[str, list[str]] = {} + for edge in edges: + pointing.setdefault(edge["to"], []).append(edge["from"]) + themes = [] + for key, title in THEMES.items(): + ideas = [r for r in records.values() if r.type == "idea" and r.fields["theme"] == key] + themes.append({"id": key, "title": title, "ideas": [ + {"id": r.id, "title": r.fields["title"], "status": r.fields["status"], + "experiments": [{"id": x, "outcome": records[x].fields["outcome"]} + for x in sorted(pointing.get(r.id, [])) + if records[x].type == "experiment"]} + for r in sorted(ideas, key=lambda r: r.id)]}) + commit = subprocess.run(["git", "-C", str(PROJECT), "rev-parse", "HEAD"], + capture_output=True, text=True, check=False).stdout.strip() + return {"commit": commit or None, + "records": [{**r.fields, "body": r.body} for _, r in sorted(records.items())], + "edges": edges, "themes": themes} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--root", type=Path, default=RESEARCH, help=argparse.SUPPRESS) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("check", help="validate every record and its references") + query = sub.add_parser("query", help="list records matching every filter") + query.add_argument("--type", choices=PREFIX) + query.add_argument("--where", action="append", default=[], metavar="FIELD=V1,V2") + query.add_argument("--ref", metavar="ID", help="records that point at ID") + nxt = sub.add_parser("next", help="print the next free id for a record type") + nxt.add_argument("type", choices=PREFIX) + sub.add_parser("export", help="write records, edges and the theme tree as JSON") + args = parser.parse_args(argv) + + records, errors = load(args.root) + if args.command == "check": + schema = json.loads((args.root / SCHEMA.name).read_text(encoding="utf-8")) + errors += check(records, schema) + print("\n".join(errors) if errors else f"OK: {len(records)} records") + return 1 if errors else 0 + if errors: + print("\n".join(errors), file=sys.stderr) + return 1 + if args.command == "query": + for record in sorted(records.values(), key=lambda r: r.id): + if matches(record, args.where, args.ref, args.type): + extra = record.fields.get("outcome") or record.fields.get("state") or \ + record.fields.get("status") or record.fields.get("confidence") or "" + print(f"{record.id}\t{record.type}\t{extra}\t{record.fields['title']}") + elif args.command == "next": + print(next_id(records, args.type)) + else: + json.dump(export(records), sys.stdout, indent=1, ensure_ascii=False) + print() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dokimi/tests/test_research_register.py b/dokimi/tests/test_research_register.py new file mode 100644 index 00000000..ec931311 --- /dev/null +++ b/dokimi/tests/test_research_register.py @@ -0,0 +1,103 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for scripts/research_register.py and the records in research/records/.""" + +from __future__ import annotations + +import importlib.util +import json +import shutil +import sys +from pathlib import Path + +import pytest + +PROJECT = Path(__file__).resolve().parents[1] +_SPEC = importlib.util.spec_from_file_location( + "research_register", PROJECT / "scripts" / "research_register.py") +assert _SPEC and _SPEC.loader +rr = importlib.util.module_from_spec(_SPEC) +# dataclasses look the module up in sys.modules while the class body runs. +sys.modules[_SPEC.name] = rr +_SPEC.loader.exec_module(rr) + + +def _write(root: Path, kind: str, rid: str, front: str, body: str = "") -> None: + path = root / "records" / kind / f"{rid}.md" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"---\nid: {rid}\ntype: {kind}\ndate: 2026-10-14\n{front}---\n{body}\n") + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + shutil.copy(rr.SCHEMA, tmp_path / rr.SCHEMA.name) + _write(tmp_path, "idea", "I001", "title: Compress\ntheme: O\nquestion: Q?\nstatus: open\n") + _write(tmp_path, "experiment", "X001", + "title: Run 1\nidea: I001\nhypothesis: H\noutcome: refuted\n", + "\n## Result\nworse") + _write(tmp_path, "experiment", "X002", + "title: Run 2\nidea: I001\nhypothesis: H\noutcome: supported\n") + _write(tmp_path, "finding", "F001", "title: F\nexperiments: [X001]\nconfidence: med\n") + _write(tmp_path, "decision", "D001", "title: D\nfindings: [F001]\nstate: rejected\n") + return tmp_path + + +def test_repository_register_is_valid() -> None: + assert rr.main(["check"]) == 0 + + +def test_valid_register_passes(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "check"]) == 0 + assert "OK: 5 records" in capsys.readouterr().out + + +@pytest.mark.parametrize(("kind", "rid", "front", "message"), [ + ("experiment", "X003", "title: T\nidea: I009\nhypothesis: H\noutcome: refuted\n", + "no record I009"), + ("experiment", "X003", "title: T\nidea: F001\nhypothesis: H\noutcome: refuted\n", + "F001 is finding, want idea"), + ("experiment", "X003", "title: T\nidea: I001\nhypothesis: H\noutcome: maybe\n", + "'maybe' is not one of"), + ("experiment", "X003", "title: T\nidea: I001\nhypothesis: H\noutcome: refuted\nextra: 1\n", + "Unevaluated properties"), + ("decision", "D002", "title: T\nstate: superseded\n", "names its superseded_by"), + ("finding", "X009", "title: T\nexperiments: [X001]\nconfidence: low\n", + "a finding id starts with F"), + ("idea", "I002", "title: T\ntheme: O\nquestion: Q\nstatus: open\nlinks: [https://example.com/x]\n", + "does not match"), +]) +def test_bad_record_fails(root: Path, capsys: pytest.CaptureFixture[str], + kind: str, rid: str, front: str, message: str) -> None: + _write(root, kind, rid, front) + assert rr.main(["--root", str(root), "check"]) == 1 + assert message in capsys.readouterr().out + + +def test_duplicate_id_fails(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + (root / "records" / "idea" / "I001.md").rename(root / "records" / "idea" / "copy.md") + _write(root, "idea", "I001", "title: Again\ntheme: O\nquestion: Q?\nstatus: open\n") + assert rr.main(["--root", str(root), "check"]) == 1 + assert "duplicate id" in capsys.readouterr().out + + +def test_query_negative_outcomes_for_idea(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "query", "--type", "experiment", "--ref", "I001", + "--where", "outcome=refuted,inconclusive"]) == 0 + out = capsys.readouterr().out.splitlines() + assert [line.split("\t")[0] for line in out] == ["X001"] + + +def test_next_id(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "next", "experiment"]) == 0 + assert capsys.readouterr().out.strip() == "X003" + + +def test_export_tree(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "export"]) == 0 + data = json.loads(capsys.readouterr().out) + assert {"from": "X001", "to": "I001", "rel": "idea"} in data["edges"] + theme = next(t for t in data["themes"] if t["id"] == "O") + assert theme["ideas"][0]["experiments"] == [ + {"id": "X001", "outcome": "refuted"}, {"id": "X002", "outcome": "supported"}] + assert next(r for r in data["records"] if r["id"] == "X001")["body"] == "## Result\nworse" From 71239676f6fde4324363651ddf3352b21ab17d90 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 20 Oct 2026 04:02:01 -0700 Subject: [PATCH 02/21] fix(research): validate before every command, reject bad superseded_by, pin more register behaviour export, query and next read the register without checking it, so a record that check rejects crashed them with a traceback, and a nested reference crashed check itself. Every command now runs the schema and reference checks first and exits 1 with the messages. superseded_by is only valid on a superseded decision and cannot name the record itself. Tests cover --ref with --type, misplaced and misnamed files, id gaps, the broken-register path for each command, and unevaluatedProperties on every record type. Signed-off-by: Nikolay Petrov --- dokimi/scripts/research_register.py | 14 ++++-- dokimi/tests/test_research_register.py | 66 ++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/dokimi/scripts/research_register.py b/dokimi/scripts/research_register.py index 38448899..b8a63197 100755 --- a/dokimi/scripts/research_register.py +++ b/dokimi/scripts/research_register.py @@ -130,13 +130,20 @@ def check(records: dict[str, Record], schema: dict[str, Any]) -> list[str]: errors.append(f"{rid}: a {kind} id starts with {PREFIX[kind]}") for field, want in REFS.get(kind, {}).items(): for target in ref_values(record, field): + if not isinstance(target, str): + continue # the schema already reports a non-id value if target not in records: errors.append(f"{rid}.{field}: no record {target}") elif records[target].type != want: errors.append(f"{rid}.{field}: {target} is {records[target].type}, " f"want {want}") - if record.fields.get("state") == "superseded" and "superseded_by" not in record.fields: + state = record.fields.get("state") + if state == "superseded" and "superseded_by" not in record.fields: errors.append(f"{rid}: a superseded decision names its superseded_by") + if "superseded_by" in record.fields and state != "superseded": + errors.append(f"{rid}: superseded_by is set but state is {state!r}, not 'superseded'") + if record.fields.get("superseded_by") == rid: + errors.append(f"{rid}: superseded_by names the record itself") return errors @@ -199,11 +206,12 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) records, errors = load(args.root) + schema = json.loads((args.root / SCHEMA.name).read_text(encoding="utf-8")) + errors += check(records, schema) if args.command == "check": - schema = json.loads((args.root / SCHEMA.name).read_text(encoding="utf-8")) - errors += check(records, schema) print("\n".join(errors) if errors else f"OK: {len(records)} records") return 1 if errors else 0 + # Every other command reads the records as valid, so refuse a broken register up front. if errors: print("\n".join(errors), file=sys.stderr) return 1 diff --git a/dokimi/tests/test_research_register.py b/dokimi/tests/test_research_register.py index ec931311..44ff4602 100644 --- a/dokimi/tests/test_research_register.py +++ b/dokimi/tests/test_research_register.py @@ -66,6 +66,19 @@ def test_valid_register_passes(root: Path, capsys: pytest.CaptureFixture[str]) - "a finding id starts with F"), ("idea", "I002", "title: T\ntheme: O\nquestion: Q\nstatus: open\nlinks: [https://example.com/x]\n", "does not match"), + ("experiment", "X003", "title: T\nidea: [[I001]]\nhypothesis: H\noutcome: refuted\n", + "is not of type 'string'"), + ("decision", "D002", "title: T\nstate: accepted\nsuperseded_by: D001\n", + "superseded_by is set but state is 'accepted'"), + ("decision", "D002", "title: T\nstate: superseded\nsuperseded_by: D002\n", + "names the record itself"), + ("idea", "I002", "title: T\ntheme: O\nquestion: Q\nstatus: open\nextra: 1\n", + "Unevaluated properties"), + ("finding", "F002", "title: T\nexperiments: [X001]\nconfidence: low\nextra: 1\n", + "Unevaluated properties"), + ("decision", "D002", "title: T\nstate: accepted\nextra: 1\n", "Unevaluated properties"), + ("implref", "R001", "title: T\ndecision: D001\nstatus: open\nlinks: []\nextra: 1\n", + "Unevaluated properties"), ]) def test_bad_record_fails(root: Path, capsys: pytest.CaptureFixture[str], kind: str, rid: str, front: str, message: str) -> None: @@ -88,11 +101,64 @@ def test_query_negative_outcomes_for_idea(root: Path, capsys: pytest.CaptureFixt assert [line.split("\t")[0] for line in out] == ["X001"] +def test_query_ref_and_type_both_filter(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "idea", "I002", "title: Other\ntheme: O\nquestion: Q?\nstatus: open\n") + _write(root, "experiment", "X003", "title: Run 3\nidea: I002\nhypothesis: H\noutcome: refuted\n") + + def listed(*argv: str) -> list[str]: + capsys.readouterr() + assert rr.main(["--root", str(root), "query", *argv]) == 0 + return [line.split("\t")[0] for line in capsys.readouterr().out.splitlines()] + + assert listed("--ref", "I001") == ["X001", "X002"] + assert listed("--ref", "I002") == ["X003"] + assert listed("--type", "idea") == ["I001", "I002"] + assert listed("--type", "experiment", "--ref", "I002") == ["X003"] + assert listed("--type", "finding", "--ref", "I001") == [] + + +def test_misplaced_record_fails(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "idea", "I002", "title: T\ntheme: O\nquestion: Q\nstatus: open\n") + (root / "records" / "idea" / "I002.md").rename(root / "records" / "finding" / "I002.md") + assert rr.main(["--root", str(root), "check"]) == 1 + assert "expected at records/idea/I002.md, found I002.md" in capsys.readouterr().out + + +def test_wrong_filename_fails(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "idea", "I002", "title: T\ntheme: O\nquestion: Q\nstatus: open\n") + (root / "records" / "idea" / "I002.md").rename(root / "records" / "idea" / "I009.md") + assert rr.main(["--root", str(root), "check"]) == 1 + assert "expected at records/idea/I002.md, found I009.md" in capsys.readouterr().out + + +def test_next_id_skips_gaps(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "experiment", "X005", "title: Run 5\nidea: I001\nhypothesis: H\noutcome: planned\n") + assert rr.main(["--root", str(root), "next", "experiment"]) == 0 + assert capsys.readouterr().out.strip() == "X006" + + def test_next_id(root: Path, capsys: pytest.CaptureFixture[str]) -> None: assert rr.main(["--root", str(root), "next", "experiment"]) == 0 assert capsys.readouterr().out.strip() == "X003" +@pytest.mark.parametrize("argv", [["query"], ["next", "experiment"], ["export"]]) +def test_commands_refuse_broken_register(root: Path, capsys: pytest.CaptureFixture[str], + argv: list[str]) -> None: + _write(root, "idea", "I002", "title: No theme\nquestion: Q\nstatus: open\n") + assert rr.main(["--root", str(root), *argv]) == 1 + captured = capsys.readouterr() + assert "I002: $: 'theme' is a required property" in captured.err + assert captured.out == "" + + +def test_export_refuses_record_without_theme_status(root: Path, + capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "idea", "I002", "title: No status\ntheme: O\nquestion: Q\n") + assert rr.main(["--root", str(root), "export"]) == 1 + assert "'status' is a required property" in capsys.readouterr().err + + def test_export_tree(root: Path, capsys: pytest.CaptureFixture[str]) -> None: assert rr.main(["--root", str(root), "export"]) == 0 data = json.loads(capsys.readouterr().out) From 7e152f4e91cd104e21b43baaa913e33bc4673418 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 20 Oct 2026 14:35:08 -0700 Subject: [PATCH 03/21] fix(research): refuse unknown --where fields, pin export and missing-field queries - X027: the hypothesis (a self-hosted tool enforces the schema) is refuted by the result text, so the outcome is refuted. - F016: drop the "~7% / lost accuracy" clause; no cited experiment has it. - query: exit 2 on a --where clause that is not FIELD=VALUE on a schema field, instead of printing nothing with exit 0. A missing field matches no value rather than the string "None". - tests: export ignores a decision that points at an idea; bad --where clauses and the missing-field case are pinned. Signed-off-by: Nikolay Petrov --- dokimi/research/records/experiment/X027.md | 2 +- dokimi/research/records/finding/F016.md | 2 +- dokimi/scripts/research_register.py | 8 +++++++ dokimi/tests/test_research_register.py | 26 ++++++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/dokimi/research/records/experiment/X027.md b/dokimi/research/records/experiment/X027.md index 5634f5d9..738ff1d5 100644 --- a/dokimi/research/records/experiment/X027.md +++ b/dokimi/research/records/experiment/X027.md @@ -5,7 +5,7 @@ title: Research workspace tool trial date: '2026-10-19' idea: I021 hypothesis: A self-hosted tool can enforce the record schema. -outcome: supported +outcome: refuted --- ## Statement -Pruning reads cut peak tokens ~10x with same accuracy; forced compaction dropped ~7% and lost accuracy once. +Pruning reads cut peak tokens ~10x with same accuracy. ## Applies to Context management diff --git a/dokimi/scripts/research_register.py b/dokimi/scripts/research_register.py index b8a63197..93d0aac4 100755 --- a/dokimi/scripts/research_register.py +++ b/dokimi/scripts/research_register.py @@ -153,6 +153,8 @@ def matches(record: Record, where: list[str], ref: str | None, kind: str | None) for clause in where: field, _, wanted = clause.partition("=") value = record.fields.get(field) + if value is None: + return False # a missing field matches no value, not the string "None" values = [str(v) for v in value] if isinstance(value, list) else [str(value)] if not set(values) & set(wanted.split(",")): return False @@ -216,6 +218,12 @@ def main(argv: list[str] | None = None) -> int: print("\n".join(errors), file=sys.stderr) return 1 if args.command == "query": + fields = {p for d in schema["$defs"].values() for p in d.get("properties", {})} + bad = [c for c in args.where if "=" not in c or c.partition("=")[0] not in fields] + if bad: + print(f"--where needs FIELD=VALUE with a schema field: {', '.join(bad)}", + file=sys.stderr) + return 2 for record in sorted(records.values(), key=lambda r: r.id): if matches(record, args.where, args.ref, args.type): extra = record.fields.get("outcome") or record.fields.get("state") or \ diff --git a/dokimi/tests/test_research_register.py b/dokimi/tests/test_research_register.py index 44ff4602..da0f2f3e 100644 --- a/dokimi/tests/test_research_register.py +++ b/dokimi/tests/test_research_register.py @@ -167,3 +167,29 @@ def test_export_tree(root: Path, capsys: pytest.CaptureFixture[str]) -> None: assert theme["ideas"][0]["experiments"] == [ {"id": "X001", "outcome": "refuted"}, {"id": "X002", "outcome": "supported"}] assert next(r for r in data["records"] if r["id"] == "X001")["body"] == "## Result\nworse" + + +def test_export_ignores_decision_pointing_at_idea(root: Path, + capsys: pytest.CaptureFixture[str]) -> None: + _write(root, "decision", "D002", "title: Uses idea\nfindings: [F001]\nideas: [I001]\n" + "state: accepted\n") + assert rr.main(["--root", str(root), "export"]) == 0 + data = json.loads(capsys.readouterr().out) + theme = next(t for t in data["themes"] if t["id"] == "O") + assert theme["ideas"][0]["experiments"] == [ + {"id": "X001", "outcome": "refuted"}, {"id": "X002", "outcome": "supported"}] + + +@pytest.mark.parametrize("clause", ["outcom=refuted", "noequals", "bogus=None"]) +def test_query_rejects_bad_where_clause(root: Path, capsys: pytest.CaptureFixture[str], + clause: str) -> None: + assert rr.main(["--root", str(root), "query", "--where", clause]) == 2 + captured = capsys.readouterr() + assert clause in captured.err + assert captured.out == "" + + +def test_query_missing_field_matches_no_value(root: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(root), "query", "--type", "experiment", + "--where", "retracted=None"]) == 0 + assert capsys.readouterr().out == "" From 154dc91064dcb880855ff0361156449d4f97066e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 20 Oct 2026 18:51:36 -0700 Subject: [PATCH 04/21] fix(research): record outcomes match their results; retracted and render rules - X006 refuted, partial retractions under Corrections; X012, X025 the same. F006 lowered to med. - X003 split: criterion wording only (refuted) and X028 criterion plus enumeration (supported). F002 and F003 cite both. - X029 holds the grader agreement F009 quotes; D004 rationale says where +0.007 comes from. X021 gains the compaction follow-up. - Outcome sweep: X002, X008, X021 inconclusive; X013, X016 refuted. - check fails a finding that cites a retracted experiment; export marks retracted experiments; query --ref on a missing id exits 2. - render writes the plans-and-research map (Mermaid mindmap and table) on demand; it is not committed or published. Signed-off-by: Nikolay Petrov --- dokimi/docs/research-register.md | 13 ++++- dokimi/research/records/decision/D004.md | 2 +- dokimi/research/records/experiment/X002.md | 2 +- dokimi/research/records/experiment/X003.md | 8 +-- dokimi/research/records/experiment/X006.md | 7 +-- dokimi/research/records/experiment/X008.md | 2 +- dokimi/research/records/experiment/X012.md | 6 ++- dokimi/research/records/experiment/X013.md | 2 +- dokimi/research/records/experiment/X016.md | 2 +- dokimi/research/records/experiment/X021.md | 4 +- dokimi/research/records/experiment/X025.md | 6 ++- dokimi/research/records/experiment/X028.md | 21 ++++++++ dokimi/research/records/experiment/X029.md | 21 ++++++++ dokimi/research/records/finding/F002.md | 1 + dokimi/research/records/finding/F003.md | 1 + dokimi/research/records/finding/F006.md | 4 +- dokimi/research/records/finding/F009.md | 1 + dokimi/scripts/research_register.py | 57 ++++++++++++++++++++-- dokimi/tests/test_research_register.py | 32 +++++++++++- 19 files changed, 165 insertions(+), 27 deletions(-) create mode 100644 dokimi/research/records/experiment/X028.md create mode 100644 dokimi/research/records/experiment/X029.md diff --git a/dokimi/docs/research-register.md b/dokimi/docs/research-register.md index 2e3e6d5e..534f9e2b 100644 --- a/dokimi/docs/research-register.md +++ b/dokimi/docs/research-register.md @@ -28,7 +28,10 @@ a decision's `state` (what the team chose) and an implementation reference's `st was built). A closed issue does not tell which of the three happened. An experiment that turns out wrong is not deleted: set `retracted: true` and say in the body -what was wrong with it. Every record has a `date` (`YYYY-MM-DD`) and optional `links`, which +what was wrong with it. `retracted` means the whole experiment is invalid, so `check` fails +any finding that still cites it; export and render keep it, marked. When only part of a +result is withdrawn, keep the outcome, leave `retracted` unset and list the withdrawn claims +under a `## Corrections` heading. Every record has a `date` (`YYYY-MM-DD`) and optional `links`, which must be public `github.com/intel/` or `github.com/napetrov/` URLs. The register is public: leave out internal hosts, paths, names and raw traces, and describe local evidence in words ("local run, 96 runs"). @@ -43,6 +46,7 @@ python scripts/research_register.py next experiment python scripts/research_register.py query --type experiment --ref I001 --where outcome=refuted,inconclusive python scripts/research_register.py query --type decision --where state=rejected python scripts/research_register.py export > register.json +python scripts/research_register.py render > map.md ``` - `check` validates every record against its schema, its path and id prefix, and every @@ -50,9 +54,14 @@ python scripts/research_register.py export > register.json - `next` prints the next free id for a type. Two PRs can claim the same id; `check` reports the duplicate on whichever merges second, which renumbers. - `query` lists the records matching every filter: `--where FIELD=V1,V2` matches any of the - values, `--ref ID` keeps the records that point at `ID`. + values, `--ref ID` keeps the records that point at `ID`. An unknown field, a clause + without `=` or a missing `ID` exits 2. - `export` writes all records, the reference edges and the ideas grouped by map theme as one JSON document, stamped with the commit, for the plans-and-research map and other views. +- `render` writes the plans-and-research map from the same tree: a Mermaid mindmap (theme, + idea, experiments marked by outcome) and a table, as one Markdown file. It is generated on + demand and not committed; open it in any Mermaid-aware viewer, such as GitHub or an editor + preview. ## Adding a record diff --git a/dokimi/research/records/decision/D004.md b/dokimi/research/records/decision/D004.md index 5a43b9b8..f9d161ee 100644 --- a/dokimi/research/records/decision/D004.md +++ b/dokimi/research/records/decision/D004.md @@ -12,4 +12,4 @@ Copyright (C) 2026 Intel Corporation SPDX-License-Identifier: Apache-2.0 --> ## Rationale -Measured gains negligible (+0.007) or refuted. +Measured gains negligible or refuted: stratified probe sampling moved agreement by +0.007, inside the grader spread F009 reports, so it does not pay for its probe budget; the LLMLingua shortener was refuted (X009), and the SliCK split showed no gain worth building either. diff --git a/dokimi/research/records/experiment/X002.md b/dokimi/research/records/experiment/X002.md index 934e6086..cfe3d0b6 100644 --- a/dokimi/research/records/experiment/X002.md +++ b/dokimi/research/records/experiment/X002.md @@ -5,7 +5,7 @@ title: Length-only baseline for judge validity date: '2026-10-19' idea: I001 hypothesis: Judge validity accuracy reflects reading content. -outcome: supported +outcome: inconclusive --- ## Setup -Haiku judge, 4 seeds; criterion only vs criterion + instruction to enumerate every figure. +Haiku judge, 4 seeds; criterion only vs the baseline prompt. The enumeration arm is X028. ## Result -Criterion only +0.045, McNemar p=0.43 (refuted). Criterion + enumeration pooled 0.599 -> 0.712, p=0.0144 over 156 pairs; still under a 0.80 gate. +Criterion only +0.045, McNemar p=0.43: within run-to-run noise. ## Evidence - local run, Haiku, 156 pairs, 4 seeds diff --git a/dokimi/research/records/experiment/X006.md b/dokimi/research/records/experiment/X006.md index ea2b2d55..9d6a0ef7 100644 --- a/dokimi/research/records/experiment/X006.md +++ b/dokimi/research/records/experiment/X006.md @@ -5,8 +5,7 @@ title: Judge ladder and cross-judge agreement date: '2026-10-19' idea: I003 hypothesis: Skill-vs-baseline verdict is judge-independent. -outcome: supported -retracted: true +outcome: refuted --- +## Setup +Haiku judge, 4 seeds; criterion plus an instruction to enumerate every figure, against the baseline prompt. + +## Result +Pooled 0.599 -> 0.712, p=0.0144 over 156 pairs. Still under the 0.80 gate, so the judge is better but not reliable on numbers (see F003). + +## Evidence +- local run, Haiku, 156 pairs, 4 seeds diff --git a/dokimi/research/records/experiment/X029.md b/dokimi/research/records/experiment/X029.md new file mode 100644 index 00000000..4a500fd7 --- /dev/null +++ b/dokimi/research/records/experiment/X029.md @@ -0,0 +1,21 @@ +--- +id: X029 +type: experiment +title: Grader agreement on the novelty judge band +date: '2026-10-19' +idea: I005 +hypothesis: Two graders agree closely enough to read small 'added' differences. +outcome: refuted +--- + +## Setup +25 probes graded by two graders on the novelty judge band. + +## Result +Cohen's kappa 0.576, fine-grained agreement 0.664; grader-to-grader spread of 8-12 percentage points. + +## Evidence +- local run, 25 dual-graded probes diff --git a/dokimi/research/records/finding/F002.md b/dokimi/research/records/finding/F002.md index a6ec5ea9..0f58a64e 100644 --- a/dokimi/research/records/finding/F002.md +++ b/dokimi/research/records/finding/F002.md @@ -5,6 +5,7 @@ title: Instructing the judge to enumerate beats describing the criterion date: '2026-10-19' experiments: - X003 +- X028 confidence: med --- ## Rationale -The five dimensions behaved as one factor; averaging diluted correctness. Done in pre-public history. +Judged from pre-public history, not from a record here: the five dimensions appeared to move together, and averaging diluted correctness. F002 is cited only for the judge-prompt context (enumeration beats description); it does not support the weights themselves. diff --git a/dokimi/research/records/decision/D004.md b/dokimi/research/records/decision/D004.md index f9d161ee..d5647eb5 100644 --- a/dokimi/research/records/decision/D004.md +++ b/dokimi/research/records/decision/D004.md @@ -12,4 +12,4 @@ Copyright (C) 2026 Intel Corporation SPDX-License-Identifier: Apache-2.0 --> ## Rationale -Measured gains negligible or refuted: stratified probe sampling moved agreement by +0.007, inside the grader spread F009 reports, so it does not pay for its probe budget; the LLMLingua shortener was refuted (X009), and the SliCK split showed no gain worth building either. +Gains were negligible or refuted. The LLMLingua shortener was refuted (X009). F009 reports the grader spread, and stratified probe sampling is judged not to pay for its probe budget. The SliCK split was dropped for the same reason. The sampling and SliCK figures come from local runs not recorded here. \ No newline at end of file diff --git a/dokimi/research/records/finding/F018.md b/dokimi/research/records/finding/F018.md index 22f9a9a8..5bb15741 100644 --- a/dokimi/research/records/finding/F018.md +++ b/dokimi/research/records/finding/F018.md @@ -5,7 +5,7 @@ title: Zero delta at ceiling is not 'no effect' date: '2026-10-19' experiments: - X024 -confidence: high +confidence: low --- ## Statement -wrong_numbers stayed 0.67-0.82 across every judge configuration measured; wrong names are caught, wrong values mostly not. +wrong_numbers detection measured 0.47-0.72 across prompt wordings (X008) and 0.599, raised to 0.712 by the X028 change, which is still under the 0.80 gate: no measured judge catches wrong values reliably. ## Applies to Judging technical answers with numbers diff --git a/dokimi/tests/test_research_register.py b/dokimi/tests/test_research_register.py index e9b4866d..cfedc618 100644 --- a/dokimi/tests/test_research_register.py +++ b/dokimi/tests/test_research_register.py @@ -375,10 +375,3 @@ def test_render_flattens_multiline_title(root: Path, capsys: pytest.CaptureFixtu assert "X003[X003 - Run split]" in out assert all(line.count("[") == line.count("]") for line in out.splitlines()) assert "| O | I002 | X003 | refuted | Run split |" in out - - -def test_agents_exception_names_every_record_type() -> None: - text = " ".join((PROJECT.parent / "AGENTS.md").read_text(encoding="utf-8").split()) - for word in ("idea", "experiment", "finding", "decision", "implementation link"): - assert word in text.split("research register", 1)[1].split("(", 1)[0], word - assert "add/add" in (PROJECT / "docs" / "research-register.md").read_text(encoding="utf-8") From 64bb550224cf1949dee3b1f1ac7dbd9dc9ded1c4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 15:26:10 -0700 Subject: [PATCH 17/21] feat(research): HTML map with search and filters; gaps report with live GitHub state - render --html writes one self-contained page (no network, no library): collapsible theme > idea > experiment > finding > decision > implref tree, search, filters by type, theme, outcome/state and implementation, a sortable table and a record panel with back-references. - gaps lists accepted decisions with nothing merged and pending implrefs; --live asks GitHub through gh for each implref link, reports recorded status that differs, and lists open issues no record links to. - R024 and R029 are merged on GitHub; their status says so now. Signed-off-by: Nikolay Petrov --- dokimi/docs/research-register.md | 14 ++ dokimi/research/records/implref/R024.md | 2 +- dokimi/research/records/implref/R029.md | 2 +- dokimi/scripts/research_map.html | 214 ++++++++++++++++++++++++ dokimi/scripts/research_register.py | 114 ++++++++++++- dokimi/tests/test_research_register.py | 64 +++++++ 6 files changed, 407 insertions(+), 3 deletions(-) create mode 100644 dokimi/scripts/research_map.html diff --git a/dokimi/docs/research-register.md b/dokimi/docs/research-register.md index 7b7a3cb3..62d888ab 100644 --- a/dokimi/docs/research-register.md +++ b/dokimi/docs/research-register.md @@ -47,6 +47,8 @@ python scripts/research_register.py query --type experiment --ref I001 --where o python scripts/research_register.py query --type decision --where state=rejected python scripts/research_register.py export > register.json python scripts/research_register.py render > map.md +python scripts/research_register.py render --html --live > map.html +python scripts/research_register.py gaps --live ``` - `check` validates every record against its schema, its path and id prefix, and every @@ -68,6 +70,18 @@ python scripts/research_register.py render > map.md idea, experiments marked by outcome) and a table, as one Markdown file. It is generated on demand and not committed; open it in any Mermaid-aware viewer, such as GitHub or an editor preview. +- `render --html` writes the same register as one self-contained page to open in a browser, + with no network and no library: a collapsible tree (theme, idea, experiment, finding, + decision, implref), a search box (`/` focuses it; every word must match the id, a field or + the body), filters for type, theme, outcome or state and implementation, a sortable table, + and a panel with the clicked record and the records that cite it. `#X003` in the URL opens + that record. +- `gaps` lists accepted decisions with nothing merged and implrefs still `open` or + `proposed`. An implref's `status` is written by hand and goes stale when its pull request + merges, so `--live` (here and on `render --html`) asks GitHub through `gh` for the state of + every link, reports implrefs whose recorded status differs, and lists the open + `intel/dokimi` issues that no record links to. It only reads: fix a stale status in a PR. + `check` stays offline. ## Adding a record diff --git a/dokimi/research/records/implref/R024.md b/dokimi/research/records/implref/R024.md index fafb3dda..c85bbc28 100644 --- a/dokimi/research/records/implref/R024.md +++ b/dokimi/research/records/implref/R024.md @@ -4,7 +4,7 @@ type: implref title: reusable repo-eval workflow (D8) date: '2026-10-19' decision: D009 -status: open +status: merged links: - https://github.com/intel/dokimi/issues/41 - https://github.com/intel/dokimi/pull/113 diff --git a/dokimi/research/records/implref/R029.md b/dokimi/research/records/implref/R029.md index bc291bb7..acad6c0b 100644 --- a/dokimi/research/records/implref/R029.md +++ b/dokimi/research/records/implref/R029.md @@ -4,7 +4,7 @@ type: implref title: User-journey catalog J1-J15 date: '2026-10-19' decision: D011 -status: open +status: merged links: - https://github.com/intel/dokimi/issues/95 - https://github.com/intel/dokimi/pull/102 diff --git a/dokimi/scripts/research_map.html b/dokimi/scripts/research_map.html new file mode 100644 index 00000000..f9faed17 --- /dev/null +++ b/dokimi/scripts/research_map.html @@ -0,0 +1,214 @@ + + + + + +Dokimi plans and research map + + + +
+

Plans and research map

+ + + + + + + + +
+
+
+ +
+
+ + + + diff --git a/dokimi/scripts/research_register.py b/dokimi/scripts/research_register.py index ebaf3ab3..31792078 100755 --- a/dokimi/scripts/research_register.py +++ b/dokimi/scripts/research_register.py @@ -205,6 +205,96 @@ def _git(root: Path, *args: str) -> str: return "" # git is not installed +_GITHUB = re.compile(r"^https://github\.com/([^/]+/[^/]+)/(issues|pull)/([0-9]+)") + + +def github_state(url: str) -> str | None: + """The live state of an issue or pull request: open, closed or merged; None if unknown.""" + match = _GITHUB.match(url) + if not match: + return None + repo, _, number = match.groups() + try: + out = subprocess.run(["gh", "api", f"repos/{repo}/issues/{number}", + "--jq", "[.state, .pull_request.merged_at] | @tsv"], + capture_output=True, text=True, check=False, timeout=60) + except (OSError, subprocess.TimeoutExpired): + return None # gh is not installed or the network is down + state, _, merged = out.stdout.strip().partition("\t") + if out.returncode or state not in ("open", "closed"): + return None + return "merged" if merged else state + + +def open_issues(repo: str) -> list[dict[str, Any]]: + """Open issues (not pull requests) of a GitHub repository, as number and title.""" + try: + out = subprocess.run(["gh", "api", "--paginate", f"repos/{repo}/issues?state=open&per_page=100", + "--jq", ".[] | select(.pull_request == null) | [.number, .title] | @tsv"], + capture_output=True, text=True, check=False, timeout=120) + except (OSError, subprocess.TimeoutExpired): + return [] + rows = [line.split("\t", 1) for line in out.stdout.splitlines() if "\t" in line] + return [{"number": int(n), "title": title} for n, title in rows] + + +def implementation(records: dict[str, Record], live: bool = False) -> dict[str, dict[str, Any]]: + """Per decision: its implrefs with recorded and (with live) GitHub state, and a summary. + + The summary is merged when any implref is merged, open when any is open or + proposed, closed when all are closed, and none without an implref. + """ + refs: dict[str, list[dict[str, Any]]] = {} + for record in sorted(records.values(), key=lambda r: r.id): + if record.type != "implref": + continue + states = [github_state(u) for u in record.fields["links"]] if live else [] + known = [s for s in states if s] + # Several links: the most advanced one says how far the work got. + live_state = next((s for s in ("merged", "open", "closed") if s in known), None) + refs.setdefault(record.fields["decision"], []).append( + {"id": record.id, "title": record.fields["title"], "status": record.fields["status"], + "live": live_state, "links": record.fields["links"]}) + result = {} + for rid, record in sorted(records.items()): + if record.type != "decision": + continue + mine = refs.get(rid, []) + effective = {(ref["live"] or ref["status"]) for ref in mine} + summary = ("merged" if "merged" in effective else + "open" if effective & {"open", "proposed"} else + "closed" if effective else "none") + result[rid] = {"state": record.fields["state"], "summary": summary, "implrefs": mine} + return result + + +def gaps(records: dict[str, Record], live: bool = False, repo: str = "intel/dokimi") -> str: + impl = implementation(records, live) + lines = [] + todo = [(d, i) for d, i in impl.items() if i["state"] == "accepted" and i["summary"] != "merged"] + lines.append(f"Accepted decisions with nothing merged: {len(todo)}") + for did, info in todo: + refs = ", ".join(f"{r['id']} {r['live'] or r['status']}" for r in info["implrefs"]) or "no implref" + lines.append(f" {did}\t{info['summary']}\t{records[did].fields['title']}\t({refs})") + pending = [r for i in impl.values() for r in i["implrefs"] + if (r["live"] or r["status"]) in ("open", "proposed")] + lines.append(f"Implrefs not done: {len(pending)}") + lines += [f" {r['id']}\t{r['live'] or r['status']}\t{r['title']}" for r in pending] + if not live: + lines.append("(recorded status only; --live asks GitHub for drift and open issues)") + return "\n".join(lines) + "\n" + drift = [r for i in impl.values() for r in i["implrefs"] if r["live"] and r["live"] != r["status"]] + lines.append(f"Implrefs whose recorded status differs from GitHub: {len(drift)}") + lines += [f" {r['id']}\trecorded {r['status']}, GitHub {r['live']}\t{r['title']}" for r in drift] + cited = {int(m.group(3)) for record in records.values() + for url in record.fields.get("links", []) + if (m := _GITHUB.match(url)) and m.group(1) == repo} + loose = [i for i in open_issues(repo) if i["number"] not in cited] + lines.append(f"Open {repo} issues no record links to: {len(loose)}") + lines += [f" #{i['number']}\t{i['title']}" for i in loose] + return "\n".join(lines) + "\n" + + def export(records: dict[str, Record], root: Path = RESEARCH) -> dict[str, Any]: edges = [{"from": rid, "to": target, "rel": field} for rid, record in sorted(records.items()) @@ -271,6 +361,17 @@ def render(data: dict[str, Any]) -> str: return "\n".join(lines) + "\n" +HTML_TEMPLATE = Path(__file__).with_name("research_map.html") + + +def render_html(data: dict[str, Any], impl: dict[str, dict[str, Any]]) -> str: + """One self-contained page: collapsible map, search, filters, table. No network needed.""" + payload = json.dumps({**data, "implementation": impl}, ensure_ascii=False) + # A record body cannot close the script element that carries it. + payload = payload.replace(" int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--root", type=Path, default=RESEARCH, help=argparse.SUPPRESS) @@ -283,7 +384,14 @@ def main(argv: list[str] | None = None) -> int: nxt = sub.add_parser("next", help="print the next free id for a record type") nxt.add_argument("type", choices=PREFIX) sub.add_parser("export", help="write records, edges and the theme tree as JSON") - sub.add_parser("render", help="write the theme tree as a Mermaid mindmap and table") + rend = sub.add_parser("render", help="write the map: Mermaid mindmap and table, or HTML") + rend.add_argument("--html", action="store_true", + help="one self-contained HTML page with search and filters") + rend.add_argument("--live", action="store_true", + help="ask GitHub (gh) for the state of every implref link") + gap = sub.add_parser("gaps", help="accepted decisions not yet built, pending implrefs") + gap.add_argument("--live", action="store_true", + help="also GitHub drift and open issues no record links to (needs gh)") args = parser.parse_args(argv) records, errors = load(args.root) @@ -315,8 +423,12 @@ def main(argv: list[str] | None = None) -> int: print(f"{record.id}\t{record.type}\t{extra}\t{record.fields['title']}") elif args.command == "next": print(next_id(records, args.type)) + elif args.command == "render" and args.html: + sys.stdout.write(render_html(export(records, args.root), implementation(records, args.live))) elif args.command == "render": sys.stdout.write(render(export(records, args.root))) + elif args.command == "gaps": + sys.stdout.write(gaps(records, args.live)) else: json.dump(export(records, args.root), sys.stdout, indent=1, ensure_ascii=False) print() diff --git a/dokimi/tests/test_research_register.py b/dokimi/tests/test_research_register.py index cfedc618..fedc7756 100644 --- a/dokimi/tests/test_research_register.py +++ b/dokimi/tests/test_research_register.py @@ -375,3 +375,67 @@ def test_render_flattens_multiline_title(root: Path, capsys: pytest.CaptureFixtu assert "X003[X003 - Run split]" in out assert all(line.count("[") == line.count("]") for line in out.splitlines()) assert "| O | I002 | X003 | refuted | Run split |" in out + + +def _implref(root: Path, rid: str, decision: str, status: str, number: int) -> None: + _write(root, "implref", rid, f"title: Build {rid}\ndecision: {decision}\nstatus: {status}\n" + f"links: [https://github.com/intel/dokimi/pull/{number}]\n") + + +@pytest.fixture +def built(root: Path) -> Path: + _write(root, "decision", "D002", "title: Do it\nfindings: [F001]\nstate: accepted\n") + _write(root, "decision", "D003", "title: Done\nfindings: [F001]\nstate: accepted\n") + _implref(root, "R001", "D002", "open", 1) + _implref(root, "R002", "D003", "merged", 2) + return root + + +def test_implementation_summary(built: Path) -> None: + records, _ = rr.load(built) + impl = rr.implementation(records) + assert {d: i["summary"] for d, i in impl.items()} == { + "D001": "none", "D002": "open", "D003": "merged"} + assert impl["D002"]["implrefs"][0]["live"] is None + + +def test_gaps_offline(built: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert rr.main(["--root", str(built), "gaps"]) == 0 + out = capsys.readouterr().out + assert "Accepted decisions with nothing merged: 1\n D002\topen\tDo it\t(R001 open)" in out + assert "Implrefs not done: 1\n R001\topen\tBuild R001" in out + assert "--live asks GitHub" in out + + +def test_gaps_live_reports_drift_and_unlinked_issues(built: Path, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(rr, "github_state", lambda url: "merged") + monkeypatch.setattr(rr, "open_issues", lambda repo: [{"number": 1, "title": "linked"}, + {"number": 7, "title": "Loose end"}]) + assert rr.main(["--root", str(built), "gaps", "--live"]) == 0 + out = capsys.readouterr().out + assert "Accepted decisions with nothing merged: 0" in out + assert "R001\trecorded open, GitHub merged\tBuild R001" in out + assert "Open intel/dokimi issues no record links to: 1\n #7\tLoose end" in out + + +def test_github_state_without_gh(monkeypatch: pytest.MonkeyPatch) -> None: + def missing(*a: object, **k: object) -> None: + raise FileNotFoundError("gh") + monkeypatch.setattr(rr.subprocess, "run", missing) + assert rr.github_state("https://github.com/intel/dokimi/pull/1") is None + assert rr.github_state("https://github.com/intel/dokimi/blob/main/x") is None + assert rr.open_issues("intel/dokimi") == [] + + +def test_render_html_embeds_data_safely(built: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(built, "experiment", "X009", "title: Run 9\nidea: I001\nhypothesis: H\noutcome: planned\n", + "") + assert rr.main(["--root", str(built), "render", "--html"]) == 0 + page = capsys.readouterr().out + assert page.count("") == 2 # the data block and the code block, nothing from a body + data = page.split('type="application/json">', 1)[1].split("", 1)[0] + loaded = json.loads(data) + assert loaded["implementation"]["D002"]["summary"] == "open" + assert next(r for r in loaded["records"] if r["id"] == "X009")["body"].startswith("") + assert "/*DATA*/" not in page From c874618d2573a2cb8e165e4ffdc490163e1da623 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 16:20:51 -0700 Subject: [PATCH 18/21] feat(research): every open issue accounted for; untracked list for engineering issues - R031-R035: journey sub-issues #96-#100 under D011. - D014/R036 catalog integration (#138), D015/R037 CLI restructure deferred (#94), D016/R038 a run records only the conditions it enforced (#148). - R039 links D012 to this PR; R040 links D002 to the judge aggregate code. - D001 is rejected like D004: a 'do not ship' decision is not work to build. - research/untracked.yaml lists open issues that are engineering, not research (#19, #30, #61, #136); gaps --live skips them and reports an entry whose issue closed. A proposed implref on an open issue is not drift. Signed-off-by: Nikolay Petrov --- dokimi/docs/research-register.md | 5 ++++ dokimi/research/records/decision/D001.md | 2 +- dokimi/research/records/decision/D014.md | 15 +++++++++++ dokimi/research/records/decision/D015.md | 15 +++++++++++ dokimi/research/records/decision/D016.md | 15 +++++++++++ dokimi/research/records/implref/R031.md | 16 +++++++++++ dokimi/research/records/implref/R032.md | 16 +++++++++++ dokimi/research/records/implref/R033.md | 16 +++++++++++ dokimi/research/records/implref/R034.md | 16 +++++++++++ dokimi/research/records/implref/R035.md | 16 +++++++++++ dokimi/research/records/implref/R036.md | 16 +++++++++++ dokimi/research/records/implref/R037.md | 16 +++++++++++ dokimi/research/records/implref/R038.md | 16 +++++++++++ dokimi/research/records/implref/R039.md | 16 +++++++++++ dokimi/research/records/implref/R040.md | 16 +++++++++++ dokimi/research/untracked.yaml | 15 +++++++++++ dokimi/scripts/research_register.py | 34 +++++++++++++++++++++--- dokimi/tests/test_research_register.py | 21 +++++++++++++++ 18 files changed, 277 insertions(+), 5 deletions(-) create mode 100644 dokimi/research/records/decision/D014.md create mode 100644 dokimi/research/records/decision/D015.md create mode 100644 dokimi/research/records/decision/D016.md create mode 100644 dokimi/research/records/implref/R031.md create mode 100644 dokimi/research/records/implref/R032.md create mode 100644 dokimi/research/records/implref/R033.md create mode 100644 dokimi/research/records/implref/R034.md create mode 100644 dokimi/research/records/implref/R035.md create mode 100644 dokimi/research/records/implref/R036.md create mode 100644 dokimi/research/records/implref/R037.md create mode 100644 dokimi/research/records/implref/R038.md create mode 100644 dokimi/research/records/implref/R039.md create mode 100644 dokimi/research/records/implref/R040.md create mode 100644 dokimi/research/untracked.yaml diff --git a/dokimi/docs/research-register.md b/dokimi/docs/research-register.md index 62d888ab..59409fa9 100644 --- a/dokimi/docs/research-register.md +++ b/dokimi/docs/research-register.md @@ -82,6 +82,11 @@ python scripts/research_register.py gaps --live every link, reports implrefs whose recorded status differs, and lists the open `intel/dokimi` issues that no record links to. It only reads: fix a stale status in a PR. `check` stays offline. +- An open issue that is engineering work with no research behind it (a dependency + dashboard, a build chore, a bug) goes in `research/untracked.yaml` with its reason + instead of a record; `gaps --live` skips those and reports an entry whose issue has closed. + An implref may also point at the file that carries the change when it landed before the + public history and has no pull request. ## Adding a record diff --git a/dokimi/research/records/decision/D001.md b/dokimi/research/records/decision/D001.md index d69a6ff2..a88f667b 100644 --- a/dokimi/research/records/decision/D001.md +++ b/dokimi/research/records/decision/D001.md @@ -6,7 +6,7 @@ date: '2026-10-19' findings: - F001 - F004 -state: accepted +state: rejected --- +## Rationale +A catalog entry should move from published to validated on per-skill evidence, not on review alone. The catalog consumes the generic engine (plan, result, compare, gate policy) planned in #121, so the catalog work lists only what a catalog adds on top. diff --git a/dokimi/research/records/decision/D015.md b/dokimi/research/records/decision/D015.md new file mode 100644 index 00000000..63b638ab --- /dev/null +++ b/dokimi/research/records/decision/D015.md @@ -0,0 +1,15 @@ +--- +id: D015 +type: decision +date: '2026-10-21' +title: CLI restructure into task-oriented groups waits for the user loop +ideas: +- I020 +state: deferred +--- + +## Rationale +The proposal regroups the 64 existing operations under task-oriented roots. The groups should follow the plan/validate/compare/gate/record loop (R030), so the restructure waits until that loop is settled. diff --git a/dokimi/research/records/decision/D016.md b/dokimi/research/records/decision/D016.md new file mode 100644 index 00000000..2e05e9bf --- /dev/null +++ b/dokimi/research/records/decision/D016.md @@ -0,0 +1,15 @@ +--- +id: D016 +type: decision +date: '2026-10-21' +title: A run records only the conditions it enforced +ideas: +- I018 +state: accepted +--- + +## Rationale +A cell record that names a condition the run did not enforce overstates what was measured. Example: `--delegation none` swaps tool arguments, but a skill with `context: fork` still starts a subagent. Either enforce the condition or report the cell as not meeting it. diff --git a/dokimi/research/records/implref/R031.md b/dokimi/research/records/implref/R031.md new file mode 100644 index 00000000..52dbf000 --- /dev/null +++ b/dokimi/research/records/implref/R031.md @@ -0,0 +1,16 @@ +--- +id: R031 +type: implref +date: '2026-10-21' +title: 'Journeys: bind available but untraced criteria to tests' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/100 +--- + +## What +AC-J03-002, AC-J03-005, AC-J12-001, AC-J14-005 already hold; bind them to test ids in the journeys catalog. diff --git a/dokimi/research/records/implref/R032.md b/dokimi/research/records/implref/R032.md new file mode 100644 index 00000000..dcef840e --- /dev/null +++ b/dokimi/research/records/implref/R032.md @@ -0,0 +1,16 @@ +--- +id: R032 +type: implref +date: '2026-10-21' +title: 'J11/J12: distinct inconclusive verdict and blocking policy' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/99 +--- + +## What +A sample cap or budget stop is reported as inconclusive, never as pass, and a policy decides whether it blocks. diff --git a/dokimi/research/records/implref/R033.md b/dokimi/research/records/implref/R033.md new file mode 100644 index 00000000..3e103e12 --- /dev/null +++ b/dokimi/research/records/implref/R033.md @@ -0,0 +1,16 @@ +--- +id: R033 +type: implref +date: '2026-10-21' +title: 'J2: join the novelty diagnosis to a task-level effect' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/98 +--- + +## What +Report novelty next to a measured task effect; re-measure an edited skill against its original in one step. diff --git a/dokimi/research/records/implref/R034.md b/dokimi/research/records/implref/R034.md new file mode 100644 index 00000000..e52a24f1 --- /dev/null +++ b/dokimi/research/records/implref/R034.md @@ -0,0 +1,16 @@ +--- +id: R034 +type: implref +date: '2026-10-21' +title: 'J7: draft tasks, negative controls and coverage gaps' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/97 +--- + +## What +Draft tasks as hypotheses with their source; list the areas an eval set has no task for. diff --git a/dokimi/research/records/implref/R035.md b/dokimi/research/records/implref/R035.md new file mode 100644 index 00000000..16d0ea79 --- /dev/null +++ b/dokimi/research/records/implref/R035.md @@ -0,0 +1,16 @@ +--- +id: R035 +type: implref +date: '2026-10-21' +title: 'J4: adherence and task-impact evidence for instructions' +decision: D011 +status: open +links: +- https://github.com/intel/dokimi/issues/96 +--- + +## What +Measure adherence to specific instructions apart from reach; report task impact of guidance arms. diff --git a/dokimi/research/records/implref/R036.md b/dokimi/research/records/implref/R036.md new file mode 100644 index 00000000..0ba8621b --- /dev/null +++ b/dokimi/research/records/implref/R036.md @@ -0,0 +1,16 @@ +--- +id: R036 +type: implref +date: '2026-10-21' +title: 'Skill catalog integration (intel/skills)' +decision: D014 +status: open +links: +- https://github.com/intel/dokimi/issues/138 +--- + +## What +Per-skill evals in catalog CI, published results, promotion to validated. diff --git a/dokimi/research/records/implref/R037.md b/dokimi/research/records/implref/R037.md new file mode 100644 index 00000000..d23165b6 --- /dev/null +++ b/dokimi/research/records/implref/R037.md @@ -0,0 +1,16 @@ +--- +id: R037 +type: implref +date: '2026-10-21' +title: 'CLI restructure: nine task-oriented groups' +decision: D015 +status: proposed +links: +- https://github.com/intel/dokimi/issues/94 +--- + +## What +Proposal and migration plan; no code yet. diff --git a/dokimi/research/records/implref/R038.md b/dokimi/research/records/implref/R038.md new file mode 100644 index 00000000..6198167b --- /dev/null +++ b/dokimi/research/records/implref/R038.md @@ -0,0 +1,16 @@ +--- +id: R038 +type: implref +date: '2026-10-21' +title: 'repo run: block fork-context skills under --delegation none' +decision: D016 +status: open +links: +- https://github.com/intel/dokimi/issues/148 +--- + +## What +Refuse or flag `context: fork` skills in a `--delegation none` cell. diff --git a/dokimi/research/records/implref/R039.md b/dokimi/research/records/implref/R039.md new file mode 100644 index 00000000..67e666d3 --- /dev/null +++ b/dokimi/research/records/implref/R039.md @@ -0,0 +1,16 @@ +--- +id: R039 +type: implref +title: 'Research register: records, validator, map' +date: '2026-10-21' +decision: D012 +status: open +links: +- https://github.com/intel/dokimi/pull/139 +--- + +## What +This register: schema, records, `check`/`query`/`export`/`render`/`gaps`, and the HTML map. diff --git a/dokimi/research/records/implref/R040.md b/dokimi/research/records/implref/R040.md new file mode 100644 index 00000000..abaf2c30 --- /dev/null +++ b/dokimi/research/records/implref/R040.md @@ -0,0 +1,16 @@ +--- +id: R040 +type: implref +title: 'Judge aggregate recomputed in code with correctness weighted x2' +date: '2026-10-21' +decision: D002 +status: merged +links: +- https://github.com/intel/dokimi/blob/main/dokimi/dokimi/eval/judge.py +--- + +## What +The aggregate is computed from the dimension scores in `eval/judge.py`, correctness at weight 2.0 and specificity outside the single-judge aggregate. Landed before the public history, so there is no pull request to link. diff --git a/dokimi/research/untracked.yaml b/dokimi/research/untracked.yaml new file mode 100644 index 00000000..065a1d1f --- /dev/null +++ b/dokimi/research/untracked.yaml @@ -0,0 +1,15 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# Open intel/dokimi issues that are engineering work, not research: no idea, +# experiment or decision behind them. `research_register.py gaps --live` does +# not list them as missing from the register, and reports an entry here whose +# issue has closed so the list does not go stale. +- issue: 19 + reason: docs build hook rewrites links on the element tree; docs tooling +- issue: 30 + reason: Renovate dependency dashboard +- issue: 61 + reason: hash the dependency lock; supply-chain hardening +- issue: 136 + reason: eval cross-judge drops the provider from the judge spec; a bug diff --git a/dokimi/scripts/research_register.py b/dokimi/scripts/research_register.py index 31792078..9f6aa39a 100755 --- a/dokimi/scripts/research_register.py +++ b/dokimi/scripts/research_register.py @@ -268,7 +268,22 @@ def implementation(records: dict[str, Record], live: bool = False) -> dict[str, return result -def gaps(records: dict[str, Record], live: bool = False, repo: str = "intel/dokimi") -> str: +def untracked(root: Path = RESEARCH) -> dict[int, str]: + """Issue number -> reason, from research/untracked.yaml: open issues that are not research.""" + path = root / "untracked.yaml" + if not path.exists(): + return {} + entries = yaml.safe_load(path.read_text(encoding="utf-8")) or [] + if not isinstance(entries, list) or not all( + isinstance(e, dict) and isinstance(e.get("issue"), int) and isinstance(e.get("reason"), str) + for e in entries): + raise ValueError(f"{path}: want a list of {{issue: , reason: }}") + return {e["issue"]: e["reason"] for e in entries} + + +def gaps(records: dict[str, Record], live: bool = False, repo: str = "intel/dokimi", + root: Path = RESEARCH) -> str: + skip = untracked(root) impl = implementation(records, live) lines = [] todo = [(d, i) for d, i in impl.items() if i["state"] == "accepted" and i["summary"] != "merged"] @@ -283,15 +298,22 @@ def gaps(records: dict[str, Record], live: bool = False, repo: str = "intel/doki if not live: lines.append("(recorded status only; --live asks GitHub for drift and open issues)") return "\n".join(lines) + "\n" - drift = [r for i in impl.values() for r in i["implrefs"] if r["live"] and r["live"] != r["status"]] + # "proposed" is how an open issue with no code yet is recorded, not drift. + drift = [r for i in impl.values() for r in i["implrefs"] if r["live"] and r["live"] != r["status"] + and (r["live"], r["status"]) != ("open", "proposed")] lines.append(f"Implrefs whose recorded status differs from GitHub: {len(drift)}") lines += [f" {r['id']}\trecorded {r['status']}, GitHub {r['live']}\t{r['title']}" for r in drift] cited = {int(m.group(3)) for record in records.values() for url in record.fields.get("links", []) if (m := _GITHUB.match(url)) and m.group(1) == repo} - loose = [i for i in open_issues(repo) if i["number"] not in cited] + issues = open_issues(repo) + loose = [i for i in issues if i["number"] not in cited and i["number"] not in skip] lines.append(f"Open {repo} issues no record links to: {len(loose)}") lines += [f" #{i['number']}\t{i['title']}" for i in loose] + if issues: # an empty list means gh failed, not that every issue closed + gone = sorted(set(skip) - {i["number"] for i in issues}) + lines.append(f"research/untracked.yaml entries no longer open: {len(gone)}") + lines += [f" #{n}\t{skip[n]}" for n in gone] return "\n".join(lines) + "\n" @@ -428,7 +450,11 @@ def main(argv: list[str] | None = None) -> int: elif args.command == "render": sys.stdout.write(render(export(records, args.root))) elif args.command == "gaps": - sys.stdout.write(gaps(records, args.live)) + try: + sys.stdout.write(gaps(records, args.live, root=args.root)) + except (ValueError, yaml.YAMLError) as exc: + print(exc, file=sys.stderr) + return 1 else: json.dump(export(records, args.root), sys.stdout, indent=1, ensure_ascii=False) print() diff --git a/dokimi/tests/test_research_register.py b/dokimi/tests/test_research_register.py index fedc7756..24322539 100644 --- a/dokimi/tests/test_research_register.py +++ b/dokimi/tests/test_research_register.py @@ -419,6 +419,27 @@ def test_gaps_live_reports_drift_and_unlinked_issues(built: Path, capsys: pytest assert "Open intel/dokimi issues no record links to: 1\n #7\tLoose end" in out +def test_gaps_live_skips_untracked_and_reports_stale_entries( + built: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch) -> None: + (built / "untracked.yaml").write_text("- issue: 7\n reason: chore\n- issue: 9\n reason: done\n") + monkeypatch.setattr(rr, "github_state", lambda url: None) + monkeypatch.setattr(rr, "open_issues", lambda repo: [{"number": 7, "title": "Loose end"}]) + assert rr.main(["--root", str(built), "gaps", "--live"]) == 0 + out = capsys.readouterr().out + assert "no record links to: 0" in out + assert "untracked.yaml entries no longer open: 1\n #9\tdone" in out + + +def test_gaps_rejects_malformed_untracked(built: Path, capsys: pytest.CaptureFixture[str]) -> None: + (built / "untracked.yaml").write_text("- issue: seven\n") + assert rr.main(["--root", str(built), "gaps"]) == 1 + assert "want a list of" in capsys.readouterr().err + + +def test_repository_untracked_list_is_valid() -> None: + assert rr.untracked() + + def test_github_state_without_gh(monkeypatch: pytest.MonkeyPatch) -> None: def missing(*a: object, **k: object) -> None: raise FileNotFoundError("gh") From 4b1d83f8c11898e5fc6ae7dfb8498f5497e00188 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 20:42:21 -0700 Subject: [PATCH 19/21] fix(research): escape every < in the HTML payload, reject render --live without --html, warn on empty live state Signed-off-by: Nikolay Petrov --- dokimi/scripts/research_register.py | 20 +++++++++++++++---- dokimi/tests/test_research_register.py | 27 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/dokimi/scripts/research_register.py b/dokimi/scripts/research_register.py index 9f6aa39a..d8f1faf7 100755 --- a/dokimi/scripts/research_register.py +++ b/dokimi/scripts/research_register.py @@ -217,7 +217,7 @@ def github_state(url: str) -> str | None: try: out = subprocess.run(["gh", "api", f"repos/{repo}/issues/{number}", "--jq", "[.state, .pull_request.merged_at] | @tsv"], - capture_output=True, text=True, check=False, timeout=60) + capture_output=True, text=True, check=False, timeout=20) except (OSError, subprocess.TimeoutExpired): return None # gh is not installed or the network is down state, _, merged = out.stdout.strip().partition("\t") @@ -245,16 +245,25 @@ def implementation(records: dict[str, Record], live: bool = False) -> dict[str, proposed, closed when all are closed, and none without an implref. """ refs: dict[str, list[dict[str, Any]]] = {} + cache: dict[str, str | None] = {} + asked = 0 for record in sorted(records.values(), key=lambda r: r.id): if record.type != "implref": continue - states = [github_state(u) for u in record.fields["links"]] if live else [] + for url in record.fields["links"] if live else []: + if url not in cache: + cache[url] = github_state(url) + asked += 1 + states = [cache[u] for u in record.fields["links"]] if live else [] known = [s for s in states if s] # Several links: the most advanced one says how far the work got. live_state = next((s for s in ("merged", "open", "closed") if s in known), None) refs.setdefault(record.fields["decision"], []).append( {"id": record.id, "title": record.fields["title"], "status": record.fields["status"], "live": live_state, "links": record.fields["links"]}) + if live and asked and not any(cache.values()): + print("warning: --live got no state from GitHub (is gh installed and logged in?); " + "the recorded status is shown instead", file=sys.stderr) result = {} for rid, record in sorted(records.items()): if record.type != "decision": @@ -389,8 +398,9 @@ def render(data: dict[str, Any]) -> str: def render_html(data: dict[str, Any], impl: dict[str, dict[str, Any]]) -> str: """One self-contained page: collapsible map, search, filters, table. No network needed.""" payload = json.dumps({**data, "implementation": impl}, ensure_ascii=False) - # A record body cannot close the script element that carries it. - payload = payload.replace(" int: help="also GitHub drift and open issues no record links to (needs gh)") args = parser.parse_args(argv) + if args.command == "render" and args.live and not args.html: + parser.error("render --live needs --html (only the HTML map shows live state)") records, errors = load(args.root) schema = json.loads((args.root / SCHEMA.name).read_text(encoding="utf-8")) errors += check(records, schema) diff --git a/dokimi/tests/test_research_register.py b/dokimi/tests/test_research_register.py index 24322539..0d530c3a 100644 --- a/dokimi/tests/test_research_register.py +++ b/dokimi/tests/test_research_register.py @@ -460,3 +460,30 @@ def test_render_html_embeds_data_safely(built: Path, capsys: pytest.CaptureFixtu assert loaded["implementation"]["D002"]["summary"] == "open" assert next(r for r in loaded["records"] if r["id"] == "X009")["body"].startswith("") assert "/*DATA*/" not in page + + +def test_render_html_escapes_comment_opener(built: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write(built, "experiment", "X009", "title: Run 9\nidea: I001\nhypothesis: H\noutcome: planned\n", + "