From 900814cebf66fbc5ea4334b4598900d3ec62666a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 21:53:52 -0700 Subject: [PATCH 1/8] feat(novelty): state the measured share of readable words on the analyze headline The headline printed the added and already-knew counts but not how much of the artifact the run read, so an entry-point run of a progressive-disclosure skill quoted rates that describe the entry point as if they described the skill. The headline now prints the measured share of readable words on stdout, and warns on stdout when a run read half or less of them. The novelty.v1 bundle record stamps the share as bundle.measured_share. The stdout warning uses the line the analyzer has always warned at (an outside share of 0.5 or more), so the stdout and stderr warnings agree and the coverage parity golden keeps its value. The settings parity module leaves out the new stdout lines when it compares with main; the journey test pins them instead. Signed-off-by: Nikolay Petrov --- dokimi/docs/skill-novelty.md | 6 +++-- dokimi/dokimi/commands/novelty.py | 28 ++++++++++++++++++++ dokimi/dokimi/novelty/bundle.py | 21 ++++++++++++++- dokimi/tests/test_novelty_bundle.py | 21 +++++++++++++++ dokimi/tests/test_novelty_cli.py | 27 +++++++++++++++++++ dokimi/tests/test_novelty_journey.py | 24 +++++++++++++++++ dokimi/tests/test_novelty_settings_parity.py | 13 ++++++++- 7 files changed, 136 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 5dac13bb..b0124cc1 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -490,8 +490,10 @@ Things to know before reading a bundle result: at runtime is the agentic track's question (`skill-agent:`), not this one. - **The two modes are different cells.** `include_resources` is recorded in `run_config`; an entry-point run and a bundle run of the same skill are not comparable numbers, and an - entry-point run whose *measurable* outside share is ≥ 0.5 says so in `warnings` - (`measurable_outside_share` nets out the role decisions below; `outside_share` is the + run that read half or less of the readable words says so on stdout (the `coverage:` line + and a `WARNING`); an entry-point run also says so in `warnings`. The share is + `bundle.measured_share` (for an entry-point run, `1 - measurable_outside_share`, + which nets out the role decisions below; `outside_share` is the on-disk figure, and the two differ for a skill whose only resources are its evals). - **A skill's own `evals/` are not content.** They are the question/answer keys for testing the skill, so extracting atoms from them would credit the skill with delivering diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index adbdc943..7caa33bd 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -823,5 +823,33 @@ def _print_headline(artifact: dict) -> None: f"degraded: {paired['lenient']['degraded']}, " f"dead tokens: {artifact['spans'].get('frac_carrying_nothing')}" ) + _print_coverage(artifact.get("bundle") or {}) for warning in artifact.get("warnings", []): print(f"WARNING {warning}", file=sys.stderr) + + +def _print_coverage(bundle: dict) -> None: + """How much of the artifact's readable words this run read, on stdout. + + The share is stated on stdout because the headline is what a user copies. A run under + the floor also says so on stdout: a stderr-only warning is one a user piping the + headline never sees, and a rate quoted off half-read skill is not a property of it. + """ + from dokimi.novelty.bundle import MEASURED_SHARE_WARN_AT, Coverage + + coverage = Coverage.from_record(bundle) + if coverage is None or not coverage.denominator("readable"): + return + share = coverage.measured_share + if share is None: + print("coverage: unknown (a bundled file did not decode, so its words are not counted)") + return + # Truncated, not rounded: 49.99% must not print as "50%" beside a "below 50%" warning. + pct = f"{int(share * 100)}%" + scope = "" if coverage.include_resources else " (entry point only)" + print(f"coverage: {pct} of the artifact's readable words were measured{scope}") + if share <= MEASURED_SHARE_WARN_AT: + rerun = ("" if coverage.include_resources or not coverage.measurable_resource_words + else "; pass --include-resources to measure the bundled files") + print(f"WARNING: only {pct} of the artifact's readable words were measured, so every " + f"rate here describes that part, not the artifact{rerun}") diff --git a/dokimi/dokimi/novelty/bundle.py b/dokimi/dokimi/novelty/bundle.py index cc07493f..581f6d00 100644 --- a/dokimi/dokimi/novelty/bundle.py +++ b/dokimi/dokimi/novelty/bundle.py @@ -191,6 +191,11 @@ def named_exclusions(rows: list[dict] | None) -> tuple[list[dict], int]: #: How a record without ``coverage_basis`` is read: it was written when the gate judged #: all bundled words, and re-reading it on another basis would move its stored value. LEGACY_COVERAGE_BASIS = "all_bundled_words" +#: A run that read this share of the readable words or less is warned, on stdout and in the +#: artifact: its rates describe the measured part, not the artifact. This is the line the +#: analyzer has always warned at (an entry-point run whose outside share is 0.5 or more), and +#: the gate's ``max_unmeasured_outside_share`` is 0.50 on the same readable basis. +MEASURED_SHARE_WARN_AT = 0.5 def _share(part: int, whole: int) -> float: @@ -333,6 +338,19 @@ def unmeasured_share(self, basis: str) -> float | None: whole = self.denominator(basis) return self.unmeasured_words / whole if whole else 0.0 + @property + def measured_share(self) -> float | None: + """Share of the readable words this run read, to 4 places; ``None`` when unknowable. + + The complement of :meth:`unmeasured_share` on the readable basis, stated as what was + read rather than what was missed. ``None`` for the same reasons: a file did not + decode, or there are no readable words to be a share of. + """ + unmeasured = self.unmeasured_share("readable") + if unmeasured is None or not self.denominator("readable"): + return None + return round(1 - unmeasured, 4) + @dataclass class BundleFile: @@ -440,7 +458,7 @@ def coverage_warnings(self) -> list[str]: # evals has nothing a bundle run would add, so pointing at the flag would be advice # that cannot be followed. The gate judges the same basis (`CURRENT_COVERAGE_BASIS`). share = coverage.outside_share_on("readable") - if not self.include_resources and share >= 0.5: + if not self.include_resources and share >= 1 - MEASURED_SHARE_WARN_AT: out.append( f"{share:.0%} of this artifact's words are in " "bundled files that were not measured (--include-resources); every rate " @@ -504,6 +522,7 @@ def summary(self) -> dict: "measured_digest": self.measured_digest(), "outside_share": coverage.outside_share, "measurable_outside_share": coverage.measurable_outside_share, + "measured_share": coverage.measured_share, "unmeasured_words": coverage.unmeasured_words, "max_file_words": self.max_file_words, "max_total_words": self.max_total_words, diff --git a/dokimi/tests/test_novelty_bundle.py b/dokimi/tests/test_novelty_bundle.py index 0e10847a..05d49204 100644 --- a/dokimi/tests/test_novelty_bundle.py +++ b/dokimi/tests/test_novelty_bundle.py @@ -990,3 +990,24 @@ def test_measured_digest_format_is_pinned(tmp_path): ref.write_text("Modified reference text.", encoding="utf-8") bundle3 = load_bundle(f"skill:{skill.parent}", include_resources=True) assert bundle3.measured_digest() != digest + + +def test_the_record_states_the_share_of_readable_words_this_run_read(tmp_path): + """100 entry words and 300 measurable reference words: an entry-point run read a quarter.""" + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=100, refs=300)}") + assert bundle.coverage().measured_share == 0.25 + assert bundle.summary()["measured_share"] == 0.25 + assert any(w.startswith("75% ") for w in bundle.coverage_warnings()) + + +def test_exactly_half_the_readable_words_read_is_warned_as_the_analyzer_always_has_been(tmp_path): + """The warning line is unchanged: a run that read half the readable words still warns.""" + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=300, refs=300)}") + assert bundle.coverage().measured_share == 0.5 + assert any(w.startswith("50% ") for w in bundle.coverage_warnings()) + + +def test_a_bundle_run_that_read_everything_it_could_measures_all_of_it(tmp_path): + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=100, refs=300)}", + include_resources=True) + assert bundle.summary()["measured_share"] == 1.0 diff --git a/dokimi/tests/test_novelty_cli.py b/dokimi/tests/test_novelty_cli.py index b2469117..71833cf7 100644 --- a/dokimi/tests/test_novelty_cli.py +++ b/dokimi/tests/test_novelty_cli.py @@ -614,3 +614,30 @@ def test_view_and_probe_repeat_the_unread_warning(tmp_path, capsys): args.func(args) assert "render as `unread`" in capsys.readouterr().err assert f"`{spans[2].id}` [unread/hatched]" in out.read_text() + + +def test_the_headline_states_the_measured_share_and_warns_on_stdout_at_half_or_less(capsys): + record = {"include_resources": False, "n_files": 1, "entry_point_words": 10, + "resource_words_measured": 0, "resource_words_on_disk": 30, + "policy_excluded_words": 0, "unreadable_files": 0, "excluded": [], + "coverage_basis": "readable"} + artifact = dict(_minimal_artifact(), bundle=record) + novelty_cmd._print_headline(artifact) + out = capsys.readouterr().out + assert "coverage: 25% of the artifact's readable words were measured (entry point only)" in out + assert "WARNING: only 25% of the artifact's readable words" in out + assert "pass --include-resources" in out + + # The line is the analyzer's own: exactly half warns, just over half does not. + novelty_cmd._print_headline(dict(artifact, bundle=dict(record, entry_point_words=30))) + out = capsys.readouterr().out + assert "coverage: 50%" in out and "WARNING: only" in out + novelty_cmd._print_headline(dict(artifact, bundle=dict(record, entry_point_words=31))) + out = capsys.readouterr().out + assert "coverage: 50%" in out and "WARNING: only" not in out + + # A file that did not decode leaves the share unknowable, and the headline says so. + novelty_cmd._print_headline(dict(artifact, bundle=dict(record, unreadable_files=1))) + out = capsys.readouterr().out + assert "coverage: unknown" in out + assert "WARNING: only" not in out diff --git a/dokimi/tests/test_novelty_journey.py b/dokimi/tests/test_novelty_journey.py index de00ee7d..6ebbad1c 100644 --- a/dokimi/tests/test_novelty_journey.py +++ b/dokimi/tests/test_novelty_journey.py @@ -155,6 +155,30 @@ def test_a_cheap_tier_run_writes_its_forfeits_into_the_artifact(tmp_path, skill, assert any("--include-resources" in w for w in artifact["warnings"]) +def test_the_stdout_headline_says_how_much_of_the_artifact_was_read(tmp_path, skill, + stub_models, capsys): + # The fixture's reference is larger than the body: an entry-point run reads under half. + entry_out = tmp_path / "entry" + _run("novelty", "analyze", "--artifact", f"skill:{skill}", "--tier", "triage", + "--model", "model-a", "--judge-model", "judge-m", "--out-dir", str(entry_out)) + printed = capsys.readouterr().out + measured = json.loads((entry_out / "novelty.json").read_text(encoding="utf-8"))[ + "bundle"]["measured_share"] + assert measured < 0.5 + assert f"coverage: {int(measured * 100)}% of the artifact's readable words were measured " \ + "(entry point only)" in printed + assert "WARNING: only" in printed and "pass --include-resources" in printed + + # A bundle run that reads everything it can says so, and does not warn. + bundle_out = tmp_path / "bundle" + _run("novelty", "analyze", "--artifact", f"skill:{skill}", "--tier", "triage", + "--include-resources", "--model", "model-a", "--judge-model", "judge-m", + "--out-dir", str(bundle_out)) + printed = capsys.readouterr().out + assert "coverage: 100% of the artifact's readable words were measured\n" in printed + assert "WARNING: only" not in printed + + @pytest.mark.outcome("cjm-novelty-stale-inventory") def test_probing_atoms_from_a_changed_artifact_is_refused(tmp_path, skill, stub_models): atoms_out = tmp_path / "atoms.json" diff --git a/dokimi/tests/test_novelty_settings_parity.py b/dokimi/tests/test_novelty_settings_parity.py index d42e6c36..9c9eb57e 100644 --- a/dokimi/tests/test_novelty_settings_parity.py +++ b/dokimi/tests/test_novelty_settings_parity.py @@ -111,6 +111,16 @@ def _kept(argv: list[str], work: Path) -> dict[str, Any]: return {} +def _without_coverage_headline(text: str) -> str: + """Drop the stdout lines the coverage headline added after main. + + They say how much of the artifact was read, which is neither a setting, a bill nor a + tier record, so this module does not compare them; ``test_novelty_journey`` pins them. + """ + return "".join(line for line in text.splitlines(keepends=True) + if not line.startswith(("coverage: ", "WARNING: only "))) + + def _capture(name: str, tmp_path: Path) -> list[dict[str, Any]]: work = tmp_path / name.replace("/", "-") (work / "accel-tuning").mkdir(parents=True) @@ -143,7 +153,8 @@ def scripted(model, provider, role, args): status = exc.code if isinstance(exc.code, int) else str(exc.code) steps.append({ "status": status, - "stdout": out.getvalue() if argv[1] == "analyze" else None, + "stdout": (_without_coverage_headline(out.getvalue()) + if argv[1] == "analyze" else None), "stderr": err.getvalue().replace(str(work), ""), "calls": [[c.role, c.calls] for c in clients], **_kept(argv, work), From db55ba00430b317a3fac39dd97573c0686bbfb85 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:11:11 -0700 Subject: [PATCH 2/8] fix(novelty): measured_share in the schema, a changelog fragment, parity checks the headline - novelty.v1 bundle: declare measured_share (number or null, 0 to 1) beside measurable_outside_share; the record already stamps it. - Add the scriv fragment the package change needs: Added for the headline coverage line and bundle.measured_share, Changed for the stdout warning. - docs/skill-novelty.md: "a run that read half or less ... says so on stdout" was ungrammatical ("an run"); split it into its own sentence. - settings parity: the coverage and WARNING lines are no longer dropped unchecked. Main's golden cannot hold them, so each analyze step checks its stdout coverage lines against the measured_share its own novelty.json stamped: the percentage, and the WARNING exactly when the share is 0.5 or less. Signed-off-by: Nikolay Petrov --- ..._215500_napetrov_novelty-measured-share.md | 15 ++++++ dokimi/docs/skill-novelty.md | 4 +- dokimi/dokimi/schemas/novelty.v1.json | 1 + dokimi/tests/test_novelty_settings_parity.py | 50 +++++++++++++++++-- 4 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md diff --git a/dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md b/dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md new file mode 100644 index 00000000..f4650adc --- /dev/null +++ b/dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md @@ -0,0 +1,15 @@ +### Added + +- **`novelty analyze` states the measured share on stdout** — the headline now prints + `coverage: N% of the artifact's readable words were measured`, with `(entry point only)` + for a run without `--include-resources`. The novelty.v1 bundle record stamps the same figure + as `bundle.measured_share`, so a script can read it without recomputing it from + `measurable_outside_share`. + +### Changed + +- **A run that read half or less of the readable words warns on stdout** — the `WARNING: only + N% ...` line goes to stdout as well as the stderr warning, because a headline piped + elsewhere never shows stderr. The threshold is unchanged, so the same runs warn as before; a + rate quoted from an entry-point run of a progressive-disclosure skill now carries that + caveat where it is copied. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index b0124cc1..f514f052 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -489,8 +489,8 @@ Things to know before reading a bundle result: deliver if the agent read all of it. Whether the agent chooses to load `references/x.md` at runtime is the agentic track's question (`skill-agent:`), not this one. - **The two modes are different cells.** `include_resources` is recorded in `run_config`; - an entry-point run and a bundle run of the same skill are not comparable numbers, and an - run that read half or less of the readable words says so on stdout (the `coverage:` line + an entry-point run and a bundle run of the same skill are not comparable numbers. A run + that read half or less of the readable words says so on stdout (the `coverage:` line and a `WARNING`); an entry-point run also says so in `warnings`. The share is `bundle.measured_share` (for an entry-point run, `1 - measurable_outside_share`, which nets out the role decisions below; `outside_share` is the diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index dc41beec..d9ae1063 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -155,6 +155,7 @@ "resource_words_on_disk": {"type": "integer", "minimum": 0}, "outside_share": {"type": "number", "minimum": 0, "maximum": 1}, "measurable_outside_share": {"type": "number", "minimum": 0, "maximum": 1}, + "measured_share": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, "policy_excluded_words": {"type": "integer", "minimum": 0}, "unmeasured_words": {"type": "integer", "minimum": 0}, "coverage_basis": { diff --git a/dokimi/tests/test_novelty_settings_parity.py b/dokimi/tests/test_novelty_settings_parity.py index 9c9eb57e..bf96b8b4 100644 --- a/dokimi/tests/test_novelty_settings_parity.py +++ b/dokimi/tests/test_novelty_settings_parity.py @@ -111,14 +111,46 @@ def _kept(argv: list[str], work: Path) -> dict[str, Any]: return {} +COVERAGE_LINES = ("coverage: ", "WARNING: only ") + + def _without_coverage_headline(text: str) -> str: """Drop the stdout lines the coverage headline added after main. They say how much of the artifact was read, which is neither a setting, a bill nor a - tier record, so this module does not compare them; ``test_novelty_journey`` pins them. + tier record, so the comparison with main leaves them out. They cannot be in main's + golden, so ``_headline_coverage`` checks them against the record they were printed from. """ return "".join(line for line in text.splitlines(keepends=True) - if not line.startswith(("coverage: ", "WARNING: only "))) + if not line.startswith(COVERAGE_LINES)) + + +def _headline_coverage(text: str, work: Path) -> dict[str, Any]: + """The coverage lines stdout printed, beside the ``bundle`` figures in the record.""" + printed = [line for line in text.splitlines() if line.startswith(COVERAGE_LINES)] + record = work / "out" / "novelty.json" + bundle = ( + json.loads(record.read_text(encoding="utf-8")).get("bundle") or {} + if record.exists() + else {} + ) + return {"printed": printed, "measured_share": bundle.get("measured_share")} + + +def _check_headline(coverage: dict[str, Any]) -> None: + """Each dropped line agrees with the measured share the same run stamped in its record.""" + printed, share = coverage["printed"], coverage["measured_share"] + if share is None: + # No record to check against (an estimate, or a run that wrote none): nothing printed. + assert printed == [] + return + pct = f"{int(share * 100)}%" + assert printed, "a run with a record must print its coverage line" + assert printed[0].startswith( + f"coverage: {pct} of the artifact's readable words were measured" + ), printed + warned = [line.startswith(f"WARNING: only {pct} ") for line in printed[1:]] + assert warned == ([True] if share <= 0.5 else []), printed def _capture(name: str, tmp_path: Path) -> list[dict[str, Any]]: @@ -155,6 +187,8 @@ def scripted(model, provider, role, args): "status": status, "stdout": (_without_coverage_headline(out.getvalue()) if argv[1] == "analyze" else None), + "coverage": (_headline_coverage(out.getvalue(), work) + if argv[1] == "analyze" else None), "stderr": err.getvalue().replace(str(work), ""), "calls": [[c.role, c.calls] for c in clients], **_kept(argv, work), @@ -168,7 +202,12 @@ def scripted(model, provider, role, args): @pytest.mark.parametrize("name", list(CASES)) def test_no_setting_bill_or_tier_record_differs_from_main(name, tmp_path): golden = json.loads(GOLDEN.read_text(encoding="utf-8")) - assert _capture(name, tmp_path) == golden[name] + steps = _capture(name, tmp_path) + coverage = [step.pop("coverage") for step in steps] + assert steps == golden[name] + for step_coverage in coverage: + if step_coverage is not None: + _check_headline(step_coverage) def test_the_golden_cases_are_the_ones_this_module_runs(): @@ -180,7 +219,10 @@ def test_the_golden_cases_are_the_ones_this_module_runs(): import tempfile tmp = Path(tempfile.mkdtemp()) - captured = {name: _capture(name, tmp) for name in CASES} + captured = { + name: [{k: v for k, v in step.items() if k != "coverage"} for step in _capture(name, tmp)] + for name in CASES + } GOLDEN.write_text(json.dumps(captured, indent=1, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") print(f"{len(captured)} cases -> {GOLDEN}") From 3872e1c4b379e822cfdabc4b8fc5734671bf895e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 11:41:45 -0700 Subject: [PATCH 3/8] fix(novelty): headline percentage floors the stored share; unknown share keeps scope and warning 0.29, 0.57 and 0.58 printed one too low (float product). An undecodable file printed coverage: unknown with no scope and no warning. Docs, changelog and the schema description now cover the bundle-run warning and the null share. Signed-off-by: Nikolay Petrov --- ..._215500_napetrov_novelty-measured-share.md | 11 +++-- dokimi/docs/skill-novelty.md | 5 ++- dokimi/dokimi/commands/novelty.py | 21 +++++++--- dokimi/dokimi/schemas/novelty.v1.json | 2 +- dokimi/tests/test_novelty_bundle.py | 15 +++++++ dokimi/tests/test_novelty_cli.py | 42 +++++++++++++++++-- dokimi/tests/test_novelty_journey.py | 3 +- dokimi/tests/test_novelty_settings_parity.py | 4 +- 8 files changed, 86 insertions(+), 17 deletions(-) diff --git a/dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md b/dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md index f4650adc..6dcb78e8 100644 --- a/dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md +++ b/dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md @@ -9,7 +9,10 @@ ### Changed - **A run that read half or less of the readable words warns on stdout** — the `WARNING: only - N% ...` line goes to stdout as well as the stderr warning, because a headline piped - elsewhere never shows stderr. The threshold is unchanged, so the same runs warn as before; a - rate quoted from an entry-point run of a progressive-disclosure skill now carries that - caveat where it is copied. + N% ...` line goes to stdout, because a headline piped elsewhere never shows stderr. The + threshold of the stderr warning is unchanged, and an entry-point run warns on both channels + as before. The stdout line is wider than the stderr one in two cases: a `--include-resources` + run whose word budgets left half or more of the readable words unread now warns on stdout + (its `warnings` list is unchanged), and `coverage: unknown` (a bundled file did not decode) + keeps the `(entry point only)` scope and warns when the entry point is half or less of the + readable words. `bundle.measured_share` is `null` when the share is unknowable. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 30761e31..bd91d18b 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -492,7 +492,10 @@ Things to know before reading a bundle result: - **The two modes are different cells.** `include_resources` is recorded in `run_config`; an entry-point run and a bundle run of the same skill are not comparable numbers. A run that read half or less of the readable words says so on stdout (the `coverage:` line - and a `WARNING`); an entry-point run also says so in `warnings`. The share is + and a `WARNING`), a bundle run held back by the word budgets included; an entry-point run + also says so in `warnings`. When a bundled file did not decode the share is unknowable: + stdout prints `coverage: unknown` (still warning if the entry point is half or less of + the readable words) and `bundle.measured_share` is `null`. The share is `bundle.measured_share` (for an entry-point run, `1 - measurable_outside_share`, which nets out the role decisions below; `outside_share` is the on-disk figure, and the two differ for a skill whose only resources are its evals). diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 7caa33bd..4af13efa 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -841,15 +841,24 @@ def _print_coverage(bundle: dict) -> None: if coverage is None or not coverage.denominator("readable"): return share = coverage.measured_share + scope = "" if coverage.include_resources else " (entry point only)" + rerun = ("" if coverage.include_resources or not coverage.measurable_resource_words + else "; pass --include-resources to measure the bundled files") if share is None: - print("coverage: unknown (a bundled file did not decode, so its words are not counted)") + print("coverage: unknown (a bundled file did not decode, so its words are not " + f"counted){scope}") + # The share is unknowable, but the entry point's part of the readable words is not: + # the stderr warning fires on it, so stdout says it too. + if (not coverage.include_resources + and coverage.outside_share_on("readable") >= 1 - MEASURED_SHARE_WARN_AT): + print("WARNING: the entry point is half or less of the artifact's readable words, " + f"so every rate here describes that part, not the artifact{rerun}") return - # Truncated, not rounded: 49.99% must not print as "50%" beside a "below 50%" warning. - pct = f"{int(share * 100)}%" - scope = "" if coverage.include_resources else " (entry point only)" + # Floored on the 4-place share, in integers: a float product floors 0.29 to 28, and + # 49.99% must not print as "50%" beside a "below 50%" warning. A true 49.996% is already + # 0.5 by the time it is stored. + pct = f"{round(share * 10000) // 100}%" print(f"coverage: {pct} of the artifact's readable words were measured{scope}") if share <= MEASURED_SHARE_WARN_AT: - rerun = ("" if coverage.include_resources or not coverage.measurable_resource_words - else "; pass --include-resources to measure the bundled files") print(f"WARNING: only {pct} of the artifact's readable words were measured, so every " f"rate here describes that part, not the artifact{rerun}") diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index d9ae1063..e6a01d4e 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -146,7 +146,7 @@ }, "bundle": { "type": "object", - "description": "Which files of the artifact were measured. A skill is a bundle: measured over the 33-skill study corpus 0.72 of all words live outside SKILL.md (per skill median 0.43, worst 0.99), so an entry-point-only run (include_resources false) is not comparable with a bundle run and must not be quoted as describing the artifact. outside_share is the on-disk share outside the entry point whether or not it was read; measurable_outside_share nets out policy_excluded_words (role decisions such as the skill\u0027s own evals, the same in both modes) and is the figure the warnings and report quote, since a rerun with --include-resources cannot read those; unmeasured_words is what such a rerun would add; excluded lists every file left out with its reason and a kind (symlink, not_text, unreadable, policy, budget, not_requested), and every reader interprets a row by the kind's class (novelty.bundle.EXCLUSION_CLASS): budget and not_requested are a gap (counted in unmeasured_words and warned about), policy is a decision (never charged as unread, and outside the readable-words denominator the warnings, the report and, for a record stamped coverage_basis readable, the gate uses), unreadable is unknown (the gate reports no artifact_coverage value while one exists; the warnings, note and report still quote a figure that leaves its words out), symlink and not_text are not content; pointer_spans names the spans whose only content is a path into the bundle, which carry no knowledge by design and are therefore not waste; by_source cuts the paired outcomes by the file that carried each atom. A bundle run measures what the artifact COULD deliver if the agent read all of it -- an upper bound; whether the agent loads a file at runtime is the agentic track\u0027s question.", + "description": "Which files of the artifact were measured. A skill is a bundle: measured over the 33-skill study corpus 0.72 of all words live outside SKILL.md (per skill median 0.43, worst 0.99), so an entry-point-only run (include_resources false) is not comparable with a bundle run and must not be quoted as describing the artifact. outside_share is the on-disk share outside the entry point whether or not it was read; measurable_outside_share nets out policy_excluded_words (role decisions such as the skill\u0027s own evals, the same in both modes) and is the figure the warnings and report quote, since a rerun with --include-resources cannot read those; measured_share is the share of the readable words this run read (null when a file did not decode); unmeasured_words is what such a rerun would add; excluded lists every file left out with its reason and a kind (symlink, not_text, unreadable, policy, budget, not_requested), and every reader interprets a row by the kind's class (novelty.bundle.EXCLUSION_CLASS): budget and not_requested are a gap (counted in unmeasured_words and warned about), policy is a decision (never charged as unread, and outside the readable-words denominator the warnings, the report and, for a record stamped coverage_basis readable, the gate uses), unreadable is unknown (the gate reports no artifact_coverage value while one exists; the warnings, note and report still quote a figure that leaves its words out), symlink and not_text are not content; pointer_spans names the spans whose only content is a path into the bundle, which carry no knowledge by design and are therefore not waste; by_source cuts the paired outcomes by the file that carried each atom. A bundle run measures what the artifact COULD deliver if the agent read all of it -- an upper bound; whether the agent loads a file at runtime is the agentic track\u0027s question.", "properties": { "include_resources": {"type": "boolean"}, "n_files": {"type": "integer", "minimum": 0}, diff --git a/dokimi/tests/test_novelty_bundle.py b/dokimi/tests/test_novelty_bundle.py index 05d49204..55c14cb3 100644 --- a/dokimi/tests/test_novelty_bundle.py +++ b/dokimi/tests/test_novelty_bundle.py @@ -684,6 +684,21 @@ def test_a_stamped_record_validates(tmp_path): schema_version="novelty.v1")) +def test_a_record_with_an_unknowable_share_validates(tmp_path): + from dokimi.artifacts import validate_artifact + + from tests.test_novelty_cli import _minimal_artifact + + record = dict(load_bundle(f"skill:{_skill_with_evals(tmp_path)}").summary(), + measured_share=None) + validate_artifact("novelty", dict(_minimal_artifact(), bundle=record, + schema_version="novelty.v1")) + with pytest.raises(Exception, match="measured_share"): + validate_artifact("novelty", dict(_minimal_artifact(), + bundle=dict(record, measured_share="most"), + schema_version="novelty.v1")) + + def test_a_note_before_measuring_says_what_will_not_be_read(skill): assert "will not be measured" in load_bundle(f"skill:{skill}").coverage_note() assert "--include-resources" in load_bundle(f"skill:{skill}").coverage_note() diff --git a/dokimi/tests/test_novelty_cli.py b/dokimi/tests/test_novelty_cli.py index 71833cf7..41297612 100644 --- a/dokimi/tests/test_novelty_cli.py +++ b/dokimi/tests/test_novelty_cli.py @@ -636,8 +636,44 @@ def test_the_headline_states_the_measured_share_and_warns_on_stdout_at_half_or_l out = capsys.readouterr().out assert "coverage: 50%" in out and "WARNING: only" not in out - # A file that did not decode leaves the share unknowable, and the headline says so. + # A file that did not decode leaves the share unknowable, and the headline says so, + # with the scope and, while the entry point is half or less of the words, the warning. novelty_cmd._print_headline(dict(artifact, bundle=dict(record, unreadable_files=1))) out = capsys.readouterr().out - assert "coverage: unknown" in out - assert "WARNING: only" not in out + assert "coverage: unknown" in out and out.count("(entry point only)") == 1 + assert "WARNING: the entry point is half or less" in out + assert "pass --include-resources" in out + # An entry point that is most of the words does not warn, as the stderr warning does not. + novelty_cmd._print_headline(dict(artifact, bundle=dict(record, unreadable_files=1, + entry_point_words=90, + resource_words_on_disk=10))) + out = capsys.readouterr().out + assert "coverage: unknown" in out and "WARNING" not in out + + +@pytest.mark.parametrize("entry, printed", [(29, "29%"), (57, "57%"), (58, "58%"), + (49, "49%"), (1, "1%")]) +def test_the_printed_percentage_is_the_share_not_a_float_product(capsys, entry, printed): + # 29 entry words in 100: 0.29 * 100 is 28.999999999999996, which int() floors to 28. + record = {"include_resources": False, "n_files": 1, "entry_point_words": entry, + "resource_words_measured": 0, "resource_words_on_disk": 100 - entry, + "policy_excluded_words": 0, "unreadable_files": 0, "excluded": [], + "coverage_basis": "readable"} + novelty_cmd._print_headline(dict(_minimal_artifact(), bundle=record)) + out = capsys.readouterr().out + assert f"coverage: {printed} of the artifact's readable words" in out + + +def test_a_bundle_run_capped_by_the_budget_warns_without_advising_the_flag_it_passed(capsys): + # --include-resources was passed; the budget left most of the reference unread. The + # stderr warning list is silent about this, the stdout line is not, and it must not tell + # the user to pass a flag they already passed. + record = {"include_resources": True, "n_files": 2, "entry_point_words": 100, + "resource_words_measured": 50, "resource_words_on_disk": 500, + "policy_excluded_words": 0, "unreadable_files": 0, "excluded": [], + "coverage_basis": "readable"} + novelty_cmd._print_headline(dict(_minimal_artifact(), bundle=record)) + out = capsys.readouterr().out + assert "coverage: 25% of the artifact's readable words were measured\n" in out + assert "WARNING: only 25%" in out + assert "--include-resources" not in out diff --git a/dokimi/tests/test_novelty_journey.py b/dokimi/tests/test_novelty_journey.py index 6ebbad1c..3bddae70 100644 --- a/dokimi/tests/test_novelty_journey.py +++ b/dokimi/tests/test_novelty_journey.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +from decimal import ROUND_FLOOR, Decimal import re import shlex from pathlib import Path @@ -165,7 +166,7 @@ def test_the_stdout_headline_says_how_much_of_the_artifact_was_read(tmp_path, sk measured = json.loads((entry_out / "novelty.json").read_text(encoding="utf-8"))[ "bundle"]["measured_share"] assert measured < 0.5 - assert f"coverage: {int(measured * 100)}% of the artifact's readable words were measured " \ + assert f"coverage: {int(Decimal(str(measured)).scaleb(2).to_integral_value(ROUND_FLOOR))}% of the artifact's readable words were measured " \ "(entry point only)" in printed assert "WARNING: only" in printed and "pass --include-resources" in printed diff --git a/dokimi/tests/test_novelty_settings_parity.py b/dokimi/tests/test_novelty_settings_parity.py index bf96b8b4..5923ac2e 100644 --- a/dokimi/tests/test_novelty_settings_parity.py +++ b/dokimi/tests/test_novelty_settings_parity.py @@ -23,6 +23,7 @@ import json import os import sys +from decimal import ROUND_FLOOR, Decimal from pathlib import Path from typing import Any @@ -144,7 +145,8 @@ def _check_headline(coverage: dict[str, Any]) -> None: # No record to check against (an estimate, or a run that wrote none): nothing printed. assert printed == [] return - pct = f"{int(share * 100)}%" + # An integer floor of the decimal share, not a float product (0.29 * 100 < 29). + pct = f"{int(Decimal(str(share)).scaleb(2).to_integral_value(ROUND_FLOOR))}%" assert printed, "a run with a record must print its coverage line" assert printed[0].startswith( f"coverage: {pct} of the artifact's readable words were measured" From 96fa63069726b3e36e73b98823ac580f36b7648f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:03:21 -0700 Subject: [PATCH 4/8] refactor(tests): split the answer-key tests out of test_novelty_bundle; chore(growth): raise commands/novelty.py for W5 Signed-off-by: Nikolay Petrov --- dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_novelty_bundle.py | 295 ----------------------- dokimi/tests/test_novelty_bundle_key.py | 306 ++++++++++++++++++++++++ 3 files changed, 307 insertions(+), 296 deletions(-) create mode 100644 dokimi/tests/test_novelty_bundle_key.py diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 2b0f5fe8..00db72ea 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 827}, + "dokimi/commands/novelty.py": {"lines": 864, "rationale": "W5 adds a novelty flag; file split tracked separately to avoid conflicts across the W-series"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1112}, diff --git a/dokimi/tests/test_novelty_bundle.py b/dokimi/tests/test_novelty_bundle.py index 55c14cb3..04d3fdbd 100644 --- a/dokimi/tests/test_novelty_bundle.py +++ b/dokimi/tests/test_novelty_bundle.py @@ -731,298 +731,3 @@ def test_a_text_artifact_gets_a_measured_digest_too(tmp_path): path = tmp_path / "notes.md" path.write_text("# Notes\n", encoding="utf-8") assert load_bundle(f"text:{path}").measured_digest().startswith("sha256:") - - -# ── Is a saved answer key still valid for the artifact as it is now? ──────── - -_KEY_SKILL = """\ ---- -name: accel-tuning -description: Configure the accelerator ---- - -# Accelerator tuning - -Enable the accelerator with `--accel`. -""" - - -def _key_skill(tmp_path): - root = tmp_path / "accel-tuning" - (root / "references").mkdir(parents=True) - (root / "SKILL.md").write_text(_KEY_SKILL, encoding="utf-8") - (root / "references" / "tuning.md").write_text( - "# Tuning\n\nNever exceed 64 threads.\n", encoding="utf-8") - return root - - -def _recorded(root, **kwargs): - return load_bundle(f"skill:{root}", include_resources=True, **kwargs).summary() - - -def _verdict(root, record): - from dokimi.novelty.bundle import check_answer_key - - return check_answer_key(f"skill:{root}", record, source="a.json") - - -def test_the_file_set_comes_from_the_record_not_from_flags(tmp_path): - root = _key_skill(tmp_path) - # A record from before budgets, roles and digests were written. - verdict = _verdict(root, {"include_resources": True, "n_files": 2, - "resource_words_measured": 6, - "files": [{"path": "SKILL.md", "role": "entry_point"}, - {"path": "references/tuning.md", "role": "reference"}]}) - assert verdict.ok - # The reference is injected although no flag asked for it: the key was extracted from it. - assert "Never exceed 64 threads" in verdict.text - assert any("a.json: 2 files" in note for note in verdict.notes) - - -def test_a_key_without_provenance_is_probed_against_the_entry_point_and_says_so(tmp_path): - root = _key_skill(tmp_path) - verdict = _verdict(root, {}) - assert verdict.ok and verdict.bundle is None - assert "Never exceed 64 threads" not in verdict.text - assert any("predates bundle provenance" in note for note in verdict.notes) - - -def test_a_key_for_an_artifact_that_cannot_be_loaded_is_refused(tmp_path): - from dokimi.novelty.bundle import check_answer_key - - for record in ({}, {"files": [{"path": "SKILL.md", "role": "entry_point"}]}): - verdict = check_answer_key(f"skill:{tmp_path / 'missing'}", record) - assert (verdict.ok, verdict.problem) == (False, "unloadable") - - -def test_a_record_without_usable_file_rows_is_refused(tmp_path): - root = _key_skill(tmp_path) - for broken in ({"include_resources": True, "files": []}, - {"include_resources": True, "files": [{"path": "SKILL.md"}]}): - verdict = _verdict(root, broken) - assert verdict.problem == "no_file_rows" and "provenance" in verdict.message - - -def test_a_removed_file_is_a_changed_file_set(tmp_path): - root = _key_skill(tmp_path) - verdict = _verdict(root, {"include_resources": True, "n_files": 3, "files": [ - {"path": "SKILL.md", "role": "entry_point"}, - {"path": "references/tuning.md", "role": "reference"}, - {"path": "references/gone.md", "role": "reference"}]}) - assert verdict.problem == "files_changed" - assert "references/gone.md" in verdict.recorded - assert "references/gone.md" not in verdict.rebuilt - - -def test_the_selection_is_replayed_with_the_recorded_budgets(tmp_path): - root = _key_skill(tmp_path) - (root / "references" / "big.md").write_text("word " * 5000, encoding="utf-8") - bundle = load_bundle(f"skill:{root}", include_resources=True, max_file_words=6000) - assert _verdict(root, bundle.summary()).text == bundle.text - - -@pytest.mark.parametrize("edit", [ - "# Tuning\n\nNever exceed 128 threads on driver 1.0.\n", - # Same path, same word count, different fact: only the digest sees this one. - "# Tuning\n\nNever exceed 128 threads.\n", -]) -def test_an_edited_bundled_file_is_a_stale_key(tmp_path, edit): - root = _key_skill(tmp_path) - record = _recorded(root) - (root / "references" / "tuning.md").write_text(edit, encoding="utf-8") - verdict = _verdict(root, record) - assert (verdict.problem, verdict.edited) == ("files_changed", ("references/tuning.md",)) - - -def test_an_edited_entry_point_is_the_re_measure_step_and_says_so(tmp_path): - root = _key_skill(tmp_path) - record = _recorded(root) - (root / "SKILL.md").write_text( - "---\nname: accel-tuning\ndescription: Configure the accelerator\n---\n\n" - "# Accelerator tuning\n\nShortened: the flag is `--accel`.\n", encoding="utf-8") - verdict = _verdict(root, record) - assert verdict.ok and verdict.entry_point_changed - assert "Shortened: the flag is" in verdict.text and "Never exceed 64" in verdict.text - assert any("has changed since these atoms were extracted" in n for n in verdict.notes) - - -@pytest.mark.parametrize("added", ["notes.md", "scripts/run.sh"]) -def test_a_new_file_in_a_role_not_seen_at_extraction_is_a_stale_key(tmp_path, added): - root = _key_skill(tmp_path) - record = _recorded(root) - # The extraction had only a reference, so the rebuild is filtered to that role and a - # new file in any other role never reached the file-set comparison. - (root / added).parent.mkdir(parents=True, exist_ok=True) - (root / added).write_text("Always pass --force.\n", encoding="utf-8") - verdict = _verdict(root, record) - assert (verdict.problem, verdict.added) == ("files_changed", (added,)) - - -def test_a_new_file_that_is_not_content_leaves_the_key_valid(tmp_path): - root = _key_skill(tmp_path) - record = _recorded(root) - (root / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n") - # A symlink is recorded, never followed, in every mode, so re-extracting would not - # measure it either. (It used to count as a change; an image never did.) - (root / "references" / "shared.md").symlink_to(root / "references" / "tuning.md") - assert _verdict(root, record).ok - - -def test_an_answer_key_from_another_skill_is_refused(tmp_path): - other = tmp_path / "other" - other.mkdir() - (other / "SKILL.md").write_text( - "---\nname: other-skill\ndescription: Tune a database\n---\n\n# Other\n", - encoding="utf-8") - root = _key_skill(tmp_path) - # Both sides are just ["SKILL.md"], so the file-set check could never catch this. - verdict = _verdict(root, load_bundle(f"skill:{other}").summary()) - assert verdict.problem == "wrong_artifact" and "belongs to one artifact" in verdict.message - - -def test_a_budget_gap_stays_a_gap_when_replayed(tmp_path): - """Rebuilding with the selected files' roles relabelled an over-budget script `policy`.""" - root = _key_skill(tmp_path) - (root / "scripts").mkdir() - (root / "scripts" / "big.py").write_text("x = 1 # w\n" * 6000, encoding="utf-8") - recorded = load_bundle(f"skill:{root}", include_resources=True) - assert recorded.coverage().unmeasured_words > 0 - probed = _verdict(root, recorded.summary()).bundle - assert probed.coverage() == recorded.coverage() - assert probed.coverage().policy_excluded_words == 0 - - -def test_requested_roles_are_replayed_when_none_was_selected(tmp_path): - root = _key_skill(tmp_path) - (root / "scripts").mkdir() - (root / "scripts" / "big.py").write_text("x = 1 # w\n" * 6000, encoding="utf-8") - recorded = load_bundle(f"skill:{root}", include_resources=True, roles=("script",)) - assert [f.path for f in recorded.files] == ["SKILL.md"] - assert _verdict(root, recorded.summary()).ok # the unchanged tree is not "changed" - - -def test_a_new_eval_case_changes_the_key_only_when_evals_were_requested(tmp_path): - root = _key_skill(tmp_path) - by_default = _recorded(root) - (root / "evals").mkdir() - (root / "evals" / "a.md").write_text("Q: one?\n", encoding="utf-8") - # Evals are not measured by default; the text is unchanged. - assert _verdict(root, by_default).ok - with_evals = _recorded(root, roles=("reference", "eval")) - (root / "evals" / "q.md").write_text("Q: two?\n", encoding="utf-8") - verdict = _verdict(root, with_evals) - assert verdict.problem == "files_changed" and verdict.added == ("evals/q.md",) - - -# ── Shortening rewrites one file ───────────────────────────────────────────── - -def test_shortening_a_bundle_key_keeps_only_the_entry_point(tmp_path): - from dokimi.novelty.atoms import Atom - from dokimi.novelty.bundle import entry_point_only - - root = _key_skill(tmp_path) - bundle = load_bundle(f"skill:{root}", include_resources=True) - spans = segment(bundle.text) - entry_end = bundle.files[0].end - inside = next(s for s in spans if s.start < entry_end and "`--accel`" in s.text) - outside = next(s for s in spans if s.start >= entry_end and "64" in s.text) - atoms = [ - Atom(statement="entry", question="q", answer="a", span_ids=(inside.id,)), - Atom(statement="reference", question="q", answer="a", span_ids=(outside.id,)), - # Half the evidence cannot be scored against a candidate that holds one file. - Atom(statement="both", question="q", answer="a", span_ids=(inside.id, outside.id)), - ] - kept_spans, kept_atoms, note = entry_point_only(bundle.summary(), spans, atoms) - assert all(span.start < entry_end for span in kept_spans) - assert [atom.statement for atom in kept_atoms] == ["entry"] - assert "bundle spans" in note and "SKILL.md" in note - # An entry-point key passes through untouched. - assert entry_point_only(load_bundle(f"skill:{root}").summary(), spans, atoms) == ( - spans, atoms, None) - - -def test_shortening_refuses_a_bundle_record_it_cannot_slice(): - from dokimi.novelty.bundle import entry_point_only - - with pytest.raises(ValueError, match="file rows"): - entry_point_only({"include_resources": True, "files": []}, [], []) - with pytest.raises(ValueError, match="span range"): - entry_point_only({"include_resources": True, "files": [{"path": "SKILL.md"}]}, [], []) - - -def test_the_report_names_excluded_files_and_counts_non_text_ones(tmp_path): - from dokimi.novelty.report import render_novelty_markdown - - from tests.test_novelty_cli import _minimal_artifact - from tests.test_novelty_gate import _artifact as _gate_artifact - - def _artifact(bundle): - # The gate's fixture, with the blocks the renderer reads. - minimal = _minimal_artifact() - return dict(_gate_artifact(), spans=minimal["spans"], artifact=minimal["artifact"], - shortening=[], - bundle=bundle) - - root = _skill_with_evals(tmp_path, refs=50) - (root / "references" / "big.md").write_text("w " * 30, encoding="utf-8") - (root / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n") - (root / "references" / "linked.md").symlink_to(root / "references" / "r.md") - bundle = load_bundle(f"skill:{root}", include_resources=True, max_file_words=40) - rendered = render_novelty_markdown(_artifact(bundle=bundle.summary())) - assert "3 files excluded: `evals/cases.md`" in rendered - assert "`references/linked.md` (symlink), `references/r.md` (over max_file_words=40)." in rendered - assert "1 non-text files not read." in rendered - partial = render_novelty_markdown(_artifact(bundle={"outside_share": 0.99})) - assert "lacks its word counts" in partial - - -def test_measured_digest_format_is_pinned(tmp_path): - """Measured_digest format is stable.""" - from dokimi.novelty.bundle import load_bundle - - # Create a test skill with two files - skill = tmp_path / "s" / "SKILL.md" - skill.parent.mkdir(parents=True) - skill.write_text( - "---\nname: test\ndescription: d\n---\n\nEntry point text.\n", - encoding="utf-8", - ) - ref = skill.parent / "references" / "ref.md" - ref.parent.mkdir() - ref.write_text("Reference text.", encoding="utf-8") - - bundle = load_bundle(f"skill:{skill.parent}", include_resources=True) - digest = bundle.measured_digest() - - # Pin the exact literal so edits to measured_digest() cannot silently invalidate stored ref_digest values - assert digest == "sha256:f0eb5f66f8b343a03bc6f5fbcd233e3bbf591db23ef075502db96bb35ddd5bf5" - - # The digest should be deterministic - same files, same digest - bundle2 = load_bundle(f"skill:{skill.parent}", include_resources=True) - assert bundle2.measured_digest() == digest - - # Modifying a file should change the digest - ref.write_text("Modified reference text.", encoding="utf-8") - bundle3 = load_bundle(f"skill:{skill.parent}", include_resources=True) - assert bundle3.measured_digest() != digest - - -def test_the_record_states_the_share_of_readable_words_this_run_read(tmp_path): - """100 entry words and 300 measurable reference words: an entry-point run read a quarter.""" - bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=100, refs=300)}") - assert bundle.coverage().measured_share == 0.25 - assert bundle.summary()["measured_share"] == 0.25 - assert any(w.startswith("75% ") for w in bundle.coverage_warnings()) - - -def test_exactly_half_the_readable_words_read_is_warned_as_the_analyzer_always_has_been(tmp_path): - """The warning line is unchanged: a run that read half the readable words still warns.""" - bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=300, refs=300)}") - assert bundle.coverage().measured_share == 0.5 - assert any(w.startswith("50% ") for w in bundle.coverage_warnings()) - - -def test_a_bundle_run_that_read_everything_it_could_measures_all_of_it(tmp_path): - bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=100, refs=300)}", - include_resources=True) - assert bundle.summary()["measured_share"] == 1.0 diff --git a/dokimi/tests/test_novelty_bundle_key.py b/dokimi/tests/test_novelty_bundle_key.py new file mode 100644 index 00000000..3f626f26 --- /dev/null +++ b/dokimi/tests/test_novelty_bundle_key.py @@ -0,0 +1,306 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A saved answer key against the bundle as it is now, and shortening a bundle's entry point.""" + +from __future__ import annotations + +import pytest + +from dokimi.novelty.bundle import load_bundle +from dokimi.novelty.spans import segment +from tests.test_novelty_bundle import _skill_with_evals + +# ── Is a saved answer key still valid for the artifact as it is now? ──────── + +_KEY_SKILL = """\ +--- +name: accel-tuning +description: Configure the accelerator +--- + +# Accelerator tuning + +Enable the accelerator with `--accel`. +""" + + +def _key_skill(tmp_path): + root = tmp_path / "accel-tuning" + (root / "references").mkdir(parents=True) + (root / "SKILL.md").write_text(_KEY_SKILL, encoding="utf-8") + (root / "references" / "tuning.md").write_text( + "# Tuning\n\nNever exceed 64 threads.\n", encoding="utf-8") + return root + + +def _recorded(root, **kwargs): + return load_bundle(f"skill:{root}", include_resources=True, **kwargs).summary() + + +def _verdict(root, record): + from dokimi.novelty.bundle import check_answer_key + + return check_answer_key(f"skill:{root}", record, source="a.json") + + +def test_the_file_set_comes_from_the_record_not_from_flags(tmp_path): + root = _key_skill(tmp_path) + # A record from before budgets, roles and digests were written. + verdict = _verdict(root, {"include_resources": True, "n_files": 2, + "resource_words_measured": 6, + "files": [{"path": "SKILL.md", "role": "entry_point"}, + {"path": "references/tuning.md", "role": "reference"}]}) + assert verdict.ok + # The reference is injected although no flag asked for it: the key was extracted from it. + assert "Never exceed 64 threads" in verdict.text + assert any("a.json: 2 files" in note for note in verdict.notes) + + +def test_a_key_without_provenance_is_probed_against_the_entry_point_and_says_so(tmp_path): + root = _key_skill(tmp_path) + verdict = _verdict(root, {}) + assert verdict.ok and verdict.bundle is None + assert "Never exceed 64 threads" not in verdict.text + assert any("predates bundle provenance" in note for note in verdict.notes) + + +def test_a_key_for_an_artifact_that_cannot_be_loaded_is_refused(tmp_path): + from dokimi.novelty.bundle import check_answer_key + + for record in ({}, {"files": [{"path": "SKILL.md", "role": "entry_point"}]}): + verdict = check_answer_key(f"skill:{tmp_path / 'missing'}", record) + assert (verdict.ok, verdict.problem) == (False, "unloadable") + + +def test_a_record_without_usable_file_rows_is_refused(tmp_path): + root = _key_skill(tmp_path) + for broken in ({"include_resources": True, "files": []}, + {"include_resources": True, "files": [{"path": "SKILL.md"}]}): + verdict = _verdict(root, broken) + assert verdict.problem == "no_file_rows" and "provenance" in verdict.message + + +def test_a_removed_file_is_a_changed_file_set(tmp_path): + root = _key_skill(tmp_path) + verdict = _verdict(root, {"include_resources": True, "n_files": 3, "files": [ + {"path": "SKILL.md", "role": "entry_point"}, + {"path": "references/tuning.md", "role": "reference"}, + {"path": "references/gone.md", "role": "reference"}]}) + assert verdict.problem == "files_changed" + assert "references/gone.md" in verdict.recorded + assert "references/gone.md" not in verdict.rebuilt + + +def test_the_selection_is_replayed_with_the_recorded_budgets(tmp_path): + root = _key_skill(tmp_path) + (root / "references" / "big.md").write_text("word " * 5000, encoding="utf-8") + bundle = load_bundle(f"skill:{root}", include_resources=True, max_file_words=6000) + assert _verdict(root, bundle.summary()).text == bundle.text + + +@pytest.mark.parametrize("edit", [ + "# Tuning\n\nNever exceed 128 threads on driver 1.0.\n", + # Same path, same word count, different fact: only the digest sees this one. + "# Tuning\n\nNever exceed 128 threads.\n", +]) +def test_an_edited_bundled_file_is_a_stale_key(tmp_path, edit): + root = _key_skill(tmp_path) + record = _recorded(root) + (root / "references" / "tuning.md").write_text(edit, encoding="utf-8") + verdict = _verdict(root, record) + assert (verdict.problem, verdict.edited) == ("files_changed", ("references/tuning.md",)) + + +def test_an_edited_entry_point_is_the_re_measure_step_and_says_so(tmp_path): + root = _key_skill(tmp_path) + record = _recorded(root) + (root / "SKILL.md").write_text( + "---\nname: accel-tuning\ndescription: Configure the accelerator\n---\n\n" + "# Accelerator tuning\n\nShortened: the flag is `--accel`.\n", encoding="utf-8") + verdict = _verdict(root, record) + assert verdict.ok and verdict.entry_point_changed + assert "Shortened: the flag is" in verdict.text and "Never exceed 64" in verdict.text + assert any("has changed since these atoms were extracted" in n for n in verdict.notes) + + +@pytest.mark.parametrize("added", ["notes.md", "scripts/run.sh"]) +def test_a_new_file_in_a_role_not_seen_at_extraction_is_a_stale_key(tmp_path, added): + root = _key_skill(tmp_path) + record = _recorded(root) + # The extraction had only a reference, so the rebuild is filtered to that role and a + # new file in any other role never reached the file-set comparison. + (root / added).parent.mkdir(parents=True, exist_ok=True) + (root / added).write_text("Always pass --force.\n", encoding="utf-8") + verdict = _verdict(root, record) + assert (verdict.problem, verdict.added) == ("files_changed", (added,)) + + +def test_a_new_file_that_is_not_content_leaves_the_key_valid(tmp_path): + root = _key_skill(tmp_path) + record = _recorded(root) + (root / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n") + # A symlink is recorded, never followed, in every mode, so re-extracting would not + # measure it either. (It used to count as a change; an image never did.) + (root / "references" / "shared.md").symlink_to(root / "references" / "tuning.md") + assert _verdict(root, record).ok + + +def test_an_answer_key_from_another_skill_is_refused(tmp_path): + other = tmp_path / "other" + other.mkdir() + (other / "SKILL.md").write_text( + "---\nname: other-skill\ndescription: Tune a database\n---\n\n# Other\n", + encoding="utf-8") + root = _key_skill(tmp_path) + # Both sides are just ["SKILL.md"], so the file-set check could never catch this. + verdict = _verdict(root, load_bundle(f"skill:{other}").summary()) + assert verdict.problem == "wrong_artifact" and "belongs to one artifact" in verdict.message + + +def test_a_budget_gap_stays_a_gap_when_replayed(tmp_path): + """Rebuilding with the selected files' roles relabelled an over-budget script `policy`.""" + root = _key_skill(tmp_path) + (root / "scripts").mkdir() + (root / "scripts" / "big.py").write_text("x = 1 # w\n" * 6000, encoding="utf-8") + recorded = load_bundle(f"skill:{root}", include_resources=True) + assert recorded.coverage().unmeasured_words > 0 + probed = _verdict(root, recorded.summary()).bundle + assert probed.coverage() == recorded.coverage() + assert probed.coverage().policy_excluded_words == 0 + + +def test_requested_roles_are_replayed_when_none_was_selected(tmp_path): + root = _key_skill(tmp_path) + (root / "scripts").mkdir() + (root / "scripts" / "big.py").write_text("x = 1 # w\n" * 6000, encoding="utf-8") + recorded = load_bundle(f"skill:{root}", include_resources=True, roles=("script",)) + assert [f.path for f in recorded.files] == ["SKILL.md"] + assert _verdict(root, recorded.summary()).ok # the unchanged tree is not "changed" + + +def test_a_new_eval_case_changes_the_key_only_when_evals_were_requested(tmp_path): + root = _key_skill(tmp_path) + by_default = _recorded(root) + (root / "evals").mkdir() + (root / "evals" / "a.md").write_text("Q: one?\n", encoding="utf-8") + # Evals are not measured by default; the text is unchanged. + assert _verdict(root, by_default).ok + with_evals = _recorded(root, roles=("reference", "eval")) + (root / "evals" / "q.md").write_text("Q: two?\n", encoding="utf-8") + verdict = _verdict(root, with_evals) + assert verdict.problem == "files_changed" and verdict.added == ("evals/q.md",) + + +# ── Shortening rewrites one file ───────────────────────────────────────────── + +def test_shortening_a_bundle_key_keeps_only_the_entry_point(tmp_path): + from dokimi.novelty.atoms import Atom + from dokimi.novelty.bundle import entry_point_only + + root = _key_skill(tmp_path) + bundle = load_bundle(f"skill:{root}", include_resources=True) + spans = segment(bundle.text) + entry_end = bundle.files[0].end + inside = next(s for s in spans if s.start < entry_end and "`--accel`" in s.text) + outside = next(s for s in spans if s.start >= entry_end and "64" in s.text) + atoms = [ + Atom(statement="entry", question="q", answer="a", span_ids=(inside.id,)), + Atom(statement="reference", question="q", answer="a", span_ids=(outside.id,)), + # Half the evidence cannot be scored against a candidate that holds one file. + Atom(statement="both", question="q", answer="a", span_ids=(inside.id, outside.id)), + ] + kept_spans, kept_atoms, note = entry_point_only(bundle.summary(), spans, atoms) + assert all(span.start < entry_end for span in kept_spans) + assert [atom.statement for atom in kept_atoms] == ["entry"] + assert "bundle spans" in note and "SKILL.md" in note + # An entry-point key passes through untouched. + assert entry_point_only(load_bundle(f"skill:{root}").summary(), spans, atoms) == ( + spans, atoms, None) + + +def test_shortening_refuses_a_bundle_record_it_cannot_slice(): + from dokimi.novelty.bundle import entry_point_only + + with pytest.raises(ValueError, match="file rows"): + entry_point_only({"include_resources": True, "files": []}, [], []) + with pytest.raises(ValueError, match="span range"): + entry_point_only({"include_resources": True, "files": [{"path": "SKILL.md"}]}, [], []) + + +def test_the_report_names_excluded_files_and_counts_non_text_ones(tmp_path): + from dokimi.novelty.report import render_novelty_markdown + + from tests.test_novelty_cli import _minimal_artifact + from tests.test_novelty_gate import _artifact as _gate_artifact + + def _artifact(bundle): + # The gate's fixture, with the blocks the renderer reads. + minimal = _minimal_artifact() + return dict(_gate_artifact(), spans=minimal["spans"], artifact=minimal["artifact"], + shortening=[], + bundle=bundle) + + root = _skill_with_evals(tmp_path, refs=50) + (root / "references" / "big.md").write_text("w " * 30, encoding="utf-8") + (root / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n") + (root / "references" / "linked.md").symlink_to(root / "references" / "r.md") + bundle = load_bundle(f"skill:{root}", include_resources=True, max_file_words=40) + rendered = render_novelty_markdown(_artifact(bundle=bundle.summary())) + assert "3 files excluded: `evals/cases.md`" in rendered + assert "`references/linked.md` (symlink), `references/r.md` (over max_file_words=40)." in rendered + assert "1 non-text files not read." in rendered + partial = render_novelty_markdown(_artifact(bundle={"outside_share": 0.99})) + assert "lacks its word counts" in partial + + +def test_measured_digest_format_is_pinned(tmp_path): + """Measured_digest format is stable.""" + from dokimi.novelty.bundle import load_bundle + + # Create a test skill with two files + skill = tmp_path / "s" / "SKILL.md" + skill.parent.mkdir(parents=True) + skill.write_text( + "---\nname: test\ndescription: d\n---\n\nEntry point text.\n", + encoding="utf-8", + ) + ref = skill.parent / "references" / "ref.md" + ref.parent.mkdir() + ref.write_text("Reference text.", encoding="utf-8") + + bundle = load_bundle(f"skill:{skill.parent}", include_resources=True) + digest = bundle.measured_digest() + + # Pin the exact literal so edits to measured_digest() cannot silently invalidate stored ref_digest values + assert digest == "sha256:f0eb5f66f8b343a03bc6f5fbcd233e3bbf591db23ef075502db96bb35ddd5bf5" + + # The digest should be deterministic - same files, same digest + bundle2 = load_bundle(f"skill:{skill.parent}", include_resources=True) + assert bundle2.measured_digest() == digest + + # Modifying a file should change the digest + ref.write_text("Modified reference text.", encoding="utf-8") + bundle3 = load_bundle(f"skill:{skill.parent}", include_resources=True) + assert bundle3.measured_digest() != digest + + +def test_the_record_states_the_share_of_readable_words_this_run_read(tmp_path): + """100 entry words and 300 measurable reference words: an entry-point run read a quarter.""" + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=100, refs=300)}") + assert bundle.coverage().measured_share == 0.25 + assert bundle.summary()["measured_share"] == 0.25 + assert any(w.startswith("75% ") for w in bundle.coverage_warnings()) + + +def test_exactly_half_the_readable_words_read_is_warned_as_the_analyzer_always_has_been(tmp_path): + """The warning line is unchanged: a run that read half the readable words still warns.""" + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=300, refs=300)}") + assert bundle.coverage().measured_share == 0.5 + assert any(w.startswith("50% ") for w in bundle.coverage_warnings()) + + +def test_a_bundle_run_that_read_everything_it_could_measures_all_of_it(tmp_path): + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=100, refs=300)}", + include_resources=True) + assert bundle.summary()["measured_share"] == 1.0 From d7db0f435d8a272e825c85254c8671cfe325f464 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 23 Oct 2026 09:49:25 +0000 Subject: [PATCH 5/8] Print <1% for a non-zero measured share under one percent Signed-off-by: Anton Volkov --- dokimi/dokimi/commands/novelty.py | 2 +- dokimi/tests/test_novelty_cli.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 4af13efa..b9ac3070 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -857,7 +857,7 @@ def _print_coverage(bundle: dict) -> None: # Floored on the 4-place share, in integers: a float product floors 0.29 to 28, and # 49.99% must not print as "50%" beside a "below 50%" warning. A true 49.996% is already # 0.5 by the time it is stored. - pct = f"{round(share * 10000) // 100}%" + pct = "<1%" if 0 < share < 0.01 else f"{round(share * 10000) // 100}%" print(f"coverage: {pct} of the artifact's readable words were measured{scope}") if share <= MEASURED_SHARE_WARN_AT: print(f"WARNING: only {pct} of the artifact's readable words were measured, so every " diff --git a/dokimi/tests/test_novelty_cli.py b/dokimi/tests/test_novelty_cli.py index 41297612..57f5b247 100644 --- a/dokimi/tests/test_novelty_cli.py +++ b/dokimi/tests/test_novelty_cli.py @@ -664,6 +664,17 @@ def test_the_printed_percentage_is_the_share_not_a_float_product(capsys, entry, assert f"coverage: {printed} of the artifact's readable words" in out +def test_a_tiny_non_zero_share_prints_as_under_one_percent(capsys): + record = {"include_resources": False, "n_files": 1, "entry_point_words": 1, + "resource_words_measured": 0, "resource_words_on_disk": 9999, + "policy_excluded_words": 0, "unreadable_files": 0, "excluded": [], + "coverage_basis": "readable"} + novelty_cmd._print_headline(dict(_minimal_artifact(), bundle=record)) + out = capsys.readouterr().out + assert "coverage: <1% of the artifact's readable words were measured (entry point only)" in out + assert "WARNING: only <1% of the artifact's readable words" in out + + def test_a_bundle_run_capped_by_the_budget_warns_without_advising_the_flag_it_passed(capsys): # --include-resources was passed; the budget left most of the reference unread. The # stderr warning list is silent about this, the stdout line is not, and it must not tell From 95b8380538c3e2b7cc2488694cf09cd25862b3f8 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 23 Oct 2026 09:49:52 +0000 Subject: [PATCH 6/8] Describe the novelty.py growth in the ratchet rationale Signed-off-by: Anton Volkov --- dokimi/scripts/growth_baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index e9356094..eaa35db1 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 864, "rationale": "W5 adds a novelty flag; file split tracked separately to avoid conflicts across the W-series"}, + "dokimi/commands/novelty.py": {"lines": 864, "rationale": "stdout coverage line and warning (_print_coverage); split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main after the #93 seed and before it merged; split it in its own PR"}, From c9e5a77f34febf16639d98d4408d8be2e359c7bb Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 23 Oct 2026 09:50:22 +0000 Subject: [PATCH 7/8] Say measured_share is rounded and null with no readable words Signed-off-by: Anton Volkov --- dokimi/docs/skill-novelty.md | 4 ++-- dokimi/dokimi/schemas/novelty.v1.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index bd91d18b..474c2805 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -496,8 +496,8 @@ Things to know before reading a bundle result: also says so in `warnings`. When a bundled file did not decode the share is unknowable: stdout prints `coverage: unknown` (still warning if the entry point is half or less of the readable words) and `bundle.measured_share` is `null`. The share is - `bundle.measured_share` (for an entry-point run, `1 - measurable_outside_share`, - which nets out the role decisions below; `outside_share` is the + `bundle.measured_share` (for an entry-point run, `1 - measurable_outside_share` to 4 + places, which nets out the role decisions below; `outside_share` is the on-disk figure, and the two differ for a skill whose only resources are its evals). - **A skill's own `evals/` are not content.** They are the question/answer keys for testing the skill, so extracting atoms from them would credit the skill with delivering diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index e6a01d4e..47358cad 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -146,7 +146,7 @@ }, "bundle": { "type": "object", - "description": "Which files of the artifact were measured. A skill is a bundle: measured over the 33-skill study corpus 0.72 of all words live outside SKILL.md (per skill median 0.43, worst 0.99), so an entry-point-only run (include_resources false) is not comparable with a bundle run and must not be quoted as describing the artifact. outside_share is the on-disk share outside the entry point whether or not it was read; measurable_outside_share nets out policy_excluded_words (role decisions such as the skill\u0027s own evals, the same in both modes) and is the figure the warnings and report quote, since a rerun with --include-resources cannot read those; measured_share is the share of the readable words this run read (null when a file did not decode); unmeasured_words is what such a rerun would add; excluded lists every file left out with its reason and a kind (symlink, not_text, unreadable, policy, budget, not_requested), and every reader interprets a row by the kind's class (novelty.bundle.EXCLUSION_CLASS): budget and not_requested are a gap (counted in unmeasured_words and warned about), policy is a decision (never charged as unread, and outside the readable-words denominator the warnings, the report and, for a record stamped coverage_basis readable, the gate uses), unreadable is unknown (the gate reports no artifact_coverage value while one exists; the warnings, note and report still quote a figure that leaves its words out), symlink and not_text are not content; pointer_spans names the spans whose only content is a path into the bundle, which carry no knowledge by design and are therefore not waste; by_source cuts the paired outcomes by the file that carried each atom. A bundle run measures what the artifact COULD deliver if the agent read all of it -- an upper bound; whether the agent loads a file at runtime is the agentic track\u0027s question.", + "description": "Which files of the artifact were measured. A skill is a bundle: measured over the 33-skill study corpus 0.72 of all words live outside SKILL.md (per skill median 0.43, worst 0.99), so an entry-point-only run (include_resources false) is not comparable with a bundle run and must not be quoted as describing the artifact. outside_share is the on-disk share outside the entry point whether or not it was read; measurable_outside_share nets out policy_excluded_words (role decisions such as the skill\u0027s own evals, the same in both modes) and is the figure the warnings and report quote, since a rerun with --include-resources cannot read those; measured_share is the share of the readable words this run read (null when a file did not decode or there are no readable words); unmeasured_words is what such a rerun would add; excluded lists every file left out with its reason and a kind (symlink, not_text, unreadable, policy, budget, not_requested), and every reader interprets a row by the kind's class (novelty.bundle.EXCLUSION_CLASS): budget and not_requested are a gap (counted in unmeasured_words and warned about), policy is a decision (never charged as unread, and outside the readable-words denominator the warnings, the report and, for a record stamped coverage_basis readable, the gate uses), unreadable is unknown (the gate reports no artifact_coverage value while one exists; the warnings, note and report still quote a figure that leaves its words out), symlink and not_text are not content; pointer_spans names the spans whose only content is a path into the bundle, which carry no knowledge by design and are therefore not waste; by_source cuts the paired outcomes by the file that carried each atom. A bundle run measures what the artifact COULD deliver if the agent read all of it -- an upper bound; whether the agent loads a file at runtime is the agentic track\u0027s question.", "properties": { "include_resources": {"type": "boolean"}, "n_files": {"type": "integer", "minimum": 0}, From 8739308357723adf3dbd51b0d86182a855dd6143 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 12:47:14 -0700 Subject: [PATCH 8/8] test(novelty): pin the unknown-share stdout guards and the 4-place measured_share Cover the include-resources guard and the exact-half boundary of the coverage: unknown warning, and the rounding of the stored share. Drop the unreachable measurable_resource_words term from the rerun advice. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty.py | 4 ++-- dokimi/tests/test_novelty_bundle_key.py | 6 ++++++ dokimi/tests/test_novelty_cli.py | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 2d8a2cf6..f3448ba5 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -878,8 +878,8 @@ def _print_coverage(bundle: dict) -> None: return share = coverage.measured_share scope = "" if coverage.include_resources else " (entry point only)" - rerun = ("" if coverage.include_resources or not coverage.measurable_resource_words - else "; pass --include-resources to measure the bundled files") + # The warnings below need readable words outside the entry point, so the flag always adds some. + rerun = "" if coverage.include_resources else "; pass --include-resources to measure the bundled files" if share is None: print("coverage: unknown (a bundled file did not decode, so its words are not " f"counted){scope}") diff --git a/dokimi/tests/test_novelty_bundle_key.py b/dokimi/tests/test_novelty_bundle_key.py index 3f626f26..12cd11a9 100644 --- a/dokimi/tests/test_novelty_bundle_key.py +++ b/dokimi/tests/test_novelty_bundle_key.py @@ -304,3 +304,9 @@ def test_a_bundle_run_that_read_everything_it_could_measures_all_of_it(tmp_path) bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=100, refs=300)}", include_resources=True) assert bundle.summary()["measured_share"] == 1.0 + + +def test_the_stored_measured_share_is_rounded_to_four_places(tmp_path): + # 2 entry words and 7 reference words: 2/9 is 0.2222..., stored as 0.2222, not 0.2222222. + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=2, refs=7)}") + assert bundle.summary()["measured_share"] == 0.2222 diff --git a/dokimi/tests/test_novelty_cli.py b/dokimi/tests/test_novelty_cli.py index 57f5b247..db439aa8 100644 --- a/dokimi/tests/test_novelty_cli.py +++ b/dokimi/tests/test_novelty_cli.py @@ -651,6 +651,28 @@ def test_the_headline_states_the_measured_share_and_warns_on_stdout_at_half_or_l assert "coverage: unknown" in out and "WARNING" not in out +def _readable_record(**over): + return dict({"include_resources": False, "n_files": 1, "entry_point_words": 10, + "resource_words_measured": 0, "resource_words_on_disk": 30, + "policy_excluded_words": 0, "unreadable_files": 0, "excluded": [], + "coverage_basis": "readable"}, **over) + + +def test_an_undecodable_file_in_a_bundle_run_does_not_claim_an_entry_point_only_warning(capsys): + # --include-resources was passed: the unknown-share line must not say the entry point is + # half or less of the words, which is false for a bundle run. + record = _readable_record(include_resources=True, unreadable_files=1) + novelty_cmd._print_headline(dict(_minimal_artifact(), bundle=record)) + out = capsys.readouterr().out + assert "coverage: unknown" in out and "WARNING" not in out + + +def test_an_unknown_share_warns_at_exactly_half_as_the_stderr_warning_does(capsys): + record = _readable_record(unreadable_files=1, entry_point_words=50, resource_words_on_disk=50) + novelty_cmd._print_headline(dict(_minimal_artifact(), bundle=record)) + assert "WARNING: the entry point is half or less" in capsys.readouterr().out + + @pytest.mark.parametrize("entry, printed", [(29, "29%"), (57, "57%"), (58, "58%"), (49, "49%"), (1, "1%")]) def test_the_printed_percentage_is_the_share_not_a_float_product(capsys, entry, printed):