diff --git a/dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md b/dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md new file mode 100644 index 00000000..6dcb78e8 --- /dev/null +++ b/dokimi/changelog.d/20261021_215500_napetrov_novelty-measured-share.md @@ -0,0 +1,18 @@ +### Added + +- **`novelty analyze` states the measured share on stdout** — the headline now prints + `coverage: N% of the artifact's readable words were measured`, with `(entry point only)` + for a run without `--include-resources`. The novelty.v1 bundle record stamps the same figure + as `bundle.measured_share`, so a script can read it without recomputing it from + `measurable_outside_share`. + +### Changed + +- **A run that read half or less of the readable words warns on stdout** — the `WARNING: only + N% ...` line goes to stdout, because a headline piped elsewhere never shows stderr. The + threshold of the stderr warning is unchanged, and an entry-point run warns on both channels + as before. The stdout line is wider than the stderr one in two cases: a `--include-resources` + run whose word budgets left half or more of the readable words unread now warns on stdout + (its `warnings` list is unchanged), and `coverage: unknown` (a bundled file did not decode) + keeps the `(entry point only)` scope and warns when the entry point is half or less of the + readable words. `bundle.measured_share` is `null` when the share is unknowable. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index f0ef48df..41de08ef 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -490,9 +490,14 @@ Things to know before reading a bundle result: deliver if the agent read all of it. Whether the agent chooses to load `references/x.md` at runtime is the agentic track's question (`skill-agent:`), not this one. - **The two modes are different cells.** `include_resources` is recorded in `run_config`; - an entry-point run and a bundle run of the same skill are not comparable numbers, and an - entry-point run whose *measurable* outside share is ≥ 0.5 says so in `warnings` - (`measurable_outside_share` nets out the role decisions below; `outside_share` is the + an entry-point run and a bundle run of the same skill are not comparable numbers. A run + that read half or less of the readable words says so on stdout (the `coverage:` line + and a `WARNING`), a bundle run held back by the word budgets included; an entry-point run + also says so in `warnings`. When a bundled file did not decode the share is unknowable: + stdout prints `coverage: unknown` (still warning if the entry point is half or less of + the readable words) and `bundle.measured_share` is `null`. The share is + `bundle.measured_share` (for an entry-point run, `1 - measurable_outside_share` to 4 + places, which nets out the role decisions below; `outside_share` is the on-disk figure, and the two differ for a skill whose only resources are its evals). - **A skill's own `evals/` are not content.** They are the question/answer keys for testing the skill, so extracting atoms from them would credit the skill with delivering diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 6afb318f..667f9b7b 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -860,5 +860,42 @@ def _print_headline(artifact: dict) -> None: f"degraded: {paired['lenient']['degraded']}, " f"dead tokens: {artifact['spans'].get('frac_carrying_nothing')}" ) + _print_coverage(artifact.get("bundle") or {}) for warning in artifact.get("warnings", []): print(f"WARNING {warning}", file=sys.stderr) + + +def _print_coverage(bundle: dict) -> None: + """How much of the artifact's readable words this run read, on stdout. + + The share is stated on stdout because the headline is what a user copies. A run under + the floor also says so on stdout: a stderr-only warning is one a user piping the + headline never sees, and a rate quoted off half-read skill is not a property of it. + """ + from dokimi.novelty.bundle import MEASURED_SHARE_WARN_AT, Coverage + + coverage = Coverage.from_record(bundle) + if coverage is None or not coverage.denominator("readable"): + return + share = coverage.measured_share + scope = "" if coverage.include_resources else " (entry point only)" + # The warnings below need readable words outside the entry point, so the flag always adds some. + rerun = "" if coverage.include_resources else "; pass --include-resources to measure the bundled files" + if share is None: + print("coverage: unknown (a bundled file did not decode, so its words are not " + f"counted){scope}") + # The share is unknowable, but the entry point's part of the readable words is not: + # the stderr warning fires on it, so stdout says it too. + if (not coverage.include_resources + and coverage.outside_share_on("readable") >= 1 - MEASURED_SHARE_WARN_AT): + print("WARNING: the entry point is half or less of the artifact's readable words, " + f"so every rate here describes that part, not the artifact{rerun}") + return + # Floored on the 4-place share, in integers: a float product floors 0.29 to 28, and + # 49.99% must not print as "50%" beside a "below 50%" warning. A true 49.996% is already + # 0.5 by the time it is stored. + pct = "<1%" if 0 < share < 0.01 else f"{round(share * 10000) // 100}%" + print(f"coverage: {pct} of the artifact's readable words were measured{scope}") + if share <= MEASURED_SHARE_WARN_AT: + print(f"WARNING: only {pct} of the artifact's readable words were measured, so every " + f"rate here describes that part, not the artifact{rerun}") diff --git a/dokimi/dokimi/metrics/bundle_coverage.py b/dokimi/dokimi/metrics/bundle_coverage.py index 42be0508..e8d4937a 100644 --- a/dokimi/dokimi/metrics/bundle_coverage.py +++ b/dokimi/dokimi/metrics/bundle_coverage.py @@ -119,6 +119,11 @@ def named_exclusions(rows: list[dict] | None) -> tuple[list[dict], int]: #: How a record without ``coverage_basis`` is read: it was written when the gate judged #: all bundled words, and re-reading it on another basis would move its stored value. LEGACY_COVERAGE_BASIS = "all_bundled_words" +#: A run that read this share of the readable words or less is warned, on stdout and in the +#: artifact: its rates describe the measured part, not the artifact. This is the line the +#: analyzer has always warned at (an entry-point run whose outside share is 0.5 or more), and +#: the gate's ``max_unmeasured_outside_share`` is 0.50 on the same readable basis. +MEASURED_SHARE_WARN_AT = 0.5 def _share(part: int, whole: int) -> float: @@ -260,3 +265,16 @@ def unmeasured_share(self, basis: str) -> float | None: return None whole = self.denominator(basis) return self.unmeasured_words / whole if whole else 0.0 + + @property + def measured_share(self) -> float | None: + """Share of the readable words this run read, to 4 places; ``None`` when unknowable. + + The complement of :meth:`unmeasured_share` on the readable basis, stated as what was + read rather than what was missed. ``None`` for the same reasons: a file did not + decode, or there are no readable words to be a share of. + """ + unmeasured = self.unmeasured_share("readable") + if unmeasured is None or not self.denominator("readable"): + return None + return round(1 - unmeasured, 4) diff --git a/dokimi/dokimi/novelty/bundle.py b/dokimi/dokimi/novelty/bundle.py index d7128e54..5ec74aa9 100644 --- a/dokimi/dokimi/novelty/bundle.py +++ b/dokimi/dokimi/novelty/bundle.py @@ -48,6 +48,7 @@ EXCLUSION_CLASS, EXCLUSION_CLASSES, LEGACY_COVERAGE_BASIS, + MEASURED_SHARE_WARN_AT, Coverage, _safe_int, _share, @@ -220,7 +221,7 @@ def coverage_warnings(self) -> list[str]: # evals has nothing a bundle run would add, so pointing at the flag would be advice # that cannot be followed. The gate judges the same basis (`CURRENT_COVERAGE_BASIS`). share = coverage.outside_share_on("readable") - if not self.include_resources and share >= 0.5: + if not self.include_resources and share >= 1 - MEASURED_SHARE_WARN_AT: out.append( f"{share:.0%} of this artifact's words are in " "bundled files that were not measured (--include-resources); every rate " @@ -284,6 +285,7 @@ def summary(self) -> dict: "measured_digest": self.measured_digest(), "outside_share": coverage.outside_share, "measurable_outside_share": coverage.measurable_outside_share, + "measured_share": coverage.measured_share, "unmeasured_words": coverage.unmeasured_words, "max_file_words": self.max_file_words, "max_total_words": self.max_total_words, diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index dc41beec..47358cad 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -146,7 +146,7 @@ }, "bundle": { "type": "object", - "description": "Which files of the artifact were measured. A skill is a bundle: measured over the 33-skill study corpus 0.72 of all words live outside SKILL.md (per skill median 0.43, worst 0.99), so an entry-point-only run (include_resources false) is not comparable with a bundle run and must not be quoted as describing the artifact. outside_share is the on-disk share outside the entry point whether or not it was read; measurable_outside_share nets out policy_excluded_words (role decisions such as the skill\u0027s own evals, the same in both modes) and is the figure the warnings and report quote, since a rerun with --include-resources cannot read those; unmeasured_words is what such a rerun would add; excluded lists every file left out with its reason and a kind (symlink, not_text, unreadable, policy, budget, not_requested), and every reader interprets a row by the kind's class (novelty.bundle.EXCLUSION_CLASS): budget and not_requested are a gap (counted in unmeasured_words and warned about), policy is a decision (never charged as unread, and outside the readable-words denominator the warnings, the report and, for a record stamped coverage_basis readable, the gate uses), unreadable is unknown (the gate reports no artifact_coverage value while one exists; the warnings, note and report still quote a figure that leaves its words out), symlink and not_text are not content; pointer_spans names the spans whose only content is a path into the bundle, which carry no knowledge by design and are therefore not waste; by_source cuts the paired outcomes by the file that carried each atom. A bundle run measures what the artifact COULD deliver if the agent read all of it -- an upper bound; whether the agent loads a file at runtime is the agentic track\u0027s question.", + "description": "Which files of the artifact were measured. A skill is a bundle: measured over the 33-skill study corpus 0.72 of all words live outside SKILL.md (per skill median 0.43, worst 0.99), so an entry-point-only run (include_resources false) is not comparable with a bundle run and must not be quoted as describing the artifact. outside_share is the on-disk share outside the entry point whether or not it was read; measurable_outside_share nets out policy_excluded_words (role decisions such as the skill\u0027s own evals, the same in both modes) and is the figure the warnings and report quote, since a rerun with --include-resources cannot read those; measured_share is the share of the readable words this run read (null when a file did not decode or there are no readable words); unmeasured_words is what such a rerun would add; excluded lists every file left out with its reason and a kind (symlink, not_text, unreadable, policy, budget, not_requested), and every reader interprets a row by the kind's class (novelty.bundle.EXCLUSION_CLASS): budget and not_requested are a gap (counted in unmeasured_words and warned about), policy is a decision (never charged as unread, and outside the readable-words denominator the warnings, the report and, for a record stamped coverage_basis readable, the gate uses), unreadable is unknown (the gate reports no artifact_coverage value while one exists; the warnings, note and report still quote a figure that leaves its words out), symlink and not_text are not content; pointer_spans names the spans whose only content is a path into the bundle, which carry no knowledge by design and are therefore not waste; by_source cuts the paired outcomes by the file that carried each atom. A bundle run measures what the artifact COULD deliver if the agent read all of it -- an upper bound; whether the agent loads a file at runtime is the agentic track\u0027s question.", "properties": { "include_resources": {"type": "boolean"}, "n_files": {"type": "integer", "minimum": 0}, @@ -155,6 +155,7 @@ "resource_words_on_disk": {"type": "integer", "minimum": 0}, "outside_share": {"type": "number", "minimum": 0, "maximum": 1}, "measurable_outside_share": {"type": "number", "minimum": 0, "maximum": 1}, + "measured_share": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, "policy_excluded_words": {"type": "integer", "minimum": 0}, "unmeasured_words": {"type": "integer", "minimum": 0}, "coverage_basis": { diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index c0456236..3d2d3eea 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 864, "rationale": "W3 adds routing no_queries handling, W10 a --catalog help line; file split tracked separately to avoid conflicts across the W-series"}, + "dokimi/commands/novelty.py": {"lines": 901, "rationale": "stdout coverage line and warning in novelty CLI; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_bundle.py b/dokimi/tests/test_novelty_bundle.py index 0e10847a..04d3fdbd 100644 --- a/dokimi/tests/test_novelty_bundle.py +++ b/dokimi/tests/test_novelty_bundle.py @@ -684,6 +684,21 @@ def test_a_stamped_record_validates(tmp_path): schema_version="novelty.v1")) +def test_a_record_with_an_unknowable_share_validates(tmp_path): + from dokimi.artifacts import validate_artifact + + from tests.test_novelty_cli import _minimal_artifact + + record = dict(load_bundle(f"skill:{_skill_with_evals(tmp_path)}").summary(), + measured_share=None) + validate_artifact("novelty", dict(_minimal_artifact(), bundle=record, + schema_version="novelty.v1")) + with pytest.raises(Exception, match="measured_share"): + validate_artifact("novelty", dict(_minimal_artifact(), + bundle=dict(record, measured_share="most"), + schema_version="novelty.v1")) + + def test_a_note_before_measuring_says_what_will_not_be_read(skill): assert "will not be measured" in load_bundle(f"skill:{skill}").coverage_note() assert "--include-resources" in load_bundle(f"skill:{skill}").coverage_note() @@ -716,277 +731,3 @@ def test_a_text_artifact_gets_a_measured_digest_too(tmp_path): path = tmp_path / "notes.md" path.write_text("# Notes\n", encoding="utf-8") assert load_bundle(f"text:{path}").measured_digest().startswith("sha256:") - - -# ── Is a saved answer key still valid for the artifact as it is now? ──────── - -_KEY_SKILL = """\ ---- -name: accel-tuning -description: Configure the accelerator ---- - -# Accelerator tuning - -Enable the accelerator with `--accel`. -""" - - -def _key_skill(tmp_path): - root = tmp_path / "accel-tuning" - (root / "references").mkdir(parents=True) - (root / "SKILL.md").write_text(_KEY_SKILL, encoding="utf-8") - (root / "references" / "tuning.md").write_text( - "# Tuning\n\nNever exceed 64 threads.\n", encoding="utf-8") - return root - - -def _recorded(root, **kwargs): - return load_bundle(f"skill:{root}", include_resources=True, **kwargs).summary() - - -def _verdict(root, record): - from dokimi.novelty.bundle import check_answer_key - - return check_answer_key(f"skill:{root}", record, source="a.json") - - -def test_the_file_set_comes_from_the_record_not_from_flags(tmp_path): - root = _key_skill(tmp_path) - # A record from before budgets, roles and digests were written. - verdict = _verdict(root, {"include_resources": True, "n_files": 2, - "resource_words_measured": 6, - "files": [{"path": "SKILL.md", "role": "entry_point"}, - {"path": "references/tuning.md", "role": "reference"}]}) - assert verdict.ok - # The reference is injected although no flag asked for it: the key was extracted from it. - assert "Never exceed 64 threads" in verdict.text - assert any("a.json: 2 files" in note for note in verdict.notes) - - -def test_a_key_without_provenance_is_probed_against_the_entry_point_and_says_so(tmp_path): - root = _key_skill(tmp_path) - verdict = _verdict(root, {}) - assert verdict.ok and verdict.bundle is None - assert "Never exceed 64 threads" not in verdict.text - assert any("predates bundle provenance" in note for note in verdict.notes) - - -def test_a_key_for_an_artifact_that_cannot_be_loaded_is_refused(tmp_path): - from dokimi.novelty.bundle import check_answer_key - - for record in ({}, {"files": [{"path": "SKILL.md", "role": "entry_point"}]}): - verdict = check_answer_key(f"skill:{tmp_path / 'missing'}", record) - assert (verdict.ok, verdict.problem) == (False, "unloadable") - - -def test_a_record_without_usable_file_rows_is_refused(tmp_path): - root = _key_skill(tmp_path) - for broken in ({"include_resources": True, "files": []}, - {"include_resources": True, "files": [{"path": "SKILL.md"}]}): - verdict = _verdict(root, broken) - assert verdict.problem == "no_file_rows" and "provenance" in verdict.message - - -def test_a_removed_file_is_a_changed_file_set(tmp_path): - root = _key_skill(tmp_path) - verdict = _verdict(root, {"include_resources": True, "n_files": 3, "files": [ - {"path": "SKILL.md", "role": "entry_point"}, - {"path": "references/tuning.md", "role": "reference"}, - {"path": "references/gone.md", "role": "reference"}]}) - assert verdict.problem == "files_changed" - assert "references/gone.md" in verdict.recorded - assert "references/gone.md" not in verdict.rebuilt - - -def test_the_selection_is_replayed_with_the_recorded_budgets(tmp_path): - root = _key_skill(tmp_path) - (root / "references" / "big.md").write_text("word " * 5000, encoding="utf-8") - bundle = load_bundle(f"skill:{root}", include_resources=True, max_file_words=6000) - assert _verdict(root, bundle.summary()).text == bundle.text - - -@pytest.mark.parametrize("edit", [ - "# Tuning\n\nNever exceed 128 threads on driver 1.0.\n", - # Same path, same word count, different fact: only the digest sees this one. - "# Tuning\n\nNever exceed 128 threads.\n", -]) -def test_an_edited_bundled_file_is_a_stale_key(tmp_path, edit): - root = _key_skill(tmp_path) - record = _recorded(root) - (root / "references" / "tuning.md").write_text(edit, encoding="utf-8") - verdict = _verdict(root, record) - assert (verdict.problem, verdict.edited) == ("files_changed", ("references/tuning.md",)) - - -def test_an_edited_entry_point_is_the_re_measure_step_and_says_so(tmp_path): - root = _key_skill(tmp_path) - record = _recorded(root) - (root / "SKILL.md").write_text( - "---\nname: accel-tuning\ndescription: Configure the accelerator\n---\n\n" - "# Accelerator tuning\n\nShortened: the flag is `--accel`.\n", encoding="utf-8") - verdict = _verdict(root, record) - assert verdict.ok and verdict.entry_point_changed - assert "Shortened: the flag is" in verdict.text and "Never exceed 64" in verdict.text - assert any("has changed since these atoms were extracted" in n for n in verdict.notes) - - -@pytest.mark.parametrize("added", ["notes.md", "scripts/run.sh"]) -def test_a_new_file_in_a_role_not_seen_at_extraction_is_a_stale_key(tmp_path, added): - root = _key_skill(tmp_path) - record = _recorded(root) - # The extraction had only a reference, so the rebuild is filtered to that role and a - # new file in any other role never reached the file-set comparison. - (root / added).parent.mkdir(parents=True, exist_ok=True) - (root / added).write_text("Always pass --force.\n", encoding="utf-8") - verdict = _verdict(root, record) - assert (verdict.problem, verdict.added) == ("files_changed", (added,)) - - -def test_a_new_file_that_is_not_content_leaves_the_key_valid(tmp_path): - root = _key_skill(tmp_path) - record = _recorded(root) - (root / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n") - # A symlink is recorded, never followed, in every mode, so re-extracting would not - # measure it either. (It used to count as a change; an image never did.) - (root / "references" / "shared.md").symlink_to(root / "references" / "tuning.md") - assert _verdict(root, record).ok - - -def test_an_answer_key_from_another_skill_is_refused(tmp_path): - other = tmp_path / "other" - other.mkdir() - (other / "SKILL.md").write_text( - "---\nname: other-skill\ndescription: Tune a database\n---\n\n# Other\n", - encoding="utf-8") - root = _key_skill(tmp_path) - # Both sides are just ["SKILL.md"], so the file-set check could never catch this. - verdict = _verdict(root, load_bundle(f"skill:{other}").summary()) - assert verdict.problem == "wrong_artifact" and "belongs to one artifact" in verdict.message - - -def test_a_budget_gap_stays_a_gap_when_replayed(tmp_path): - """Rebuilding with the selected files' roles relabelled an over-budget script `policy`.""" - root = _key_skill(tmp_path) - (root / "scripts").mkdir() - (root / "scripts" / "big.py").write_text("x = 1 # w\n" * 6000, encoding="utf-8") - recorded = load_bundle(f"skill:{root}", include_resources=True) - assert recorded.coverage().unmeasured_words > 0 - probed = _verdict(root, recorded.summary()).bundle - assert probed.coverage() == recorded.coverage() - assert probed.coverage().policy_excluded_words == 0 - - -def test_requested_roles_are_replayed_when_none_was_selected(tmp_path): - root = _key_skill(tmp_path) - (root / "scripts").mkdir() - (root / "scripts" / "big.py").write_text("x = 1 # w\n" * 6000, encoding="utf-8") - recorded = load_bundle(f"skill:{root}", include_resources=True, roles=("script",)) - assert [f.path for f in recorded.files] == ["SKILL.md"] - assert _verdict(root, recorded.summary()).ok # the unchanged tree is not "changed" - - -def test_a_new_eval_case_changes_the_key_only_when_evals_were_requested(tmp_path): - root = _key_skill(tmp_path) - by_default = _recorded(root) - (root / "evals").mkdir() - (root / "evals" / "a.md").write_text("Q: one?\n", encoding="utf-8") - # Evals are not measured by default; the text is unchanged. - assert _verdict(root, by_default).ok - with_evals = _recorded(root, roles=("reference", "eval")) - (root / "evals" / "q.md").write_text("Q: two?\n", encoding="utf-8") - verdict = _verdict(root, with_evals) - assert verdict.problem == "files_changed" and verdict.added == ("evals/q.md",) - - -# ── Shortening rewrites one file ───────────────────────────────────────────── - -def test_shortening_a_bundle_key_keeps_only_the_entry_point(tmp_path): - from dokimi.novelty.atoms import Atom - from dokimi.novelty.bundle import entry_point_only - - root = _key_skill(tmp_path) - bundle = load_bundle(f"skill:{root}", include_resources=True) - spans = segment(bundle.text) - entry_end = bundle.files[0].end - inside = next(s for s in spans if s.start < entry_end and "`--accel`" in s.text) - outside = next(s for s in spans if s.start >= entry_end and "64" in s.text) - atoms = [ - Atom(statement="entry", question="q", answer="a", span_ids=(inside.id,)), - Atom(statement="reference", question="q", answer="a", span_ids=(outside.id,)), - # Half the evidence cannot be scored against a candidate that holds one file. - Atom(statement="both", question="q", answer="a", span_ids=(inside.id, outside.id)), - ] - kept_spans, kept_atoms, note = entry_point_only(bundle.summary(), spans, atoms) - assert all(span.start < entry_end for span in kept_spans) - assert [atom.statement for atom in kept_atoms] == ["entry"] - assert "bundle spans" in note and "SKILL.md" in note - # An entry-point key passes through untouched. - assert entry_point_only(load_bundle(f"skill:{root}").summary(), spans, atoms) == ( - spans, atoms, None) - - -def test_shortening_refuses_a_bundle_record_it_cannot_slice(): - from dokimi.novelty.bundle import entry_point_only - - with pytest.raises(ValueError, match="file rows"): - entry_point_only({"include_resources": True, "files": []}, [], []) - with pytest.raises(ValueError, match="span range"): - entry_point_only({"include_resources": True, "files": [{"path": "SKILL.md"}]}, [], []) - - -def test_the_report_names_excluded_files_and_counts_non_text_ones(tmp_path): - from dokimi.novelty.report import render_novelty_markdown - - from tests.test_novelty_cli import _minimal_artifact - from tests.test_novelty_gate import _artifact as _gate_artifact - - def _artifact(bundle): - # The gate's fixture, with the blocks the renderer reads. - minimal = _minimal_artifact() - return dict(_gate_artifact(), spans=minimal["spans"], artifact=minimal["artifact"], - shortening=[], - bundle=bundle) - - root = _skill_with_evals(tmp_path, refs=50) - (root / "references" / "big.md").write_text("w " * 30, encoding="utf-8") - (root / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n") - (root / "references" / "linked.md").symlink_to(root / "references" / "r.md") - bundle = load_bundle(f"skill:{root}", include_resources=True, max_file_words=40) - rendered = render_novelty_markdown(_artifact(bundle=bundle.summary())) - assert "3 files excluded: `evals/cases.md`" in rendered - assert "`references/linked.md` (symlink), `references/r.md` (over max_file_words=40)." in rendered - assert "1 non-text files not read." in rendered - partial = render_novelty_markdown(_artifact(bundle={"outside_share": 0.99})) - assert "lacks its word counts" in partial - - -def test_measured_digest_format_is_pinned(tmp_path): - """Measured_digest format is stable.""" - from dokimi.novelty.bundle import load_bundle - - # Create a test skill with two files - skill = tmp_path / "s" / "SKILL.md" - skill.parent.mkdir(parents=True) - skill.write_text( - "---\nname: test\ndescription: d\n---\n\nEntry point text.\n", - encoding="utf-8", - ) - ref = skill.parent / "references" / "ref.md" - ref.parent.mkdir() - ref.write_text("Reference text.", encoding="utf-8") - - bundle = load_bundle(f"skill:{skill.parent}", include_resources=True) - digest = bundle.measured_digest() - - # Pin the exact literal so edits to measured_digest() cannot silently invalidate stored ref_digest values - assert digest == "sha256:f0eb5f66f8b343a03bc6f5fbcd233e3bbf591db23ef075502db96bb35ddd5bf5" - - # The digest should be deterministic - same files, same digest - bundle2 = load_bundle(f"skill:{skill.parent}", include_resources=True) - assert bundle2.measured_digest() == digest - - # Modifying a file should change the digest - ref.write_text("Modified reference text.", encoding="utf-8") - bundle3 = load_bundle(f"skill:{skill.parent}", include_resources=True) - assert bundle3.measured_digest() != digest diff --git a/dokimi/tests/test_novelty_bundle_key.py b/dokimi/tests/test_novelty_bundle_key.py new file mode 100644 index 00000000..12cd11a9 --- /dev/null +++ b/dokimi/tests/test_novelty_bundle_key.py @@ -0,0 +1,312 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A saved answer key against the bundle as it is now, and shortening a bundle's entry point.""" + +from __future__ import annotations + +import pytest + +from dokimi.novelty.bundle import load_bundle +from dokimi.novelty.spans import segment +from tests.test_novelty_bundle import _skill_with_evals + +# ── Is a saved answer key still valid for the artifact as it is now? ──────── + +_KEY_SKILL = """\ +--- +name: accel-tuning +description: Configure the accelerator +--- + +# Accelerator tuning + +Enable the accelerator with `--accel`. +""" + + +def _key_skill(tmp_path): + root = tmp_path / "accel-tuning" + (root / "references").mkdir(parents=True) + (root / "SKILL.md").write_text(_KEY_SKILL, encoding="utf-8") + (root / "references" / "tuning.md").write_text( + "# Tuning\n\nNever exceed 64 threads.\n", encoding="utf-8") + return root + + +def _recorded(root, **kwargs): + return load_bundle(f"skill:{root}", include_resources=True, **kwargs).summary() + + +def _verdict(root, record): + from dokimi.novelty.bundle import check_answer_key + + return check_answer_key(f"skill:{root}", record, source="a.json") + + +def test_the_file_set_comes_from_the_record_not_from_flags(tmp_path): + root = _key_skill(tmp_path) + # A record from before budgets, roles and digests were written. + verdict = _verdict(root, {"include_resources": True, "n_files": 2, + "resource_words_measured": 6, + "files": [{"path": "SKILL.md", "role": "entry_point"}, + {"path": "references/tuning.md", "role": "reference"}]}) + assert verdict.ok + # The reference is injected although no flag asked for it: the key was extracted from it. + assert "Never exceed 64 threads" in verdict.text + assert any("a.json: 2 files" in note for note in verdict.notes) + + +def test_a_key_without_provenance_is_probed_against_the_entry_point_and_says_so(tmp_path): + root = _key_skill(tmp_path) + verdict = _verdict(root, {}) + assert verdict.ok and verdict.bundle is None + assert "Never exceed 64 threads" not in verdict.text + assert any("predates bundle provenance" in note for note in verdict.notes) + + +def test_a_key_for_an_artifact_that_cannot_be_loaded_is_refused(tmp_path): + from dokimi.novelty.bundle import check_answer_key + + for record in ({}, {"files": [{"path": "SKILL.md", "role": "entry_point"}]}): + verdict = check_answer_key(f"skill:{tmp_path / 'missing'}", record) + assert (verdict.ok, verdict.problem) == (False, "unloadable") + + +def test_a_record_without_usable_file_rows_is_refused(tmp_path): + root = _key_skill(tmp_path) + for broken in ({"include_resources": True, "files": []}, + {"include_resources": True, "files": [{"path": "SKILL.md"}]}): + verdict = _verdict(root, broken) + assert verdict.problem == "no_file_rows" and "provenance" in verdict.message + + +def test_a_removed_file_is_a_changed_file_set(tmp_path): + root = _key_skill(tmp_path) + verdict = _verdict(root, {"include_resources": True, "n_files": 3, "files": [ + {"path": "SKILL.md", "role": "entry_point"}, + {"path": "references/tuning.md", "role": "reference"}, + {"path": "references/gone.md", "role": "reference"}]}) + assert verdict.problem == "files_changed" + assert "references/gone.md" in verdict.recorded + assert "references/gone.md" not in verdict.rebuilt + + +def test_the_selection_is_replayed_with_the_recorded_budgets(tmp_path): + root = _key_skill(tmp_path) + (root / "references" / "big.md").write_text("word " * 5000, encoding="utf-8") + bundle = load_bundle(f"skill:{root}", include_resources=True, max_file_words=6000) + assert _verdict(root, bundle.summary()).text == bundle.text + + +@pytest.mark.parametrize("edit", [ + "# Tuning\n\nNever exceed 128 threads on driver 1.0.\n", + # Same path, same word count, different fact: only the digest sees this one. + "# Tuning\n\nNever exceed 128 threads.\n", +]) +def test_an_edited_bundled_file_is_a_stale_key(tmp_path, edit): + root = _key_skill(tmp_path) + record = _recorded(root) + (root / "references" / "tuning.md").write_text(edit, encoding="utf-8") + verdict = _verdict(root, record) + assert (verdict.problem, verdict.edited) == ("files_changed", ("references/tuning.md",)) + + +def test_an_edited_entry_point_is_the_re_measure_step_and_says_so(tmp_path): + root = _key_skill(tmp_path) + record = _recorded(root) + (root / "SKILL.md").write_text( + "---\nname: accel-tuning\ndescription: Configure the accelerator\n---\n\n" + "# Accelerator tuning\n\nShortened: the flag is `--accel`.\n", encoding="utf-8") + verdict = _verdict(root, record) + assert verdict.ok and verdict.entry_point_changed + assert "Shortened: the flag is" in verdict.text and "Never exceed 64" in verdict.text + assert any("has changed since these atoms were extracted" in n for n in verdict.notes) + + +@pytest.mark.parametrize("added", ["notes.md", "scripts/run.sh"]) +def test_a_new_file_in_a_role_not_seen_at_extraction_is_a_stale_key(tmp_path, added): + root = _key_skill(tmp_path) + record = _recorded(root) + # The extraction had only a reference, so the rebuild is filtered to that role and a + # new file in any other role never reached the file-set comparison. + (root / added).parent.mkdir(parents=True, exist_ok=True) + (root / added).write_text("Always pass --force.\n", encoding="utf-8") + verdict = _verdict(root, record) + assert (verdict.problem, verdict.added) == ("files_changed", (added,)) + + +def test_a_new_file_that_is_not_content_leaves_the_key_valid(tmp_path): + root = _key_skill(tmp_path) + record = _recorded(root) + (root / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n") + # A symlink is recorded, never followed, in every mode, so re-extracting would not + # measure it either. (It used to count as a change; an image never did.) + (root / "references" / "shared.md").symlink_to(root / "references" / "tuning.md") + assert _verdict(root, record).ok + + +def test_an_answer_key_from_another_skill_is_refused(tmp_path): + other = tmp_path / "other" + other.mkdir() + (other / "SKILL.md").write_text( + "---\nname: other-skill\ndescription: Tune a database\n---\n\n# Other\n", + encoding="utf-8") + root = _key_skill(tmp_path) + # Both sides are just ["SKILL.md"], so the file-set check could never catch this. + verdict = _verdict(root, load_bundle(f"skill:{other}").summary()) + assert verdict.problem == "wrong_artifact" and "belongs to one artifact" in verdict.message + + +def test_a_budget_gap_stays_a_gap_when_replayed(tmp_path): + """Rebuilding with the selected files' roles relabelled an over-budget script `policy`.""" + root = _key_skill(tmp_path) + (root / "scripts").mkdir() + (root / "scripts" / "big.py").write_text("x = 1 # w\n" * 6000, encoding="utf-8") + recorded = load_bundle(f"skill:{root}", include_resources=True) + assert recorded.coverage().unmeasured_words > 0 + probed = _verdict(root, recorded.summary()).bundle + assert probed.coverage() == recorded.coverage() + assert probed.coverage().policy_excluded_words == 0 + + +def test_requested_roles_are_replayed_when_none_was_selected(tmp_path): + root = _key_skill(tmp_path) + (root / "scripts").mkdir() + (root / "scripts" / "big.py").write_text("x = 1 # w\n" * 6000, encoding="utf-8") + recorded = load_bundle(f"skill:{root}", include_resources=True, roles=("script",)) + assert [f.path for f in recorded.files] == ["SKILL.md"] + assert _verdict(root, recorded.summary()).ok # the unchanged tree is not "changed" + + +def test_a_new_eval_case_changes_the_key_only_when_evals_were_requested(tmp_path): + root = _key_skill(tmp_path) + by_default = _recorded(root) + (root / "evals").mkdir() + (root / "evals" / "a.md").write_text("Q: one?\n", encoding="utf-8") + # Evals are not measured by default; the text is unchanged. + assert _verdict(root, by_default).ok + with_evals = _recorded(root, roles=("reference", "eval")) + (root / "evals" / "q.md").write_text("Q: two?\n", encoding="utf-8") + verdict = _verdict(root, with_evals) + assert verdict.problem == "files_changed" and verdict.added == ("evals/q.md",) + + +# ── Shortening rewrites one file ───────────────────────────────────────────── + +def test_shortening_a_bundle_key_keeps_only_the_entry_point(tmp_path): + from dokimi.novelty.atoms import Atom + from dokimi.novelty.bundle import entry_point_only + + root = _key_skill(tmp_path) + bundle = load_bundle(f"skill:{root}", include_resources=True) + spans = segment(bundle.text) + entry_end = bundle.files[0].end + inside = next(s for s in spans if s.start < entry_end and "`--accel`" in s.text) + outside = next(s for s in spans if s.start >= entry_end and "64" in s.text) + atoms = [ + Atom(statement="entry", question="q", answer="a", span_ids=(inside.id,)), + Atom(statement="reference", question="q", answer="a", span_ids=(outside.id,)), + # Half the evidence cannot be scored against a candidate that holds one file. + Atom(statement="both", question="q", answer="a", span_ids=(inside.id, outside.id)), + ] + kept_spans, kept_atoms, note = entry_point_only(bundle.summary(), spans, atoms) + assert all(span.start < entry_end for span in kept_spans) + assert [atom.statement for atom in kept_atoms] == ["entry"] + assert "bundle spans" in note and "SKILL.md" in note + # An entry-point key passes through untouched. + assert entry_point_only(load_bundle(f"skill:{root}").summary(), spans, atoms) == ( + spans, atoms, None) + + +def test_shortening_refuses_a_bundle_record_it_cannot_slice(): + from dokimi.novelty.bundle import entry_point_only + + with pytest.raises(ValueError, match="file rows"): + entry_point_only({"include_resources": True, "files": []}, [], []) + with pytest.raises(ValueError, match="span range"): + entry_point_only({"include_resources": True, "files": [{"path": "SKILL.md"}]}, [], []) + + +def test_the_report_names_excluded_files_and_counts_non_text_ones(tmp_path): + from dokimi.novelty.report import render_novelty_markdown + + from tests.test_novelty_cli import _minimal_artifact + from tests.test_novelty_gate import _artifact as _gate_artifact + + def _artifact(bundle): + # The gate's fixture, with the blocks the renderer reads. + minimal = _minimal_artifact() + return dict(_gate_artifact(), spans=minimal["spans"], artifact=minimal["artifact"], + shortening=[], + bundle=bundle) + + root = _skill_with_evals(tmp_path, refs=50) + (root / "references" / "big.md").write_text("w " * 30, encoding="utf-8") + (root / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n") + (root / "references" / "linked.md").symlink_to(root / "references" / "r.md") + bundle = load_bundle(f"skill:{root}", include_resources=True, max_file_words=40) + rendered = render_novelty_markdown(_artifact(bundle=bundle.summary())) + assert "3 files excluded: `evals/cases.md`" in rendered + assert "`references/linked.md` (symlink), `references/r.md` (over max_file_words=40)." in rendered + assert "1 non-text files not read." in rendered + partial = render_novelty_markdown(_artifact(bundle={"outside_share": 0.99})) + assert "lacks its word counts" in partial + + +def test_measured_digest_format_is_pinned(tmp_path): + """Measured_digest format is stable.""" + from dokimi.novelty.bundle import load_bundle + + # Create a test skill with two files + skill = tmp_path / "s" / "SKILL.md" + skill.parent.mkdir(parents=True) + skill.write_text( + "---\nname: test\ndescription: d\n---\n\nEntry point text.\n", + encoding="utf-8", + ) + ref = skill.parent / "references" / "ref.md" + ref.parent.mkdir() + ref.write_text("Reference text.", encoding="utf-8") + + bundle = load_bundle(f"skill:{skill.parent}", include_resources=True) + digest = bundle.measured_digest() + + # Pin the exact literal so edits to measured_digest() cannot silently invalidate stored ref_digest values + assert digest == "sha256:f0eb5f66f8b343a03bc6f5fbcd233e3bbf591db23ef075502db96bb35ddd5bf5" + + # The digest should be deterministic - same files, same digest + bundle2 = load_bundle(f"skill:{skill.parent}", include_resources=True) + assert bundle2.measured_digest() == digest + + # Modifying a file should change the digest + ref.write_text("Modified reference text.", encoding="utf-8") + bundle3 = load_bundle(f"skill:{skill.parent}", include_resources=True) + assert bundle3.measured_digest() != digest + + +def test_the_record_states_the_share_of_readable_words_this_run_read(tmp_path): + """100 entry words and 300 measurable reference words: an entry-point run read a quarter.""" + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=100, refs=300)}") + assert bundle.coverage().measured_share == 0.25 + assert bundle.summary()["measured_share"] == 0.25 + assert any(w.startswith("75% ") for w in bundle.coverage_warnings()) + + +def test_exactly_half_the_readable_words_read_is_warned_as_the_analyzer_always_has_been(tmp_path): + """The warning line is unchanged: a run that read half the readable words still warns.""" + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=300, refs=300)}") + assert bundle.coverage().measured_share == 0.5 + assert any(w.startswith("50% ") for w in bundle.coverage_warnings()) + + +def test_a_bundle_run_that_read_everything_it_could_measures_all_of_it(tmp_path): + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=100, refs=300)}", + include_resources=True) + assert bundle.summary()["measured_share"] == 1.0 + + +def test_the_stored_measured_share_is_rounded_to_four_places(tmp_path): + # 2 entry words and 7 reference words: 2/9 is 0.2222..., stored as 0.2222, not 0.2222222. + bundle = load_bundle(f"skill:{_skill_with_evals(tmp_path, entry=2, refs=7)}") + assert bundle.summary()["measured_share"] == 0.2222 diff --git a/dokimi/tests/test_novelty_cli.py b/dokimi/tests/test_novelty_cli.py index b2469117..db439aa8 100644 --- a/dokimi/tests/test_novelty_cli.py +++ b/dokimi/tests/test_novelty_cli.py @@ -614,3 +614,99 @@ def test_view_and_probe_repeat_the_unread_warning(tmp_path, capsys): args.func(args) assert "render as `unread`" in capsys.readouterr().err assert f"`{spans[2].id}` [unread/hatched]" in out.read_text() + + +def test_the_headline_states_the_measured_share_and_warns_on_stdout_at_half_or_less(capsys): + record = {"include_resources": False, "n_files": 1, "entry_point_words": 10, + "resource_words_measured": 0, "resource_words_on_disk": 30, + "policy_excluded_words": 0, "unreadable_files": 0, "excluded": [], + "coverage_basis": "readable"} + artifact = dict(_minimal_artifact(), bundle=record) + novelty_cmd._print_headline(artifact) + out = capsys.readouterr().out + assert "coverage: 25% of the artifact's readable words were measured (entry point only)" in out + assert "WARNING: only 25% of the artifact's readable words" in out + assert "pass --include-resources" in out + + # The line is the analyzer's own: exactly half warns, just over half does not. + novelty_cmd._print_headline(dict(artifact, bundle=dict(record, entry_point_words=30))) + out = capsys.readouterr().out + assert "coverage: 50%" in out and "WARNING: only" in out + novelty_cmd._print_headline(dict(artifact, bundle=dict(record, entry_point_words=31))) + out = capsys.readouterr().out + assert "coverage: 50%" in out and "WARNING: only" not in out + + # A file that did not decode leaves the share unknowable, and the headline says so, + # with the scope and, while the entry point is half or less of the words, the warning. + novelty_cmd._print_headline(dict(artifact, bundle=dict(record, unreadable_files=1))) + out = capsys.readouterr().out + assert "coverage: unknown" in out and out.count("(entry point only)") == 1 + assert "WARNING: the entry point is half or less" in out + assert "pass --include-resources" in out + # An entry point that is most of the words does not warn, as the stderr warning does not. + novelty_cmd._print_headline(dict(artifact, bundle=dict(record, unreadable_files=1, + entry_point_words=90, + resource_words_on_disk=10))) + out = capsys.readouterr().out + assert "coverage: unknown" in out and "WARNING" not in out + + +def _readable_record(**over): + return dict({"include_resources": False, "n_files": 1, "entry_point_words": 10, + "resource_words_measured": 0, "resource_words_on_disk": 30, + "policy_excluded_words": 0, "unreadable_files": 0, "excluded": [], + "coverage_basis": "readable"}, **over) + + +def test_an_undecodable_file_in_a_bundle_run_does_not_claim_an_entry_point_only_warning(capsys): + # --include-resources was passed: the unknown-share line must not say the entry point is + # half or less of the words, which is false for a bundle run. + record = _readable_record(include_resources=True, unreadable_files=1) + novelty_cmd._print_headline(dict(_minimal_artifact(), bundle=record)) + out = capsys.readouterr().out + assert "coverage: unknown" in out and "WARNING" not in out + + +def test_an_unknown_share_warns_at_exactly_half_as_the_stderr_warning_does(capsys): + record = _readable_record(unreadable_files=1, entry_point_words=50, resource_words_on_disk=50) + novelty_cmd._print_headline(dict(_minimal_artifact(), bundle=record)) + assert "WARNING: the entry point is half or less" in capsys.readouterr().out + + +@pytest.mark.parametrize("entry, printed", [(29, "29%"), (57, "57%"), (58, "58%"), + (49, "49%"), (1, "1%")]) +def test_the_printed_percentage_is_the_share_not_a_float_product(capsys, entry, printed): + # 29 entry words in 100: 0.29 * 100 is 28.999999999999996, which int() floors to 28. + record = {"include_resources": False, "n_files": 1, "entry_point_words": entry, + "resource_words_measured": 0, "resource_words_on_disk": 100 - entry, + "policy_excluded_words": 0, "unreadable_files": 0, "excluded": [], + "coverage_basis": "readable"} + novelty_cmd._print_headline(dict(_minimal_artifact(), bundle=record)) + out = capsys.readouterr().out + assert f"coverage: {printed} of the artifact's readable words" in out + + +def test_a_tiny_non_zero_share_prints_as_under_one_percent(capsys): + record = {"include_resources": False, "n_files": 1, "entry_point_words": 1, + "resource_words_measured": 0, "resource_words_on_disk": 9999, + "policy_excluded_words": 0, "unreadable_files": 0, "excluded": [], + "coverage_basis": "readable"} + novelty_cmd._print_headline(dict(_minimal_artifact(), bundle=record)) + out = capsys.readouterr().out + assert "coverage: <1% of the artifact's readable words were measured (entry point only)" in out + assert "WARNING: only <1% of the artifact's readable words" in out + + +def test_a_bundle_run_capped_by_the_budget_warns_without_advising_the_flag_it_passed(capsys): + # --include-resources was passed; the budget left most of the reference unread. The + # stderr warning list is silent about this, the stdout line is not, and it must not tell + # the user to pass a flag they already passed. + record = {"include_resources": True, "n_files": 2, "entry_point_words": 100, + "resource_words_measured": 50, "resource_words_on_disk": 500, + "policy_excluded_words": 0, "unreadable_files": 0, "excluded": [], + "coverage_basis": "readable"} + novelty_cmd._print_headline(dict(_minimal_artifact(), bundle=record)) + out = capsys.readouterr().out + assert "coverage: 25% of the artifact's readable words were measured\n" in out + assert "WARNING: only 25%" in out + assert "--include-resources" not in out diff --git a/dokimi/tests/test_novelty_journey.py b/dokimi/tests/test_novelty_journey.py index de00ee7d..3bddae70 100644 --- a/dokimi/tests/test_novelty_journey.py +++ b/dokimi/tests/test_novelty_journey.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +from decimal import ROUND_FLOOR, Decimal import re import shlex from pathlib import Path @@ -155,6 +156,30 @@ def test_a_cheap_tier_run_writes_its_forfeits_into_the_artifact(tmp_path, skill, assert any("--include-resources" in w for w in artifact["warnings"]) +def test_the_stdout_headline_says_how_much_of_the_artifact_was_read(tmp_path, skill, + stub_models, capsys): + # The fixture's reference is larger than the body: an entry-point run reads under half. + entry_out = tmp_path / "entry" + _run("novelty", "analyze", "--artifact", f"skill:{skill}", "--tier", "triage", + "--model", "model-a", "--judge-model", "judge-m", "--out-dir", str(entry_out)) + printed = capsys.readouterr().out + measured = json.loads((entry_out / "novelty.json").read_text(encoding="utf-8"))[ + "bundle"]["measured_share"] + assert measured < 0.5 + assert f"coverage: {int(Decimal(str(measured)).scaleb(2).to_integral_value(ROUND_FLOOR))}% of the artifact's readable words were measured " \ + "(entry point only)" in printed + assert "WARNING: only" in printed and "pass --include-resources" in printed + + # A bundle run that reads everything it can says so, and does not warn. + bundle_out = tmp_path / "bundle" + _run("novelty", "analyze", "--artifact", f"skill:{skill}", "--tier", "triage", + "--include-resources", "--model", "model-a", "--judge-model", "judge-m", + "--out-dir", str(bundle_out)) + printed = capsys.readouterr().out + assert "coverage: 100% of the artifact's readable words were measured\n" in printed + assert "WARNING: only" not in printed + + @pytest.mark.outcome("cjm-novelty-stale-inventory") def test_probing_atoms_from_a_changed_artifact_is_refused(tmp_path, skill, stub_models): atoms_out = tmp_path / "atoms.json" diff --git a/dokimi/tests/test_novelty_settings_parity.py b/dokimi/tests/test_novelty_settings_parity.py index d42e6c36..5923ac2e 100644 --- a/dokimi/tests/test_novelty_settings_parity.py +++ b/dokimi/tests/test_novelty_settings_parity.py @@ -23,6 +23,7 @@ import json import os import sys +from decimal import ROUND_FLOOR, Decimal from pathlib import Path from typing import Any @@ -111,6 +112,49 @@ def _kept(argv: list[str], work: Path) -> dict[str, Any]: return {} +COVERAGE_LINES = ("coverage: ", "WARNING: only ") + + +def _without_coverage_headline(text: str) -> str: + """Drop the stdout lines the coverage headline added after main. + + They say how much of the artifact was read, which is neither a setting, a bill nor a + tier record, so the comparison with main leaves them out. They cannot be in main's + golden, so ``_headline_coverage`` checks them against the record they were printed from. + """ + return "".join(line for line in text.splitlines(keepends=True) + if not line.startswith(COVERAGE_LINES)) + + +def _headline_coverage(text: str, work: Path) -> dict[str, Any]: + """The coverage lines stdout printed, beside the ``bundle`` figures in the record.""" + printed = [line for line in text.splitlines() if line.startswith(COVERAGE_LINES)] + record = work / "out" / "novelty.json" + bundle = ( + json.loads(record.read_text(encoding="utf-8")).get("bundle") or {} + if record.exists() + else {} + ) + return {"printed": printed, "measured_share": bundle.get("measured_share")} + + +def _check_headline(coverage: dict[str, Any]) -> None: + """Each dropped line agrees with the measured share the same run stamped in its record.""" + printed, share = coverage["printed"], coverage["measured_share"] + if share is None: + # No record to check against (an estimate, or a run that wrote none): nothing printed. + assert printed == [] + return + # An integer floor of the decimal share, not a float product (0.29 * 100 < 29). + pct = f"{int(Decimal(str(share)).scaleb(2).to_integral_value(ROUND_FLOOR))}%" + assert printed, "a run with a record must print its coverage line" + assert printed[0].startswith( + f"coverage: {pct} of the artifact's readable words were measured" + ), printed + warned = [line.startswith(f"WARNING: only {pct} ") for line in printed[1:]] + assert warned == ([True] if share <= 0.5 else []), printed + + def _capture(name: str, tmp_path: Path) -> list[dict[str, Any]]: work = tmp_path / name.replace("/", "-") (work / "accel-tuning").mkdir(parents=True) @@ -143,7 +187,10 @@ def scripted(model, provider, role, args): status = exc.code if isinstance(exc.code, int) else str(exc.code) steps.append({ "status": status, - "stdout": out.getvalue() if argv[1] == "analyze" else None, + "stdout": (_without_coverage_headline(out.getvalue()) + if argv[1] == "analyze" else None), + "coverage": (_headline_coverage(out.getvalue(), work) + if argv[1] == "analyze" else None), "stderr": err.getvalue().replace(str(work), ""), "calls": [[c.role, c.calls] for c in clients], **_kept(argv, work), @@ -157,7 +204,12 @@ def scripted(model, provider, role, args): @pytest.mark.parametrize("name", list(CASES)) def test_no_setting_bill_or_tier_record_differs_from_main(name, tmp_path): golden = json.loads(GOLDEN.read_text(encoding="utf-8")) - assert _capture(name, tmp_path) == golden[name] + steps = _capture(name, tmp_path) + coverage = [step.pop("coverage") for step in steps] + assert steps == golden[name] + for step_coverage in coverage: + if step_coverage is not None: + _check_headline(step_coverage) def test_the_golden_cases_are_the_ones_this_module_runs(): @@ -169,7 +221,10 @@ def test_the_golden_cases_are_the_ones_this_module_runs(): import tempfile tmp = Path(tempfile.mkdtemp()) - captured = {name: _capture(name, tmp) for name in CASES} + captured = { + name: [{k: v for k, v in step.items() if k != "coverage"} for step in _capture(name, tmp)] + for name in CASES + } GOLDEN.write_text(json.dumps(captured, indent=1, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") print(f"{len(captured)} cases -> {GOLDEN}")