From 5fc51e8d7c4b84ddfb8a79c4557eea921845f4f4 Mon Sep 17 00:00:00 2001 From: Sanskar Date: Fri, 30 Oct 2026 14:33:31 +0530 Subject: [PATCH 1/7] feat(model): aggregate actual generation token counts with latency in HF smoke v5 --- model/train/smoke_report.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/model/train/smoke_report.py b/model/train/smoke_report.py index 732012d..abf5147 100644 --- a/model/train/smoke_report.py +++ b/model/train/smoke_report.py @@ -11,7 +11,7 @@ from pathlib import Path LANGUAGES = ("rust", "c", "cpp", "python", "typescript") -PROTOCOL = "lineloom-hf-synthetic-smoke-v4" +PROTOCOL = "lineloom-hf-synthetic-smoke-v5" GENERATION_POLICY = "greedy-fim-v2" PUBLIC_FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "completion-v1.json" @@ -97,24 +97,32 @@ def fingerprints(fixtures: list[dict], max_new_tokens: int) -> dict: -def generation_latency_summary(results: list[dict]) -> dict: - """Aggregate completed local generations; never expose per-probe timings. +def generation_latency_summary(results: list[dict], max_new_tokens: int = 128) -> dict: + """Source-free generation timing and actual output-length accounting. - Duration covers model.generate only. It excludes model loading, prompt - tokenization and decoding, so it is NOT end-to-end editor latency. - CUDA callers must synchronize around generation before recording times. + Count actual generated token IDs, including a terminal stop token when + present, not UTF-8 characters or decoded text. This avoids interpreting + a slower but longer completion as a like-for-like latency regression. + Duration covers model.generate only; it is NOT editor latency. """ - if not isinstance(results, list) or not 1 <= len(results) <= 30: + if (not isinstance(results, list) or not 1 <= len(results) <= 30 or + type(max_new_tokens) is not int or not 1 <= max_new_tokens <= 128): raise SmokeReportRejected("invalid-generation-measurements") buckets: dict[str, list[int]] = {language: [] for language in LANGUAGES} + token_buckets: dict[str, list[int]] = {language: [] for language in LANGUAGES} all_times: list[int] = [] for item in results: if not isinstance(item, dict) or item.get("language") not in buckets: raise SmokeReportRejected("invalid-generation-language") duration = item.get("generationMs") + generated_tokens = item.get("generationTokens") if type(duration) is not int or not 0 <= duration <= 3_600_000: raise SmokeReportRejected("invalid-generation-duration") + if (type(generated_tokens) is not int or + not 1 <= generated_tokens <= max_new_tokens): + raise SmokeReportRejected("invalid-generation-token-count") buckets[item["language"]].append(duration) + token_buckets[item["language"]].append(generated_tokens) all_times.append(duration) def nearest_rank(values: list[int], fraction: float) -> int | None: @@ -134,6 +142,14 @@ def nearest_rank(values: list[int], fraction: float) -> int | None: "p95Ms": nearest_rank(buckets[language], 0.95), } for language in LANGUAGES }, + "generatedTokenAccounting": "new-token-ids-including-stop-v1", + "generatedTokenCount": sum(sum(values) for values in token_buckets.values()), + "generatedTokensByLanguage": { + language: { + "attempts": len(token_buckets[language]), + "totalTokens": sum(token_buckets[language]), + } for language in LANGUAGES + }, } From 55b9ea00fcbda7379490b8eee1ee4797cbcdef8a Mon Sep 17 00:00:00 2001 From: Sanskar Date: Fri, 30 Oct 2026 14:33:43 +0530 Subject: [PATCH 2/7] feat(model): record generated IDs per local HF synthetic probe without persisting source --- model/train/smoke.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/model/train/smoke.py b/model/train/smoke.py index d98f828..b868a35 100644 --- a/model/train/smoke.py +++ b/model/train/smoke.py @@ -184,7 +184,8 @@ def run_local(fixture_set: list[dict], args) -> dict: completion = tokenizer.decode(tokens, skip_special_tokens=False) exact = normalize_completion(completion) == fixture["expected"] result.append({"language": fixture["language"], "exact": exact, - "generationMs": generation_ms}) + "generationMs": generation_ms, + "generationTokens": len(tokens)}) verify_smoke_inputs_unchanged( base_dir, base_identity, Path(args.adapter_dir) if args.adapter_dir else None, @@ -207,7 +208,7 @@ def run_local(fixture_set: list[dict], args) -> dict: "baseWeightFiles": base_identity["baseWeightFiles"], "maxNewTokens": args.max_new_tokens, **summarize(result), - **generation_latency_summary(result), + **generation_latency_summary(result, args.max_new_tokens), "limitation": "Strict original-synthetic equality only; no human, compiler, editor or deployment QA." } From 68ca2a1acd843dea41ecb3dd55e0b3230eaa9f54 Mon Sep 17 00:00:00 2001 From: Sanskar Date: Fri, 30 Oct 2026 14:34:11 +0530 Subject: [PATCH 3/7] feat(model): validate generated-token totals and report length confounders in HF comparisons --- model/train/compare_smoke.py | 43 ++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/model/train/compare_smoke.py b/model/train/compare_smoke.py index c702420..7df3889 100644 --- a/model/train/compare_smoke.py +++ b/model/train/compare_smoke.py @@ -141,6 +141,29 @@ def checked_latency(value) -> bool: raise SmokeReportRejected("invalid-generation-language-latency") elif group["p50Ms"] is not None or group["p95Ms"] is not None: raise SmokeReportRejected("unexpected-generation-language-latency") + if report.get("generatedTokenAccounting") != "new-token-ids-including-stop-v1": + raise SmokeReportRejected("invalid-generated-token-method") + total = report.get("generatedTokenCount") + if (type(total) is not int or + not attempts <= total <= attempts * tokens): + raise SmokeReportRejected("invalid-generated-token-total") + token_groups = report.get("generatedTokensByLanguage") + if not isinstance(token_groups, dict) or set(token_groups) != set(LANGUAGES): + raise SmokeReportRejected("invalid-generated-token-groups") + counted_tokens = 0 + for name in LANGUAGES: + group = token_groups[name] + expected_count = groups[name]["attempts"] + if (not isinstance(group, dict) or + set(group) != {"attempts", "totalTokens"} or + type(group["attempts"]) is not int or + group["attempts"] != expected_count or + type(group["totalTokens"]) is not int or + not expected_count <= group["totalTokens"] <= expected_count * tokens): + raise SmokeReportRejected("invalid-generated-token-language") + counted_tokens += group["totalTokens"] + if counted_tokens != total: + raise SmokeReportRejected("mismatched-generated-token-total") return report @@ -163,11 +186,16 @@ def compare_reports(base: dict, adapter: dict) -> dict: after = right["perLanguage"][language]["strictExactMatches"] left_timing = left["generationLatencyByLanguage"][language] right_timing = right["generationLatencyByLanguage"][language] + left_tokens = left["generatedTokensByLanguage"][language]["totalTokens"] + right_tokens = right["generatedTokensByLanguage"][language]["totalTokens"] by_language[language] = { "attemptsPerModel": left["perLanguage"][language]["attempts"], "baseStrictExactMatches": before, "adapterStrictExactMatches": after, "adapterMinusBase": after - before, + "baseGeneratedTokens": left_tokens, + "adapterGeneratedTokens": right_tokens, + "generatedTokenDelta": right_tokens - left_tokens, "baseGenerationP50Ms": left_timing["p50Ms"], "adapterGenerationP50Ms": right_timing["p50Ms"], "generationP50DeltaMs": ( @@ -182,13 +210,22 @@ def compare_reports(base: dict, adapter: dict) -> dict: ), } return { - "comparisonProtocol": "lineloom-hf-synthetic-comparison-v3", + "comparisonProtocol": "lineloom-hf-synthetic-comparison-v4", "benchmarkProtocol": PROTOCOL, "scope": left["selectedLanguage"], "syntheticAttemptsPerModel": left["syntheticAttempts"], "baseStrictExactMatches": left["strictExactMatches"], "adapterStrictExactMatches": right["strictExactMatches"], "adapterMinusBase": delta, + "generatedTokenAccounting": "new-token-ids-including-stop-v1", + "baseGeneratedTokens": left["generatedTokenCount"], + "adapterGeneratedTokens": right["generatedTokenCount"], + "generatedTokenDelta": right["generatedTokenCount"] - left["generatedTokenCount"], + "generatedTokenCountStatus": ( + "AGGREGATE_TOKEN_TOTALS_MATCH_ONLY" + if left["generatedTokenCount"] == right["generatedTokenCount"] + else "DIFFERENT_GENERATED_TOKEN_TOTALS" + ), "latencyMeasurement": "monotonic-generation-only-v1", "baseGenerationP50Ms": left["generationP50Ms"], "adapterGenerationP50Ms": right["generationP50Ms"], @@ -202,7 +239,9 @@ def compare_reports(base: dict, adapter: dict) -> dict: "do not verify real weights, compute environment, code " "correctness, hardware parity, editor latency, acceptance " "or release readiness. Generation-only timings are " - "descriptive, and small samples are unstable."), + "descriptive, and small samples are unstable. " + "Different generated token counts confound absolute latency; " + "even equal totals do not prove per-probe length parity."), } From 508452de91dd78b7c646320c5dcf1777dc6dea15 Mon Sep 17 00:00:00 2001 From: Sanskar Date: Fri, 30 Oct 2026 14:34:37 +0530 Subject: [PATCH 4/7] test(model): exercise token-count summaries bounds and v5 offline smoke plan --- model/train/test_smoke.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/model/train/test_smoke.py b/model/train/test_smoke.py index 1f86a97..2b9c8b4 100644 --- a/model/train/test_smoke.py +++ b/model/train/test_smoke.py @@ -51,10 +51,10 @@ def test_aggregates_have_only_fixed_language_counts(self): def test_generation_timing_nearest_rank_and_no_source_output(self): from_smoke = smoke.generation_latency_summary([ - {"language": "rust", "generationMs": 8}, - {"language": "rust", "generationMs": 20}, - {"language": "rust", "generationMs": 11}, - {"language": "python", "generationMs": 3}, + {"language": "rust", "generationMs": 8, "generationTokens": 2}, + {"language": "rust", "generationMs": 20, "generationTokens": 3}, + {"language": "rust", "generationMs": 11, "generationTokens": 4}, + {"language": "python", "generationMs": 3, "generationTokens": 5}, ]) self.assertEqual(from_smoke["latencyMeasurement"], "monotonic-generation-only-v1") self.assertEqual(from_smoke["generationP50Ms"], 8) @@ -63,15 +63,33 @@ def test_generation_timing_nearest_rank_and_no_source_output(self): {"attempts": 3, "p50Ms": 11, "p95Ms": 20}) self.assertEqual(from_smoke["generationLatencyByLanguage"]["c"], {"attempts": 0, "p50Ms": None, "p95Ms": None}) + self.assertEqual(from_smoke["generatedTokenAccounting"], "new-token-ids-including-stop-v1") + self.assertEqual(from_smoke["generatedTokenCount"], 14) + self.assertEqual(from_smoke["generatedTokensByLanguage"]["rust"], + {"attempts": 3, "totalTokens": 9}) + self.assertEqual(from_smoke["generatedTokensByLanguage"]["c"], + {"attempts": 0, "totalTokens": 0}) self.assertNotIn("prefix", json.dumps(from_smoke)) for invalid in (True, 1.5, -1, 3_600_001, float("nan"), "10"): with self.subTest(invalid=invalid), self.assertRaises(ValueError): smoke.generation_latency_summary([ - {"language": "rust", "generationMs": invalid} + {"language": "rust", "generationMs": invalid, + "generationTokens": 1} ]) - for invalid in ([], "invalid", [{"language": "other", "generationMs": 10}]): + for invalid in ([], "invalid", [{"language": "other", "generationMs": 10, + "generationTokens": 1}]): with self.assertRaises(ValueError): smoke.generation_latency_summary(invalid) + for invalid in (None, True, 0, -1, 129, 1.5, float("nan"), "10"): + with self.subTest(token_count=invalid), self.assertRaises(ValueError): + smoke.generation_latency_summary([ + {"language": "rust", "generationMs": 10, + "generationTokens": invalid} + ]) + with self.assertRaises(ValueError): + smoke.generation_latency_summary([ + {"language": "rust", "generationMs": 10, "generationTokens": 3} + ], max_new_tokens=2) def test_adapter_validation_requires_local_peft_files(self): with tempfile.TemporaryDirectory() as temp: @@ -141,7 +159,7 @@ def test_default_smoke_is_offline_plan_without_checkpoint_or_framework_import(se self.assertEqual(result["perLanguage"]["cpp"], 6) self.assertEqual(result["networkCalls"], 0) self.assertFalse(result["realModelInferenceExecuted"]) - self.assertEqual(result["benchmarkProtocol"], "lineloom-hf-synthetic-smoke-v4") + self.assertEqual(result["benchmarkProtocol"], "lineloom-hf-synthetic-smoke-v5") self.assertEqual(result["generationPolicy"], "greedy-fim-v2") self.assertEqual(result["selectedLanguage"], "cpp") self.assertEqual(len(result["fixtureSetSha256"]), 64) From 95a37a96afb5beefe1ebe136b884e6c6ba2a38f8 Mon Sep 17 00:00:00 2001 From: Sanskar Date: Fri, 30 Oct 2026 14:35:04 +0530 Subject: [PATCH 5/7] test(model): validate generated-token lengths deltas and false parity claims --- model/train/test_smoke_compare.py | 54 +++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/model/train/test_smoke_compare.py b/model/train/test_smoke_compare.py index 01c1c5b..55c0c04 100644 --- a/model/train/test_smoke_compare.py +++ b/model/train/test_smoke_compare.py @@ -37,7 +37,8 @@ def simulated_report(kind: str, language: str = "all", tokens: int = 64, } timings = generation_latency_summary([ {"language": item["language"], - "generationMs": (index + 1) * (2 if kind == "local-lora-adapter" else 1)} + "generationMs": (index + 1) * (2 if kind == "local-lora-adapter" else 1), + "generationTokens": (index % 3) + (2 if kind == "local-lora-adapter" else 1)} for index, item in enumerate(selected) ]) return { @@ -126,11 +127,17 @@ def test_compatible_counts_are_only_source_free_descriptive_deltas(self): self.assertEqual(result["perLanguage"]["rust"]["adapterMinusBase"], 1) self.assertEqual(result["perLanguage"]["cpp"]["attemptsPerModel"], 6) self.assertEqual(result["verdict"], "DESCRIPTIVE_SYNTHETIC_COUNTS_ONLY") - self.assertEqual(result["comparisonProtocol"], "lineloom-hf-synthetic-comparison-v3") + self.assertEqual(result["comparisonProtocol"], "lineloom-hf-synthetic-comparison-v4") self.assertEqual(result["latencyMeasurement"], "monotonic-generation-only-v1") self.assertEqual(result["generationP50DeltaMs"], 15) self.assertEqual(result["generationP95DeltaMs"], 29) self.assertEqual(result["perLanguage"]["rust"]["generationP95DeltaMs"], 14) + self.assertEqual(result["generatedTokenAccounting"], "new-token-ids-including-stop-v1") + self.assertEqual(result["baseGeneratedTokens"], 60) + self.assertEqual(result["adapterGeneratedTokens"], 90) + self.assertEqual(result["generatedTokenDelta"], 30) + self.assertEqual(result["generatedTokenCountStatus"], "DIFFERENT_GENERATED_TOKEN_TOTALS") + self.assertEqual(result["perLanguage"]["rust"]["generatedTokenDelta"], 6) self.assertNotIn("fixtureSetSha256", result) self.assertNotIn("baseRevisionDeclared", result) @@ -145,6 +152,8 @@ def test_single_language_scope_and_no_false_improvement(self): self.assertEqual(result["perLanguage"]["cpp"]["attemptsPerModel"], 6) self.assertEqual(result["perLanguage"]["rust"]["generationP50DeltaMs"], None) self.assertEqual(result["perLanguage"]["cpp"]["generationP95DeltaMs"], 6) + self.assertEqual(result["generatedTokenDelta"], 6) + self.assertEqual(result["perLanguage"]["rust"]["generatedTokenDelta"], 0) def test_mismatched_identity_scope_and_generation_rejected(self): for field, other in ( @@ -235,6 +244,47 @@ def test_latency_fields_are_bounded_and_language_scoped(self): simulated_report("local-base-model", language="cpp"), cpp_only ) + def test_generated_token_accounting_catches_inconsistent_or_malformed_reports(self): + cases = [ + ("generatedTokenAccounting", "unverified"), + ("generatedTokenCount", True), + ("generatedTokenCount", -1), + ("generatedTokenCount", 30 * 129), + ("generatedTokensByLanguage", {}), + ] + for field, value in cases: + candidate = copy.deepcopy(self.adapter) + candidate[field] = value + with self.subTest(field=field), self.assertRaises(SmokeReportRejected): + compare_smoke.compare_reports(self.base, candidate) + for lang in ("rust", "python"): + for change in ( + {"totalTokens": True}, + {"totalTokens": -1}, + {"totalTokens": 6 * 65}, + {"attempts": 7}, + {"extra": "private source"}, + ): + candidate = copy.deepcopy(self.adapter) + candidate["generatedTokensByLanguage"][lang].update(change) + with self.subTest(language=lang, change=change), self.assertRaises(SmokeReportRejected): + compare_smoke.compare_reports(self.base, candidate) + candidate = copy.deepcopy(self.adapter) + candidate["generatedTokensByLanguage"]["cpp"]["totalTokens"] += 1 + with self.assertRaises(SmokeReportRejected): + compare_smoke.compare_reports(self.base, candidate) + + def test_equal_aggregate_token_totals_do_not_claim_per_probe_parity(self): + candidate = copy.deepcopy(self.adapter) + candidate["generatedTokenCount"] = self.base["generatedTokenCount"] + candidate["generatedTokensByLanguage"] = copy.deepcopy( + self.base["generatedTokensByLanguage"] + ) + result = compare_smoke.compare_reports(self.base, candidate) + self.assertEqual(result["generatedTokenCountStatus"], "AGGREGATE_TOKEN_TOTALS_MATCH_ONLY") + self.assertEqual(result["generatedTokenDelta"], 0) + self.assertIn("even equal totals", result["limitation"]) + def test_wrong_base_and_adapter_positions_rejected(self): with self.assertRaises(SmokeReportRejected): compare_smoke.compare_reports(self.adapter, self.base) From 07efa2fc78c555357632ba468878acbc3f9d2494 Mon Sep 17 00:00:00 2001 From: Sanskar Date: Fri, 30 Oct 2026 14:35:51 +0530 Subject: [PATCH 6/7] docs(model): define HF smoke v5 generated-token accounting and latency caveats --- model/EVALUATION_PROTOCOL.md | 53 ++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/model/EVALUATION_PROTOCOL.md b/model/EVALUATION_PROTOCOL.md index 8e50dfd..1e32437 100644 --- a/model/EVALUATION_PROTOCOL.md +++ b/model/EVALUATION_PROTOCOL.md @@ -274,28 +274,39 @@ and a checksum says nothing about code usefulness, correctness, legality or safety. The real-model and human-review gates remain. -## 14. Local HF synthetic generation timing (v4 current protocol) +## 14. Local HF synthetic generation timing and output length (v5 current protocol) For actual reviewed local `smoke.py --run` trials, protocol -`lineloom-hf-synthetic-smoke-v4` adds nearest-rank **p50/p95 generation +`lineloom-hf-synthetic-smoke-v5` reports nearest-rank **p50/p95 generation latency in milliseconds**, overall and per target language. Measurement uses a monotonic clock surrounding only the `model.generate()` call, -with explicit GPU synchronization before and after when using CUDA. -Model loading, prompt tokenization, decoding, VS Code integration and all -editor-side delays are **outside this timing**. Unselected languages report -zero attempts and null percentiles. - -Offline `compare_smoke.py` now requires the same fixed timing scheme and -matching fixture/request/checkpoint scope and reports base, adapter and -adapter-minus-base latency differences. A **positive difference** is slower -generation in that particular run, but is not an independently verified -regression; compare only with controlled hardware, load, thermal state, -execution order and reproducible conditions. With only six examples per -language, these p95 measurements are highly unstable. - -This v4 contract intentionally rejects older v1–v3 HF reports in the -comparator; rerun both from the same reviewed checkout. The separate -Ollama benchmark uses `lineloom-synthetic-v4` and remains incomparable. -Offline unit tests use invented timings, without GPU, training or any -model-generation measurements. No real HF inference has been executed -or claimed by this development change. +with explicit CUDA synchronization on both sides. Loading, prompt +tokenization, decoding, VS Code integration and all editor-side delays +are **outside this timing**. Unselected languages report zero attempts +and null latency percentiles. + +V5 additionally counts **actual generated token IDs** (including a stop +token when generated), aggregated globally and per language, with at +most `maxNewTokens` IDs per attempt. It does **not** print tokens, decoded +text, fixture identities or individual token counts. The source-free +comparator validates count bounds and five-language sums, then reports +base, adapter and adapter-minus-base generation-token totals. If token +totals differ, raw generation-duration deltas are especially unsuitable +for a speed comparison: one model may simply produce more code. Even +equal totals do **not** establish matching per-probe output lengths. +The runner intentionally does not claim throughput, first-token latency, +identical hardware, compiler correctness or editor acceptance. + +Offline `compare_smoke.py` requires identical fixture/request/checkpoint +scope and reports only descriptive base-versus-adapter differences. A +**positive duration difference** is slower in that particular run, not +an independently verified regression. Compare only on controlled +hardware, load, thermal state, quantization and execution order. +With six examples per language, p95 is highly unstable. + +V5 rejects previous v1–v4 HF smoke protocols in the comparator; rerun +both base and adapter from the same reviewed checkout. The separate +Ollama benchmark uses `lineloom-synthetic-v4` and is not directly +comparable. Offline tests use invented timing and token counts without +GPU, model inference, training, generated private source or network +access. No real HF inference is claimed by this development change. From ea5b9a56ad7a3b89c9cd69403c430ea78a2e859c Mon Sep 17 00:00:00 2001 From: Sanskar Date: Fri, 30 Oct 2026 14:36:07 +0530 Subject: [PATCH 7/7] docs(model): describe output length accounting in local base-vs-adapter comparisons --- model/train/README.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/model/train/README.md b/model/train/README.md index c0417b5..a0f10d4 100644 --- a/model/train/README.md +++ b/model/train/README.md @@ -271,7 +271,7 @@ Python standard library. They check seeded repeatability, distribution, bucket coverage, rejection of validation/test contamination, nonfinite losses and independence from the process-global PRNG. -## Comparing local HF base-versus-adapter smoke reports (protocol v4) +## Comparing local HF base-versus-adapter smoke reports (protocol v5) After **independently approved** local base and adapter runs on the same machine with the *same* base revision, tokenizer, fixtures and generation @@ -293,7 +293,7 @@ than 32 KiB. Only one of the fixed checked-in synthetic scopes (`all`, `rust`, `c`, `cpp`, `python`, `typescript`) is valid. Real local `smoke.py --run` reports now include the -`lineloom-hf-synthetic-smoke-v4` protocol, deterministic +`lineloom-hf-synthetic-smoke-v5` protocol, deterministic `fixtureSetSha256`, `requestSpecSha256`, checked-in fixture scope, greedy generation contract, fixed local base config/tokenizer SHA-256, and the **user-declared** 40-character base revision. The comparator @@ -302,14 +302,20 @@ and refuses plans, legacy reports, corrupted counts, mixed base revisions, different settings, or comparisons without a local base report followed by a local adapter report. -The comparison output contains **allowlisted strict-match counts and -generation-only latency p50/p95** by language and total, including -adapter-minus-base deltas, plus fixed caveats. The timer uses a monotonic -clock around `model.generate()` only (and synchronizes CUDA on each side). +The comparison output contains **allowlisted strict-match counts, +generation-only latency p50/p95, and generated-token totals** by language +and overall, with adapter-minus-base deltas. Generated-token totals +count actual output token IDs, including a stop token when present, but +never include raw tokens, private source, individual probe identities +or decoded text. The timer uses a monotonic clock around +`model.generate()` only (and synchronizes CUDA on each side). It excludes model loading, tokenization, decoding, and VS Code/editor time. Positive latency deltas mean the adapter was slower in the recorded run; -negative deltas mean faster. Six probes per language produce unstable p95 -values. Compare only on the same reviewed machine in controlled conditions: +negative deltas mean faster. If the models produce **different numbers of +generated tokens**, those latency differences do not represent a +same-length throughput comparison. Even identical aggregate token totals +cannot prove matching lengths for individual probes. Six probes per +language produce unstable p95 values. Compare only on controlled conditions: the report hashes do **not** prove hardware or runtime equivalence. It never prints prompts, completions, model checkpoint paths, private metadata, filenames or user-supplied report fields. @@ -321,7 +327,7 @@ GPU settings, tokenizer runtime behavior, hidden sampling defaults, compiler correctness, editor acceptance, equivalent timing conditions or whether reports were truthfully generated. **The six probes per language are not a production model evaluation**, and a strict exact-match improvement does not establish -quality. Prior HF v1–v3 reports must be rerun; HF protocol v4 remains +quality. Prior HF v1–v4 reports must be rerun; HF protocol v5 remains incomparable with the Ollama `lineloom-synthetic-v4` protocol. Tests are run automatically by `npm run test:model-training` using Python @@ -352,7 +358,8 @@ checkpoint weights cannot now be compared as if they were the same base. receipts and HF smoke `lineloom-hf-synthetic-smoke-v2` reports are not accepted by the new versioned verification/comparator workflow. New receipts use `lineloom-local-adapter-receipt-v2`; new HF reports use -`lineloom-hf-synthetic-smoke-v4` with generation-only timing. Do not alter old receipts to make them +`lineloom-hf-synthetic-smoke-v5` with generation-only timing and +generated-token totals. Do not alter old receipts to make them pass or imply that a local SHA-256 digest authenticates a declared upstream revision, provenance, permissions or loaded model state. Independent checkpoint review, safe isolation, and actual inference/quality checks