固定ratioモデルのincremental差分を分離して観測

models/experimental/RATIO_CHUNKS.ja.md

@@ -0,0 +1,59 @@
+<!-- SPDX-License-Identifier: MIT -->
+# 固定ratioモデルのchunk比較
+
+感度実験で選んだ因子0.90を動かさず、同じtuning 8録音のfull cp1252/UTF-8全16入力を
+whole / 1 / 7 / 64 / 1024-byte chunkで観測する。各入力4 KiB以下・各process10秒上限。
+標準legacy、identity/filteredの因子1/0.90を比較し、全400観測を保存する。
+因子やtableを調整する試験ではなく、incrementalの採用gateを確認する試験。
+大入力、追加fuzz、P01対象の停止調査は行わない。
+
+## 比較の分離
+
+- 同じmodelのwholeとの差。
+- 同じchunk scheduleのlegacyとの差。
+- 同じprofile・同じchunk scheduleの因子1との差。
+
+各比較で候補数、encoding/language順序、confidence bitsを含む候補一覧、final done、
+first done offsetを別に記録する。feed回数はchunkにより必然的に変わるので差分指標には
+含めず、生観測に保持する。first done offsetはfeed後に観測した位置で、内部の厳密な
+最初の判定byte位置を表すとは限らない。完全一致を近似float比較へ置き換えない。
+
+## 結果
+
+cp1252のstrict decode-equivalent件数（各8入力）:
+
+| chunk | legacy | identity 1 | identity 0.90 | filtered 1 | filtered 0.90 |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| whole | 8 | 4 | 8 | 5 | 8 |
+| 1 | 0 | 0 | 0 | 0 | 0 |
+| 7 | 6 | 4 | 4 | 4 | 4 |
+| 64 | 8 | 4 | 7 | 5 | 8 |
+| 1024 | 8 | 4 | 8 | 5 | 8 |
+
+UTF-8 exactは全model・全scheduleで8/8。全出力confidenceは有限かつ[0,1]内だった。
+wholeに対する候補完全一致の不一致件数は、1/7/64-byteで各modelとも16/16、
+1024-byteで各modelとも4/16。encoding/language順序の不一致とは分けてreportへ保持する。
+final done / first done offsetのwholeとの差は今回すべて0件。
+
+1-byteの悪化はlegacyにも存在し、今回のratio変更だけに起因するとは言えない。
+一方、7-byteでは生成modelがlegacyより悪く、wholeの8/8だけで標準採用する根拠にはならない。
+因子0.90は小規模whole-inputの候補のままとし、incremental gateを達成したとは扱わない。
+この結果を見てtuning gridを増やさない。具体的な原因は別途追跡する。
+
+## 再現
+
+```sh
+uv run --no-project python models/experimental/ratio_chunks.py \
+  IDENTITY FILTERED MANIFEST SWEEP BUILDS OUTPUT
+```
+
+入力は24録音版の親artifact、training/tuning manifest、固定ratio sweep report、
+その検証済みbuild directory。sweep hash、親model、入力hash、source/binaryを照合する。
+wholeの4生成model観測は保存sweepと完全一致を要求する。
+複数buildの標準targetも各入力・chunkで完全一致を要求する。
+
+private report: `archives/v3-corpus/paris24-ratio-chunks-v1.json`。
+content hash: `5d28f112f1ed472e1a438df9790f9e2f50c9aa0e216943059e2bb51621d0c8c1`。
+生観測、3種類の差分、accuracy、buildの参照元sweep hashを保存している。
+全条件を2回観測してreport全体のbyte一致を確認した。
+今回の評価対象はtuningであり、独立汎化精度の主張ではない。

models/experimental/ratio_chunks.py

@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: MIT
+"""Bounded fixed-factor chunk observations; never tune against chunk differences."""
+import argparse
+import json
+from pathlib import Path
+
+import engine_comparison
+import engine_probe
+import paired_controls
+import ratio_sweep
+import ratio_variant
+from model import canonical, digest, write_idempotent
+from sequence_contract import content_hash
+
+SWEEP_HASH = "8ebbe3a8e939488bf89aa1fa2c3d6169a494c99e2cf23faf9dc37419f93cd6d6"
+CHUNKS = (0, 1, 7, 64, 1024)
+
+
+def differences(observed, reference):
+    def names(value):
+        return [(c["encoding"], c["language"]) for c in value["candidates"]]
+    return dict(candidate_count=observed["candidate_count"] != reference["candidate_count"],
+                encoding_language_order=names(observed) != names(reference),
+                exact_candidates=observed["candidates"] != reference["candidates"],
+                final_done=observed["final_done"] != reference["final_done"],
+                first_done_offset=observed["first_done_offset"] != reference["first_done_offset"])
+
+
+def run(identity, filtered, manifest, root, sweep, directory):
+    if sweep.get("content_hash") != SWEEP_HASH or content_hash(sweep) != SWEEP_HASH:
+        raise ValueError("requires the frozen ratio sweep")
+    if manifest["content_hash"] != identity["contract"]["provenance"]["corpus_content_hash"]:
+        raise ValueError("requires the fixed training/tuning manifest")
+    records, _ = paired_controls.select_pairs(identity, filtered, manifest, root, "tuning")
+    if len(records) != 16 or any(len(data) > engine_probe.LIMIT for _, _, data in records):
+        raise ValueError("requires all sixteen bounded tuning inputs")
+    builds, saved = {}, {}
+    for name, parent in (("identity", identity), ("filtered", filtered)):
+        for factor in ("1", "0.90"):
+            key = f"{name}-{factor}"
+            previous = sweep["results"][name][factor]
+            variant = previous["variant"]
+            ratio_variant.validate(variant, parent)
+            path = directory / f"build-{key}"
+            binaries, provenance = engine_comparison.verified_build(path, variant)
+            if provenance != previous["build"]:
+                raise ValueError("build differs from frozen sweep")
+            builds[key] = (path, variant, binaries, provenance)
+            saved[key] = {row["sample_id"]: row for row in previous["documents"]}
+    documents = []
+    for source, sample, data in records:
+        schedules = {}
+        for chunk in CHUNKS:
+            observations = {}
+            for key, (_, _, binaries, _) in builds.items():
+                normal = engine_probe.observe(binaries["uchardet-conformance"], data, chunk)
+                if "legacy" in observations and normal != observations["legacy"]:
+                    raise ValueError("standard targets differ")
+                observations["legacy"] = normal
+                value = engine_probe.observe(binaries["uchardet-conformance-experimental"], data, chunk)
+                old = saved[key][sample["id"]]
+                if old["sample_sha256"] != digest(data):
+                    raise ValueError("tuning input differs from sweep")
+                if chunk == 0 and value != old["observation"]:
+                    raise ValueError("whole-input observation differs from sweep")
+                observations[key] = value
+            schedules[str(chunk)] = observations
+        comparisons = {}
+        scores = {}
+        for chunk, observations in schedules.items():
+            comparisons[chunk], scores[chunk] = {}, {}
+            for key, value in observations.items():
+                parent_key = key.replace("-0.90", "-1")
+                comparisons[chunk][key] = dict(
+                    vs_whole=differences(value, schedules["0"][key]),
+                    vs_legacy_same_chunk=differences(value, observations["legacy"]),
+                    vs_unscaled_same_chunk=differences(value, observations[parent_key]))
+                scores[chunk][key] = engine_comparison.score(value, data, sample["encoding"], source["language"])
+        documents.append(dict(sample_id=sample["id"], sample_sha256=sample["sha256"],
+                              encoding=sample["encoding"], byte_length=len(data),
+                              observations=schedules, comparisons=comparisons, scores=scores))
+    for path, variant, _, provenance in builds.values():
+        if engine_comparison.verified_build(path, variant)[1] != provenance:
+            raise ValueError("build changed during observation")
+    summary = {}
+    for chunk in map(str, CHUNKS):
+        summary[chunk] = {}
+        for key in ("legacy", *builds):
+            rows = [dict(encoding=row["encoding"], score=row["scores"][chunk][key],
+                         observation=row["observations"][chunk][key]) for row in documents]
+            value = ratio_sweep.summarize(rows)
+            value["differences"] = {
+                comparison: {field: sum(row["comparisons"][chunk][key][comparison][field] for row in documents)
+                             for field in documents[0]["comparisons"][chunk][key][comparison]}
+                for comparison in ("vs_whole", "vs_legacy_same_chunk", "vs_unscaled_same_chunk")}
+            summary[chunk][key] = value
+    result = dict(schema="paris24-fixed-ratio-chunks-v1", sweep_hash=SWEEP_HASH,
+                  manifest_hash=manifest["content_hash"], chunks=list(CHUNKS),
+                  documents=documents, summary=summary,
+                  dependencies={p.name: digest(p.read_bytes()) for p in (Path(__file__),
+                      Path(engine_probe.__file__), Path(engine_comparison.__file__),
+                      Path(ratio_sweep.__file__), Path(ratio_variant.__file__), Path(paired_controls.__file__))})
+    result["content_hash"] = content_hash(result)
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name in ("identity", "filtered", "manifest", "sweep", "builds", "output"):
+        parser.add_argument(name, type=Path)
+    args = parser.parse_args()
+    def load(path):
+        return json.loads(path.read_text(encoding="utf-8"))
+    result = run(load(args.identity), load(args.filtered), load(args.manifest),
+                 args.manifest.parent, load(args.sweep), args.builds)
+    write_idempotent(args.output, canonical(result))
+
+
+if __name__ == "__main__":
+    main()

models/experimental/test_ratio_chunks.py

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: MIT
+import copy
+import unittest
+from unittest.mock import patch
+
+import ratio_chunks
+
+
+class RatioChunkTests(unittest.TestCase):
+    def test_confidence_order_and_lifecycle_are_separate(self):
+        base = dict(candidate_count=2, candidates=[
+            dict(encoding="cp1252", language="fr", confidence_bits="3f000000"),
+            dict(encoding="UTF-8", language=None, confidence_bits="3e000000")],
+            final_done=True, first_done_offset=128, feed_calls=1)
+        changed = copy.deepcopy(base)
+        changed["feed_calls"] = 128
+        self.assertFalse(any(ratio_chunks.differences(changed, base).values()))
+        changed["candidates"][0]["confidence_bits"] = "3f000001"
+        result = ratio_chunks.differences(changed, base)
+        self.assertTrue(result["exact_candidates"])
+        self.assertFalse(result["encoding_language_order"])
+        changed["candidates"].reverse()
+        self.assertTrue(ratio_chunks.differences(changed, base)["encoding_language_order"])
+        changed["first_done_offset"] = 64
+        self.assertTrue(ratio_chunks.differences(changed, base)["first_done_offset"])
+
+    def test_wrong_sweep_rejected_before_process(self):
+        with patch.object(ratio_chunks.engine_probe, "observe") as observe:
+            with self.assertRaisesRegex(ValueError, "frozen ratio"):
+                ratio_chunks.run(None, None, None, None, {}, None)
+            observe.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()