Add paired cp1252 and UTF-8 native model controls

models/experimental/PAIRED_CONTROLS.ja.md

@@ -0,0 +1,48 @@
+<!-- SPDX-License-Identifier: MIT -->
+# 同一文章の cp1252 / UTF-8 対照評価
+
+`paired_controls.py` はFrench validation/tuning文書のcp1252とUTF-8を対にする。
+同じUnicode本文を違うencodingへ変換したときの、単一proberの反応差を見るtoolである。
+全detectorのfalse-positive rateやencoding accuracyではない。
+
+```sh
+uv run --no-project python models/experimental/paired_controls.py \
+  /disk/identity-training.json /disk/filtered-training.json \
+  /disk/validation/manifest.json validation \
+  build/release/src/libuchardet.a /disk/paired.json
+```
+
+## 選択と独立性
+
+既存のnative比較のtraining検証・split/SHA/origin監査を先に実施する。
+independent sourceがあれば本文読取り前に拒否。corpus全体のhash/codecも再検証する。
+full / complete / text / cp1252の各sourceに、同じsourceのfull / complete / text / UTF-8が
+ちょうど1件必要。欠損や重複を黙って除外しない。strict decodeした本文の一致も確認する。
+両variantとも65536 bytes以下。HTML/切り詰めvariantを比較へ混ぜない。
+
+## 対照の分類
+
+- `bytes_identical`: ASCII等で同じbytesなら、識別可能な負例へ数えない。
+- `control_cp1252_decodable`: UTF-8のbytesがPythonのstrict cp1252 codecでもdecode可能か。
+  decode可能でも、元のUnicode本文と同じ意味とは限らない。
+- `stopped_before_filtered_end`: proberの消費文字数がfilter後の長さより小さい文書数。
+  confidenceが全文の統計に基づくという誤解を防ぐ。停止理由をこの値だけで確定しない。
+
+UTF-8として妥当なbytesでもcp1252としては不正なbyteを含み得る。
+構造的な拒否と、両方にdecodeできるbytes上の統計的な反応差を分離する。
+decodabilityはraw bytesについてのcodec検証、途中終了はfilter後のprober観測であり、
+同じ条件ではない。最終byteでの拒否等は消費長だけでは判断できない。
+
+## 観測と集計
+
+legacy/identity/filteredを同じlibrary・harnessでbuildし、各variantを新規proberへ一回feedする。
+reportの各文書にはsample encoding、counter/state/confidence bitと由来を記録する。
+encoding別のsummaryを保ち、正例と対照例のconfidenceやstateを混ぜて平均しない。
+
+有限かつbytesが異なるpairに限り、cp1252側が高い／同値／UTF-8側が高い件数を集計する。
+非有限・bytes同一は別枠。cp1252へdecode可能かによる層別集計も行う。
+この順序比較は任意のthresholdを後付けで選ぶものではないが、正解率でもない。
+単一proberはUTF-8候補とのランキングを行わず、内部confidenceは確率ではない。
+
+既存modelのtraining overlapはUNKNOWN。生成modelの採用・再学習・ratio変更は行わない。
+P01の全detector/大入力停止調査やfuzzを再開するtoolではない。

models/experimental/native_comparison.py

@@ -72,8 +72,7 @@ def summarize(documents):
     )
 
 
-def compare(identity, filtered, manifest, root, split, library, compiler="c++"):
-    records = select_records(identity, filtered, manifest, root, split)
+def observe_models(identity, filtered, records, library, compiler="c++"):
     library = Path(library).resolve(strict=True)
     library_hash = digest(library.read_bytes())
     profiles, lengths = {}, {}
@@ -91,14 +90,16 @@ def compare(identity, filtered, manifest, root, split, library, compiler="c++"):
             for source, sample, data in records:
                 observed = sequence_probe.observe(binary, data)
                 current = (observed["raw_bytes"], observed["filtered_bytes"])
-                if source["id"] in lengths and current != lengths[source["id"]]:
+                key = (source["id"], sample["id"])
+                if key in lengths and current != lengths[key]:
                     raise ValueError("filter lengths differ between models")
-                lengths[source["id"]] = current
+                lengths[key] = current
                 documents.append(
                     dict(
                         source=source,
                         sample_sha256=sample["sha256"],
                         sample_id=sample["id"],
+                        sample_encoding=sample["encoding"],
                         encoder=sample["encoder"],
                         encoder_version=sample["encoder_version"],
                         observation=observed,
@@ -112,6 +113,12 @@ def compare(identity, filtered, manifest, root, split, library, compiler="c++"):
         )
     if digest(library.read_bytes()) != library_hash:
         raise ValueError("static library changed during comparison")
+    return profiles
+
+
+def compare(identity, filtered, manifest, root, split, library, compiler="c++"):
+    records = select_records(identity, filtered, manifest, root, split)
+    profiles = observe_models(identity, filtered, records, library, compiler)
     report = dict(
         schema="native-french-model-comparison-v1",
         deployment_status="NOT_ENGINE_CALIBRATED",

models/experimental/paired_controls.py

@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: MIT
+"""Same-source cp1252/UTF-8 controls for a single French prober, not detector accuracy."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import struct
+from pathlib import Path
+
+import native_comparison
+import sequence_training
+from model import canonical, digest, safe_path, write_idempotent
+from sequence_contract import content_hash
+
+
+def select_pairs(identity, filtered, manifest, root, split):
+    # Includes full corpus verification and metadata-only independent/leakage refusal.
+    positives = native_comparison.select_records(identity, filtered, manifest, root, split)
+    candidates = {}
+    for sample in manifest["samples"]:
+        if (
+            sample["encoding"] == "utf-8"
+            and sample["format"] == "text"
+            and sample["boundary"] == "complete"
+            and sample["byte_limit"] is None
+        ):
+            candidates.setdefault(sample["source_id"], []).append(sample)
+    records, pairs = [], []
+    for source, positive, data in positives:
+        matches = candidates.get(source["id"], [])
+        if len(matches) != 1:
+            raise ValueError(
+                "each selected cp1252 document needs exactly one full UTF-8 counterpart"
+            )
+        control = matches[0]
+        other = safe_path(root, control["path"]).read_bytes()
+        if len(other) > 65536:
+            raise ValueError("UTF-8 control exceeds 65536 byte limit")
+        if data.decode("cp1252", errors="strict") != other.decode("utf-8", errors="strict"):
+            raise ValueError("paired encodings differ in decoded text")
+        try:
+            other.decode("cp1252", errors="strict")
+            decodable = True
+        except UnicodeDecodeError:
+            decodable = False
+        records.extend(((source, positive, data), (source, control, other)))
+        pairs.append(
+            dict(
+                source_id=source["id"],
+                positive_sample=positive["id"],
+                control_sample=control["id"],
+                bytes_identical=data == other,
+                control_cp1252_decodable=decodable,
+            )
+        )
+    return records, pairs
+
+
+def separation(documents, pairs):
+    indexed = {document["sample_id"]: document for document in documents}
+    result = dict(
+        total_pairs=len(pairs),
+        byte_identical_pairs=0,
+        finite_distinct_pairs=0,
+        nonfinite_distinct_pairs=0,
+        positive_higher=0,
+        equal=0,
+        control_higher=0,
+    )
+    for pair in pairs:
+        if pair["bytes_identical"]:
+            result["byte_identical_pairs"] += 1
+            continue
+        values = [
+            struct.unpack(
+                "!f",
+                bytes.fromhex(indexed[pair[field]]["observation"]["snapshot"]["confidence_bits"]),
+            )[0]
+            for field in ("positive_sample", "control_sample")
+        ]
+        if not all(math.isfinite(value) for value in values):
+            result["nonfinite_distinct_pairs"] += 1
+            continue
+        result["finite_distinct_pairs"] += 1
+        key = (
+            "positive_higher"
+            if values[0] > values[1]
+            else "control_higher"
+            if values[0] < values[1]
+            else "equal"
+        )
+        result[key] += 1
+    return result
+
+
+def compare(identity, filtered, manifest, root, split, library, compiler="c++"):
+    records, pairs = select_pairs(identity, filtered, manifest, root, split)
+    profiles = native_comparison.observe_models(identity, filtered, records, library, compiler)
+    for profile in profiles.values():
+        documents = profile["documents"]
+        profile["summary"] = {
+            encoding: native_comparison.summarize(
+                [document for document in documents if document["sample_encoding"] == encoding]
+            )
+            for encoding in ("cp1252", "utf-8")
+        }
+        profile["paired_separation"] = separation(documents, pairs)
+        profile["control_decodability_strata"] = {
+            name: separation(
+                documents, [pair for pair in pairs if pair["control_cp1252_decodable"] == value]
+            )
+            for name, value in (("cp1252_decodable", True), ("cp1252_undecodable", False))
+        }
+        for encoding, summary in profile["summary"].items():
+            summary["stopped_before_filtered_end"] = sum(
+                document["observation"]["snapshot"]["total_characters"]
+                < document["observation"]["filtered_bytes"]
+                for document in documents
+                if document["sample_encoding"] == encoding
+            )
+    dependencies = {
+        path.name: digest(path.read_bytes())
+        for path in (Path(__file__), Path(native_comparison.__file__))
+    }
+    report = dict(
+        schema="paired-french-encoding-controls-v1",
+        deployment_status="NOT_ENGINE_CALIBRATED",
+        metric="same-text prober response contrast; NOT false-positive rate or encoding accuracy",
+        corpus_content_hash=manifest["content_hash"],
+        split=split,
+        pairs=pairs,
+        legacy_training_overlap="UNKNOWN",
+        runtime=sequence_training.runtime(),
+        driver_dependencies=dependencies,
+        profiles=profiles,
+    )
+    report["content_hash"] = content_hash(report)
+    return report
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("identity", type=Path)
+    parser.add_argument("filtered", type=Path)
+    parser.add_argument("manifest", type=Path)
+    parser.add_argument("split", choices=("tuning", "validation"))
+    parser.add_argument("library", type=Path)
+    parser.add_argument("output", type=Path)
+    parser.add_argument("--cxx", default="c++")
+    args = parser.parse_args()
+
+    def load(path):
+        return json.loads(path.read_text(encoding="utf-8"))
+
+    report = compare(
+        load(args.identity),
+        load(args.filtered),
+        load(args.manifest),
+        args.manifest.parent,
+        args.split,
+        args.library,
+        args.cxx,
+    )
+    write_idempotent(args.output, canonical(report))
+
+
+if __name__ == "__main__":
+    main()