diff --git a/models/experimental/PAIRED_CONTROLS.ja.md b/models/experimental/PAIRED_CONTROLS.ja.md new file mode 100644 index 0000000..25cdc94 --- /dev/null +++ b/models/experimental/PAIRED_CONTROLS.ja.md @@ -0,0 +1,48 @@ + +# 同一文章の cp1252 / UTF-8 対照評価 + +`paired_controls.py` はFrench validation/tuning文書のcp1252とUTF-8を対にする。 +同じUnicode本文を違うencodingへ変換したときの、単一proberの反応差を見るtoolである。 +全detectorのfalse-positive rateやencoding accuracyではない。 + +```sh +uv run --no-project python models/experimental/paired_controls.py \ + /disk/identity-training.json /disk/filtered-training.json \ + /disk/validation/manifest.json validation \ + build/release/src/libuchardet.a /disk/paired.json +``` + +## 選択と独立性 + +既存のnative比較のtraining検証・split/SHA/origin監査を先に実施する。 +independent sourceがあれば本文読取り前に拒否。corpus全体のhash/codecも再検証する。 +full / complete / text / cp1252の各sourceに、同じsourceのfull / complete / text / UTF-8が +ちょうど1件必要。欠損や重複を黙って除外しない。strict decodeした本文の一致も確認する。 +両variantとも65536 bytes以下。HTML/切り詰めvariantを比較へ混ぜない。 + +## 対照の分類 + +- `bytes_identical`: ASCII等で同じbytesなら、識別可能な負例へ数えない。 +- `control_cp1252_decodable`: UTF-8のbytesがPythonのstrict cp1252 codecでもdecode可能か。 + decode可能でも、元のUnicode本文と同じ意味とは限らない。 +- `stopped_before_filtered_end`: proberの消費文字数がfilter後の長さより小さい文書数。 + confidenceが全文の統計に基づくという誤解を防ぐ。停止理由をこの値だけで確定しない。 + +UTF-8として妥当なbytesでもcp1252としては不正なbyteを含み得る。 +構造的な拒否と、両方にdecodeできるbytes上の統計的な反応差を分離する。 +decodabilityはraw bytesについてのcodec検証、途中終了はfilter後のprober観測であり、 +同じ条件ではない。最終byteでの拒否等は消費長だけでは判断できない。 + +## 観測と集計 + +legacy/identity/filteredを同じlibrary・harnessでbuildし、各variantを新規proberへ一回feedする。 +reportの各文書にはsample encoding、counter/state/confidence bitと由来を記録する。 +encoding別のsummaryを保ち、正例と対照例のconfidenceやstateを混ぜて平均しない。 + +有限かつbytesが異なるpairに限り、cp1252側が高い/同値/UTF-8側が高い件数を集計する。 +非有限・bytes同一は別枠。cp1252へdecode可能かによる層別集計も行う。 +この順序比較は任意のthresholdを後付けで選ぶものではないが、正解率でもない。 +単一proberはUTF-8候補とのランキングを行わず、内部confidenceは確率ではない。 + +既存modelのtraining overlapはUNKNOWN。生成modelの採用・再学習・ratio変更は行わない。 +P01の全detector/大入力停止調査やfuzzを再開するtoolではない。 diff --git a/models/experimental/native_comparison.py b/models/experimental/native_comparison.py index 179f7d7..a8515b1 100644 --- a/models/experimental/native_comparison.py +++ b/models/experimental/native_comparison.py @@ -72,8 +72,7 @@ def summarize(documents): ) -def compare(identity, filtered, manifest, root, split, library, compiler="c++"): - records = select_records(identity, filtered, manifest, root, split) +def observe_models(identity, filtered, records, library, compiler="c++"): library = Path(library).resolve(strict=True) library_hash = digest(library.read_bytes()) profiles, lengths = {}, {} @@ -91,14 +90,16 @@ def compare(identity, filtered, manifest, root, split, library, compiler="c++"): for source, sample, data in records: observed = sequence_probe.observe(binary, data) current = (observed["raw_bytes"], observed["filtered_bytes"]) - if source["id"] in lengths and current != lengths[source["id"]]: + key = (source["id"], sample["id"]) + if key in lengths and current != lengths[key]: raise ValueError("filter lengths differ between models") - lengths[source["id"]] = current + lengths[key] = current documents.append( dict( source=source, sample_sha256=sample["sha256"], sample_id=sample["id"], + sample_encoding=sample["encoding"], encoder=sample["encoder"], encoder_version=sample["encoder_version"], observation=observed, @@ -112,6 +113,12 @@ def compare(identity, filtered, manifest, root, split, library, compiler="c++"): ) if digest(library.read_bytes()) != library_hash: raise ValueError("static library changed during comparison") + return profiles + + +def compare(identity, filtered, manifest, root, split, library, compiler="c++"): + records = select_records(identity, filtered, manifest, root, split) + profiles = observe_models(identity, filtered, records, library, compiler) report = dict( schema="native-french-model-comparison-v1", deployment_status="NOT_ENGINE_CALIBRATED", diff --git a/models/experimental/paired_controls.py b/models/experimental/paired_controls.py new file mode 100644 index 0000000..5fcca5f --- /dev/null +++ b/models/experimental/paired_controls.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: MIT +"""Same-source cp1252/UTF-8 controls for a single French prober, not detector accuracy.""" + +from __future__ import annotations + +import argparse +import json +import math +import struct +from pathlib import Path + +import native_comparison +import sequence_training +from model import canonical, digest, safe_path, write_idempotent +from sequence_contract import content_hash + + +def select_pairs(identity, filtered, manifest, root, split): + # Includes full corpus verification and metadata-only independent/leakage refusal. + positives = native_comparison.select_records(identity, filtered, manifest, root, split) + candidates = {} + for sample in manifest["samples"]: + if ( + sample["encoding"] == "utf-8" + and sample["format"] == "text" + and sample["boundary"] == "complete" + and sample["byte_limit"] is None + ): + candidates.setdefault(sample["source_id"], []).append(sample) + records, pairs = [], [] + for source, positive, data in positives: + matches = candidates.get(source["id"], []) + if len(matches) != 1: + raise ValueError( + "each selected cp1252 document needs exactly one full UTF-8 counterpart" + ) + control = matches[0] + other = safe_path(root, control["path"]).read_bytes() + if len(other) > 65536: + raise ValueError("UTF-8 control exceeds 65536 byte limit") + if data.decode("cp1252", errors="strict") != other.decode("utf-8", errors="strict"): + raise ValueError("paired encodings differ in decoded text") + try: + other.decode("cp1252", errors="strict") + decodable = True + except UnicodeDecodeError: + decodable = False + records.extend(((source, positive, data), (source, control, other))) + pairs.append( + dict( + source_id=source["id"], + positive_sample=positive["id"], + control_sample=control["id"], + bytes_identical=data == other, + control_cp1252_decodable=decodable, + ) + ) + return records, pairs + + +def separation(documents, pairs): + indexed = {document["sample_id"]: document for document in documents} + result = dict( + total_pairs=len(pairs), + byte_identical_pairs=0, + finite_distinct_pairs=0, + nonfinite_distinct_pairs=0, + positive_higher=0, + equal=0, + control_higher=0, + ) + for pair in pairs: + if pair["bytes_identical"]: + result["byte_identical_pairs"] += 1 + continue + values = [ + struct.unpack( + "!f", + bytes.fromhex(indexed[pair[field]]["observation"]["snapshot"]["confidence_bits"]), + )[0] + for field in ("positive_sample", "control_sample") + ] + if not all(math.isfinite(value) for value in values): + result["nonfinite_distinct_pairs"] += 1 + continue + result["finite_distinct_pairs"] += 1 + key = ( + "positive_higher" + if values[0] > values[1] + else "control_higher" + if values[0] < values[1] + else "equal" + ) + result[key] += 1 + return result + + +def compare(identity, filtered, manifest, root, split, library, compiler="c++"): + records, pairs = select_pairs(identity, filtered, manifest, root, split) + profiles = native_comparison.observe_models(identity, filtered, records, library, compiler) + for profile in profiles.values(): + documents = profile["documents"] + profile["summary"] = { + encoding: native_comparison.summarize( + [document for document in documents if document["sample_encoding"] == encoding] + ) + for encoding in ("cp1252", "utf-8") + } + profile["paired_separation"] = separation(documents, pairs) + profile["control_decodability_strata"] = { + name: separation( + documents, [pair for pair in pairs if pair["control_cp1252_decodable"] == value] + ) + for name, value in (("cp1252_decodable", True), ("cp1252_undecodable", False)) + } + for encoding, summary in profile["summary"].items(): + summary["stopped_before_filtered_end"] = sum( + document["observation"]["snapshot"]["total_characters"] + < document["observation"]["filtered_bytes"] + for document in documents + if document["sample_encoding"] == encoding + ) + dependencies = { + path.name: digest(path.read_bytes()) + for path in (Path(__file__), Path(native_comparison.__file__)) + } + report = dict( + schema="paired-french-encoding-controls-v1", + deployment_status="NOT_ENGINE_CALIBRATED", + metric="same-text prober response contrast; NOT false-positive rate or encoding accuracy", + corpus_content_hash=manifest["content_hash"], + split=split, + pairs=pairs, + legacy_training_overlap="UNKNOWN", + runtime=sequence_training.runtime(), + driver_dependencies=dependencies, + profiles=profiles, + ) + report["content_hash"] = content_hash(report) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("identity", type=Path) + parser.add_argument("filtered", type=Path) + parser.add_argument("manifest", type=Path) + parser.add_argument("split", choices=("tuning", "validation")) + parser.add_argument("library", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--cxx", default="c++") + args = parser.parse_args() + + def load(path): + return json.loads(path.read_text(encoding="utf-8")) + + report = compare( + load(args.identity), + load(args.filtered), + load(args.manifest), + args.manifest.parent, + args.split, + args.library, + args.cxx, + ) + write_idempotent(args.output, canonical(report)) + + +if __name__ == "__main__": + main() diff --git a/models/experimental/test_paired_controls.py b/models/experimental/test_paired_controls.py new file mode 100644 index 0000000..00f3d06 --- /dev/null +++ b/models/experimental/test_paired_controls.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: MIT +import copy +import os +import unittest +from pathlib import Path +from unittest.mock import patch + +import paired_controls +import test_filtered_evaluation as fixture +from framework import generate +from model import digest + + +class PairedControlTests(unittest.TestCase): + setUp = fixture.FilteredEvaluationTests.setUp + baseline = fixture.FilteredEvaluationTests.baseline + + def validation(self, text="outside thé café", encodings=("cp1252", "utf-8")): + data = text.encode() + (self.root / "heldout.txt").write_bytes(data) + source = dict( + id="heldout", + path="heldout.txt", + language="fr", + license="MIT", + license_reference="corpus/LICENSES/MIT.txt", + revision="synthetic-v1", + origin="synthetic:paired-heldout", + kind="synthetic", + sha256=digest(data), + split="validation", + ) + root = self.root / "validation" + manifest = generate( + dict( + sources=[source], + encodings=list(encodings), + byte_limits=[None, 3], + formats=["text", "html-clean"], + ), + self.root, + root, + ) + return manifest, root + + def test_pairs_are_same_source_same_text_full_documents(self): + manifest, root = self.validation() + records, pairs = paired_controls.select_pairs( + self.baseline(), self.artifact, manifest, root, "validation" + ) + self.assertEqual(len(records), 2) + self.assertEqual(len(pairs), 1) + self.assertEqual(records[0][0], records[1][0]) + self.assertEqual(records[0][2].decode("cp1252"), records[1][2].decode("utf-8")) + self.assertFalse(pairs[0]["bytes_identical"]) + self.assertEqual([s["encoding"] for _, s, _ in records], ["cp1252", "utf-8"]) + + def test_ascii_marked_ambiguous_not_negative(self): + manifest, root = self.validation("plain ASCII only") + _, pairs = paired_controls.select_pairs( + self.baseline(), self.artifact, manifest, root, "validation" + ) + self.assertTrue(pairs[0]["bytes_identical"]) + summary = paired_controls.separation([], pairs) + self.assertEqual(summary["byte_identical_pairs"], 1) + self.assertEqual(summary["finite_distinct_pairs"], 0) + + def test_utf8_validity_does_not_imply_cp1252_decodability(self): + manifest, root = self.validation("bonjour ” café") + _, pairs = paired_controls.select_pairs( + self.baseline(), self.artifact, manifest, root, "validation" + ) + self.assertFalse(pairs[0]["control_cp1252_decodable"]) + + def test_missing_counterpart_is_error(self): + manifest, root = self.validation(encodings=("cp1252",)) + with self.assertRaisesRegex(ValueError, "exactly one"): + paired_controls.select_pairs( + self.baseline(), self.artifact, manifest, root, "validation" + ) + + def test_control_bytes_revalidated(self): + manifest, root = self.validation() + control = next(s for s in manifest["samples"] if s["encoding"] == "utf-8") + (root / control["path"]).write_bytes(b"changed") + with self.assertRaises(ValueError): + paired_controls.select_pairs( + self.baseline(), self.artifact, manifest, root, "validation" + ) + + def test_independent_refused_before_sample_read(self): + manifest, root = self.validation() + manifest = copy.deepcopy(manifest) + manifest["sources"][0]["split"] = "independent" + with patch.object(paired_controls, "safe_path") as read: + with self.assertRaisesRegex(ValueError, "sealed"): + paired_controls.select_pairs( + self.baseline(), self.artifact, manifest, root, "validation" + ) + read.assert_not_called() + + def test_separation_nonfinite_ties_and_direction(self): + pairs, documents = [], [] + for index, (positive, control) in enumerate( + ( + ("3f800000", "00000000"), + ("00000000", "3f800000"), + ("00000000", "80000000"), + ("7fc00000", "00000000"), + ) + ): + names = [f"{index}-positive", f"{index}-control"] + pairs.append( + dict(positive_sample=names[0], control_sample=names[1], bytes_identical=False) + ) + documents.extend( + dict(sample_id=name, observation=dict(snapshot=dict(confidence_bits=bits))) + for name, bits in zip(names, (positive, control)) + ) + self.assertEqual( + paired_controls.separation(documents, pairs), + dict( + total_pairs=4, + byte_identical_pairs=0, + finite_distinct_pairs=3, + nonfinite_distinct_pairs=1, + positive_higher=1, + equal=1, + control_higher=1, + ), + ) + + @unittest.skipUnless(os.environ.get("UCHARDET_STATIC_LIBRARY"), "native library not configured") + def test_actual_paired_native_observations(self): + manifest, root = self.validation() + report = paired_controls.compare( + self.baseline(), + self.artifact, + manifest, + root, + "validation", + Path(os.environ["UCHARDET_STATIC_LIBRARY"]), + os.environ.get("UCHARDET_PROBE_CXX", "c++"), + ) + for profile in report["profiles"].values(): + self.assertEqual(len(profile["documents"]), 2) + self.assertEqual(profile["summary"]["cp1252"]["documents"], 1) + self.assertEqual(profile["summary"]["utf-8"]["documents"], 1) + self.assertEqual(profile["paired_separation"]["finite_distinct_pairs"], 1) + self.assertEqual( + profile["control_decodability_strata"]["cp1252_decodable"]["total_pairs"], 1 + ) + self.assertEqual(profile["summary"]["utf-8"]["stopped_before_filtered_end"], 0) + + +if __name__ == "__main__": + unittest.main()