From d6e97b1f896edac2e36d53b762c74dc2e8730a21 Mon Sep 17 00:00:00 2001 From: Yoshihiro Misawa Date: Mon, 21 Sep 2026 08:09:56 +0900 Subject: [PATCH] Add opt-in generated model candidate comparison --- .github/workflows/native-dev.yml | 1 + benchmark/CMakeLists.txt | 6 + benchmark/uchardet-conformance.cpp | 8 ++ models/experimental/ENGINE_PROBE.ja.md | 79 +++++++++++ models/experimental/engine_comparison.py | 121 ++++++++++++++++ models/experimental/engine_probe.py | 132 ++++++++++++++++++ models/experimental/test_engine_comparison.py | 37 +++++ models/experimental/test_engine_probe.py | 75 ++++++++++ src/CMakeLists.txt | 20 +++ src/nsSBCSGroupProber.cpp | 8 ++ 10 files changed, 487 insertions(+) create mode 100644 models/experimental/ENGINE_PROBE.ja.md create mode 100644 models/experimental/engine_comparison.py create mode 100644 models/experimental/engine_probe.py create mode 100644 models/experimental/test_engine_comparison.py create mode 100644 models/experimental/test_engine_probe.py diff --git a/.github/workflows/native-dev.yml b/.github/workflows/native-dev.yml index d47a99a..fbdb1fa 100644 --- a/.github/workflows/native-dev.yml +++ b/.github/workflows/native-dev.yml @@ -83,6 +83,7 @@ jobs: env: UCHARDET_FILTER_PROFILE: build/release/benchmark/uchardet-filter-profile UCHARDET_STATIC_LIBRARY: build/release/src/libuchardet.a + UCHARDET_ENGINE_EXPERIMENT: '1' run: | uv run --no-project --python 3.11 python -m unittest discover -s corpus -p 'test_*.py' uv run --no-project --python 3.11 python -m unittest discover -s corpus/sources -p 'test_*.py' diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 53b1871..a955394 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -7,6 +7,12 @@ target_link_libraries(uchardet-output ${UCHARDET_LIBRARY}) add_executable(uchardet-conformance uchardet-conformance.cpp) target_link_libraries(uchardet-conformance ${UCHARDET_LIBRARY}) +if(TARGET libuchardet_experimental) + add_executable(uchardet-conformance-experimental EXCLUDE_FROM_ALL uchardet-conformance.cpp) + target_compile_definitions(uchardet-conformance-experimental PRIVATE UCHARDET_EXPERIMENTAL_INPUT_LIMIT=4096) + target_link_libraries(uchardet-conformance-experimental libuchardet_experimental) +endif() + option(BUILD_INTROSPECTION "Build internal diagnostic observer (static builds only)" OFF) if(BUILD_INTROSPECTION) if(BUILD_SHARED_LIBS) diff --git a/benchmark/uchardet-conformance.cpp b/benchmark/uchardet-conformance.cpp index a85f039..1edeb1a 100644 --- a/benchmark/uchardet-conformance.cpp +++ b/benchmark/uchardet-conformance.cpp @@ -41,7 +41,15 @@ int main(int argc, char** argv) { for (int file = 3; file < argc; ++file) { std::ifstream stream(argv[file], std::ios::binary); if (!stream) throw std::runtime_error("cannot open input"); +#ifdef UCHARDET_EXPERIMENTAL_INPUT_LIMIT + std::vector bytes(UCHARDET_EXPERIMENTAL_INPUT_LIMIT + 1); + stream.read(bytes.data(), static_cast(bytes.size())); + bytes.resize(static_cast(stream.gcount())); + if (bytes.size() > UCHARDET_EXPERIMENTAL_INPUT_LIMIT) + throw std::runtime_error("experimental input exceeds 4096 bytes"); +#else const std::vector bytes((std::istreambuf_iterator(stream)), std::istreambuf_iterator()); +#endif if (stream.bad()) throw std::runtime_error("cannot read input"); if (!detector || mode == "fresh") detector.reset(uchardet_new()); else uchardet_reset(detector.get()); diff --git a/models/experimental/ENGINE_PROBE.ja.md b/models/experimental/ENGINE_PROBE.ja.md new file mode 100644 index 0000000..3cc10a3 --- /dev/null +++ b/models/experimental/ENGINE_PROBE.ja.md @@ -0,0 +1,79 @@ + +# 生成Frenchモデルを候補群へ接続する非デフォルトtarget + +単一proberの内部スコアだけでは候補競合の結果を評価できないため、French cp1252の +1 slotだけを差し替えた別static libraryと候補観測実行fileをbuildする。 +標準`libuchardet`・CLI・install・Python wheelのmodelを置き換える機能ではない。 +通常のCMake設定では実験target自体が存在せず、明示した場合も`ALL`/installから除外する。 + +## 範囲 + +- `nsSBCSGroupProber`の`Windows_1252FrenchModel`参照だけを実験target内で置き換える。 +- 同じgroupのISO-8859-1/15 French、他言語、UTF-8のlanguage model、ranking処理は維持する。 +- したがってFrenchモデル全体の置換ではなく、1 slotのみ異なるhybrid engineの比較になる。 +- 出力は既存conformance toolの候補数・順序・encoding・language・confidence bit列とdone観測。 +- modelのencoding名が`cp1252`なら、その表記も出力差になる。exact名とcodec互換性を分ける。 +- 評価入口は4 KiB以下、既定はone-shot/fresh。Python側と実験実行file側で上限を確認する。 +- 大入力の停止調査、追加fuzz、P01依存のBOM試作を再開しない。 + +## Build + +driverは現在Linuxのみ。CMake/C++ compiler以外の新しいdependencyは不要。 + +```sh +uv run --no-project python models/experimental/engine_probe.py \ + --reference /disk/engine-reference +uv run --no-project python models/experimental/engine_probe.py \ + --training /disk/frozen-training.json /disk/engine-generated +``` + +`--training`は既存identity/filtered training artifactを、そのhash・依存revision・runtimeを含め +検証してからheaderへ変換する。言語fr、codec cp1252以外は拒否する。 +`--reference`はlegacy tableをコピーせず同じmodelへの参照を接続し、adapterだけで結果が +変わらないことを検査する対照。任意の既存出力directoryは上書きしない。 + +driverは標準buildで実験実行fileが生成されないことを確認した後、明示targetをbuildする。 +両方の実行fileとheader、compiler/CMake情報、compile設定、source hashを記録する。 +生model/headerは私的build artifactとして扱い、repositoryへ追加しない。 + +CMakeを直接使う場合は`BUILD_SHARED_LIBS=OFF`、`BUILD_BENCHMARK=ON`、 +`UCHARDET_EXPERIMENTAL_MODEL_HEADER=/absolute/path/model.hpp`を明示し、 +`uchardet-conformance-experimental`を指定してbuildする。 +この低レベル経路は信頼済みC++ header用で、JSON検証を代行しない。 + +## 比較時の注意 + +`engine_probe.observe(binary, data, chunk=0)`は4 KiBを超える入力を実行前に拒否し、 +各processの実行を10秒で打ち切る。timeoutを正解率・一致として数えない。 +標準conformance実行file自体にはこの専用上限がないため、pilotでは必ずこの入口を使う。 + +trainingと評価dataの重複監査・manifest検証は評価driver側で行う。 +低レベルbuild/observe helperを使っただけでcorpus独立性が保証されるわけではない。 +モデルの較正・encoding accuracy・性能・権利のgateも別であり、接続成功は採用承認ではない。 + +CIは明示環境変数`UCHARDET_ENGINE_EXPERIMENT=1`でreference一致と人工modelの結果変化、 +標準target不変、default build/installからの分離、入力上限を検証する。 +人工modelはこのtestで作った小tableで、自然言語modelの品質を示さない。 + +## Paired corpus評価 + +```sh +uv run --no-project python models/experimental/engine_comparison.py \ + /disk/identity-training.json /disk/filtered-training.json \ + /disk/validation/manifest.json \ + /disk/engine-identity /disk/engine-filtered /disk/comparison.json +``` + +既存paired control validatorでtrainingとの重複、manifest、同一Unicode文書の +full cp1252/UTF-8対を検証する。全入力が4 KiB以下でなければ評価全体を拒否し、 +切り詰めたり大きな文書だけを集計から外したりしない。 +固定training contractとbuild header・source・実行fileのhashを照合し、 +両buildの標準target出力が一致しなければ結果を生成しない。 + +結果には候補一覧とconfidence bit列を保持する。集計ではcodec aliasを正規化した +先頭候補のexact codec、strict decode後の文字列一致、language一致、正解codecの +候補内存在を分離する。decode-equivalentはその入力だけの性質であり、encoding全体の +互換性・superset関係を意味しない。compatible/superset指標は未評価と明記する。 +候補内に正解がないことだけでmodel欠落と断定せず、group内部の選抜も考慮する。 +legacy modelのtraining overlapは不明であり、現行生成modelとの公平な独立学習比較を +保証するものではない。 diff --git a/models/experimental/engine_comparison.py b/models/experimental/engine_comparison.py new file mode 100644 index 0000000..2e9dbb2 --- /dev/null +++ b/models/experimental/engine_comparison.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: MIT +"""Small paired-corpus candidate observations; no default model replacement.""" +import argparse +import codecs +import json +from pathlib import Path + +import engine_probe +import paired_controls +from model import canonical, digest, safe_path, write_idempotent +from sequence_contract import content_hash, emit_cpp + + +def score(record, data, encoding, language): + expected = codecs.lookup(encoding).name + candidates = record["candidates"] + names = [] + for candidate in candidates: + try: + names.append(codecs.lookup(candidate["encoding"]).name) + except (LookupError, TypeError): + names.append(None) + ranks = [i + 1 for i, name in enumerate(names) if name == expected] + status = "no_candidate" + if candidates: + status = "unknown_codec" + if names[0] is not None: + try: + decoded = data.decode(names[0], errors="strict") + status = "equal" if decoded == data.decode(expected, errors="strict") else "different" + except UnicodeError: + status = "decode_error" + return dict(expected_codec=expected, top1_exact_codec=bool(names and names[0] == expected), + top1_decode_status=status, expected_candidate_ranks=ranks, + top1_language_match=bool(candidates and candidates[0]["language"] == language)) + + +def verified_build(directory, training): + directory = Path(directory).resolve(strict=True) + provenance = json.loads((directory / "provenance.json").read_text(encoding="utf-8")) + if provenance["mode"] != "generated" or provenance["contract_hash"] != training["contract"]["content_hash"]: + raise ValueError("build does not match frozen training contract") + header = (directory / "model.hpp").read_bytes() + if header != emit_cpp(training["contract"]) or digest(header) != provenance["model_header_sha256"]: + raise ValueError("generated build header mismatch") + for path, sha in provenance["dependencies"].items(): + if digest(safe_path(engine_probe.BASE, path).read_bytes()) != sha: + raise ValueError("build source revision mismatch") + binaries = {} + for name in ("uchardet-conformance", "uchardet-conformance-experimental"): + binary = directory / "build/benchmark" / name + if digest(binary.read_bytes()) != provenance["binaries"][name]: + raise ValueError("build executable hash mismatch") + binaries[name] = binary + return binaries, provenance + + +def compare(identity, filtered, manifest, root, split, identity_build, filtered_build): + records, pairs = paired_controls.select_pairs(identity, filtered, manifest, root, split) + if any(len(data) > engine_probe.LIMIT for _, _, data in records): + raise ValueError("paired full-engine evaluation requires every input <=4096 bytes") + inputs = (("identity", identity, identity_build), ("filtered", filtered, filtered_build)) + builds = {name: verified_build(directory, training) for name, training, directory in inputs} + documents = [] + for source, sample, data in records: + observed = {} + for name, _, _ in inputs: + binaries, _ = builds[name] + baseline = engine_probe.observe(binaries["uchardet-conformance"], data) + if "legacy" in observed and baseline != observed["legacy"]: + raise ValueError("normal target differs between generated builds") + observed["legacy"] = baseline + observed[name] = engine_probe.observe(binaries["uchardet-conformance-experimental"], data) + documents.append(dict(source=source, sample_id=sample["id"], + sample_sha256=sample["sha256"], sample_encoding=sample["encoding"], + byte_length=len(data), observations=observed, + scores={name: score(value, data, sample["encoding"], source["language"]) + for name, value in observed.items()})) + for name, training, directory in inputs: + _, final = verified_build(directory, training) + if final != builds[name][1]: + raise ValueError("build changed during evaluation") + summary = {} + for encoding in ("cp1252", "utf-8"): + selected = [row for row in documents if row["sample_encoding"] == encoding] + summary[encoding] = {} + for name in ("legacy", "identity", "filtered"): + summary[encoding][name] = dict( + samples=len(selected), exact_codec=sum(d["scores"][name]["top1_exact_codec"] for d in selected), + language_match=sum(d["scores"][name]["top1_language_match"] for d in selected), + expected_candidate_present=sum(bool(d["scores"][name]["expected_candidate_ranks"]) for d in selected), + decode_status={status: sum(d["scores"][name]["top1_decode_status"] == status for d in selected) + for status in ("equal", "different", "decode_error", "unknown_codec", "no_candidate")}, + ) + result = dict(schema="small-paired-engine-comparison-v1", corpus_content_hash=manifest["content_hash"], + split=split, scope="one French cp1252 slot; all other engine/model code unchanged", + input_limit=engine_probe.LIMIT, feed="whole/fresh", pairs=pairs, documents=documents, + summary=summary, builds={name: provenance for name, (_, provenance) in builds.items()}, + training_hashes={"identity": identity["content_hash"], "filtered": filtered["content_hash"]}, + legacy_training_overlap="UNKNOWN", compatible_superset_metric="NOT_EVALUATED", + driver_dependencies={p.name: digest(p.read_bytes()) for p in + (Path(__file__), Path(engine_probe.__file__), Path(paired_controls.__file__))}) + result["content_hash"] = content_hash(result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("identity", "filtered", "manifest", "identity_build", "filtered_build", "output"): + parser.add_argument(name, type=Path) + parser.add_argument("--split", choices=("tuning", "validation"), default="validation") + args = parser.parse_args() + def load(path): + return json.loads(path.read_text(encoding="utf-8")) + result = compare(load(args.identity), load(args.filtered), load(args.manifest), args.manifest.parent, + args.split, args.identity_build, args.filtered_build) + write_idempotent(args.output, canonical(result)) + + +if __name__ == "__main__": + main() diff --git a/models/experimental/engine_probe.py b/models/experimental/engine_probe.py new file mode 100644 index 0000000..3a7b36a --- /dev/null +++ b/models/experimental/engine_probe.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: MIT +"""Opt-in French model connection to a separate, non-installed full-engine target.""" +import argparse +import codecs +import json +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile + +import filtered_training +import sequence_training +from model import canonical, digest, write_idempotent +from sequence_contract import emit_cpp, validate + +BASE = Path(__file__).resolve().parents[2] +LIMIT = 4096 + + +def training_contract(artifact): + if artifact.get("profile") == sequence_training.PROFILE: + sequence_training.validate(artifact) + elif artifact.get("profile") == filtered_training.PROFILE: + filtered_training.validate(artifact) + else: + raise ValueError("unsupported frozen training profile") + return artifact["contract"] + + +def build(contract, directory, compiler="c++"): + """None is a legacy-reference adapter, not a generated replacement.""" + if sys.platform != "linux": + raise ValueError("experimental build driver currently supports Linux only") + if contract is not None: + validate(contract) + if contract["language"] != "fr" or codecs.lookup(contract["encoding"]).name != "cp1252": + raise ValueError("only the French cp1252 slot can be replaced") + header = emit_cpp(contract) + else: + header = (b'#pragma once\n#include "nsSBCharSetProber.h"\n' + b'#include "nsSBCharSetProber-generated.h"\n' + b'namespace uchardet_sequence_pilot {\n' + b'static const SequenceModel& model = Windows_1252FrenchModel;\n}\n') + cmake, cxx = shutil.which("cmake"), shutil.which(compiler) + if not cmake or not cxx: + raise ValueError("CMake and a C++ compiler are required") + directory = Path(directory).absolute() + if directory.exists(): + raise ValueError("use a new experiment directory") + paths = [BASE / "CMakeLists.txt", BASE / "benchmark/CMakeLists.txt", + BASE / "benchmark/uchardet-conformance.cpp", Path(__file__), + Path(__file__).with_name("sequence_contract.py"), Path(__file__).with_name("model.py")] + paths += [p for p in (BASE / "src").rglob("*") if p.is_file() and + (p.suffix in (".cpp", ".h", ".cmake") or p.name == "CMakeLists.txt")] + dependencies = {str(p.relative_to(BASE)): digest(p.read_bytes()) for p in sorted(paths)} + directory.mkdir(parents=True) + model_path = directory / "model.hpp" + write_idempotent(model_path, header) + build_dir = directory / "build" + options = ["-DCMAKE_BUILD_TYPE=Release", "-DBUILD_SHARED_LIBS=OFF", "-DBUILD_BINARY=OFF", + "-DBUILD_TESTING=OFF", "-DBUILD_BENCHMARK=ON", f"-DCMAKE_CXX_COMPILER={cxx}", + f"-DUCHARDET_EXPERIMENTAL_MODEL_HEADER={model_path}"] + subprocess.run([cmake, "-S", str(BASE), "-B", str(build_dir), *options], check=True, + capture_output=True, timeout=60) + names = ("uchardet-conformance", "uchardet-conformance-experimental") + subprocess.run([cmake, "--build", str(build_dir), "--parallel", "2"], + check=True, capture_output=True, timeout=180) + if (build_dir / "benchmark" / names[1]).exists(): + raise ValueError("experimental executable unexpectedly included in default build") + subprocess.run([cmake, "--build", str(build_dir), "--target", names[1], "--parallel", "2"], + check=True, capture_output=True, timeout=180) + if any(digest((BASE / path).read_bytes()) != sha for path, sha in dependencies.items()): + raise ValueError("source changed during build") + binaries = [build_dir / "benchmark" / name for name in names] + provenance = dict( + slot="Windows_1252FrenchModel", mode="reference" if contract is None else "generated", + contract_hash=None if contract is None else contract["content_hash"], + model_header_sha256=digest(header), dependencies=dependencies, + cmake_version=subprocess.run([cmake, "--version"], check=True, capture_output=True, + text=True, timeout=10).stdout, + compiler_version=subprocess.run([cxx, "--version"], check=True, capture_output=True, + text=True, timeout=10).stdout, + compiler_sha256=digest(Path(cxx).read_bytes()), options=options[:-1], + cache_configuration={line.split("=", 1)[0]: line.split("=", 1)[1] + for line in (build_dir / "CMakeCache.txt").read_text().splitlines() + if line.startswith(("CMAKE_CXX_FLAGS", "CMAKE_GENERATOR:", + "CHECK_SSE2:", "TARGET_ARCHITECTURE:"))}, + binaries={p.name: digest(p.read_bytes()) for p in binaries}, + ) + write_idempotent(directory / "provenance.json", canonical(provenance)) + return binaries, provenance + + +def observe(binary, data, chunk=0): + if len(data) > LIMIT: + raise ValueError("full-engine pilot input exceeds 4096 bytes") + if type(chunk) is not int or chunk not in (0, 1, 7, 64, 1024): + raise ValueError("unsupported pilot chunk schedule") + with tempfile.TemporaryDirectory(prefix="uchardet-engine-input-") as temporary: + path = Path(temporary) / "input.bin" + path.write_bytes(data) + result = subprocess.run([str(binary), "fresh", str(chunk), str(path)], check=True, + capture_output=True, timeout=10) + record = json.loads(result.stdout) + if (record.get("schema_version") != 1 or record["input_index"] != 0 or + record["byte_length"] != len(data) or + type(record["candidate_count"]) is not int or + record["candidate_count"] != len(record["candidates"])): + raise ValueError("unexpected candidate observation") + for candidate in record["candidates"]: + if not re.fullmatch(r"[0-9a-f]{8}", candidate["confidence_bits"]): + raise ValueError("invalid confidence bit pattern") + return record + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--training", type=Path) + source.add_argument("--reference", action="store_true") + parser.add_argument("directory", type=Path) + parser.add_argument("--cxx", default="c++") + args = parser.parse_args() + contract = None if args.reference else training_contract( + json.loads(args.training.read_text(encoding="utf-8"))) + build(contract, args.directory, args.cxx) + + +if __name__ == "__main__": + main() diff --git a/models/experimental/test_engine_comparison.py b/models/experimental/test_engine_comparison.py new file mode 100644 index 0000000..c73979b --- /dev/null +++ b/models/experimental/test_engine_comparison.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: MIT +import unittest +from unittest.mock import patch + +import engine_comparison as comparison + + +class EngineComparisonTests(unittest.TestCase): + def test_alias_exact_and_decode_equivalence_are_separate(self): + record = {"candidates": [{"encoding": "ISO-8859-1", "language": "fr"}, + {"encoding": "WINDOWS-1252", "language": "fr"}]} + result = comparison.score(record, b"caf\xe9", "cp1252", "fr") + self.assertFalse(result["top1_exact_codec"]) + self.assertEqual(result["expected_candidate_ranks"], [2]) + self.assertEqual(result["top1_decode_status"], "equal") + self.assertTrue(result["top1_language_match"]) + self.assertEqual(comparison.score(record, b"\x80", "cp1252", "fr")["top1_decode_status"], "different") + record["candidates"].reverse() + self.assertTrue(comparison.score(record, b"\x80", "cp1252", "fr")["top1_exact_codec"]) + + def test_unknown_codec_empty_and_decode_error(self): + for candidates, expected in (([], "no_candidate"), + ([{"encoding": "X-UNKNOWN", "language": None}], "unknown_codec"), + ([{"encoding": "UTF-8", "language": "fr"}], "decode_error")): + result = comparison.score({"candidates": candidates}, b"\xe9", "cp1252", "fr") + self.assertEqual(result["top1_decode_status"], expected) + + def test_large_input_rejected_before_any_build_or_process(self): + with patch.object(comparison.paired_controls, "select_pairs", return_value=([({}, {}, b"a" * 4097)], [])): + with patch.object(comparison, "verified_build") as build: + with self.assertRaisesRegex(ValueError, "4096"): + comparison.compare(None, None, None, None, "validation", None, None) + build.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/models/experimental/test_engine_probe.py b/models/experimental/test_engine_probe.py new file mode 100644 index 0000000..5147aa6 --- /dev/null +++ b/models/experimental/test_engine_probe.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: MIT +import os +import subprocess +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import engine_probe +from sequence_contract import content_hash +from test_sequence_contract import fixture + + +class EngineProbeGuards(unittest.TestCase): + def test_rejects_large_input_and_unknown_schedule_before_execution(self): + with patch.object(engine_probe.subprocess, "run") as run: + for data, chunk in ((b"x" * 4097, 0), (b"", True), (b"", 2)): + with self.assertRaises(ValueError): + engine_probe.observe(Path("unused"), data, chunk) + run.assert_not_called() + + def test_unknown_training_profile_is_not_emitted(self): + with self.assertRaises(ValueError): + engine_probe.training_contract({"profile": "unknown"}) + + def test_rejects_other_language_slot(self): + contract = fixture() + contract["language"] = "de" + contract["provenance"]["sources"][0]["language"] = "de" + contract["content_hash"] = content_hash(contract) + with patch.object(engine_probe.sys, "platform", "linux"): + with self.assertRaises(ValueError): + engine_probe.build(contract, "unused") + + +@unittest.skipUnless(os.environ.get("UCHARDET_ENGINE_EXPERIMENT"), "opt-in engine build disabled") +class NativeEngineProbeTests(unittest.TestCase): + def test_reference_adapter_and_generated_slot(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + reference, _ = engine_probe.build(None, root / "reference") + for file in sorted((engine_probe.BASE / "test/fr").iterdir()): + for chunk in (0, 7): + records = [engine_probe.observe(binary, file.read_bytes(), chunk) for binary in reference] + self.assertEqual(*records) + contract = fixture() + contract["byte_to_order"][0xe9] = 0 + contract["content_hash"] = content_hash(contract) + generated, provenance = engine_probe.build(contract, root / "generated") + self.assertEqual(provenance["contract_hash"], contract["content_hash"]) + # A deliberate low-quality artificial table must affect the candidate records. + data = (engine_probe.BASE / "test/fr/windows-1252.txt").read_bytes() + self.assertNotEqual(*(engine_probe.observe(binary, data) for binary in generated)) + # The normal target in the same build must remain the reference detector. + self.assertEqual(engine_probe.observe(reference[0], data), + engine_probe.observe(generated[0], data)) + oversized = root / "oversized.bin" + oversized.write_bytes(b"a" * 4097) + result = subprocess.run([str(generated[1]), "fresh", "0", str(oversized)], + check=False, capture_output=True, timeout=10) + self.assertNotEqual(result.returncode, 0) + self.assertIn(b"4096", result.stderr) + subprocess.run(["cmake", "--install", str(root / "generated/build"), + "--prefix", str(root / "installed")], check=True, + capture_output=True, timeout=30) + installed = (root / "generated/build/install_manifest.txt").read_text().splitlines() + self.assertTrue(installed) + self.assertFalse(any("experimental" in Path(path).name for path in installed)) + self.assertFalse(any(Path(path).name == "model.hpp" for path in installed)) + with self.assertRaises(ValueError): + engine_probe.build(None, root / "reference") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cd7b3d3..c4c8d48 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -177,6 +177,26 @@ install( include(symbols.cmake) +# Explicitly requested diagnostic library: never installed or part of ALL. +set(UCHARDET_EXPERIMENTAL_MODEL_HEADER "" CACHE FILEPATH + "Private generated French cp1252 model header for experimental diagnostics") +if(UCHARDET_EXPERIMENTAL_MODEL_HEADER) + if(NOT BUILD_BENCHMARK OR BUILD_SHARED_LIBS) + message(FATAL_ERROR "Experimental model requires BUILD_BENCHMARK=ON and BUILD_SHARED_LIBS=OFF") + endif() + if(NOT EXISTS "${UCHARDET_EXPERIMENTAL_MODEL_HEADER}") + message(FATAL_ERROR "Experimental model header does not exist") + endif() + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/experimental") + configure_file("${UCHARDET_EXPERIMENTAL_MODEL_HEADER}" + "${CMAKE_CURRENT_BINARY_DIR}/experimental/uchardet-experimental-model.hpp" COPYONLY) + add_library(libuchardet_experimental STATIC EXCLUDE_FROM_ALL ${UCHARDET_SOURCES}) + target_compile_definitions(libuchardet_experimental PRIVATE + BUILDING_UCHARDET UCHARDET_EXPERIMENTAL_FRENCH_MODEL) + target_include_directories(libuchardet_experimental PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" + PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/experimental") +endif() + if (BUILD_BINARY) add_subdirectory(tools) endif (BUILD_BINARY) diff --git a/src/nsSBCSGroupProber.cpp b/src/nsSBCSGroupProber.cpp index 2eec74a..4af2a98 100644 --- a/src/nsSBCSGroupProber.cpp +++ b/src/nsSBCSGroupProber.cpp @@ -46,6 +46,10 @@ #include "nsHebrewProber.h" +#ifdef UCHARDET_EXPERIMENTAL_FRENCH_MODEL +#include "uchardet-experimental-model.hpp" +#endif + nsSBCSGroupProber::nsSBCSGroupProber() { nsHebrewProber *hebprober = new nsHebrewProber(); @@ -112,7 +116,11 @@ nsSBCSGroupProber::nsSBCSGroupProber() mProbers[n++] = new nsSingleByteCharSetProber(&Iso_8859_1FrenchModel); mProbers[n++] = new nsSingleByteCharSetProber(&Iso_8859_15FrenchModel); +#ifdef UCHARDET_EXPERIMENTAL_FRENCH_MODEL + mProbers[n++] = new nsSingleByteCharSetProber(&uchardet_sequence_pilot::model); +#else mProbers[n++] = new nsSingleByteCharSetProber(&Windows_1252FrenchModel); +#endif mProbers[n++] = new nsSingleByteCharSetProber(&Iso_8859_1SpanishModel); mProbers[n++] = new nsSingleByteCharSetProber(&Iso_8859_15SpanishModel);