From 76a839dc6dfe66dc8cbfbe30fb013159d8cd73bc Mon Sep 17 00:00:00 2001 From: Yoshihiro Misawa Date: Mon, 21 Sep 2026 17:45:44 +0900 Subject: [PATCH] diagnostics: expose cached SBCS selection without score queries --- benchmark/introspection.ja.md | 9 ++++ benchmark/test_selection_trace.py | 72 +++++++++++++++++++++++++++++++ benchmark/uchardet-trace.cpp | 18 +++++++- 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 benchmark/test_selection_trace.py diff --git a/benchmark/introspection.ja.md b/benchmark/introspection.ja.md index 67835c7..dcd2dd5 100644 --- a/benchmark/introspection.ja.md +++ b/benchmark/introspection.ja.md @@ -23,6 +23,15 @@ chunkは0（全体）、1、7、64、1024を指定できます。性能測定に ## 観測できること +`singlebyte_selection`はSBCS groupの既存キャッシュを読み取ります。 +`cached_best_index`は`children.singlebyte_group`およびmodel統計のindexと対応し、 +未選択ならnull。`active_count`はgroupが保持するactive数です。 +`selection_path`はgroupがfoundなら`found_shortcut`、rejectedなら`all_rejected`、 +それ以外は`unknown`です。detecting状態のキャッシュはconfidence照会でも名前取得時の +fallbackでも更新され得るため、indexがあるだけで最大scoreによる選択と断定しません。 +GetConfidence/GetCharSetNameを追加で呼ばず、engineの計算や状態を変更しません。 +SBCS内部の選択とC API全候補の先頭選択も別です。 + JSONLの `initial`／`after_feed`／`after_end` eventには次を記録します。 - `input_state`：0=ASCII、1=escape系、2=high-byte入力。 diff --git a/benchmark/test_selection_trace.py b/benchmark/test_selection_trace.py new file mode 100644 index 0000000..835136a --- /dev/null +++ b/benchmark/test_selection_trace.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: MIT +"""Observe the SBCS choice without triggering an extra score/name query.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +@unittest.skipUnless(os.environ.get("UCHARDET_TRACE"), "set UCHARDET_TRACE") +class SelectionTraceTests(unittest.TestCase): + def trace(self, data, chunk, tool=None): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "input" + path.write_bytes(data) + result = subprocess.run( + [tool or os.environ["UCHARDET_TRACE"], str(chunk), str(path)], + check=True, capture_output=True, text=True, timeout=10, + ) + return [json.loads(line) for line in result.stdout.splitlines()] + + def test_cache_and_active_count_match_children(self): + for data in (b"", b"plain ASCII", "café et thé".encode("cp1252"), + "日本語の文章です。".encode() * 5): + for chunk in (0, 1, 7, 64, 1024): + for record in self.trace(data, chunk): + if record["event"] == "raw_report": + continue + choice = record["singlebyte_selection"] + children = record["children"]["singlebyte_group"] + if children is None: + self.assertIsNone(choice) + continue + self.assertEqual(choice["active_count"], sum( + child["current"]["active"] for child in children)) + index = choice["cached_best_index"] + if index is not None: + self.assertGreaterEqual(index, 0) + self.assertLess(index, len(children)) + self.assertTrue(children[index]["current"]["present"]) + state = record["probers"]["singlebyte_group"] + expected = {"found": "found_shortcut", "rejected": "all_rejected"} + self.assertEqual(choice["selection_path"], expected.get(state, "unknown")) + + def test_final_choice_identifies_reported_statistical_model(self): + records = self.trace("café et thé".encode("cp1252"), 0) + final = records[-1] + self.assertEqual(final["event"], "after_end") + index = final["singlebyte_selection"]["cached_best_index"] + self.assertIsNotNone(index) + models = final["prober_evidence"]["singlebyte_models"] + selected = next(model for model in models if model["prober_index"] == index) + reported = {(r["encoding"], r["language"]) for r in records + if r["event"] == "raw_report"} + self.assertIn((selected["model_encoding"], selected["model_language"]), reported) + + @unittest.skipUnless(os.environ.get("UCHARDET_SELECTION_TRACE_BASELINE"), + "set selection trace baseline") + def test_previous_observations_unchanged(self): + for data in (b"", b"plain ASCII", "café et thé".encode("cp1252"), + "日本語の文章です。".encode() * 5): + for chunk in (0, 1, 7, 64, 1024): + current = self.trace(data, chunk) + for record in current: + record.pop("singlebyte_selection", None) + self.assertEqual(current, self.trace( + data, chunk, os.environ["UCHARDET_SELECTION_TRACE_BASELINE"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmark/uchardet-trace.cpp b/benchmark/uchardet-trace.cpp index 4055c5e..0e2b5de 100644 --- a/benchmark/uchardet-trace.cpp +++ b/benchmark/uchardet-trace.cpp @@ -49,6 +49,20 @@ struct ChildState { // state (Hebrew delegates to the two model probers); do not query scores/names. class UchardetTraceAccess { public: + static void Selection(const nsSBCSGroupProber* group) { + if (!group) { std::cout << "null"; return; } + // Reset initializes this cache. Do not call GetConfidence/GetCharSetName: + // both can update it, destroying the state we intend to observe. + std::cout << "{\"cached_best_index\":"; + if (group->mBestGuess < 0) std::cout << "null"; + else std::cout << group->mBestGuess; + std::cout << ",\"active_count\":" << group->mActiveNum + << ",\"selection_path\":"; + if (group->mState == eFoundIt) quoted("found_shortcut"); + else if (group->mState == eNotMe) quoted("all_rejected"); + else quoted("unknown"); // Could be a score query or a name fallback. + std::cout << '}'; + } static void Machines(const nsMBCSGroupProber* group) { if (!group) { std::cout << "null"; return; } std::cout << '['; @@ -174,7 +188,9 @@ class Observer : public nsUniversalDetector { Children(dynamic_cast(mCharSetProbers[0]), previous_multibyte_); std::cout << ",\"singlebyte_group\":"; Children(dynamic_cast(mCharSetProbers[1]), previous_singlebyte_); - std::cout << "},\"language_detectors\":"; + std::cout << "},\"singlebyte_selection\":"; + UchardetTraceAccess::Selection(dynamic_cast(mCharSetProbers[1])); + std::cout << ",\"language_detectors\":"; UchardetTraceAccess::Languages(dynamic_cast(mCharSetProbers[0])); std::cout << ",\"prober_evidence\":{\"multibyte_machines\":"; UchardetTraceAccess::Machines(dynamic_cast(mCharSetProbers[0]));