diagnostics: SBCSの選択cacheとmodel統計を対応付ける

benchmark/introspection.ja.md

@@ -23,6 +23,15 @@ chunkは0（全体）、1、7、64、1024を指定できます。性能測定に
 
 ## 観測できること
 
+`singlebyte_selection`はSBCS groupの既存キャッシュを読み取ります。
+`cached_best_index`は`children.singlebyte_group`およびmodel統計のindexと対応し、
+未選択ならnull。`active_count`はgroupが保持するactive数です。
+`selection_path`はgroupがfoundなら`found_shortcut`、rejectedなら`all_rejected`、
+それ以外は`unknown`です。detecting状態のキャッシュはconfidence照会でも名前取得時の
+fallbackでも更新され得るため、indexがあるだけで最大scoreによる選択と断定しません。
+GetConfidence/GetCharSetNameを追加で呼ばず、engineの計算や状態を変更しません。
+SBCS内部の選択とC API全候補の先頭選択も別です。
+
 JSONLの `initial`／`after_feed`／`after_end` eventには次を記録します。
 
 - `input_state`：0=ASCII、1=escape系、2=high-byte入力。

benchmark/test_selection_trace.py

@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: MIT
+"""Observe the SBCS choice without triggering an extra score/name query."""
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+
+@unittest.skipUnless(os.environ.get("UCHARDET_TRACE"), "set UCHARDET_TRACE")
+class SelectionTraceTests(unittest.TestCase):
+    def trace(self, data, chunk, tool=None):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "input"
+            path.write_bytes(data)
+            result = subprocess.run(
+                [tool or os.environ["UCHARDET_TRACE"], str(chunk), str(path)],
+                check=True, capture_output=True, text=True, timeout=10,
+            )
+        return [json.loads(line) for line in result.stdout.splitlines()]
+
+    def test_cache_and_active_count_match_children(self):
+        for data in (b"", b"plain ASCII", "café et thé".encode("cp1252"),
+                     "日本語の文章です。".encode() * 5):
+            for chunk in (0, 1, 7, 64, 1024):
+                for record in self.trace(data, chunk):
+                    if record["event"] == "raw_report":
+                        continue
+                    choice = record["singlebyte_selection"]
+                    children = record["children"]["singlebyte_group"]
+                    if children is None:
+                        self.assertIsNone(choice)
+                        continue
+                    self.assertEqual(choice["active_count"], sum(
+                        child["current"]["active"] for child in children))
+                    index = choice["cached_best_index"]
+                    if index is not None:
+                        self.assertGreaterEqual(index, 0)
+                        self.assertLess(index, len(children))
+                        self.assertTrue(children[index]["current"]["present"])
+                    state = record["probers"]["singlebyte_group"]
+                    expected = {"found": "found_shortcut", "rejected": "all_rejected"}
+                    self.assertEqual(choice["selection_path"], expected.get(state, "unknown"))
+
+    def test_final_choice_identifies_reported_statistical_model(self):
+        records = self.trace("café et thé".encode("cp1252"), 0)
+        final = records[-1]
+        self.assertEqual(final["event"], "after_end")
+        index = final["singlebyte_selection"]["cached_best_index"]
+        self.assertIsNotNone(index)
+        models = final["prober_evidence"]["singlebyte_models"]
+        selected = next(model for model in models if model["prober_index"] == index)
+        reported = {(r["encoding"], r["language"]) for r in records
+                    if r["event"] == "raw_report"}
+        self.assertIn((selected["model_encoding"], selected["model_language"]), reported)
+
+    @unittest.skipUnless(os.environ.get("UCHARDET_SELECTION_TRACE_BASELINE"),
+                         "set selection trace baseline")
+    def test_previous_observations_unchanged(self):
+        for data in (b"", b"plain ASCII", "café et thé".encode("cp1252"),
+                     "日本語の文章です。".encode() * 5):
+            for chunk in (0, 1, 7, 64, 1024):
+                current = self.trace(data, chunk)
+                for record in current:
+                    record.pop("singlebyte_selection", None)
+                self.assertEqual(current, self.trace(
+                    data, chunk, os.environ["UCHARDET_SELECTION_TRACE_BASELINE"]))
+
+
+if __name__ == "__main__":
+    unittest.main()

benchmark/uchardet-trace.cpp

@@ -49,6 +49,20 @@ struct ChildState {
 // state (Hebrew delegates to the two model probers); do not query scores/names.
 class UchardetTraceAccess {
 public:
+  static void Selection(const nsSBCSGroupProber* group) {
+    if (!group) { std::cout << "null"; return; }
+    // Reset initializes this cache. Do not call GetConfidence/GetCharSetName:
+    // both can update it, destroying the state we intend to observe.
+    std::cout << "{\"cached_best_index\":";
+    if (group->mBestGuess < 0) std::cout << "null";
+    else std::cout << group->mBestGuess;
+    std::cout << ",\"active_count\":" << group->mActiveNum
+              << ",\"selection_path\":";
+    if (group->mState == eFoundIt) quoted("found_shortcut");
+    else if (group->mState == eNotMe) quoted("all_rejected");
+    else quoted("unknown"); // Could be a score query or a name fallback.
+    std::cout << '}';
+  }
   static void Machines(const nsMBCSGroupProber* group) {
     if (!group) { std::cout << "null"; return; }
     std::cout << '[';
@@ -174,7 +188,9 @@ class Observer : public nsUniversalDetector {
     Children(dynamic_cast<nsMBCSGroupProber*>(mCharSetProbers[0]), previous_multibyte_);
     std::cout << ",\"singlebyte_group\":";
     Children(dynamic_cast<nsSBCSGroupProber*>(mCharSetProbers[1]), previous_singlebyte_);
-    std::cout << "},\"language_detectors\":";
+    std::cout << "},\"singlebyte_selection\":";
+    UchardetTraceAccess::Selection(dynamic_cast<nsSBCSGroupProber*>(mCharSetProbers[1]));
+    std::cout << ",\"language_detectors\":";
     UchardetTraceAccess::Languages(dynamic_cast<nsMBCSGroupProber*>(mCharSetProbers[0]));
     std::cout << ",\"prober_evidence\":{\"multibyte_machines\":";
     UchardetTraceAccess::Machines(dynamic_cast<nsMBCSGroupProber*>(mCharSetProbers[0]));