Add opt-in native SequenceModel counter probe

.github/workflows/native-dev.yml

@@ -82,6 +82,7 @@ jobs:
       - name: Corpus and model tool tests (offline)
         env:
           UCHARDET_FILTER_PROFILE: build/release/benchmark/uchardet-filter-profile
+          UCHARDET_STATIC_LIBRARY: build/release/src/libuchardet.a
         run: |
           uv run --no-project --python 3.11 python -m unittest discover -s corpus -p 'test_*.py'
           uv run --no-project --python 3.11 python -m unittest discover -s corpus/sources -p 'test_*.py'

models/experimental/SEQUENCE_PROBE.ja.md

@@ -0,0 +1,68 @@
+<!-- SPDX-License-Identifier: MIT -->
+# 非デフォルトの native SequenceModel probe
+
+`sequence_probe.py` / `sequence-probe.cpp` は、明示契約から生成したモデルを
+実際の `nsSingleByteCharSetProber` に渡す診断用harness。
+通常のdetectorに登録せず、libraryのsource/既定model/公開APIを変更しない。
+filterやprober処理の再実装ではなく、既存static libraryの関数を呼ぶ。
+
+## 実行
+
+```sh
+cmake --preset release
+cmake --build --preset release --parallel 2
+uv run --no-project python models/experimental/sequence_probe.py \
+  /disk/contract.json build/release/src/libuchardet.a /disk/sample.bin /disk/probe.json
+```
+
+入力のcontractはtraining artifact全体ではなく、その`contract`フィールド。
+既存SequenceModel validatorとemitterで検証したheaderを一時directoryに生成し、
+trustedなstatic libraryへリンクする。GCC/Clang系driverのC++11 CLIを対象とする。
+`--cxx clang++`で切替可能。MSVC driver用のbuild機構は未実装。
+`clang++`をsymlink実体の`clang`名で起動してlink条件を変えない。
+
+CLIの一時build/inputは最大64 KiBの入力を扱うための小規模診断用。
+compile timeoutは60秒、実行は10秒。自然文生成modelを含むheader/binaryを公開・配布しない。
+Pythonから複数文書を扱う場合は、fresh directoryで`build()`を一度実行し、
+返されたbinaryに対して`observe(binary, bytes)`を文書ごとに呼べる。
+既存binaryの上書きは拒否する。reportの保存には排他的・べき等helperを使う。
+
+## 観測範囲
+
+- 一文書を一回 `FilterWithoutEnglishLettersToBuffer` に通す。
+- filterが空ならproberへfeedしない。非空なら一回だけfeedする。
+- 単一model、non-reversed、補助name proberなし。
+- `state`（0 detecting / 1 found / 2 rejected）、処理文字数、頻出/低頻度/制御文字数、
+  sequence総数、4カテゴリcounter、最後のorderを取得する。
+- confidenceはbinary32 bitをhex文字列として保存する。内部値は負値や1超にもなり得る。
+  確率・公開C APIの最終confidence・較正済みscoreとして表示しない。
+- 同じinstanceをresetした後のcounter/state/confidenceも記録する。
+
+観測用subclassからprotected counterを読むだけで、演算・threshold・state遷移を変更しない。
+実装は独立したMIT wrapper、呼び出すlibraryと生成modelの権利は別。
+新規fileのMITをlibraryやmodelの再ライセンスと扱わない。
+
+## 再現情報と限界
+
+reportは入力/contract/header/library/compiler/binaryのhash、driver名/version、flags、
+wrapper/emitterおよびnative headerのhashを持つ。
+static libraryが申告したsource revisionから作られたことまで自動証明するものではない。
+compiler内部の実行program・標準library・OSを全て固定するbuild manifestでもない。
+build directory/環境が変わるとbinary hashが変わり得るため、観測値の一致とbinaryの一致は
+別々に確認する。生成物を勝手に同一視しない。
+
+このCLIはraw bytesとcontractの低水準診断であり、corpusのsplit/leakage監査は行わない。
+自然文評価では呼出し側がtraining artifact/corpusを検証し、tuning/validationだけを選択する。
+独立holdout、全detectorの候補競合、incremental feed、性能、fuzz、安全性網羅を検証しない。
+P01の調査を再開するための代替toolではない。
+
+## テスト
+
+```sh
+UCHARDET_STATIC_LIBRARY=build/release/src/libuchardet.a \
+  uv run --no-project python -m unittest discover -s models/experimental -p 'test_sequence_probe.py'
+```
+
+人工モデルでcategory0〜3、低頻度letterのnegative加算、空filter、reset、
+1024 sequenceのshortcut境界を検証する。native library未指定ならnative5件はskip。
+CIはLinux diagnosticsで実行する。他OSで実行したとは主張しない。

models/experimental/sequence-probe.cpp

@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: MIT
+// Independent observation harness. Detector/filter implementations stay in the library.
+#include "sequence-model.hpp"
+#include <algorithm>
+#include <cstdint>
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <stdexcept>
+#include <vector>
+
+class SequenceProbe : public nsSingleByteCharSetProber {
+ public:
+  explicit SequenceProbe(const SequenceModel* model) : nsSingleByteCharSetProber(model) {}
+
+  void print() {
+    const float confidence = GetConfidence(0);
+    std::uint32_t bits = 0;
+    static_assert(sizeof(bits) == sizeof(confidence), "binary32 required");
+    std::memcpy(&bits, &confidence, sizeof(bits));
+    std::cout << "{\"state\":" << static_cast<int>(GetState())
+              << ",\"total_characters\":" << mTotalChar
+              << ",\"control_characters\":" << mCtrlChar
+              << ",\"frequent_characters\":" << mFreqChar
+              << ",\"out_characters\":" << mOutChar
+              << ",\"total_sequences\":" << mTotalSeqs
+              << ",\"last_order\":" << static_cast<unsigned>(mLastOrder)
+              << ",\"categories\":[";
+    for (unsigned i = 0; i < NUMBER_OF_SEQ_CAT; ++i) {
+      if (i) std::cout << ',';
+      std::cout << mSeqCounters[i];
+    }
+    // A bit string remains JSON-valid even if a model produces nonfinite confidence.
+    std::cout << "],\"confidence_bits\":\"" << std::hex << std::setw(8)
+              << std::setfill('0') << bits << std::dec << "\"}";
+  }
+};
+
+int main(int argc, char** argv) {
+  try {
+    if (argc != 2) throw std::runtime_error("usage: sequence-probe FILE");
+    std::ifstream input(argv[1], std::ios::binary);
+    if (!input) throw std::runtime_error("cannot open input");
+    const std::size_t limit = 65536;
+    std::vector<char> data(limit + 1);
+    input.read(data.data(), static_cast<std::streamsize>(data.size()));
+    if (input.bad()) throw std::runtime_error("input read failed");
+    data.resize(static_cast<std::size_t>(input.gcount()));
+    if (data.size() > limit) throw std::runtime_error("input exceeds 65536 byte diagnostic limit");
+    std::vector<char> buffer(std::max<std::size_t>(1, data.size()));
+    PRUint32 retained = 0;
+    if (!data.empty()) {
+      nsCharSetProber::FilterWithoutEnglishLettersToBuffer(
+          data.data(), static_cast<PRUint32>(data.size()), buffer.data(), retained);
+    }
+    if (retained > data.size()) throw std::runtime_error("unexpected filter length");
+    SequenceProbe probe(&uchardet_sequence_pilot::model);
+    // Match the SBCS group's empty-filter behavior. This is a single, non-reversed prober.
+    if (retained) probe.HandleData(buffer.data(), retained, nullptr, nullptr);
+    std::cout << "{\"schema\":\"sequence-native-probe-v1\",\"raw_bytes\":" << data.size()
+              << ",\"filtered_bytes\":" << retained << ",\"snapshot\":";
+    probe.print();
+    std::cout << ",\"after_reset\":";
+    probe.Reset();
+    probe.print();
+    std::cout << "}\n";
+    return std::cout ? 0 : 1;
+  } catch (const std::exception& error) {
+    std::cerr << error.what() << '\n';
+    return 1;
+  }
+}

models/experimental/sequence_probe.py

@@ -0,0 +1,122 @@
+# SPDX-License-Identifier: MIT
+"""Build/run an opt-in probe using a validated model and a trusted native static library."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+from model import canonical, digest, write_idempotent
+from sequence_contract import content_hash, emit_cpp, validate
+
+FLAGS = ["-std=c++11", "-O2", "-Wall", "-Wextra", "-Wpedantic"]
+
+
+def build(contract, library, directory, compiler="c++"):
+    validate(contract)
+    executable = shutil.which(compiler)
+    if not executable:
+        raise ValueError("C++ compiler not found")
+    # Keep the driver name: resolving clang++ to clang changes C++ runtime linkage.
+    compiler = Path(executable).absolute()
+    library = Path(library).resolve(strict=True)
+    directory = Path(directory).resolve(strict=True)
+    base = Path(__file__).resolve().parents[2]
+    source = Path(__file__).with_name("sequence-probe.cpp")
+    header = emit_cpp(contract)
+    write_idempotent(directory / "sequence-model.hpp", header)
+    binary = directory / "sequence-probe"
+    if binary.exists():
+        raise ValueError("probe output already exists; use a fresh build directory")
+    dependencies = {
+        str(path.relative_to(base)): digest(path.read_bytes())
+        for path in (
+            source,
+            Path(__file__),
+            Path(__file__).with_name("sequence_contract.py"),
+            Path(__file__).with_name("model.py"),
+            base / "corpus/artifact.py",
+            base / "corpus/framework.py",
+            *sorted((base / "src").glob("*.h")),
+        )
+    }
+    provenance = dict(
+        contract_hash=contract["content_hash"],
+        header_sha256=digest(header),
+        static_library_sha256=digest(library.read_bytes()),
+        compiler_sha256=digest(compiler.read_bytes()),
+        compiler_driver_name=compiler.name,
+        compiler_version=subprocess.run(
+            [str(compiler), "--version"], check=True, capture_output=True, text=True, timeout=10
+        ).stdout,
+        flags=FLAGS.copy(),
+        dependencies=dependencies,
+    )
+    subprocess.run(
+        [
+            str(compiler),
+            *FLAGS,
+            "-I",
+            str(base / "src"),
+            "-I",
+            str(directory),
+            str(source),
+            str(library),
+            "-o",
+            str(binary),
+        ],
+        check=True,
+        capture_output=True,
+        timeout=60,
+    )
+    if digest(library.read_bytes()) != provenance["static_library_sha256"]:
+        raise ValueError("native library changed during build")
+    provenance["probe_binary_sha256"] = digest(binary.read_bytes())
+    return binary, provenance
+
+
+def observe(binary, data):
+    if len(data) > 65536:
+        raise ValueError("probe input exceeds 65536 bytes")
+    with tempfile.TemporaryDirectory(prefix="uchardet-sequence-input-") as temporary:
+        path = Path(temporary) / "input.bin"
+        path.write_bytes(data)
+        result = subprocess.run(
+            [str(binary), str(path)], check=True, capture_output=True, timeout=10
+        )
+    observation = json.loads(result.stdout)
+    if observation.get("schema") != "sequence-native-probe-v1" or observation["raw_bytes"] != len(
+        data
+    ):
+        raise ValueError("unexpected native observation")
+    return observation
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("contract", type=Path)
+    parser.add_argument("library", type=Path)
+    parser.add_argument("input", type=Path)
+    parser.add_argument("output", type=Path)
+    parser.add_argument("--cxx", default="c++")
+    args = parser.parse_args()
+    contract = json.loads(args.contract.read_text(encoding="utf-8"))
+    with args.input.open("rb") as stream:
+        data = stream.read(65537)
+    if len(data) > 65536:
+        parser.error("probe input exceeds 65536 bytes")
+    with tempfile.TemporaryDirectory(prefix="uchardet-sequence-build-") as directory:
+        binary, provenance = build(contract, args.library, directory, args.cxx)
+        result = dict(
+            provenance=provenance, input_sha256=digest(data), observation=observe(binary, data)
+        )
+    result["content_hash"] = content_hash(result)
+    write_idempotent(args.output, canonical(result))
+
+
+if __name__ == "__main__":
+    main()

models/experimental/test_sequence_probe.py

@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: MIT
+import os
+import struct
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import sequence_probe
+from sequence_contract import content_hash
+from test_sequence_contract import fixture
+
+
+class SequenceProbeGuards(unittest.TestCase):
+    def test_input_limit_before_native_execution(self):
+        with patch.object(sequence_probe.subprocess, "run") as run:
+            with self.assertRaisesRegex(ValueError, "65536"):
+                sequence_probe.observe(Path("unused"), b"a" * 65537)
+            run.assert_not_called()
+
+
+@unittest.skipUnless(
+    os.environ.get("UCHARDET_STATIC_LIBRARY"), "native static library not configured"
+)
+class NativeSequenceProbeTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.temporary = tempfile.TemporaryDirectory()
+        cls.addClassCleanup(cls.temporary.cleanup)
+        cls.contract = fixture()
+        # Original artificial model: high-byte frequent letters survive the native filter.
+        cls.contract["byte_to_order"][0xE9] = 0
+        cls.contract["byte_to_order"][0xE0] = 1
+        cls.contract["byte_to_order"][0xE8] = 2  # Valid but outside the frequent matrix.
+        cls.contract["content_hash"] = content_hash(cls.contract)
+        cls.binary, cls.provenance = sequence_probe.build(
+            cls.contract,
+            Path(os.environ["UCHARDET_STATIC_LIBRARY"]),
+            cls.temporary.name,
+            os.environ.get("UCHARDET_PROBE_CXX", "c++"),
+        )
+
+    def observed(self, data):
+        return sequence_probe.observe(self.binary, data)
+
+    def test_native_category_counters_and_negative_confidence(self):
+        observation = self.observed(b"\xe9\xe9\xe0\xe0\xe9")
+        state = observation["snapshot"]
+        self.assertEqual(observation["filtered_bytes"], 5)
+        self.assertEqual(state["total_characters"], 5)
+        self.assertEqual(state["frequent_characters"], 5)
+        self.assertEqual(state["out_characters"], 0)
+        self.assertEqual(state["control_characters"], 0)
+        self.assertEqual(state["total_sequences"], 4)
+        self.assertEqual(state["categories"], [1, 1, 1, 1])
+        self.assertEqual(state["last_order"], 0)
+        self.assertEqual(state["state"], 0)  # Short evidence does not invoke the shortcut.
+        confidence = struct.unpack("!f", bytes.fromhex(state["confidence_bits"]))[0]
+        self.assertAlmostEqual(confidence, -11 / 12, places=6)
+
+    def test_rare_letters_count_as_negative_sequences(self):
+        state = self.observed(b"\xe9\xe8\xe0")["snapshot"]
+        self.assertEqual(state["total_sequences"], 2)
+        self.assertEqual(state["categories"], [2, 0, 0, 0])
+        self.assertEqual(state["frequent_characters"], 2)
+        self.assertEqual(state["out_characters"], 1)
+
+    def test_empty_ascii_and_reset(self):
+        expected = dict(
+            state=0,
+            total_characters=0,
+            control_characters=0,
+            frequent_characters=0,
+            out_characters=0,
+            total_sequences=0,
+            last_order=255,
+            categories=[0, 0, 0, 0],
+            confidence_bits="3c23d70a",
+        )
+        for data in (b"", b"plain ASCII words", b"\xe9\xe0"):
+            observation = self.observed(data)
+            self.assertEqual(observation["after_reset"], expected)
+            if not any(b >= 128 for b in data):
+                self.assertEqual(observation["snapshot"], expected)
+
+    def test_shortcut_threshold_is_strict_and_confidence_is_not_clamped(self):
+        before = self.observed(b"\xe0" * 1025)["snapshot"]
+        after = self.observed(b"\xe0" * 1026)["snapshot"]
+        self.assertEqual(before["total_sequences"], 1024)
+        self.assertEqual(before["state"], 0)
+        self.assertEqual(after["total_sequences"], 1025)
+        self.assertEqual(after["state"], 1)
+        confidence = struct.unpack("!f", bytes.fromhex(after["confidence_bits"]))[0]
+        self.assertGreater(confidence, 1)
+
+    def test_repeat_and_build_provenance(self):
+        self.assertEqual(self.observed(b"caf\xe9"), self.observed(b"caf\xe9"))
+        self.assertEqual(self.provenance["contract_hash"], self.contract["content_hash"])
+        for name in (
+            "header_sha256",
+            "static_library_sha256",
+            "compiler_sha256",
+            "probe_binary_sha256",
+        ):
+            self.assertEqual(len(self.provenance[name]), 64)
+        with self.assertRaisesRegex(ValueError, "already exists"):
+            sequence_probe.build(
+                self.contract, Path(os.environ["UCHARDET_STATIC_LIBRARY"]), self.temporary.name
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()