From f69e46c2095ad06243e9884c2822cd14fec493b0 Mon Sep 17 00:00:00 2001 From: Yoshihiro Misawa Date: Mon, 21 Sep 2026 06:09:02 +0900 Subject: [PATCH] Add opt-in native SequenceModel observation harness --- .github/workflows/native-dev.yml | 1 + models/experimental/SEQUENCE_PROBE.ja.md | 68 ++++++++++++ models/experimental/sequence-probe.cpp | 73 ++++++++++++ models/experimental/sequence_probe.py | 122 +++++++++++++++++++++ models/experimental/test_sequence_probe.py | 113 +++++++++++++++++++ 5 files changed, 377 insertions(+) create mode 100644 models/experimental/SEQUENCE_PROBE.ja.md create mode 100644 models/experimental/sequence-probe.cpp create mode 100644 models/experimental/sequence_probe.py create mode 100644 models/experimental/test_sequence_probe.py diff --git a/.github/workflows/native-dev.yml b/.github/workflows/native-dev.yml index acfcabd..d47a99a 100644 --- a/.github/workflows/native-dev.yml +++ b/.github/workflows/native-dev.yml @@ -82,6 +82,7 @@ jobs: - name: Corpus and model tool tests (offline) env: UCHARDET_FILTER_PROFILE: build/release/benchmark/uchardet-filter-profile + UCHARDET_STATIC_LIBRARY: build/release/src/libuchardet.a run: | uv run --no-project --python 3.11 python -m unittest discover -s corpus -p 'test_*.py' uv run --no-project --python 3.11 python -m unittest discover -s corpus/sources -p 'test_*.py' diff --git a/models/experimental/SEQUENCE_PROBE.ja.md b/models/experimental/SEQUENCE_PROBE.ja.md new file mode 100644 index 0000000..df0fae4 --- /dev/null +++ b/models/experimental/SEQUENCE_PROBE.ja.md @@ -0,0 +1,68 @@ + +# 非デフォルトの native SequenceModel probe + +`sequence_probe.py` / `sequence-probe.cpp` は、明示契約から生成したモデルを +実際の `nsSingleByteCharSetProber` に渡す診断用harness。 +通常のdetectorに登録せず、libraryのsource/既定model/公開APIを変更しない。 +filterやprober処理の再実装ではなく、既存static libraryの関数を呼ぶ。 + +## 実行 + +```sh +cmake --preset release +cmake --build --preset release --parallel 2 +uv run --no-project python models/experimental/sequence_probe.py \ + /disk/contract.json build/release/src/libuchardet.a /disk/sample.bin /disk/probe.json +``` + +入力のcontractはtraining artifact全体ではなく、その`contract`フィールド。 +既存SequenceModel validatorとemitterで検証したheaderを一時directoryに生成し、 +trustedなstatic libraryへリンクする。GCC/Clang系driverのC++11 CLIを対象とする。 +`--cxx clang++`で切替可能。MSVC driver用のbuild機構は未実装。 +`clang++`をsymlink実体の`clang`名で起動してlink条件を変えない。 + +CLIの一時build/inputは最大64 KiBの入力を扱うための小規模診断用。 +compile timeoutは60秒、実行は10秒。自然文生成modelを含むheader/binaryを公開・配布しない。 +Pythonから複数文書を扱う場合は、fresh directoryで`build()`を一度実行し、 +返されたbinaryに対して`observe(binary, bytes)`を文書ごとに呼べる。 +既存binaryの上書きは拒否する。reportの保存には排他的・べき等helperを使う。 + +## 観測範囲 + +- 一文書を一回 `FilterWithoutEnglishLettersToBuffer` に通す。 +- filterが空ならproberへfeedしない。非空なら一回だけfeedする。 +- 単一model、non-reversed、補助name proberなし。 +- `state`（0 detecting / 1 found / 2 rejected）、処理文字数、頻出/低頻度/制御文字数、 + sequence総数、4カテゴリcounter、最後のorderを取得する。 +- confidenceはbinary32 bitをhex文字列として保存する。内部値は負値や1超にもなり得る。 + 確率・公開C APIの最終confidence・較正済みscoreとして表示しない。 +- 同じinstanceをresetした後のcounter/state/confidenceも記録する。 + +観測用subclassからprotected counterを読むだけで、演算・threshold・state遷移を変更しない。 +実装は独立したMIT wrapper、呼び出すlibraryと生成modelの権利は別。 +新規fileのMITをlibraryやmodelの再ライセンスと扱わない。 + +## 再現情報と限界 + +reportは入力/contract/header/library/compiler/binaryのhash、driver名/version、flags、 +wrapper/emitterおよびnative headerのhashを持つ。 +static libraryが申告したsource revisionから作られたことまで自動証明するものではない。 +compiler内部の実行program・標準library・OSを全て固定するbuild manifestでもない。 +build directory/環境が変わるとbinary hashが変わり得るため、観測値の一致とbinaryの一致は +別々に確認する。生成物を勝手に同一視しない。 + +このCLIはraw bytesとcontractの低水準診断であり、corpusのsplit/leakage監査は行わない。 +自然文評価では呼出し側がtraining artifact/corpusを検証し、tuning/validationだけを選択する。 +独立holdout、全detectorの候補競合、incremental feed、性能、fuzz、安全性網羅を検証しない。 +P01の調査を再開するための代替toolではない。 + +## テスト + +```sh +UCHARDET_STATIC_LIBRARY=build/release/src/libuchardet.a \ + uv run --no-project python -m unittest discover -s models/experimental -p 'test_sequence_probe.py' +``` + +人工モデルでcategory0〜3、低頻度letterのnegative加算、空filter、reset、 +1024 sequenceのshortcut境界を検証する。native library未指定ならnative5件はskip。 +CIはLinux diagnosticsで実行する。他OSで実行したとは主張しない。 diff --git a/models/experimental/sequence-probe.cpp b/models/experimental/sequence-probe.cpp new file mode 100644 index 0000000..f52ab11 --- /dev/null +++ b/models/experimental/sequence-probe.cpp @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: MIT +// Independent observation harness. Detector/filter implementations stay in the library. +#include "sequence-model.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +class SequenceProbe : public nsSingleByteCharSetProber { + public: + explicit SequenceProbe(const SequenceModel* model) : nsSingleByteCharSetProber(model) {} + + void print() { + const float confidence = GetConfidence(0); + std::uint32_t bits = 0; + static_assert(sizeof(bits) == sizeof(confidence), "binary32 required"); + std::memcpy(&bits, &confidence, sizeof(bits)); + std::cout << "{\"state\":" << static_cast(GetState()) + << ",\"total_characters\":" << mTotalChar + << ",\"control_characters\":" << mCtrlChar + << ",\"frequent_characters\":" << mFreqChar + << ",\"out_characters\":" << mOutChar + << ",\"total_sequences\":" << mTotalSeqs + << ",\"last_order\":" << static_cast(mLastOrder) + << ",\"categories\":["; + for (unsigned i = 0; i < NUMBER_OF_SEQ_CAT; ++i) { + if (i) std::cout << ','; + std::cout << mSeqCounters[i]; + } + // A bit string remains JSON-valid even if a model produces nonfinite confidence. + std::cout << "],\"confidence_bits\":\"" << std::hex << std::setw(8) + << std::setfill('0') << bits << std::dec << "\"}"; + } +}; + +int main(int argc, char** argv) { + try { + if (argc != 2) throw std::runtime_error("usage: sequence-probe FILE"); + std::ifstream input(argv[1], std::ios::binary); + if (!input) throw std::runtime_error("cannot open input"); + const std::size_t limit = 65536; + std::vector data(limit + 1); + input.read(data.data(), static_cast(data.size())); + if (input.bad()) throw std::runtime_error("input read failed"); + data.resize(static_cast(input.gcount())); + if (data.size() > limit) throw std::runtime_error("input exceeds 65536 byte diagnostic limit"); + std::vector buffer(std::max(1, data.size())); + PRUint32 retained = 0; + if (!data.empty()) { + nsCharSetProber::FilterWithoutEnglishLettersToBuffer( + data.data(), static_cast(data.size()), buffer.data(), retained); + } + if (retained > data.size()) throw std::runtime_error("unexpected filter length"); + SequenceProbe probe(&uchardet_sequence_pilot::model); + // Match the SBCS group's empty-filter behavior. This is a single, non-reversed prober. + if (retained) probe.HandleData(buffer.data(), retained, nullptr, nullptr); + std::cout << "{\"schema\":\"sequence-native-probe-v1\",\"raw_bytes\":" << data.size() + << ",\"filtered_bytes\":" << retained << ",\"snapshot\":"; + probe.print(); + std::cout << ",\"after_reset\":"; + probe.Reset(); + probe.print(); + std::cout << "}\n"; + return std::cout ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/models/experimental/sequence_probe.py b/models/experimental/sequence_probe.py new file mode 100644 index 0000000..7d3fa24 --- /dev/null +++ b/models/experimental/sequence_probe.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: MIT +"""Build/run an opt-in probe using a validated model and a trusted native static library.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import tempfile +from pathlib import Path + +from model import canonical, digest, write_idempotent +from sequence_contract import content_hash, emit_cpp, validate + +FLAGS = ["-std=c++11", "-O2", "-Wall", "-Wextra", "-Wpedantic"] + + +def build(contract, library, directory, compiler="c++"): + validate(contract) + executable = shutil.which(compiler) + if not executable: + raise ValueError("C++ compiler not found") + # Keep the driver name: resolving clang++ to clang changes C++ runtime linkage. + compiler = Path(executable).absolute() + library = Path(library).resolve(strict=True) + directory = Path(directory).resolve(strict=True) + base = Path(__file__).resolve().parents[2] + source = Path(__file__).with_name("sequence-probe.cpp") + header = emit_cpp(contract) + write_idempotent(directory / "sequence-model.hpp", header) + binary = directory / "sequence-probe" + if binary.exists(): + raise ValueError("probe output already exists; use a fresh build directory") + dependencies = { + str(path.relative_to(base)): digest(path.read_bytes()) + for path in ( + source, + Path(__file__), + Path(__file__).with_name("sequence_contract.py"), + Path(__file__).with_name("model.py"), + base / "corpus/artifact.py", + base / "corpus/framework.py", + *sorted((base / "src").glob("*.h")), + ) + } + provenance = dict( + contract_hash=contract["content_hash"], + header_sha256=digest(header), + static_library_sha256=digest(library.read_bytes()), + compiler_sha256=digest(compiler.read_bytes()), + compiler_driver_name=compiler.name, + compiler_version=subprocess.run( + [str(compiler), "--version"], check=True, capture_output=True, text=True, timeout=10 + ).stdout, + flags=FLAGS.copy(), + dependencies=dependencies, + ) + subprocess.run( + [ + str(compiler), + *FLAGS, + "-I", + str(base / "src"), + "-I", + str(directory), + str(source), + str(library), + "-o", + str(binary), + ], + check=True, + capture_output=True, + timeout=60, + ) + if digest(library.read_bytes()) != provenance["static_library_sha256"]: + raise ValueError("native library changed during build") + provenance["probe_binary_sha256"] = digest(binary.read_bytes()) + return binary, provenance + + +def observe(binary, data): + if len(data) > 65536: + raise ValueError("probe input exceeds 65536 bytes") + with tempfile.TemporaryDirectory(prefix="uchardet-sequence-input-") as temporary: + path = Path(temporary) / "input.bin" + path.write_bytes(data) + result = subprocess.run( + [str(binary), str(path)], check=True, capture_output=True, timeout=10 + ) + observation = json.loads(result.stdout) + if observation.get("schema") != "sequence-native-probe-v1" or observation["raw_bytes"] != len( + data + ): + raise ValueError("unexpected native observation") + return observation + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("contract", type=Path) + parser.add_argument("library", type=Path) + parser.add_argument("input", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--cxx", default="c++") + args = parser.parse_args() + contract = json.loads(args.contract.read_text(encoding="utf-8")) + with args.input.open("rb") as stream: + data = stream.read(65537) + if len(data) > 65536: + parser.error("probe input exceeds 65536 bytes") + with tempfile.TemporaryDirectory(prefix="uchardet-sequence-build-") as directory: + binary, provenance = build(contract, args.library, directory, args.cxx) + result = dict( + provenance=provenance, input_sha256=digest(data), observation=observe(binary, data) + ) + result["content_hash"] = content_hash(result) + write_idempotent(args.output, canonical(result)) + + +if __name__ == "__main__": + main() diff --git a/models/experimental/test_sequence_probe.py b/models/experimental/test_sequence_probe.py new file mode 100644 index 0000000..688d812 --- /dev/null +++ b/models/experimental/test_sequence_probe.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: MIT +import os +import struct +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import sequence_probe +from sequence_contract import content_hash +from test_sequence_contract import fixture + + +class SequenceProbeGuards(unittest.TestCase): + def test_input_limit_before_native_execution(self): + with patch.object(sequence_probe.subprocess, "run") as run: + with self.assertRaisesRegex(ValueError, "65536"): + sequence_probe.observe(Path("unused"), b"a" * 65537) + run.assert_not_called() + + +@unittest.skipUnless( + os.environ.get("UCHARDET_STATIC_LIBRARY"), "native static library not configured" +) +class NativeSequenceProbeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temporary = tempfile.TemporaryDirectory() + cls.addClassCleanup(cls.temporary.cleanup) + cls.contract = fixture() + # Original artificial model: high-byte frequent letters survive the native filter. + cls.contract["byte_to_order"][0xE9] = 0 + cls.contract["byte_to_order"][0xE0] = 1 + cls.contract["byte_to_order"][0xE8] = 2 # Valid but outside the frequent matrix. + cls.contract["content_hash"] = content_hash(cls.contract) + cls.binary, cls.provenance = sequence_probe.build( + cls.contract, + Path(os.environ["UCHARDET_STATIC_LIBRARY"]), + cls.temporary.name, + os.environ.get("UCHARDET_PROBE_CXX", "c++"), + ) + + def observed(self, data): + return sequence_probe.observe(self.binary, data) + + def test_native_category_counters_and_negative_confidence(self): + observation = self.observed(b"\xe9\xe9\xe0\xe0\xe9") + state = observation["snapshot"] + self.assertEqual(observation["filtered_bytes"], 5) + self.assertEqual(state["total_characters"], 5) + self.assertEqual(state["frequent_characters"], 5) + self.assertEqual(state["out_characters"], 0) + self.assertEqual(state["control_characters"], 0) + self.assertEqual(state["total_sequences"], 4) + self.assertEqual(state["categories"], [1, 1, 1, 1]) + self.assertEqual(state["last_order"], 0) + self.assertEqual(state["state"], 0) # Short evidence does not invoke the shortcut. + confidence = struct.unpack("!f", bytes.fromhex(state["confidence_bits"]))[0] + self.assertAlmostEqual(confidence, -11 / 12, places=6) + + def test_rare_letters_count_as_negative_sequences(self): + state = self.observed(b"\xe9\xe8\xe0")["snapshot"] + self.assertEqual(state["total_sequences"], 2) + self.assertEqual(state["categories"], [2, 0, 0, 0]) + self.assertEqual(state["frequent_characters"], 2) + self.assertEqual(state["out_characters"], 1) + + def test_empty_ascii_and_reset(self): + expected = dict( + state=0, + total_characters=0, + control_characters=0, + frequent_characters=0, + out_characters=0, + total_sequences=0, + last_order=255, + categories=[0, 0, 0, 0], + confidence_bits="3c23d70a", + ) + for data in (b"", b"plain ASCII words", b"\xe9\xe0"): + observation = self.observed(data) + self.assertEqual(observation["after_reset"], expected) + if not any(b >= 128 for b in data): + self.assertEqual(observation["snapshot"], expected) + + def test_shortcut_threshold_is_strict_and_confidence_is_not_clamped(self): + before = self.observed(b"\xe0" * 1025)["snapshot"] + after = self.observed(b"\xe0" * 1026)["snapshot"] + self.assertEqual(before["total_sequences"], 1024) + self.assertEqual(before["state"], 0) + self.assertEqual(after["total_sequences"], 1025) + self.assertEqual(after["state"], 1) + confidence = struct.unpack("!f", bytes.fromhex(after["confidence_bits"]))[0] + self.assertGreater(confidence, 1) + + def test_repeat_and_build_provenance(self): + self.assertEqual(self.observed(b"caf\xe9"), self.observed(b"caf\xe9")) + self.assertEqual(self.provenance["contract_hash"], self.contract["content_hash"]) + for name in ( + "header_sha256", + "static_library_sha256", + "compiler_sha256", + "probe_binary_sha256", + ): + self.assertEqual(len(self.provenance[name]), 64) + with self.assertRaisesRegex(ValueError, "already exists"): + sequence_probe.build( + self.contract, Path(os.environ["UCHARDET_STATIC_LIBRARY"]), self.temporary.name + ) + + +if __name__ == "__main__": + unittest.main()