Measure static-link allocation calls in single-model reuse

models/experimental/MODEL_ALLOCATIONS.ja.md

@@ -0,0 +1,30 @@
+<!-- SPDX-License-Identifier: MIT -->
+# 単一モデル reuse の allocation call-site 観測
+
+64-bit Linux、GNU-compatible linker、Itanium C++ ABI向けの実験tool。
+engineや既存modelを変更せず、静的libraryへリンクする診断実行ファイルだけをinstrumentする。
+counterはこのsingle-thread tool専用であり、並列測定やthread-safeな汎用allocator監視には使わない。
+
+```sh
+uv run --no-project python models/experimental/model_allocations.py \
+  identity-training.json filtered-training.json corpus/manifest.json \
+  build/src/libuchardet.a allocation-report.json
+```
+
+同じfrozen trainingとvalidation/tuning corpusを用いる。独立holdoutを含むmanifestは拒否する。
+legacy/identity/filteredそれぞれで通常版と計測版をbuildし、最終snapshot/reset結果を照合する。
+128回のwarm-up後、1回のreset/filter/feed/confidenceについて呼び出し回数を記録する。
+input/scratch/prober確保、入出力、JSON生成は区間外。時間計測との併用は拒否する。
+
+計測対象はlinker `--wrap`で捕捉できるmalloc/calloc/realloc/freeと、throwing・unalignedの
+scalar/array new/deleteの呼び出し。起動時に全8経路を実際に呼び、期待回数との一致を確認する。
+計測の自己検証に失敗した場合、0回として成功を報告しない。
+
+**物理的なallocation数、確保byte数、peak/live memoryの測定ではない。**
+shared library内部の呼び出し、aligned/nothrow/sized/custom allocator、mmapは捕捉しない。
+libraryのLTOや独自allocatorは想定しない。全detectorのallocation gateを満たす証拠にもならない。
+0回という結果は、選択した入力と区間の対象call-siteだけに限定する。
+API呼び出しの入れ子を物理allocationとして合算しない。各counterを個別に保持する。
+
+compiler/flags、static library、診断source、generated header、実行fileのhashを保存する。
+生成modelを公開せず、計測結果を理由に既定modelへ採用しない。

models/experimental/allocation-hooks.cpp

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: MIT
+#include "allocation-hooks.hpp"
+#include <cstddef>
+#include <cstdlib>
+#include <new>
+#include <stdexcept>
+
+bool allocation_tracking = false;
+AllocationCalls allocation_calls = {};
+
+extern "C" {
+void* __real_malloc(std::size_t);
+void* __real_calloc(std::size_t, std::size_t);
+void* __real_realloc(void*, std::size_t);
+void __real_free(void*);
+void* __real__Znwm(std::size_t);
+void* __real__Znam(std::size_t);
+void __real__ZdlPv(void*);
+void __real__ZdaPv(void*);
+
+void* __wrap_malloc(std::size_t n) {
+  if (allocation_tracking) ++allocation_calls.malloc_calls;
+  return __real_malloc(n);
+}
+void* __wrap_calloc(std::size_t n, std::size_t size) {
+  if (allocation_tracking) ++allocation_calls.calloc_calls;
+  return __real_calloc(n, size);
+}
+void* __wrap_realloc(void* p, std::size_t n) {
+  if (allocation_tracking) ++allocation_calls.realloc_calls;
+  return __real_realloc(p, n);
+}
+void __wrap_free(void* p) {
+  if (allocation_tracking) ++allocation_calls.free_calls;
+  __real_free(p);
+}
+void* __wrap__Znwm(std::size_t n) {
+  if (allocation_tracking) ++allocation_calls.new_calls;
+  return __real__Znwm(n);
+}
+void* __wrap__Znam(std::size_t n) {
+  if (allocation_tracking) ++allocation_calls.new_array_calls;
+  return __real__Znam(n);
+}
+void __wrap__ZdlPv(void* p) {
+  if (allocation_tracking) ++allocation_calls.delete_calls;
+  __real__ZdlPv(p);
+}
+void __wrap__ZdaPv(void* p) {
+  if (allocation_tracking) ++allocation_calls.delete_array_calls;
+  __real__ZdaPv(p);
+}
+}
+
+void allocation_self_test() {
+  // Volatile function pointers prevent optimizing away calibration calls.
+  void* (*volatile allocate)(std::size_t) = &std::malloc;
+  void* (*volatile zero_allocate)(std::size_t, std::size_t) = &std::calloc;
+  void* (*volatile resize)(void*, std::size_t) = &std::realloc;
+  void (*volatile release)(void*) = &std::free;
+  void* (*volatile cpp_allocate)(std::size_t) = &::operator new;
+  void* (*volatile array_allocate)(std::size_t) = &::operator new[];
+  void (*volatile cpp_release)(void*) = &::operator delete;
+  void (*volatile array_release)(void*) = &::operator delete[];
+  allocation_calls = {};
+  allocation_tracking = true;
+  void* p = allocate(16);
+  void* q = zero_allocate(1, 16);
+  void* resized = resize(p, 32);
+  release(resized ? resized : p);
+  release(q);
+  cpp_release(cpp_allocate(16));
+  array_release(array_allocate(16));
+  allocation_tracking = false;
+  const AllocationCalls& c = allocation_calls;
+  if (c.malloc_calls != 1 || c.calloc_calls != 1 || c.realloc_calls != 1 ||
+      c.free_calls != 2 || c.new_calls != 1 || c.new_array_calls != 1 ||
+      c.delete_calls != 1 || c.delete_array_calls != 1)
+    throw std::runtime_error("allocation call-site calibration failed");
+  allocation_calls = {};
+}

models/experimental/allocation-hooks.hpp

@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: MIT
+// Linux/Itanium ABI, static-link call-site instrumentation; not a heap profiler.
+#pragma once
+#include <cstdint>
+
+struct AllocationCalls {
+  std::uint64_t malloc_calls, calloc_calls, realloc_calls, free_calls;
+  std::uint64_t new_calls, new_array_calls, delete_calls, delete_array_calls;
+};
+extern bool allocation_tracking;
+extern AllocationCalls allocation_calls;
+void allocation_self_test();

models/experimental/model_allocations.py

@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: MIT
+"""Observe selected static-link allocation call sites during warmed single-prober reuse."""
+
+import argparse
+import json
+import tempfile
+from pathlib import Path
+
+import native_comparison
+import sequence_probe
+from model import canonical, digest, write_idempotent
+from sequence_contract import content_hash
+
+
+def run(identity, filtered, manifest, root, split, library, compiler="c++"):
+    records = native_comparison.select_records(
+        identity, filtered, manifest, root, split
+    )
+    library = Path(library).resolve(strict=True)
+    library_hash = digest(library.read_bytes())
+    profiles = {}
+    with tempfile.TemporaryDirectory(prefix="uchardet-allocations-") as temporary:
+        for name, training in (
+            ("legacy", None),
+            ("identity", identity),
+            ("filtered", filtered),
+        ):
+            builds = []
+            for enabled in (False, True):
+                directory = Path(temporary) / f"{name}-{enabled}"
+                directory.mkdir()
+                build = (
+                    sequence_probe.build_reference(
+                        library, directory, compiler, allocations=enabled
+                    )
+                    if training is None
+                    else sequence_probe.build(
+                        training["contract"],
+                        library,
+                        directory,
+                        compiler,
+                        allocations=enabled,
+                    )
+                )
+                if build[1]["static_library_sha256"] != library_hash:
+                    raise ValueError("library changed between builds")
+                builds.append(build)
+            observations = []
+            for source, sample, data in records:
+                plain = sequence_probe.observe(builds[0][0], data)
+                counted = sequence_probe.observe(builds[1][0], data)
+                calls = counted.pop("allocation_calls")
+                if counted != plain:
+                    raise ValueError("instrumented observation differs from reference")
+                observations.append(
+                    dict(
+                        source=source,
+                        sample_sha256=sample["sha256"],
+                        byte_length=len(data),
+                        calls=calls,
+                        observation=plain,
+                    )
+                )
+            profiles[name] = dict(
+                training_artifact_hash=training["content_hash"] if training else None,
+                baseline_provenance=builds[0][1],
+                counted_provenance=builds[1][1],
+                observations=observations,
+            )
+    if digest(library.read_bytes()) != library_hash:
+        raise ValueError("library changed during observation")
+    report = dict(
+        schema="native-model-allocation-calls-v1",
+        corpus_content_hash=manifest["content_hash"],
+        split=split,
+        warmup_iterations=128,
+        observed_iterations=1,
+        scope="static-link calls to malloc/calloc/realloc/free and throwing unaligned new/delete",
+        excluded="shared-library internals, aligned/nothrow/sized/custom allocators, mmap, setup, I/O",
+        metric="API call counts, not physical allocations, live bytes, peak memory or latency",
+        driver_dependencies={
+            path.name: digest(path.read_bytes())
+            for path in (Path(__file__), Path(native_comparison.__file__))
+        },
+        profiles=profiles,
+    )
+    report["content_hash"] = content_hash(report)
+    return report
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name in ("identity", "filtered", "manifest", "library", "output"):
+        parser.add_argument(name, type=Path)
+    parser.add_argument(
+        "--split", choices=("tuning", "validation"), default="validation"
+    )
+    parser.add_argument("--cxx", default="c++")
+    args = parser.parse_args()
+
+    def load(path):
+        return json.loads(path.read_text(encoding="utf-8"))
+
+    result = run(
+        load(args.identity),
+        load(args.filtered),
+        load(args.manifest),
+        args.manifest.parent,
+        args.split,
+        args.library,
+        args.cxx,
+    )
+    write_idempotent(args.output, canonical(result))
+
+
+if __name__ == "__main__":
+    main()

models/experimental/sequence-probe.cpp

@@ -11,6 +11,9 @@
 #include <stdexcept>
 #include <string>
 #include <vector>
+#ifdef UCHARDET_ALLOCATION_PROBE
+#include "allocation-hooks.hpp"
+#endif
 
 class SequenceProbe : public nsSingleByteCharSetProber {
  public:
@@ -43,6 +46,10 @@ int main(int argc, char** argv) {
   try {
     if (argc != 2 && argc != 3)
       throw std::runtime_error("usage: sequence-probe FILE [ITERATIONS]");
+#ifdef UCHARDET_ALLOCATION_PROBE
+    if (argc != 2) throw std::runtime_error("allocation and timing are separate modes");
+    allocation_self_test();
+#endif
     std::uint64_t iterations = 0;
     if (argc == 3) {
       const std::string argument(argv[2]);
@@ -79,6 +86,12 @@ int main(int argc, char** argv) {
       return bits;
     };
     run();
+#ifdef UCHARDET_ALLOCATION_PROBE
+    for (unsigned i = 0; i < 128; ++i) run();
+    allocation_tracking = true;
+    run();
+    allocation_tracking = false;
+#endif
     std::int64_t elapsed_ns = 0;
     volatile std::uint64_t checksum = 0;
     const unsigned warmup = 128;
@@ -97,6 +110,15 @@ int main(int argc, char** argv) {
               << "\",\"model_frequent_count\":" << uchardet_sequence_pilot::model.freqCharCount
               << ",\"snapshot\":";
     probe.print();
+#ifdef UCHARDET_ALLOCATION_PROBE
+    const AllocationCalls& c = allocation_calls;
+    std::cout << ",\"allocation_calls\":{\"malloc\":" << c.malloc_calls
+              << ",\"calloc\":" << c.calloc_calls << ",\"realloc\":" << c.realloc_calls
+              << ",\"free\":" << c.free_calls << ",\"new\":" << c.new_calls
+              << ",\"new_array\":" << c.new_array_calls
+              << ",\"delete\":" << c.delete_calls
+              << ",\"delete_array\":" << c.delete_array_calls << '}';
+#endif
     std::cout << ",\"after_reset\":";
     probe.Reset();
     probe.print();