From fa32d55a431da8c6354f3052bdc94c1c77c04e62 Mon Sep 17 00:00:00 2001 From: Yoshihiro Misawa Date: Mon, 21 Sep 2026 08:40:32 +0900 Subject: [PATCH 1/2] Freeze bounded French ratio sensitivity experiment --- models/experimental/RATIO_VARIANT.ja.md | 46 +++++++++ models/experimental/ratio_sweep.py | 119 ++++++++++++++++++++++ models/experimental/ratio_variant.py | 59 +++++++++++ models/experimental/test_ratio_sweep.py | 42 ++++++++ models/experimental/test_ratio_variant.py | 50 +++++++++ 5 files changed, 316 insertions(+) create mode 100644 models/experimental/RATIO_VARIANT.ja.md create mode 100644 models/experimental/ratio_sweep.py create mode 100644 models/experimental/ratio_variant.py create mode 100644 models/experimental/test_ratio_sweep.py create mode 100644 models/experimental/test_ratio_variant.py diff --git a/models/experimental/RATIO_VARIANT.ja.md b/models/experimental/RATIO_VARIANT.ja.md new file mode 100644 index 0000000..19a5b84 --- /dev/null +++ b/models/experimental/RATIO_VARIANT.ja.md @@ -0,0 +1,46 @@ + +# French生成モデルのratio感度実験（事前固定） + +目的は、固定した生成modelのconfidence scaleが候補競合に与える影響を切り分けること。 +確率としてのconfidence calibrationや標準採用を完了する実験ではない。 +旧model・engine共通の係数・category table・byte orderは変更しない。 + +## 予測前に固定する条件 + +- training: Paris 24録音版の固定identity / filtered artifact。 +- tuning: 残り8録音、full cp1252/UTF-8の全16入力。各4 KiB以下、fresh/one-shot。 +- ratio因子: 1、19/20、9/10、4/5の4通り。追加探索はこの実験ではしない。 +- 2 profile × 4因子を全て報告し、良いものだけ掲載しない。 +- 各因子は親ratioに適用し、前のvariantへ累積適用しない。 +- factor 1はmodel contractをbyte単位で維持する対照。 +- 判定対象は先頭候補のexact codec / decode-equivalent / language、正解codec候補内存在。 +- 全candidateのconfidence bits、done、候補数・順序を保持する。 +- 各processは10秒上限。timeout/errorを成功・不一致の集計に紛れ込ませない。 +- validationと独立holdoutへ新たな予測を行わない。P01は再開しない。 + +## 次段階へ進める条件 + +同じprofileのfactor 1に対し、cp1252のdecode-equivalentが1件以上改善し、 +UTF-8 exact/decode-equivalent、両encodingのlanguage一致件数に悪化がないこと。 +全出力confidenceが有限かつ[0,1]内にあることも要求する（clampしない）。 +複数候補が満たす場合はcp1252 decode-equivalent、exact codec、factorが1に近い順で選ぶ。 +いずれも満たさなければこのgridの結果は不採択。候補を増やして同じtuningへ再挑戦しない。 +1件改善でも標準採用を意味せず、未使用data・性能・incremental等のgateは残る。 +元modelのconfidenceは保証された確率ではなく、ratioを下げる操作は負の値にも作用する。 + +## Artifact + +`ratio_variant.py PARENT FACTOR OUTPUT`で私的なvariant artifactを作る。 +親training artifactは既存generatorのvalidatorで再検証し、任意のmodel tableを受け入れない。 +親artifact hash、固定因子、変換tool hash、派生contractを記録する。 +`validate(variant, parent)`は変換を再計算して完全一致を要求する。 +licenseはUNDETERMINEDのまま、deployment statusはSENSITIVITY_ONLY_NOT_CALIBRATED。 +このartifactを元のtraining artifactとして扱ったり、元validatorを緩めたりしない。 + +`ratio_sweep.py IDENTITY FILTERED MANIFEST BASELINE BUILDS OUTPUT`で全条件を実行する。 +BASELINEは事前固定した`paris24-tuning-engine-v1.json`で、content hashを固定値と照合する。 +MANIFESTと親artifactもbaselineに記録されたhashへ固定する。 +既存buildはcontract/source/binaryの完全照合後のみ再利用する。途中で失敗したbuildを +削除・上書きして自動再開する処理はない。新しい出力report名を指定して再観測できる。 +因子1と標準targetは保存baselineと完全一致を要求する。 +この文書は結果を観測する前の仕様であり、実測結果と採否は別文書へ記録する。 diff --git a/models/experimental/ratio_sweep.py b/models/experimental/ratio_sweep.py new file mode 100644 index 0000000..d78b63b --- /dev/null +++ b/models/experimental/ratio_sweep.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: MIT +"""Run only the predeclared Paris24 tuning sensitivity experiment.""" +import argparse +import json +import math +from pathlib import Path +import struct + +import engine_comparison +import engine_probe +import paired_controls +import ratio_variant +from model import canonical, digest, write_idempotent +from sequence_contract import content_hash + +BASELINE_HASH = "d34ffb575ce241519978ff704c9da96c821f67f3cae39dff54cbbde23f6ea9a6" + + +def summarize(rows): + result = {} + for encoding in ("cp1252", "utf-8"): + selected = [row for row in rows if row["encoding"] == encoding] + result[encoding] = dict(samples=len(selected), + exact=sum(row["score"]["top1_exact_codec"] for row in selected), + decoded=sum(row["score"]["top1_decode_status"] == "equal" for row in selected), + language=sum(row["score"]["top1_language_match"] for row in selected), + candidate_present=sum(bool(row["score"]["expected_candidate_ranks"]) for row in selected)) + values = [struct.unpack("!f", bytes.fromhex(c["confidence_bits"]))[0] + for row in rows for c in row["observation"]["candidates"]] + result["invalid_confidences"] = sum(not math.isfinite(v) or not 0 <= v <= 1 for v in values) + return result + + +def eligible(candidate, baseline): + return (candidate["invalid_confidences"] == 0 and + candidate["cp1252"]["decoded"] > baseline["cp1252"]["decoded"] and + all(candidate["utf-8"][key] >= baseline["utf-8"][key] for key in ("exact", "decoded")) and + all(candidate[encoding]["language"] >= baseline[encoding]["language"] + for encoding in ("cp1252", "utf-8"))) + + +def run(identity, filtered, manifest, root, baseline, directory): + if baseline.get("content_hash") != BASELINE_HASH or content_hash(baseline) != BASELINE_HASH: + raise ValueError("requires the frozen Paris24 baseline") + parents = {"identity": identity, "filtered": filtered} + if (manifest["content_hash"] != baseline["corpus_content_hash"] or + {name: parent["content_hash"] for name, parent in parents.items()} != baseline["training_hashes"]): + raise ValueError("experiment inputs differ from preregistered baseline") + records, _ = paired_controls.select_pairs(identity, filtered, manifest, root, "tuning") + if len(records) != 16 or any(len(data) > engine_probe.LIMIT for _, _, data in records): + raise ValueError("requires all sixteen bounded tuning inputs") + saved = {row["sample_id"]: row for row in baseline["documents"]} + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + results = {} + for name, parent in parents.items(): + results[name] = {} + for factor in ratio_variant.FACTORS: + variant = ratio_variant.derive(parent, factor) + variant_path = directory / f"{name}-{factor}.json" + write_idempotent(variant_path, canonical(variant)) + build_dir = directory / f"build-{name}-{factor}" + if not build_dir.exists(): + engine_probe.build(variant["contract"], build_dir) + # Reuse is allowed only after exact contract/source/binary verification. + binaries, provenance = engine_comparison.verified_build(build_dir, variant) + rows = [] + for source, sample, data in records: + previous = saved[sample["id"]] + if previous["sample_sha256"] != digest(data): + raise ValueError("baseline input hash mismatch") + normal = engine_probe.observe(binaries["uchardet-conformance"], data) + if normal != previous["observations"]["legacy"]: + raise ValueError("standard engine differs from frozen baseline") + observed = engine_probe.observe(binaries["uchardet-conformance-experimental"], data) + if factor == "1" and observed != previous["observations"][name]: + raise ValueError("factor one differs from frozen baseline") + rows.append(dict(sample_id=sample["id"], sample_sha256=sample["sha256"], + encoding=sample["encoding"], observation=observed, + score=engine_comparison.score(observed, data, sample["encoding"], source["language"]))) + _, after = engine_comparison.verified_build(build_dir, variant) + if after != provenance: + raise ValueError("build changed during observation") + results[name][factor] = dict(variant=variant, build=provenance, + documents=rows, summary=summarize(rows)) + decisions = {} + for name, variants in results.items(): + candidates = [factor for factor, value in variants.items() + if eligible(value["summary"], variants["1"]["summary"])] + candidates.sort(key=lambda factor: ( + variants[factor]["summary"]["cp1252"]["decoded"], + variants[factor]["summary"]["cp1252"]["exact"], float(factor)), reverse=True) + decisions[name] = dict(eligible_factors=candidates, + next_stage_candidate=candidates[0] if candidates else None, + deployment_approved=False) + report = dict(schema="paris24-ratio-sensitivity-v1", baseline_hash=BASELINE_HASH, + results=results, decisions=decisions, + dependencies={path.name: digest(path.read_bytes()) for path in ( + Path(__file__), Path(ratio_variant.__file__), + Path(engine_comparison.__file__), Path(paired_controls.__file__), + Path(__file__).with_name("RATIO_VARIANT.ja.md"))}) + report["content_hash"] = content_hash(report) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("identity", "filtered", "manifest", "baseline", "builds", "output"): + parser.add_argument(name, type=Path) + args = parser.parse_args() + def load(path): + return json.loads(path.read_text(encoding="utf-8")) + report = run(load(args.identity), load(args.filtered), load(args.manifest), + args.manifest.parent, load(args.baseline), args.builds) + write_idempotent(args.output, canonical(report)) + + +if __name__ == "__main__": + main() diff --git a/models/experimental/ratio_variant.py b/models/experimental/ratio_variant.py new file mode 100644 index 0000000..d86abb9 --- /dev/null +++ b/models/experimental/ratio_variant.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: MIT +"""Frozen sensitivity variants, not calibrated probability estimates or deployment models.""" +import argparse +import copy +import json +from pathlib import Path +import struct + +from engine_probe import training_contract +from model import canonical, digest, write_idempotent +from sequence_contract import content_hash, ratio, validate as validate_contract + +FACTORS = {"1": (1, 1), "0.95": (19, 20), "0.90": (9, 10), "0.80": (4, 5)} +PROFILE = "french-ratio-sensitivity-v1" + + +def derive(parent, factor): + if not isinstance(factor, str) or factor not in FACTORS: + raise ValueError("factor must belong to the frozen sensitivity grid") + contract = copy.deepcopy(training_contract(parent)) + if contract["language"] != "fr": + raise ValueError("French-only sensitivity experiment") + numerator, denominator = FACTORS[factor] + tool_hash = digest(Path(__file__).read_bytes()) + if factor != "1": + value = ratio(contract) * numerator / denominator + contract["typical_positive_ratio_bits"] = struct.pack("!f", value).hex() + contract["generation_parameters"]["typical_positive_ratio"] = ( + f"parent binary32 ratio multiplied by {numerator}/{denominator}; rounded to binary32; sensitivity only") + contract["provenance"]["generator_version"] = PROFILE + contract["provenance"]["generator_source_sha256"] = digest(canonical({ + "parent_artifact": parent["content_hash"], "variant_tool": tool_hash})) + contract["content_hash"] = content_hash(contract) + validate_contract(contract) + result = dict(profile=PROFILE, deployment_status="SENSITIVITY_ONLY_NOT_CALIBRATED", + parent_artifact_hash=parent["content_hash"], factor=factor, + factor_fraction=[numerator, denominator], tool_sha256=tool_hash, + contract=contract) + result["content_hash"] = content_hash(result) + return result + + +def validate(artifact, parent): + if canonical(artifact) != canonical(derive(parent, artifact.get("factor"))): + raise ValueError("variant differs from the verified parent and frozen transform") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("parent", type=Path) + parser.add_argument("factor", choices=FACTORS) + parser.add_argument("output", type=Path) + args = parser.parse_args() + parent = json.loads(args.parent.read_text(encoding="utf-8")) + write_idempotent(args.output, canonical(derive(parent, args.factor))) + + +if __name__ == "__main__": + main() diff --git a/models/experimental/test_ratio_sweep.py b/models/experimental/test_ratio_sweep.py new file mode 100644 index 0000000..e700401 --- /dev/null +++ b/models/experimental/test_ratio_sweep.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: MIT +import copy +import unittest +from pathlib import Path +from unittest.mock import patch + +import ratio_sweep + + +class RatioSweepTests(unittest.TestCase): + def test_gate_requires_improvement_without_regressions(self): + base = {"cp1252": {"decoded": 4, "exact": 1, "language": 8}, + "utf-8": {"decoded": 8, "exact": 8, "language": 8}, "invalid_confidences": 0} + self.assertFalse(ratio_sweep.eligible(base, base)) + candidate = copy.deepcopy(base) + candidate["cp1252"]["decoded"] = 5 + self.assertTrue(ratio_sweep.eligible(candidate, base)) + for encoding, key in (("cp1252", "language"), ("utf-8", "language"), + ("utf-8", "exact"), ("utf-8", "decoded")): + bad = copy.deepcopy(candidate) + bad[encoding][key] -= 1 + self.assertFalse(ratio_sweep.eligible(bad, base)) + candidate["invalid_confidences"] = 1 + self.assertFalse(ratio_sweep.eligible(candidate, base)) + + def test_confidence_gate_includes_non_top_candidates(self): + score = dict(top1_exact_codec=True, top1_decode_status="equal", + top1_language_match=True, expected_candidate_ranks=[1]) + for bits in ("7fc00000", "7f800000", "bf000000", "3f800001"): + row = dict(encoding="cp1252", score=score, observation={"candidates": [ + {"confidence_bits": "3f800000"}, {"confidence_bits": bits}]}) + self.assertEqual(ratio_sweep.summarize([row])["invalid_confidences"], 1) + + def test_wrong_baseline_rejected_before_build_or_read(self): + with patch.object(ratio_sweep.engine_probe, "build") as build: + with self.assertRaisesRegex(ValueError, "frozen Paris24"): + ratio_sweep.run(None, None, None, None, {}, Path("unused")) + build.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/models/experimental/test_ratio_variant.py b/models/experimental/test_ratio_variant.py new file mode 100644 index 0000000..1bca7b2 --- /dev/null +++ b/models/experimental/test_ratio_variant.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: MIT +import copy +import unittest + +import ratio_variant as variant +from model import canonical +from sequence_contract import content_hash, ratio +import test_sequence_training as training_fixtures + + +class RatioVariantTests(unittest.TestCase): + def setUp(self): + fixture = training_fixtures.SequenceTrainingTests() + fixture.setUp() + self.addCleanup(fixture.doCleanups) + self.parent = fixture.artifact + + def test_identity_preserves_exact_contract(self): + result = variant.derive(self.parent, "1") + self.assertEqual(canonical(result["contract"]), canonical(self.parent["contract"])) + variant.validate(result, self.parent) + + def test_frozen_grid_preserves_tables_sources_and_parent(self): + before = canonical(self.parent) + for factor in variant.FACTORS: + result = variant.derive(self.parent, factor) + variant.validate(result, self.parent) + self.assertEqual(result, variant.derive(self.parent, factor)) + for field in ("byte_to_order", "pair_categories", "frequent_character_count", + "encoding", "language", "generated_model_license", "keep_english_letters"): + self.assertEqual(result["contract"][field], self.parent["contract"][field]) + self.assertEqual(result["contract"]["provenance"]["sources"], self.parent["contract"]["provenance"]["sources"]) + self.assertAlmostEqual(ratio(result["contract"]), ratio(self.parent["contract"]) * float(factor), places=6) + self.assertEqual(before, canonical(self.parent)) + + def test_arbitrary_factor_and_rehashed_tampering_rejected(self): + for factor in (None, True, 0.9, "0.5", "nan"): + with self.assertRaises(ValueError): + variant.derive(self.parent, factor) + result = variant.derive(self.parent, "0.90") + changed = copy.deepcopy(result) + changed["contract"]["pair_categories"][0] ^= 1 + changed["contract"]["content_hash"] = content_hash(changed["contract"]) + changed["content_hash"] = content_hash(changed) + with self.assertRaises(ValueError): + variant.validate(changed, self.parent) + + +if __name__ == "__main__": + unittest.main() From 24856061216794de492ea141c76ae6558b8a1bd5 Mon Sep 17 00:00:00 2001 From: Yoshihiro Misawa Date: Mon, 21 Sep 2026 08:43:11 +0900 Subject: [PATCH 2/2] Record all preregistered ratio sensitivity outcomes --- models/experimental/RATIO_RESULTS.ja.md | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 models/experimental/RATIO_RESULTS.ja.md diff --git a/models/experimental/RATIO_RESULTS.ja.md b/models/experimental/RATIO_RESULTS.ja.md new file mode 100644 index 0000000..c778f47 --- /dev/null +++ b/models/experimental/RATIO_RESULTS.ja.md @@ -0,0 +1,40 @@ + +# Paris24 ratio感度実験の結果 + +[事前固定した仕様](RATIO_VARIANT.ja.md)と実装をcommit `fa32d55`に保存してから実行した。 +係数未変更baselineのcontent hashは +`d34ffb575ce241519978ff704c9da96c821f67f3cae39dff54cbbde23f6ea9a6`。 +各modelは24録音だけで学習し、別のtuning 8録音の全16 encoding入力を評価した。 +validationの新たな予測や独立holdoutの開封は行っていない。 + +## 全条件の結果 + +cp1252の母数は各8入力。範囲外confidence件数はUTF-8も含む全16入力の全候補で数える。 +全条件でUTF-8 exact/decode-equivalentは8/8、両encodingのlanguage一致は各8/8だった。 + +| model | ratio因子 | cp1252 exact | cp1252 decode-equivalent | 範囲外confidence | 事前基準 | +| --- | ---: | ---: | ---: | ---: | --- | +| identity | 1.00 | 1 | 4 | 0 | 対照 | +| identity | 0.95 | 4 | 7 | 0 | 適格 | +| identity | 0.90 | 8 | 8 | 0 | 次段階候補 | +| identity | 0.80 | 8 | 8 | 1 | 不採択 | +| filtered | 1.00 | 2 | 5 | 0 | 対照 | +| filtered | 0.95 | 5 | 7 | 0 | 適格 | +| filtered | 0.90 | 8 | 8 | 0 | 次段階候補 | +| filtered | 0.80 | 8 | 8 | 2 | 不採択 | + +因子1と保存baseline、および各buildの標準targetとlegacy baselineの候補・bit列・done観測は +完全一致した。正解codecの候補内存在件数は今回exact件数と同じだった。 +0.80の出力はclampせず保存し、事前に定めた[0,1]条件により不採択とした。 + +両profileとも0.90が次段階候補だが、profile間の優劣や標準採用は未決定。 +この8録音に合わせた選択なので、8/8を未使用dataの汎化精度として公表しない。 +次段階では選んだ因子を固定し、未使用data、性能、incremental、confidenceの妥当性、 +他言語・encodingへの影響を別途確認する必要がある。 +同じtuningを使って候補gridを追加・微調整することはしない。 + +private report: `archives/v3-corpus/paris24-ratio-sweep-v1.json`。 +content hash: `8ebbe3a8e939488bf89aa1fa2c3d6169a494c99e2cf23faf9dc37419f93cd6d6`。 +reportは全候補、score、variant、親hash、build provenance、採否を保持する。 +同じ検証済みbuildから2回全条件を観測し、report全体のbyte一致を確認した。 +標準model・公開API・engineの共通係数は変更していない。