From 72e8f72ff1238d1491e39fcc0e9ef24a17de49d2 Mon Sep 17 00:00:00 2001 From: Yoshihiro Misawa Date: Mon, 21 Sep 2026 08:29:15 +0900 Subject: [PATCH] Separate Paris training and tuning by recording identity --- corpus/sources/PARIS_TRAINING.ja.md | 35 ++++++++++++++ corpus/sources/paris-training-tuning.json | 58 +++++++++++++++++++++++ corpus/sources/paris_training.py | 25 +++++++++- corpus/sources/test_paris_training.py | 32 +++++++++++++ 4 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 corpus/sources/paris-training-tuning.json diff --git a/corpus/sources/PARIS_TRAINING.ja.md b/corpus/sources/PARIS_TRAINING.ja.md index 86f1f67..59ae588 100644 --- a/corpus/sources/PARIS_TRAINING.ja.md +++ b/corpus/sources/PARIS_TRAINING.ja.md @@ -68,3 +68,38 @@ full textのUTF-8/cp1252をstrict往復で生成し、変換不能はframework 会話という同一domain内のvalidationであり、独立評価を代替しない。 このPRではmodel再生成・検出精度評価・独立holdout予測は行っていない。 本文やmodelをrepositoryへ公開せず、P01の保留作業も再開しない。 + +## training / tuningを分ける新しいrevision + +係数を調整する場合は既存validationを転用せず、別recipeへ +`"tuning_recordings": 8`を指定する。既知の隔離を適用した後の32録音について、 +recording identity SHA-256の昇順の先頭8録音をtuning、残り24録音をtrainingとする。 +順位は本文、文字コードへの変換可否、入力長、検出結果には依存しない。 +録音内の文を別splitに分けない。少なくとも1録音をtrainingに残す条件を検証する。 + +新profileは`paris-stories-recording-training-tuning-v1`。 +既存recipeにこのfieldがない場合は従来どおり全録音trainingで、旧profileを維持する。 +recipe・出力先を新しくし、既存corpus、validation manifest、modelを上書きしない。 +reportには録音ごとの割当と`previous_training_models_reusable: false`を記録する。 +通常のframeworkで生成した後、既存split/近重複監査を再実行する。 + +32録音すべてで学習した旧モデルは、新しいtuning録音を既に学習しているため、 +新分割での未学習比較には使えない。24録音だけからmodelを再生成する。 +この分割は「未読の独立holdoutを確保した」という主張ではない。既存trainingとして +利用済みの資料を、今後の再学習・較正のために分離する手続きである。 +話者・意味内容の独立性は録音IDの分離だけでは保証しない。 +係数候補や採否基準はtuning予測を見る前に固定し、独立holdoutは引き続き開封しない。 + +固定recipeは`corpus/sources/paris-training-tuning.json`。上記ingest/generate commandの +recipeと出力先を変更して再現する。外部再取得は不要で、既存の検証済みraw cacheを使える。 + +初回の実データ検証ではtraining 24録音 / tuning 8録音、UTF-8/cp1252の64 variantsが +全て成功した。別出力先への再取込・再生成は全fileでbyte一致した。 +tuningのfull入力は1,625〜3,331 bytesで、4 KiB上限の候補競合評価に収まる。 +長さによる録音の再選択は行っていない。 + +- 新manifest content hash: `1da3f89ee4325d1216e79aefc078def8fb4d8fec3730f3893deef4af0648bd19` +- validationを含む48 source / 1,128 pairの近似重複候補: 0 +- overlap report content hash: `b0691327f16ef9447e5b778ba62088adf70877e55175aa0e18fdfda5f1a87434` + +この段階では新modelの学習・tuning予測・係数変更はまだ実施していない。 diff --git a/corpus/sources/paris-training-tuning.json b/corpus/sources/paris-training-tuning.json new file mode 100644 index 0000000..5575e79 --- /dev/null +++ b/corpus/sources/paris-training-tuning.json @@ -0,0 +1,58 @@ +{ + "recipe_version": 1, + "repository": "UniversalDependencies/UD_French-ParisStories", + "revision": "dec76f7a1731318b033c578d534410b0a3d4ea5c", + "license": "CC-BY-SA-4.0", + "upstream_split": "train", + "quarantine_recording_ids": [ + "3d6643a3c3dc8219b68ff1a45c1d48c35635be5dea1ca7ffd61a17d8c9b0235c" + ], + "quarantine_sentence_ids": [ + "ParisStories_2020_maisonAbondonnee_1", + "ParisStories_2020_maisonAbondonnee_2", + "ParisStories_2020_maisonAbondonnee_3", + "ParisStories_2020_maisonAbondonnee_4", + "ParisStories_2020_maisonAbondonnee_5", + "ParisStories_2020_maisonAbondonnee_6", + "ParisStories_2020_maisonAbondonnee_7", + "ParisStories_2020_maisonAbondonnee_8", + "ParisStories_2020_maisonAbondonnee_9", + "ParisStories_2020_maisonAbondonnee_10", + "ParisStories_2020_maisonAbondonnee_11", + "ParisStories_2020_maisonAbondonnee_12", + "ParisStories_2020_maisonAbondonnee_13", + "ParisStories_2020_maisonAbondonnee_14", + "ParisStories_2020_maisonAbondonnee_15", + "ParisStories_2020_maisonAbondonnee_16", + "ParisStories_2020_maisonAbondonnee_17", + "ParisStories_2020_maisonAbondonnee_18", + "ParisStories_2020_maisonAbondonnee_19", + "ParisStories_2020_maisonAbondonnee_20", + "ParisStories_2020_maisonAbondonnee_21", + "ParisStories_2020_maisonAbondonnee_22", + "ParisStories_2020_maisonAbondonnee_23", + "ParisStories_2020_maisonAbondonnee_24", + "ParisStories_2020_maisonAbondonnee_25", + "ParisStories_2020_maisonAbondonnee_26", + "ParisStories_2020_maisonAbondonnee_27" + ], + "validation_manifest_content_hash": "6340aff3b3d424fced87b782d75036c64152b4840eec98a05dc9a5b4f90216aa", + "files": [ + [ + "LICENSE.txt", + "899b1804a12ebc090b96339614eede1b64b686721b650a71430b55b5235f7f79", + 202 + ], + [ + "README.md", + "3a6381529a560d09ffa9746fa6279b1d19ada33bb96386a2fa95e61e74205e91", + 6015 + ], + [ + "fr_parisstories-ud-train.conllu", + "558c125ecb84f4e16ef8d8871368e24b8c6054446a06da9a5211fc67abb65a1d", + 2208034 + ] + ], + "tuning_recordings": 8 +} diff --git a/corpus/sources/paris_training.py b/corpus/sources/paris_training.py index db72036..d6191f4 100644 --- a/corpus/sources/paris_training.py +++ b/corpus/sources/paris_training.py @@ -14,12 +14,29 @@ TEXT_FILE = "fr_parisstories-ud-train.conllu" MAX_FILE_BYTES = 4 * 1024 * 1024 PROFILE = "paris-stories-recording-training-v1" +SPLIT_PROFILE = "paris-stories-recording-training-tuning-v1" + + +def recording_splits(identities, tuning_count): + """Fixed metadata-only ordering; never inspect text or detector predictions.""" + identities = list(identities) + if (type(tuning_count) is not int or tuning_count < 0 or + tuning_count >= len(identities) or len(set(identities)) != len(identities)): + raise ValueError("tuning count must leave at least one training recording") + if any(not re.fullmatch(r"[0-9a-f]{64}", identity) for identity in identities): + raise ValueError("recording identities must be SHA-256") + tuning = set(sorted(identities)[:tuning_count]) + return {identity: "tuning" if identity in tuning else "training" + for identity in sorted(identities)} def validate_recipe(recipe): if (recipe.get("recipe_version") != 1 or recipe.get("repository") != pilot.REPOSITORY or recipe.get("license") != "CC-BY-SA-4.0" or recipe.get("upstream_split") != "train"): raise ValueError("unsupported training recipe") + count = recipe.get("tuning_recordings", 0) + if type(count) is not int or not 0 <= count <= 100: + raise ValueError("invalid tuning recording count") if not re.fullmatch(r"[0-9a-f]{40}", recipe["revision"]): raise ValueError("immutable revision required") if not re.fullmatch(r"[0-9a-f]{64}", recipe["validation_manifest_content_hash"]): @@ -112,6 +129,7 @@ def ingest(recipe, root, validation, output): reason="VALIDATION_RECORDING_OVERLAP")) if not groups: raise ValueError("no admissible training recordings") + assignments = recording_splits(groups, recipe.get("tuning_recordings", 0)) validation_ids = {sid for s in validation["sources"] for sid in s.get("sentence_ids", [])} sources, payloads = [], {} base = f"https://github.com/{pilot.REPOSITORY}/blob/{recipe['revision']}" @@ -126,7 +144,7 @@ def ingest(recipe, root, validation, output): id=source_id, path=relative, language="fr", kind="natural", license=recipe["license"], license_reference=f"{base}/LICENSE.txt", revision=recipe["revision"], origin=f"parisstories:recording:{identity}", - sha256=digest(data), split="training", source_url=f"{base}/{TEXT_FILE}", + sha256=digest(data), split=assignments[identity], source_url=f"{base}/{TEXT_FILE}", source_file_sha256=digest(cached[TEXT_FILE]), upstream_split="train", source_kind="spoken-transcript", extraction_profile=PROFILE, sentence_ids=group["sentence_ids"], recording_identity_sha256=identity, @@ -147,6 +165,11 @@ def ingest(recipe, root, validation, output): p.name: digest(p.read_bytes()) for p in (Path(__file__), Path(pilot.__file__)) }, quarantined_records=quarantine, quarantine_policy="explicit missing-recording/validation-overlap identities; raw retained") + if recipe.get("tuning_recordings", 0): + report.update(profile=SPLIT_PROFILE, source_split="training+tuning", + recording_assignments=assignments, + split_policy="ascending recording SHA-256; first N tuning; remainder training", + previous_training_models_reusable=False) output.mkdir(parents=True) for relative, data in payloads.items(): write_idempotent(safe_path(output, relative), data) diff --git a/corpus/sources/test_paris_training.py b/corpus/sources/test_paris_training.py index 485867a..79106b2 100644 --- a/corpus/sources/test_paris_training.py +++ b/corpus/sources/test_paris_training.py @@ -13,6 +13,38 @@ class ParisTrainingTests(unittest.TestCase): + def test_recording_assignment_is_order_independent(self): + identities = ["3" * 64, "1" * 64, "2" * 64] + expected = {"1" * 64: "tuning", "2" * 64: "training", "3" * 64: "training"} + self.assertEqual(training.recording_splits(identities, 1), expected) + self.assertEqual(training.recording_splits(reversed(identities), 1), expected) + self.assertEqual(set(training.recording_splits(identities, 0).values()), {"training"}) + for invalid in (True, -1, 3, 4, 1.0): + with self.assertRaises(ValueError): + training.recording_splits(identities, invalid) + with self.assertRaises(ValueError): + training.recording_splits(["1" * 64, "1" * 64], 1) + + def test_tuning_recipe_preserves_recordings_and_excludes_old_models(self): + data = self.raw + fixtures.sentence("extra_1", "Autre texte.", "separate").encode() + (self.root / training.TEXT_FILE).write_bytes(data) + self.recipe["files"][-1] = [training.TEXT_FILE, pilot.digest(data), len(data)] + self.recipe["tuning_recordings"] = 1 + report = self.ingest() + config = json.loads((self.root / "output/config.json").read_text()) + self.assertEqual({s["split"] for s in config["sources"]}, {"training", "tuning"}) + self.assertEqual(report["profile"], training.SPLIT_PROFILE) + self.assertFalse(report["previous_training_models_reusable"]) + self.assertEqual(sum(len(s["sentence_ids"]) for s in config["sources"]), 2) + for source in config["sources"]: + self.assertEqual(source["split"], report["recording_assignments"][source["recording_identity_sha256"]]) + + def test_tuning_cannot_take_all_recordings(self): + self.recipe["tuning_recordings"] = 1 + with self.assertRaisesRegex(ValueError, "leave at least one"): + self.ingest() + self.assertFalse((self.root / "output").exists()) + def setUp(self): temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup)