Paris corpusのtrainingとtuningを録音単位で分離

corpus/sources/PARIS_TRAINING.ja.md

@@ -68,3 +68,38 @@ full textのUTF-8/cp1252をstrict往復で生成し、変換不能はframework
 会話という同一domain内のvalidationであり、独立評価を代替しない。
 このPRではmodel再生成・検出精度評価・独立holdout予測は行っていない。
 本文やmodelをrepositoryへ公開せず、P01の保留作業も再開しない。
+
+## training / tuningを分ける新しいrevision
+
+係数を調整する場合は既存validationを転用せず、別recipeへ
+`"tuning_recordings": 8`を指定する。既知の隔離を適用した後の32録音について、
+recording identity SHA-256の昇順の先頭8録音をtuning、残り24録音をtrainingとする。
+順位は本文、文字コードへの変換可否、入力長、検出結果には依存しない。
+録音内の文を別splitに分けない。少なくとも1録音をtrainingに残す条件を検証する。
+
+新profileは`paris-stories-recording-training-tuning-v1`。
+既存recipeにこのfieldがない場合は従来どおり全録音trainingで、旧profileを維持する。
+recipe・出力先を新しくし、既存corpus、validation manifest、modelを上書きしない。
+reportには録音ごとの割当と`previous_training_models_reusable: false`を記録する。
+通常のframeworkで生成した後、既存split/近重複監査を再実行する。
+
+32録音すべてで学習した旧モデルは、新しいtuning録音を既に学習しているため、
+新分割での未学習比較には使えない。24録音だけからmodelを再生成する。
+この分割は「未読の独立holdoutを確保した」という主張ではない。既存trainingとして
+利用済みの資料を、今後の再学習・較正のために分離する手続きである。
+話者・意味内容の独立性は録音IDの分離だけでは保証しない。
+係数候補や採否基準はtuning予測を見る前に固定し、独立holdoutは引き続き開封しない。
+
+固定recipeは`corpus/sources/paris-training-tuning.json`。上記ingest/generate commandの
+recipeと出力先を変更して再現する。外部再取得は不要で、既存の検証済みraw cacheを使える。
+
+初回の実データ検証ではtraining 24録音 / tuning 8録音、UTF-8/cp1252の64 variantsが
+全て成功した。別出力先への再取込・再生成は全fileでbyte一致した。
+tuningのfull入力は1,625〜3,331 bytesで、4 KiB上限の候補競合評価に収まる。
+長さによる録音の再選択は行っていない。
+
+- 新manifest content hash: `1da3f89ee4325d1216e79aefc078def8fb4d8fec3730f3893deef4af0648bd19`
+- validationを含む48 source / 1,128 pairの近似重複候補: 0
+- overlap report content hash: `b0691327f16ef9447e5b778ba62088adf70877e55175aa0e18fdfda5f1a87434`
+
+この段階では新modelの学習・tuning予測・係数変更はまだ実施していない。

corpus/sources/paris-training-tuning.json

@@ -0,0 +1,58 @@
+{
+  "recipe_version": 1,
+  "repository": "UniversalDependencies/UD_French-ParisStories",
+  "revision": "dec76f7a1731318b033c578d534410b0a3d4ea5c",
+  "license": "CC-BY-SA-4.0",
+  "upstream_split": "train",
+  "quarantine_recording_ids": [
+    "3d6643a3c3dc8219b68ff1a45c1d48c35635be5dea1ca7ffd61a17d8c9b0235c"
+  ],
+  "quarantine_sentence_ids": [
+    "ParisStories_2020_maisonAbondonnee_1",
+    "ParisStories_2020_maisonAbondonnee_2",
+    "ParisStories_2020_maisonAbondonnee_3",
+    "ParisStories_2020_maisonAbondonnee_4",
+    "ParisStories_2020_maisonAbondonnee_5",
+    "ParisStories_2020_maisonAbondonnee_6",
+    "ParisStories_2020_maisonAbondonnee_7",
+    "ParisStories_2020_maisonAbondonnee_8",
+    "ParisStories_2020_maisonAbondonnee_9",
+    "ParisStories_2020_maisonAbondonnee_10",
+    "ParisStories_2020_maisonAbondonnee_11",
+    "ParisStories_2020_maisonAbondonnee_12",
+    "ParisStories_2020_maisonAbondonnee_13",
+    "ParisStories_2020_maisonAbondonnee_14",
+    "ParisStories_2020_maisonAbondonnee_15",
+    "ParisStories_2020_maisonAbondonnee_16",
+    "ParisStories_2020_maisonAbondonnee_17",
+    "ParisStories_2020_maisonAbondonnee_18",
+    "ParisStories_2020_maisonAbondonnee_19",
+    "ParisStories_2020_maisonAbondonnee_20",
+    "ParisStories_2020_maisonAbondonnee_21",
+    "ParisStories_2020_maisonAbondonnee_22",
+    "ParisStories_2020_maisonAbondonnee_23",
+    "ParisStories_2020_maisonAbondonnee_24",
+    "ParisStories_2020_maisonAbondonnee_25",
+    "ParisStories_2020_maisonAbondonnee_26",
+    "ParisStories_2020_maisonAbondonnee_27"
+  ],
+  "validation_manifest_content_hash": "6340aff3b3d424fced87b782d75036c64152b4840eec98a05dc9a5b4f90216aa",
+  "files": [
+    [
+      "LICENSE.txt",
+      "899b1804a12ebc090b96339614eede1b64b686721b650a71430b55b5235f7f79",
+      202
+    ],
+    [
+      "README.md",
+      "3a6381529a560d09ffa9746fa6279b1d19ada33bb96386a2fa95e61e74205e91",
+      6015
+    ],
+    [
+      "fr_parisstories-ud-train.conllu",
+      "558c125ecb84f4e16ef8d8871368e24b8c6054446a06da9a5211fc67abb65a1d",
+      2208034
+    ]
+  ],
+  "tuning_recordings": 8
+}

corpus/sources/paris_training.py

@@ -14,12 +14,29 @@
 TEXT_FILE = "fr_parisstories-ud-train.conllu"
 MAX_FILE_BYTES = 4 * 1024 * 1024
 PROFILE = "paris-stories-recording-training-v1"
+SPLIT_PROFILE = "paris-stories-recording-training-tuning-v1"
+
+
+def recording_splits(identities, tuning_count):
+    """Fixed metadata-only ordering; never inspect text or detector predictions."""
+    identities = list(identities)
+    if (type(tuning_count) is not int or tuning_count < 0 or
+            tuning_count >= len(identities) or len(set(identities)) != len(identities)):
+        raise ValueError("tuning count must leave at least one training recording")
+    if any(not re.fullmatch(r"[0-9a-f]{64}", identity) for identity in identities):
+        raise ValueError("recording identities must be SHA-256")
+    tuning = set(sorted(identities)[:tuning_count])
+    return {identity: "tuning" if identity in tuning else "training"
+            for identity in sorted(identities)}
 
 
 def validate_recipe(recipe):
     if (recipe.get("recipe_version") != 1 or recipe.get("repository") != pilot.REPOSITORY or
             recipe.get("license") != "CC-BY-SA-4.0" or recipe.get("upstream_split") != "train"):
         raise ValueError("unsupported training recipe")
+    count = recipe.get("tuning_recordings", 0)
+    if type(count) is not int or not 0 <= count <= 100:
+        raise ValueError("invalid tuning recording count")
     if not re.fullmatch(r"[0-9a-f]{40}", recipe["revision"]):
         raise ValueError("immutable revision required")
     if not re.fullmatch(r"[0-9a-f]{64}", recipe["validation_manifest_content_hash"]):
@@ -112,6 +129,7 @@ def ingest(recipe, root, validation, output):
                                reason="VALIDATION_RECORDING_OVERLAP"))
     if not groups:
         raise ValueError("no admissible training recordings")
+    assignments = recording_splits(groups, recipe.get("tuning_recordings", 0))
     validation_ids = {sid for s in validation["sources"] for sid in s.get("sentence_ids", [])}
     sources, payloads = [], {}
     base = f"https://github.com/{pilot.REPOSITORY}/blob/{recipe['revision']}"
@@ -126,7 +144,7 @@ def ingest(recipe, root, validation, output):
             id=source_id, path=relative, language="fr", kind="natural",
             license=recipe["license"], license_reference=f"{base}/LICENSE.txt",
             revision=recipe["revision"], origin=f"parisstories:recording:{identity}",
-            sha256=digest(data), split="training", source_url=f"{base}/{TEXT_FILE}",
+            sha256=digest(data), split=assignments[identity], source_url=f"{base}/{TEXT_FILE}",
             source_file_sha256=digest(cached[TEXT_FILE]), upstream_split="train",
             source_kind="spoken-transcript", extraction_profile=PROFILE,
             sentence_ids=group["sentence_ids"], recording_identity_sha256=identity,
@@ -147,6 +165,11 @@ def ingest(recipe, root, validation, output):
                       p.name: digest(p.read_bytes()) for p in (Path(__file__), Path(pilot.__file__))
                   }, quarantined_records=quarantine,
                   quarantine_policy="explicit missing-recording/validation-overlap identities; raw retained")
+    if recipe.get("tuning_recordings", 0):
+        report.update(profile=SPLIT_PROFILE, source_split="training+tuning",
+                      recording_assignments=assignments,
+                      split_policy="ascending recording SHA-256; first N tuning; remainder training",
+                      previous_training_models_reusable=False)
     output.mkdir(parents=True)
     for relative, data in payloads.items():
         write_idempotent(safe_path(output, relative), data)

corpus/sources/test_paris_training.py

@@ -13,6 +13,38 @@
 
 
 class ParisTrainingTests(unittest.TestCase):
+    def test_recording_assignment_is_order_independent(self):
+        identities = ["3" * 64, "1" * 64, "2" * 64]
+        expected = {"1" * 64: "tuning", "2" * 64: "training", "3" * 64: "training"}
+        self.assertEqual(training.recording_splits(identities, 1), expected)
+        self.assertEqual(training.recording_splits(reversed(identities), 1), expected)
+        self.assertEqual(set(training.recording_splits(identities, 0).values()), {"training"})
+        for invalid in (True, -1, 3, 4, 1.0):
+            with self.assertRaises(ValueError):
+                training.recording_splits(identities, invalid)
+        with self.assertRaises(ValueError):
+            training.recording_splits(["1" * 64, "1" * 64], 1)
+
+    def test_tuning_recipe_preserves_recordings_and_excludes_old_models(self):
+        data = self.raw + fixtures.sentence("extra_1", "Autre texte.", "separate").encode()
+        (self.root / training.TEXT_FILE).write_bytes(data)
+        self.recipe["files"][-1] = [training.TEXT_FILE, pilot.digest(data), len(data)]
+        self.recipe["tuning_recordings"] = 1
+        report = self.ingest()
+        config = json.loads((self.root / "output/config.json").read_text())
+        self.assertEqual({s["split"] for s in config["sources"]}, {"training", "tuning"})
+        self.assertEqual(report["profile"], training.SPLIT_PROFILE)
+        self.assertFalse(report["previous_training_models_reusable"])
+        self.assertEqual(sum(len(s["sentence_ids"]) for s in config["sources"]), 2)
+        for source in config["sources"]:
+            self.assertEqual(source["split"], report["recording_assignments"][source["recording_identity_sha256"]])
+
+    def test_tuning_cannot_take_all_recordings(self):
+        self.recipe["tuning_recordings"] = 1
+        with self.assertRaisesRegex(ValueError, "leave at least one"):
+            self.ingest()
+        self.assertFalse((self.root / "output").exists())
+
     def setUp(self):
         temporary = tempfile.TemporaryDirectory()
         self.addCleanup(temporary.cleanup)