feat: read DSL files and group their lines into cards

Package.swift

@@ -144,9 +144,10 @@ let package = Package(
         .testTarget(
             name: "HanReaderCoreTests",
             dependencies: ["HanReaderCore"],
-            // Real CC-CEDICT data, a small slice extracted by
-            // Scripts/extract-cedict-fixture.py so its provenance is
-            // documented and it can be regenerated rather than hand-edited.
+            // Small slices of the real CC-CEDICT and 大БКРС data, each
+            // extracted by a committed script (Scripts/extract-cedict-fixture.py
+            // and Scripts/extract-dsl-fixture.py) so their provenance is
+            // documented and they can be regenerated rather than hand-edited.
             //
             // Kept inside the target directory rather than at Tests/Fixtures:
             // a resource path reaching outside the target works on the current

Scripts/extract-dsl-fixture.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""Extract the committed BKRS test fixture from a full 大БКРС DSL set.
+
+The fixture is a deliberately small slice of real DSL data, chosen to cover
+every shape the importer has to handle. It is committed so the tests are
+hermetic and fast; this script exists so its provenance is documented and the
+slice can be regenerated rather than hand-maintained.
+
+    ./Scripts/extract-dsl-fixture.py path/to/dabkrs_1.dsl
+
+`#INCLUDE` directives are followed, so pointing it at the master file is
+enough. Writes Tests/HanReaderCoreTests/Fixtures/bkrs-slice.dsl.txt.
+
+The output is **UTF-8 with no byte-order mark**, where the source is
+UTF-16LE+BOM. That is on purpose twice over: a UTF-8 file stays diffable in
+git, and the tests re-encode it to UTF-16LE+BOM at runtime and assert both
+readings produce identical cards — so the committed bytes are reviewable and
+the real encoding path is still exercised.
+
+**Licensing.** 大БКРС is not freely licensed and is not ours to redistribute.
+HanReader ships an importer, not the data. This slice exists under the one
+exception CONTRIBUTING.md names: a ~200-card test fixture, regenerable by this
+script. Do not enlarge it, and do not commit the source files.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import re
+import sys
+
+# Cards picked by headword, each for a stated reason, so a future edit can tell
+# what it would be removing.
+WANTED: dict[str, str] = {
+    # The worked examples from the design: a division heading, numbered senses
+    # under it, examples, and a second division.
+    "爱": "division + numbered senses + example; the plan's worked example",
+    # Homographs and multi-reading cards. The predecessor put the whole
+    # comma-separated reading above the word; one entry per reading is what
+    # replaces that.
+    "了": "two readings, le and liǎo; 30 senses",
+    "一": "57 senses; carries its own tone-sandhi note as prose",
+    "打": "57 senses, the deepest nesting in the set",
+    # The one card in 3,434,224 whose article is split across two physical
+    # lines, inside an open [m1].
+    "乐芙兰": "article spans two lines inside an open [m1]",
+    # A run-together reading with the U+2019 apostrophe the set uses as a
+    # syllable boundary.
+    "土耳其斯坦": "run-together reading with a U+2019 syllable boundary",
+}
+
+# Cards picked by what their markup contains. The first card matching each
+# pattern is taken, so the slice stays deterministic.
+PATTERNS: dict[str, tuple[str, str]] = {
+    "escaped-brackets": (r"\\\[", "escaped \\[ — 55,414 in the set; stripDSL ate them"),
+    "colour-argument": (r"\[c [a-z]+\]", "[c brown]: the only tag taking an argument"),
+    "level-four": (r"\[m4\]", "[m4]: 344 in the set, the deepest level used"),
+    "block-no-example": (
+        r"\[\*\](?:(?!\[ex\]).)*\[/\*\]",
+        "[*] with no [ex] inside — 62 in the set, and that path must not crash",
+    ),
+    "reference": (r"\[ref\]", "[ref] cross-reference"),
+    "register": (r"\[p\](?:бот|ист|уст)\.", "a [p] field marker rather than a part of speech"),
+    "no-reading": (r"^_$", "`_` for the reading — 77% of the set"),
+}
+
+HEADER = """#NAME "HanReader test slice of 大БКРС"
+#INDEX_LANGUAGE "Chinese"
+#CONTENTS_LANGUAGE "Russian"
+"""
+
+# A run of consecutive cards taken verbatim after the anchor, on top of the
+# picks above. Curated cards cover the shapes someone thought of; a contiguous
+# run covers the ones nobody did, at a realistic density. 爱 is the anchor
+# because it is already in the slice and its position is stable.
+RUN_ANCHOR = "爱"
+RUN_LENGTH = 190
+
+# DSL has no comment syntax — a `#` line is a directive — so the reasons go in
+# a sidecar instead of into the fixture, which stays a file the importer can
+# read as-is.
+MANIFEST_HEADER = """Why each card is in bkrs-slice.dsl.txt.
+
+Generated by Scripts/extract-dsl-fixture.py; do not hand-edit. DSL has no
+comment syntax, so this cannot live inside the fixture itself.
+"""
+
+
+def read_dsl(path: pathlib.Path) -> list[str]:
+    """Decodes one DSL file into lines, honouring its byte-order mark."""
+    raw = path.read_bytes()
+    if raw[:2] == b"\xff\xfe":
+        text = raw[2:].decode("utf-16-le")
+    elif raw[:2] == b"\xfe\xff":
+        text = raw[2:].decode("utf-16-be")
+    elif raw[:3] == b"\xef\xbb\xbf":
+        text = raw[3:].decode("utf-8")
+    else:
+        text = raw.decode("utf-8")
+    return [line.rstrip("\r") for line in text.split("\n")]
+
+
+def file_set(master: pathlib.Path) -> list[pathlib.Path]:
+    """Follows #INCLUDE from the master file."""
+    files = [master]
+    for line in read_dsl(master):
+        if not line.startswith("#"):
+            break
+        match = re.match(r'#INCLUDE\s+"([^"]+)"', line)
+        if match:
+            sibling = master.parent / match.group(1)
+            if sibling.exists():
+                files.append(sibling)
+    return files
+
+
+def cards(lines: list[str]):
+    """Yields (headwords, body_lines) for each card."""
+    index, count = 0, len(lines)
+    while index < count:
+        line = lines[index]
+        if not line or line.startswith("#"):
+            index += 1
+            continue
+        headwords = []
+        while index < count and lines[index] and not lines[index][:1] in (" ", "\t", "#"):
+            headwords.append(lines[index])
+            index += 1
+        body = []
+        while index < count and lines[index][:1] in (" ", "\t"):
+            body.append(lines[index])
+            index += 1
+        if headwords and body:
+            yield headwords, body
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print(__doc__, file=sys.stderr)
+        return 2
+    master = pathlib.Path(sys.argv[1]).expanduser()
+    if not master.exists():
+        print(f"no such file: {master}", file=sys.stderr)
+        return 1
+
+    compiled = {key: re.compile(pattern) for key, (pattern, _) in PATTERNS.items()}
+    chosen: dict[str, tuple[list[str], list[str], str]] = {}
+    found_patterns: set[str] = set()
+    run_remaining = 0
+
+    for path in file_set(master):
+        for headwords, body in cards(read_dsl(path)):
+            key = headwords[0]
+            if run_remaining > 0 and key not in chosen:
+                run_remaining -= 1
+                chosen[key] = (headwords, body, f"consecutive run after {RUN_ANCHOR}")
+                continue
+            if key in WANTED and key not in chosen:
+                chosen[key] = (headwords, body, WANTED[key])
+                if key == RUN_ANCHOR:
+                    run_remaining = RUN_LENGTH
+                continue
+            stripped = [line.strip() for line in body]
+            for name, regex in compiled.items():
+                if name in found_patterns or key in chosen:
+                    continue
+                if any(regex.search(line) for line in stripped):
+                    found_patterns.add(name)
+                    chosen[key] = (headwords, body, PATTERNS[name][1])
+
+    missing = sorted(set(WANTED) - set(chosen))
+    if missing:
+        print(f"warning: headwords not found: {', '.join(missing)}", file=sys.stderr)
+    for name in sorted(set(PATTERNS) - found_patterns):
+        print(f"warning: no card matched {name}", file=sys.stderr)
+
+    root = pathlib.Path(__file__).resolve().parent.parent
+    out = root / "Tests/HanReaderCoreTests/Fixtures/bkrs-slice.dsl.txt"
+    manifest = root / "Tests/HanReaderCoreTests/Fixtures/bkrs-slice.manifest.txt"
+
+    with out.open("w", encoding="utf-8", newline="\n") as handle:
+        handle.write(HEADER)
+        handle.write("\n")
+        for key in sorted(chosen):
+            headwords, body, _ = chosen[key]
+            for headword in headwords:
+                handle.write(headword + "\n")
+            for line in body:
+                handle.write(line + "\n")
+
+    with manifest.open("w", encoding="utf-8", newline="\n") as handle:
+        handle.write(MANIFEST_HEADER)
+        handle.write("\n")
+        for key in sorted(chosen):
+            handle.write(f"{key}\t{chosen[key][2]}\n")
+
+    size = out.stat().st_size
+    print(f"wrote {out.relative_to(root)} and {manifest.relative_to(root)}")
+    print(f"  {len(chosen)} cards, {size:,} bytes")
+    if size > 256 * 1024:
+        print("  error: over the 256 KB fixture limit CI enforces", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())