From c053255e4d7c15970bee93119798736a10172c8c Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:31:45 +0300 Subject: [PATCH 1/2] feat: build senses from DSL cards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layer four: turning a card's token stream into structured senses. This is the layer the rebuild exists for. The predecessor produced **one sense per card** — 3,434,222 rows over 3,434,222 headwords, exactly one each — because it treated a line as a sense and a card's whole sense run arrives on a single physical line. It then tried to recover the structure downstream by splitting that blob on semicolons. Over the same input this produces 3,691,468 senses, of which 115,825 cards carry more than one; 了 has 30 and 一 and 打 have 57 apiece. `[mN]` is an **indent level, not a container**, and getting that wrong is the one mistake that looks right in a unit test. A card writes `[m1]…[/m][m2]…[/m]`, closing each span before opening the next, so by the time the `[m2]` arrives its parent is already closed. Hierarchy comes from the levels in document order, not from which spans happen to be open — my first version used an open/close stack and gave every nested sense a nil parent. A level-indexed map of the most recent sense at each depth is what makes `[m4]` find the right `[m2]`, and what makes a later `[m1]` start a new branch instead of joining the old one. Markup is not reliably balanced either: the set has two more `[mN]` opens than `[/m]` closes, so anything still open at the end of a card is closed there. One entry per reading, rather than one entry whose reading is the whole comma-separated string. It is what the schema says, the duplication is nothing (9,069 cards of 3,434,224), and the alternative puts `le, liǎo, liào` above the word on the page — which is precisely the prototype behaviour this replaces. `_` is not a reading. A naive split yields `[""]`, an entry claiming its pronunciation is the empty string, for 77% of the dictionary. Examples split at the end of the leading Han run, keeping `raw` always because the split is a heuristic. The CJK blocks alone are not enough: the quotation marks Chinese uses live in General Punctuation, and a block test stops at the first opening quote and hands half the sentence to the translation. Senses gained level, parent, label, part of speech, registers and examples; glosses gained style runs. Both encode only what they use — writing the empty defaults for every CC-CEDICT sense added 14 MB to a 35 MB bundled dictionary, and omitting them takes it to 32 MB, below where it started. Both decode field by field so a container compiled before these fields existed still opens, rather than a model addition forcing a re-import. Validated over the whole set: 3,434,224 cards to 3,442,623 entries and 3,691,468 senses in 70 seconds, with zero diagnostics. Co-Authored-By: Claude Opus 5 --- .../HanReaderCore/Format/DSLCardBuilder.swift | 124 ++++++ .../HanReaderCore/Format/SenseAssembly.swift | 377 ++++++++++++++++++ .../HanReaderCore/Model/DictionaryEntry.swift | 171 +++++++- .../DSLCardBuilderTests.swift | 268 +++++++++++++ .../HanReaderCoreTests/SenseCodingTests.swift | 106 +++++ 5 files changed, 1044 insertions(+), 2 deletions(-) create mode 100644 Sources/HanReaderCore/Format/DSLCardBuilder.swift create mode 100644 Sources/HanReaderCore/Format/SenseAssembly.swift create mode 100644 Tests/HanReaderCoreTests/DSLCardBuilderTests.swift create mode 100644 Tests/HanReaderCoreTests/SenseCodingTests.swift diff --git a/Sources/HanReaderCore/Format/DSLCardBuilder.swift b/Sources/HanReaderCore/Format/DSLCardBuilder.swift new file mode 100644 index 0000000..b189bee --- /dev/null +++ b/Sources/HanReaderCore/Format/DSLCardBuilder.swift @@ -0,0 +1,124 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation + +/// One card as it appears in the file: a headword, a reading, and a body. +public struct DSLCard: Hashable, Sendable { + /// Consecutive unindented lines. The DSL spec allows several spellings + /// to share one card; BKRS never uses it, but accepting it costs a line + /// and refusing it would be a parser bug on the next dictionary. + public let headwords: [String] + /// The reading line, already stripped of its indent. `_` means the + /// dictionary has no reading for this word — **77% of BKRS cards**. + public let pinyin: String + /// Everything after the reading line. + public let body: [String] + + public init(headwords: [String], pinyin: String, body: [String]) { + self.headwords = headwords + self.pinyin = pinyin + self.body = body + } +} + +/// Turns a card into entries. +/// +/// ### One entry per reading +/// +/// A card with several readings — `le, liǎo, liào` — becomes several +/// entries sharing a sense run, rather than one entry whose reading is the +/// whole comma-separated string. Three reasons, in order of weight: +/// +/// 1. It is what the schema says. An entry is keyed on headword *and* +/// reading, which is the shape CC-CEDICT already needs for 了. +/// 2. It makes the existing reading-selection rule work for BKRS. The +/// alternative puts `le, liǎo, liào` above the word on the page, which is +/// exactly the prototype's behaviour this rebuild exists to replace. +/// 3. The duplication is nothing: 9,069 of 3,434,224 cards have more than +/// one reading. +/// +/// ### The sense stack is indexed by level, not pushed and popped +/// +/// `[m4]` after an `[m3]` must nest under the right `[m2]`, and a card can +/// reopen a shallower level without closing the deeper one. A stack that +/// simply pushes on open and pops on close attaches a sense to whatever +/// happened to be last, which is wrong whenever the levels skip. +/// +/// Markup is also not reliably balanced: across the whole set there are two +/// more `[mN]` opens than `[/m]` closes. Anything left open at the end of a +/// card is closed there. +public struct DSLCardBuilder: Sendable { + /// The toneless syllables this dictionary's readings are built from, + /// used to split a run-together reading like `tǔ'ěrqísītǎn`. + private let syllableBases: Set + + public init(syllableBases: Set) { + self.syllableBases = syllableBases + } + + public func entries(from card: DSLCard) + -> (entries: [DictionaryEntry], diagnostics: [DSLDiagnostic]) + { + var assembly = SenseAssembly() + var diagnostics: [DSLDiagnostic] = [] + + for line in card.body { + let lexed = DSLLexer.tokens(in: line) + diagnostics.append(contentsOf: lexed.diagnostics) + assembly.consume(lexed.tokens) + } + let senses = assembly.finish() + + guard let simplified = card.headwords.first else { return ([], diagnostics) } + let headword = Headword( + simplified: simplified, + traditional: card.headwords.dropFirst().first, + ) + + let readings = Self.readings(in: card.pinyin) + guard !readings.isEmpty else { + return ([DictionaryEntry(headword: headword, reading: [], senses: senses)], diagnostics) + } + let entries = readings.map { reading in + DictionaryEntry( + headword: headword, + reading: tokens(forReading: reading), + senses: senses, + ) + } + return (entries, diagnostics) + } + + // MARK: - Readings + + /// Splits the reading line into individual readings. + /// + /// `_` is not a reading. Returning `[""]` for it — which a naive split + /// does — produces an entry claiming to know a pronunciation that is the + /// empty string, and 77% of this dictionary is that case. + static func readings(in pinyin: String) -> [String] { + let trimmed = pinyin.trimmingCharacters(in: .whitespaces) + guard !trimmed.isEmpty, trimmed != "_" else { return [] } + return trimmed + .split(separator: ",") + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty && $0 != "_" } + } + + /// Converts one diacritic reading into tokens. + /// + /// Falls back to a literal when the reading cannot be split into known + /// syllables. That is a degradation, not a failure: the reading still + /// displays exactly as the dictionary wrote it, and only the + /// cross-dictionary merge key is lost — so the word appears as its own + /// group instead of joining CC-CEDICT's. + func tokens(forReading reading: String) -> [PinyinToken] { + guard let syllables = PinyinSyllabifier.syllables( + fromDiacritic: reading, + bases: syllableBases, + ) else { + return [.literal(reading)] + } + return syllables.map { .syllable($0) } + } +} diff --git a/Sources/HanReaderCore/Format/SenseAssembly.swift b/Sources/HanReaderCore/Format/SenseAssembly.swift new file mode 100644 index 0000000..0ea6a8d --- /dev/null +++ b/Sources/HanReaderCore/Format/SenseAssembly.swift @@ -0,0 +1,377 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation + +/// Builds a card's senses from its token stream. +/// +/// The predecessor produced **one sense per card** — 3,434,222 rows over +/// 3,434,222 headwords, exactly one each — because it treated a line as a +/// sense and a card's whole sense run arrives on a single physical line. It +/// then tried to recover the structure downstream by splitting the blob on +/// semicolons, which is why the reader showed one unseparated string where +/// the dictionary had a numbered list. +/// +/// The structure is in the markup. This reads it. +struct SenseAssembly { + private var drafts: [Draft] = [] + /// The most recent sense seen at each level, by index into `drafts`. + /// + /// Not an open/close stack, and that distinction is the whole of the + /// nesting logic. `[mN]` is an **indent level**, not a container: a card + /// writes `[m1]…[/m][m2]…[/m]`, closing each span before opening the + /// next, so by the time the `[m2]` arrives its `[m1]` parent is already + /// closed. Hierarchy comes from the levels in document order, not from + /// which spans happen to be open. + /// + /// Opening a sense at level L therefore takes the most recent sense at + /// the nearest level above it as its parent, and forgets everything + /// deeper than L — which is what makes `[m1]` after an `[m3]` start a + /// new branch rather than attaching to the old one. + private var lastByLevel: [Int: Int] = [:] + private var current: Int? + + /// Where each emphasis began, as a UTF-16 offset into the sense's text. + private var styleStarts: [TextStyle: Int] = [:] + private var referenceStart: Int? + /// Set while inside `[p]…[/p]`, which is a label rather than prose. + private var grammar: String? + /// Set while inside `[ex]…[/ex]`. + private var example: String? + + private struct Draft { + var level: Int + var parent: Int? + var text = "" + var styles: [StyleRun] = [] + var references: [CrossReference] = [] + var partOfSpeech: [String] = [] + var registers: [String] = [] + var examples: [UsageExample] = [] + } + + // MARK: - Consuming tokens + + mutating func consume(_ tokens: [DSLToken]) { + for token in tokens { + switch token { + case let .text(text): append(text) + case let .open(tag): open(tag) + case let .close(kind): close(kind) + } + } + } + + private mutating func open(_ tag: DSLTag) { + switch tag.kind { + case .sense: + openSense(level: max(0, (tag.level ?? 1) - 1)) + case .grammar: + grammar = "" + case .example: + example = "" + case .bold: + styleStarts[.bold] = length + case .italic: + styleStarts[.italic] = length + case .colour: + styleStarts[.highlighted] = length + case .reference: + referenceStart = length + case .exampleBlock: + // `[*]` is a wrapper, and 62 of them in the set contain no + // `[ex]` at all. Treating it as transparent means that path + // needs no special case and loses nothing: the text inside + // simply joins the gloss. + break + } + } + + private mutating func close(_ kind: DSLTagKind) { + switch kind { + case .sense: closeDeepestSense() + case .grammar: closeGrammar() + case .example: closeExample() + case .bold: closeStyle(.bold) + case .italic: closeStyle(.italic) + case .colour: closeStyle(.highlighted) + case .reference: closeReference() + case .exampleBlock: break + } + } + + // MARK: - Senses + + private mutating func openSense(level: Int) { + let parent = lastByLevel + .filter { $0.key < level } + .max { $0.key < $1.key }? + .value + + drafts.append(Draft(level: level, parent: parent)) + let index = drafts.count - 1 + + // Everything deeper than this belongs to the branch just left. + for seen in lastByLevel.keys where seen > level { + lastByLevel.removeValue(forKey: seen) + } + lastByLevel[level] = index + current = index + styleStarts.removeAll() + referenceStart = nil + } + + /// `[/m]` ends the span, not the level. + /// + /// It closes any emphasis left open inside the sense and nothing more. + /// The level hierarchy outlives it — see `lastByLevel` — and text + /// between a close and the next open stays with the sense it followed + /// rather than being dropped. + private mutating func closeDeepestSense() { + styleStarts.removeAll() + referenceStart = nil + } + + // MARK: - Text + + private var length: Int { + guard let current else { return 0 } + return drafts[current].text.utf16.count + } + + private mutating func append(_ text: String) { + if grammar != nil { + grammar? += text + return + } + if example != nil { + example? += text + return + } + // Text outside any `[mN]` still belongs to the card; a sense is + // opened for it rather than letting it fall on the floor. Cards + // whose body carries no markup at all exist, and dropping them + // would lose the entry entirely. + if current == nil { + openSense(level: 0) + } + guard let current else { return } + drafts[current].text += text + } + + // MARK: - Decorations + + private mutating func closeStyle(_ style: TextStyle) { + guard let current, let start = styleStarts.removeValue(forKey: style) else { return } + let end = drafts[current].text.utf16.count + guard end > start else { return } + drafts[current].styles.append(StyleRun(style: style, range: start ..< end)) + } + + private mutating func closeReference() { + guard let current, let start = referenceStart else { return } + referenceStart = nil + let text = drafts[current].text + let end = text.utf16.count + guard end > start, + let range = Range(NSRange(location: start, length: end - start), in: text) + else { return } + drafts[current].references.append(CrossReference( + simplified: String(text[range]), + traditional: nil, + reading: nil, + range: start ..< end, + )) + } + + /// `[p]гл.[/p]` is a category, not prose. + /// + /// Unrecognised labels go to `registers` verbatim rather than being + /// dropped. A field marker this list has not heard of is still + /// information the dictionary chose to print. + private mutating func closeGrammar() { + defer { grammar = nil } + guard let text = grammar?.trimmingCharacters(in: .whitespaces), !text.isEmpty else { + return + } + if current == nil { + openSense(level: 0) + } + guard let current else { return } + if Self.partsOfSpeech.contains(text) { + drafts[current].partOfSpeech.append(text) + } else { + drafts[current].registers.append(text) + } + } + + private mutating func closeExample() { + defer { example = nil } + guard let raw = example?.trimmingCharacters(in: .whitespaces), !raw.isEmpty else { return } + if current == nil { + openSense(level: 0) + } + guard let current else { return } + drafts[current].examples.append(Self.splitExample(raw)) + } + + // MARK: - Finishing + + mutating func finish() -> [Sense] { + lastByLevel.removeAll() + current = nil + return drafts.enumerated().map { index, draft in + var draft = draft + let label = Self.liftLabel(from: &draft) + return Sense( + id: index, + kind: Self.kind(of: draft), + gloss: Gloss( + text: draft.text.trimmingCharacters(in: .whitespaces), + references: draft.references, + styles: draft.styles, + ), + level: draft.level, + parent: draft.parent, + label: label, + partOfSpeech: draft.partOfSpeech, + registers: draft.registers, + examples: draft.examples, + ) + } + } +} + +// MARK: - Text helpers + +/// Pure functions over a sense's text, kept out of the state machine above +/// so each can be read — and tested — without the assembly around it. +extension SenseAssembly { + /// Takes the dictionary's own numbering out of the gloss. + /// + /// Left in place it appears twice — once from the dictionary and once + /// from whatever numbering the reader's list applies — and it stops the + /// gloss being usable as a one-line summary. + private static func liftLabel(from draft: inout Draft) -> String? { + let text = draft.text.trimmingCharacters(in: .whitespaces) + guard let match = labelPrefix(of: text) else { return nil } + + let removed = draft.text.utf16.count - text.utf16.count + + match.utf16.count + draft.text = String(text.dropFirst(match.count)) + .trimmingCharacters(in: .whitespaces) + draft.styles = shift(draft.styles, by: removed, limit: draft.text.utf16.count) + draft.references = draft.references.compactMap { reference in + let lower = reference.range.lowerBound - removed + let upper = reference.range.upperBound - removed + guard lower >= 0, upper <= draft.text.utf16.count, lower < upper else { return nil } + return CrossReference( + simplified: reference.simplified, + traditional: reference.traditional, + reading: reference.reading, + range: lower ..< upper, + ) + } + return match.trimmingCharacters(in: .whitespaces) + } + + private static func shift(_ styles: [StyleRun], by offset: Int, limit: Int) -> [StyleRun] { + styles.compactMap { run in + let lower = max(0, run.range.lowerBound - offset) + let upper = min(limit, run.range.upperBound - offset) + guard lower < upper else { return nil } + return StyleRun(style: run.style, range: lower ..< upper) + } + } + + /// `I`, `II`, `1)`, `а)` — the forms BKRS numbers senses with. + static func labelPrefix(of text: String) -> String? { + var scalars = Substring(text) + + // A Roman numeral division heading, alone at the front. + let roman = scalars.prefix { $0 == "I" || $0 == "V" || $0 == "X" } + if !roman.isEmpty { + let rest = scalars.dropFirst(roman.count) + if rest.isEmpty || rest.first == " " { + return String(roman) + } + } + + // `12)` or `а)` — a number or a single letter, then a bracket. + let digits = scalars.prefix(while: \.isNumber) + if !digits.isEmpty, scalars.dropFirst(digits.count).first == ")" { + return String(scalars.prefix(digits.count + 1)) + } + if let first = scalars.first, first.isLetter, + scalars.dropFirst().first == ")" + { + scalars = scalars.dropFirst(2) + return String(text.prefix(2)) + } + return nil + } + + /// Splits an example into its Chinese and its translation. + /// + /// At the end of the leading run of Han characters and the punctuation + /// and spacing that belongs with them. A heuristic, which is why `raw` + /// is always kept: an example that splits badly has to stay readable in + /// full rather than be truncated to whichever half the heuristic liked. + static func splitExample(_ raw: String) -> UsageExample { + var end = raw.startIndex + var index = raw.startIndex + while index < raw.endIndex { + let character = raw[index] + let scalars = character.unicodeScalars + let isHan = scalars.allSatisfy(ScalarClass.isHan) + let isCJKPunctuation = scalars.allSatisfy(isChinesePunctuation) + if isHan || isCJKPunctuation { + index = raw.index(after: index) + end = index + continue + } + if character == " " { + index = raw.index(after: index) + continue + } + break + } + + let chinese = String(raw[raw.startIndex ..< end]).trimmingCharacters(in: .whitespaces) + let translation = String(raw[end...]).trimmingCharacters(in: .whitespaces) + return UsageExample(chinese: chinese, translation: translation, raw: raw) + } + + /// Punctuation that belongs with Chinese text. + /// + /// The CJK blocks are not enough: the quotation marks Chinese actually + /// uses — “ ” ‘ ’ — live in General Punctuation alongside the dash and + /// the ellipsis, so a block test alone stops the split at the first + /// opening quote and hands half the sentence to the translation. + private static func isChinesePunctuation(_ scalar: Unicode.Scalar) -> Bool { + switch scalar.value { + case 0x3000 ... 0x303F: true // CJK symbols and punctuation + case 0xFF00 ... 0xFF65: true // fullwidth forms + case 0x2018, 0x2019, 0x201C, 0x201D: true // the quotation marks + case 0x2014, 0x2026, 0x00B7: true // dash, ellipsis, interpunct + default: false + } + } + + private static func kind(of draft: Draft) -> SenseKind { + if !draft.references.isEmpty, draft.text.count <= 24 { + return .reference + } + if draft.text.isEmpty, !draft.partOfSpeech.isEmpty || !draft.registers.isEmpty { + return .label + } + return .definition + } + + /// Grammatical categories, as BKRS abbreviates them. Anything else is + /// treated as a register or field marker. + static let partsOfSpeech: Set = [ + "гл.", "сущ.", "прил.", "нареч.", "мест.", "числ.", "предлог", + "союз", "частица", "межд.", "служ. сл.", "счётн. сл.", "собств.", + "гл. А", "гл. Б", "гл. В", "усл.", "вспом. гл.", "послелог", + ] +} diff --git a/Sources/HanReaderCore/Model/DictionaryEntry.swift b/Sources/HanReaderCore/Model/DictionaryEntry.swift index 58fa6c1..4bd3a08 100644 --- a/Sources/HanReaderCore/Model/DictionaryEntry.swift +++ b/Sources/HanReaderCore/Model/DictionaryEntry.swift @@ -69,10 +69,91 @@ public struct Gloss: Hashable, Sendable, Codable { /// simplified form so it reads as prose. public let text: String public let references: [CrossReference] + /// Emphasis within the text, as offsets rather than inline markup. + /// + /// Kept rather than flattened because BKRS italicises 631,732 spans, + /// nearly all of them parenthetical grammatical notes — "(кого-л.)", + /// "(о человеке)" — which read as part of the definition once the + /// italics are gone. Dropping them is the difference between a gloss + /// and a gloss with its asides folded into it. + public let styles: [StyleRun] - public init(text: String, references: [CrossReference] = []) { + public init(text: String, references: [CrossReference] = [], styles: [StyleRun] = []) { self.text = text self.references = references + self.styles = styles + } + + private enum CodingKeys: String, CodingKey { + case text, references, styles + } + + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + text = try container.decode(String.self, forKey: .text) + references = try container.decodeIfPresent([CrossReference].self, forKey: .references) ?? [] + styles = try container.decodeIfPresent([StyleRun].self, forKey: .styles) ?? [] + } + + /// Empty collections are omitted rather than written as `[]`. + /// + /// Every CC-CEDICT sense has no references and no styles, and writing + /// the empty arrays anyway added 14 MB to a 35 MB bundled dictionary — + /// 40% of the file, to say "nothing here" 107,619 times. + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(text, forKey: .text) + if !references.isEmpty { + try container.encode(references, forKey: .references) + } + if !styles.isEmpty { + try container.encode(styles, forKey: .styles) + } + } +} + +/// How a span of a gloss is emphasised. +public enum TextStyle: String, Hashable, Sendable, Codable, CaseIterable { + case bold + case italic + /// A coloured span. The source names a colour; what the reader sees is + /// the app's own palette, because a dictionary's `brown` is not a + /// colour that works on both a light and a dark page. + case highlighted +} + +/// One emphasised span of a gloss. +public struct StyleRun: Hashable, Sendable, Codable { + public let style: TextStyle + /// Where the span sits in the gloss, as a UTF-16 offset range — the + /// unit `NSRange` and `AttributedString` both use, and the same unit + /// `CrossReference` uses. + public let range: Range + + public init(style: TextStyle, range: Range) { + self.style = style + self.range = range + } +} + +/// A sentence showing a word in use. +public struct UsageExample: Hashable, Sendable, Codable { + /// The Chinese, if it could be separated from the translation. + public let chinese: String + /// The translation, which for BKRS is Russian. + public let translation: String + /// The example exactly as the dictionary wrote it. + /// + /// Always kept. The split between the two scripts is a heuristic over + /// where the Han characters stop, and an example that splits badly must + /// still be readable in full rather than silently truncated to whichever + /// half the heuristic liked. + public let raw: String + + public init(chinese: String, translation: String, raw: String) { + self.chinese = chinese + self.translation = translation + self.raw = raw } } @@ -96,10 +177,96 @@ public struct Sense: Hashable, Sendable, Identifiable, Codable { public let kind: SenseKind public let gloss: Gloss - public init(id: Int, kind: SenseKind, gloss: Gloss) { + /// How deeply this sense is nested, counting from zero. + /// + /// Flat rather than a tree, with `parent` carrying the structure. A flat + /// list with an explicit level is isomorphic to a tree, encodes without + /// recursion, and renders directly as an indented list — which is all + /// the UI needs. + public let level: Int + /// The enclosing sense, or nil at the top. + public let parent: Int? + /// The dictionary's own numbering: `I`, `1)`, `а)`. + /// + /// Lifted out of the gloss rather than left at the front of it, so the + /// reader can renumber or indent without the original marker appearing + /// twice. + public let label: String? + /// Grammatical category: гл., сущ., прил. + public let partOfSpeech: [String] + /// Field or register: бот., ист., уст. + public let registers: [String] + public let examples: [UsageExample] + + public init( + id: Int, + kind: SenseKind, + gloss: Gloss, + level: Int = 0, + parent: Int? = nil, + label: String? = nil, + partOfSpeech: [String] = [], + registers: [String] = [], + examples: [UsageExample] = [], + ) { self.id = id self.kind = kind self.gloss = gloss + self.level = level + self.parent = parent + self.label = label + self.partOfSpeech = partOfSpeech + self.registers = registers + self.examples = examples + } + + private enum CodingKeys: String, CodingKey { + case id, kind, gloss, level, parent, label, partOfSpeech, registers, examples + } + + /// Decoded field by field with defaults, so a container compiled before + /// these fields existed still opens. A synthesised `init(from:)` fails + /// on any missing key, which would turn a model addition into a forced + /// re-import of a 3.4-million-entry dictionary. + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + id = try container.decode(Int.self, forKey: .id) + kind = try container.decode(SenseKind.self, forKey: .kind) + gloss = try container.decode(Gloss.self, forKey: .gloss) + level = try container.decodeIfPresent(Int.self, forKey: .level) ?? 0 + parent = try container.decodeIfPresent(Int.self, forKey: .parent) + label = try container.decodeIfPresent(String.self, forKey: .label) + partOfSpeech = try container.decodeIfPresent([String].self, forKey: .partOfSpeech) ?? [] + registers = try container.decodeIfPresent([String].self, forKey: .registers) ?? [] + examples = try container.decodeIfPresent([UsageExample].self, forKey: .examples) ?? [] + } + + /// Defaults are omitted, for the same reason as `Gloss`: a dictionary + /// with no nesting, no numbering and no examples should not pay for + /// the fields that describe them. + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(id, forKey: .id) + try container.encode(kind, forKey: .kind) + try container.encode(gloss, forKey: .gloss) + if level != 0 { + try container.encode(level, forKey: .level) + } + if let parent { + try container.encode(parent, forKey: .parent) + } + if let label { + try container.encode(label, forKey: .label) + } + if !partOfSpeech.isEmpty { + try container.encode(partOfSpeech, forKey: .partOfSpeech) + } + if !registers.isEmpty { + try container.encode(registers, forKey: .registers) + } + if !examples.isEmpty { + try container.encode(examples, forKey: .examples) + } } } diff --git a/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift b/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift new file mode 100644 index 0000000..5563855 --- /dev/null +++ b/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift @@ -0,0 +1,268 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import Testing +@testable import HanReaderCore + +/// Enough syllables to split the readings these tests use. +private let bases: Set = [ + "ai", "le", "liao", "yi", "da", "zhong", "guo", "tu", "er", "qi", "si", "tan", +] + +private func build( + headword: String = "爱", + pinyin: String = "ài", + _ body: String..., +) + -> (entries: [DictionaryEntry], diagnostics: [DSLDiagnostic]) +{ + DSLCardBuilder(syllableBases: bases).entries(from: DSLCard( + headwords: [headword], + pinyin: pinyin, + body: body, + )) +} + +private func senses(_ body: String...) -> [Sense] { + DSLCardBuilder(syllableBases: bases).entries(from: DSLCard( + headwords: ["爱"], + pinyin: "ài", + body: body, + )).entries.first?.senses ?? [] +} + +@Suite("DSL card builder") +struct DSLCardBuilderTests { + // MARK: - The bug the rebuild exists for + + /// Test number one. The predecessor produced **one sense per card** — + /// 3,434,222 rows over 3,434,222 headwords, exactly one each — because + /// it treated a line as a sense, and a card's whole sense run arrives on + /// a single physical line. The structure is in the markup. + @Test("Two sense spans on one line become two senses") + func twoSensesOneLine() { + let result = senses("[m1]первый[/m][m2]второй[/m]") + #expect(result.count == 2) + #expect(result.map(\.gloss.text) == ["первый", "второй"]) + #expect(result.map(\.level) == [0, 1]) + } + + @Test("A real card's shape comes out as a list, not a blob") + func realisticCard() { + // A division heading, two numbered senses under it, then a second + // division — the shape most multi-sense cards take. + let division = "[m1][b]I[/b] [p]гл.[/p][/m]" + let numbered = "[m2]1) любить[/m][m2]2) ценить[/m]" + let second = "[m1][b]II[/b] [p]сущ.[/p][/m]" + let result = senses(division + numbered + second) + #expect(result.count == 4) + #expect(result.map(\.level) == [0, 1, 1, 0]) + #expect(result.map(\.label) == ["I", "1)", "2)", "II"]) + #expect(result[1].gloss.text == "любить") + #expect(result[0].partOfSpeech == ["гл."]) + } + + // MARK: - Nesting + + /// `[m4]` after an `[m3]` has to find the right `[m2]`. A stack that + /// pushes on open and pops on close attaches it to whatever was last, + /// which is wrong the moment levels skip. + @Test("A deeper sense nests under the right parent") + func nesting() { + let result = senses("[m1]A[/m][m2]B[/m][m3]C[/m][m4]D[/m][m2]E[/m]") + #expect(result.map(\.level) == [0, 1, 2, 3, 1]) + #expect(result.map(\.parent) == [nil, 0, 1, 2, 0]) + } + + @Test("Reopening a shallower level closes the deeper ones") + func reopeningShallower() { + let result = senses("[m1]A[/m][m3]B[/m][m1]C[/m]") + #expect(result.map(\.level) == [0, 2, 0]) + #expect(result.map(\.parent) == [nil, 0, nil]) + } + + /// Across the whole set there are two more `[mN]` opens than `[/m]` + /// closes, so the builder cannot assume balance. + @Test("Unclosed markup still produces its senses") + func unbalancedMarkup() { + let result = senses("[m1]первый[m2]второй") + #expect(result.count == 2) + #expect(result.map(\.gloss.text) == ["первый", "второй"]) + } + + @Test("Body text with no markup at all is still a sense") + func noMarkup() { + let result = senses("просто текст") + #expect(result.map(\.gloss.text) == ["просто текст"]) + } + + // MARK: - Readings + + /// 77% of cards. Returning `[""]` — which a naive split does — produces + /// an entry claiming its pronunciation is the empty string. + @Test("An underscore is not a reading") + func underscoreReading() { + let result = build(pinyin: "_", "[m1]текст[/m]") + #expect(result.entries.count == 1) + #expect(result.entries[0].reading.isEmpty) + #expect(result.entries[0].readingDisplay.isEmpty) + } + + /// One entry per reading, so the reading-selection rule can choose + /// between them. One entry reading `le, liǎo, liào` would put all three + /// above the word, which is the prototype's behaviour. + @Test("A card with several readings becomes several entries") + func severalReadings() { + let result = build(headword: "了", pinyin: "le, liǎo", "[m1]текст[/m]") + #expect(result.entries.count == 2) + #expect(result.entries.map(\.readingDisplay) == ["le", "liǎo"]) + // They share the sense run rather than splitting it. + #expect(result.entries.allSatisfy { $0.senses.count == 1 }) + } + + @Test("A reading that cannot be split still displays") + func unsplittableReading() { + let result = build(pinyin: "zzzqqq", "[m1]текст[/m]") + #expect(result.entries.count == 1) + // Degraded to a literal: the reading shows, only the merge key is + // lost, so the word forms its own group instead of joining one. + #expect(result.entries[0].readingDisplay == "zzzqqq") + } + + @Test("A run-together reading is split into syllables") + func runTogetherReading() { + let result = build(pinyin: "àiguó", "[m1]текст[/m]") + #expect(result.entries.first?.reading.count == 2) + } + + // MARK: - Labels and categories + + @Test("A grammatical category is separated from the prose") + func partOfSpeech() { + let result = senses("[m1][p]гл.[/p]любить[/m]") + #expect(result[0].partOfSpeech == ["гл."]) + #expect(result[0].gloss.text == "любить") + } + + /// Unrecognised markers go to registers verbatim rather than being + /// dropped. A field marker this list has not heard of is still + /// something the dictionary chose to print. + @Test( + "A field marker is kept even when unrecognised", + arguments: ["бот.", "ист.", "уст.", "зззз."], + ) + func registers(marker: String) { + let result = senses("[m1][p]\(marker)[/p]текст[/m]") + #expect(result[0].registers == [marker]) + #expect(result[0].partOfSpeech.isEmpty) + } + + @Test("Numbering is lifted out of the gloss", arguments: [ + ("1) любить", "1)", "любить"), + ("12) ценить", "12)", "ценить"), + ("а) первый", "а)", "первый"), + ("II сущ", "II", "сущ"), + ]) + func labels(body: String, label: String, gloss: String) { + let result = senses("[m1]\(body)[/m]") + #expect(result[0].label == label) + #expect(result[0].gloss.text == gloss) + } + + @Test("Prose that merely starts with a letter is not a label") + func notALabel() { + let result = senses("[m1]Исторический термин[/m]") + #expect(result[0].label == nil) + #expect(result[0].gloss.text == "Исторический термин") + } + + // MARK: - Emphasis and references + + /// 631,732 italic spans, nearly all parenthetical grammatical notes. + /// Flattened, they read as part of the definition. + @Test("Italics are kept as a range, not flattened") + func italics() throws { + let result = senses("[m1]любить ([i]кого-л.[/i])[/m]") + #expect(result[0].gloss.text == "любить (кого-л.)") + // One `#require` per statement: a macro expanded inside another + // macro's argument is a recursive expansion the compiler refuses. + let run = try #require(result[0].gloss.styles.first) + #expect(run.style == .italic) + let text = result[0].gloss.text + let span = NSRange(location: run.range.lowerBound, length: run.range.count) + let range = try #require(Range(span, in: text)) + #expect(String(text[range]) == "кого-л.") + } + + @Test("A cross-reference records the word it points at") + func reference() { + let result = senses("[m1]см. [ref]磁性碰锁[/ref][/m]") + #expect(result[0].gloss.references.first?.simplified == "磁性碰锁") + #expect(result[0].gloss.text == "см. 磁性碰锁") + } + + @Test("An empty emphasis span records nothing") + func emptyStyle() { + #expect(senses("[m1][i][/i]текст[/m]")[0].gloss.styles.isEmpty) + } + + // MARK: - Examples + + @Test("An example is split into its two languages") + func exampleSplit() { + let result = senses("[m1][*][ex]爱儿童 любить ребёнка (детей)[/ex][/*][/m]") + let example = result[0].examples.first + #expect(example?.chinese == "爱儿童") + #expect(example?.translation == "любить ребёнка (детей)") + #expect(example?.raw == "爱儿童 любить ребёнка (детей)") + } + + @Test("An example with no Chinese still keeps everything") + func exampleWithoutChinese() { + let example = SenseAssembly.splitExample("только по-русски") + #expect(example.chinese.isEmpty) + #expect(example.translation == "только по-русски") + #expect(example.raw == "только по-русски") + } + + @Test("CJK punctuation stays with the Chinese") + func examplePunctuation() { + let example = SenseAssembly.splitExample("他说：“好。” он сказал") + #expect(example.chinese == "他说：“好。”") + #expect(example.translation == "он сказал") + } + + /// 62 `[*]` spans in the set contain no `[ex]`. The block is treated as + /// transparent, so that path needs no special case and loses nothing. + @Test("An example block with no example does not lose its text") + func blockWithoutExample() { + let result = senses("[m1][*]просто текст[/*][/m]") + #expect(result[0].examples.isEmpty) + #expect(result[0].gloss.text == "просто текст") + } + + // MARK: - Entry shape + + @Test("An empty card produces no entries") + func emptyCard() { + let result = DSLCardBuilder(syllableBases: bases) + .entries(from: DSLCard(headwords: [], pinyin: "_", body: [])) + #expect(result.entries.isEmpty) + } + + @Test("A second headword line is the traditional spelling") + func traditionalHeadword() { + let result = DSLCardBuilder(syllableBases: bases).entries(from: DSLCard( + headwords: ["爱", "愛"], + pinyin: "ài", + body: ["[m1]любить[/m]"], + )) + #expect(result.entries.first?.headword.traditional == "愛") + } + + @Test("Diagnostics from the lexer reach the caller") + func diagnosticsPropagate() { + let result = build("[m1][zzz]текст[/m]") + #expect(result.diagnostics.count == 1) + } +} diff --git a/Tests/HanReaderCoreTests/SenseCodingTests.swift b/Tests/HanReaderCoreTests/SenseCodingTests.swift new file mode 100644 index 0000000..04c7443 --- /dev/null +++ b/Tests/HanReaderCoreTests/SenseCodingTests.swift @@ -0,0 +1,106 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import Testing +@testable import HanReaderCore + +@Suite("Sense coding") +struct SenseCodingTests { + private enum CodingTestError: Error { case notUTF8 } + + private func encoded(_ sense: Sense) throws -> String { + let data = try JSONEncoder().encode(sense) + // A plain unwrap rather than `#require`: this helper is called from + // inside `#expect`, and a macro expanded within a macro's argument + // is a recursive expansion the compiler refuses. + guard let text = String(bytes: data, encoding: .utf8) else { + throw CodingTestError.notUTF8 + } + return text + } + + /// Writing `"level":0,"partOfSpeech":[],"registers":[],"examples":[]` + /// for every CC-CEDICT sense added 14 MB to a 35 MB dictionary — 40% of + /// the file to say "nothing here" 107,619 times. + @Test("A plain sense encodes none of the fields it does not use") + func defaultsAreOmitted() throws { + let json = try encoded(Sense(id: 0, kind: .definition, gloss: Gloss(text: "to love"))) + for absent in [ + "level", + "parent", + "label", + "partOfSpeech", + "registers", + "examples", + "styles", + "references", + ] { + #expect(!json.contains(absent), "\(absent) should not be written") + } + #expect(json.contains("to love")) + } + + @Test("A sense that uses the fields writes them") + func populatedFieldsAreWritten() throws { + let sense = Sense( + id: 1, + kind: .definition, + gloss: Gloss(text: "любить", styles: [StyleRun(style: .italic, range: 0 ..< 6)]), + level: 1, + parent: 0, + label: "1)", + partOfSpeech: ["гл."], + examples: [UsageExample(chinese: "爱", translation: "любить", raw: "爱 любить")], + ) + let json = try encoded(sense) + for present in ["level", "parent", "label", "partOfSpeech", "examples", "styles"] { + #expect(json.contains(present)) + } + #expect(!json.contains("registers"), "still omitted when empty") + } + + @Test("Everything round-trips") + func roundTrip() throws { + let sense = Sense( + id: 3, + kind: .reference, + gloss: Gloss( + text: "см. 磁性碰锁", + references: [CrossReference( + simplified: "磁性碰锁", traditional: nil, reading: nil, range: 4 ..< 9, + )], + styles: [StyleRun(style: .bold, range: 0 ..< 3)], + ), + level: 2, + parent: 1, + label: "а)", + partOfSpeech: ["сущ."], + registers: ["бот."], + examples: [UsageExample(chinese: "锁", translation: "замок", raw: "锁 замок")], + ) + let data = try JSONEncoder().encode(sense) + #expect(try JSONDecoder().decode(Sense.self, from: data) == sense) + } + + /// A container compiled before these fields existed must still open. A + /// synthesised `init(from:)` fails on any missing key, which would make + /// a model addition force a re-import of a 3.4-million-entry dictionary. + @Test("A payload written before these fields existed still decodes") + func oldPayloadDecodes() throws { + let old = #"{"id":0,"kind":0,"gloss":{"text":"to love","references":[]}}"# + let sense = try JSONDecoder().decode(Sense.self, from: Data(old.utf8)) + #expect(sense.gloss.text == "to love") + #expect(sense.level == 0) + #expect(sense.examples.isEmpty) + #expect(sense.label == nil) + } + + @Test("A gloss written before styles existed still decodes") + func oldGlossDecodes() throws { + let old = #"{"text":"to love"}"# + let gloss = try JSONDecoder().decode(Gloss.self, from: Data(old.utf8)) + #expect(gloss.text == "to love") + #expect(gloss.styles.isEmpty) + #expect(gloss.references.isEmpty) + } +} From 34720e66f854938137f53373d28636543427e167 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:21:32 +0300 Subject: [PATCH 2/2] fix: keep gloss ranges and unbalanced markup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight correctness fixes to the sense builder, seven of them raised in review on this branch and all eight measured against the full 3,434,224-card set. **Gloss ranges moved out from under the text they describe.** A style run or a cross-reference records a UTF-16 range into the sense's text, and the text was then trimmed in one function while the offsets were adjusted in another. Lifting `1)` out of a gloss also removes the space after it, which the offset arithmetic did not account for, so every range in a numbered sense pointed one character to the right: 16,957 senses with a style run and 3,974 with a cross-reference, the latter not merely shifted but dropped whenever it reached the end of its gloss. A further 85,395 unnumbered senses had the same fault from the final trim alone — anything whose prose follows a `[p]…[/p]`, which leaves a leading space, and there are 395,878 of those tags. Now the prefix is measured once, where it is removed, and every range moves by that one number and is clipped to the result. Clipped, not discarded: a run reaching into whitespace the trim took is still the run the dictionary asked for. Over the whole set, 0 of 380,683 style runs and 0 of 96,165 references now point at anything other than what they claim to; the tests assert the substring a range selects rather than its offsets, because an offset that is wrong by one still reads as a plausible number in a failure message. **Markup left open was discarded rather than settled.** `[/m]` cleared the pending style and reference offsets, so 241 sense spans that close with a style tag still open kept their text and lost their emphasis. An unclosed `[p]` or `[ex]` was worse: those divert text out of the gloss, so its contents left the sense altogether and the buffer went on collecting the *next* sense's prose. None occur in this set, but losing text is the one thing this importer must never do. Everything still open is now settled against the sense it was opened in — at `[/m]`, when the next sense opens, and at the end of the card. **A decoration opening before any `[mN]`** — 479 cards — lost its offset, because the fallback sense was created later by the first text token and cleared what the `[i]` had already recorded. Settling on sense boundaries makes the clearing unnecessary, so it is gone rather than guarded. **A `[ref]` span including the space before its closing tag** — 21 of them, `[ref]尽管 [/ref]` — recorded a headword with a trailing space, which matches no entry in any dictionary, so the link pointed nowhere. The span is trimmed along with the string, so the range and the word it names cannot disagree. **A card's article is not always one physical line.** 乐芙兰 is the one card in 3,434,224 that breaks a line inside an open `[m1]`, and most other DSL dictionaries write one sense per line. A break inside a sense is worth a space: concatenating gave `Ле Блан(чемпион из Лиги Легенд)`. **Several headword lines are several spellings**, each now its own entry over the same senses, so every spelling can be looked up. Reading the second line as the traditional form of the first was an invention — the format says consecutive headword lines are alternative headwords and says nothing about script — and it silently dropped every spelling after the second. No card in this set has more than one headword, so this changes nothing about what BKRS imports; it changes what the next dictionary imports. Counts over the whole set are unchanged — 3,434,224 cards to 3,442,623 entries and 3,691,468 senses, zero diagnostics, 17.8 s — which is the point: these fix what the senses contain, not how many there are. Each fix was mutation-tested; two of the first attempts were killed by no test at all and the tests were written until they were. Co-Authored-By: Claude Opus 5 --- .../HanReaderCore/Format/DSLCardBuilder.swift | 43 ++- .../HanReaderCore/Format/SenseAssembly.swift | 275 ++++++++++++++---- .../DSLCardBuilderTests.swift | 258 +++++++++++++++- 3 files changed, 494 insertions(+), 82 deletions(-) diff --git a/Sources/HanReaderCore/Format/DSLCardBuilder.swift b/Sources/HanReaderCore/Format/DSLCardBuilder.swift index b189bee..6aef101 100644 --- a/Sources/HanReaderCore/Format/DSLCardBuilder.swift +++ b/Sources/HanReaderCore/Format/DSLCardBuilder.swift @@ -37,6 +37,20 @@ public struct DSLCard: Hashable, Sendable { /// 3. The duplication is nothing: 9,069 of 3,434,224 cards have more than /// one reading. /// +/// ### And one entry per spelling, for the same reason +/// +/// A card may carry several headword lines. Each becomes its own entry over +/// the same sense run, so every spelling the dictionary lists can be looked +/// up. The alternative — reading the second line as the traditional form of +/// the first — is an invention: the format says consecutive headword lines +/// are alternative headwords for one article, and says nothing about which +/// script they are in. Guessing would write a wrong `traditional` into the +/// cross-dictionary merge key, and it also silently dropped every spelling +/// after the second. +/// +/// No card in this set has more than one headword, so nothing here changes +/// what BKRS imports. It changes what the next dictionary imports. +/// /// ### The sense stack is indexed by level, not pushed and popped /// /// `[m4]` after an `[m3]` must nest under the right `[m2]`, and a card can @@ -62,29 +76,26 @@ public struct DSLCardBuilder: Sendable { var assembly = SenseAssembly() var diagnostics: [DSLDiagnostic] = [] - for line in card.body { + for (index, line) in card.body.enumerated() { + if index > 0 { + assembly.beginLine() + } let lexed = DSLLexer.tokens(in: line) diagnostics.append(contentsOf: lexed.diagnostics) assembly.consume(lexed.tokens) } let senses = assembly.finish() - guard let simplified = card.headwords.first else { return ([], diagnostics) } - let headword = Headword( - simplified: simplified, - traditional: card.headwords.dropFirst().first, - ) + guard !card.headwords.isEmpty else { return ([], diagnostics) } + let headwords = card.headwords.map { Headword(simplified: $0, traditional: nil) } + let readings = Self.readings(in: card.pinyin).map { tokens(forReading: $0) } - let readings = Self.readings(in: card.pinyin) - guard !readings.isEmpty else { - return ([DictionaryEntry(headword: headword, reading: [], senses: senses)], diagnostics) - } - let entries = readings.map { reading in - DictionaryEntry( - headword: headword, - reading: tokens(forReading: reading), - senses: senses, - ) + let entries = headwords.flatMap { headword in + readings.isEmpty + ? [DictionaryEntry(headword: headword, reading: [], senses: senses)] + : readings.map { + DictionaryEntry(headword: headword, reading: $0, senses: senses) + } } return (entries, diagnostics) } diff --git a/Sources/HanReaderCore/Format/SenseAssembly.swift b/Sources/HanReaderCore/Format/SenseAssembly.swift index 0ea6a8d..3368023 100644 --- a/Sources/HanReaderCore/Format/SenseAssembly.swift +++ b/Sources/HanReaderCore/Format/SenseAssembly.swift @@ -51,6 +51,37 @@ struct SenseAssembly { // MARK: - Consuming tokens + /// Joins the next body line to the one before it. + /// + /// A card's article is usually one physical line, but a line break can + /// fall *inside* an open `[mN]`. 乐芙兰 is the one card in 3,434,224 + /// where it does — `[m1]Ле Блан` on one line, `(чемпион из Лиги + /// Легенд)[/m]` on the next — and most other DSL dictionaries write one + /// sense per line, where this matters far more often. + /// + /// The break is worth a space. Concatenating gives + /// `Ле Блан(чемпион из Лиги Легенд)`; inserting one unconditionally would + /// put a leading space on every sense that *is* closed at the end of its + /// line, which is the common case. So: a space only where there is + /// already text to separate. + mutating func beginLine() { + guard let tail = openText, !tail.isEmpty, !tail.hasSuffix(" ") else { return } + append(" ") + } + + /// The text `append` would currently add to, mirroring its precedence. + /// Nil when nothing is open, so beginning a line never opens a sense. + private var openText: String? { + if let grammar { + return grammar + } + if let example { + return example + } + guard let current else { return nil } + return drafts[current].text + } + mutating func consume(_ tokens: [DSLToken]) { for token in tokens { switch token { @@ -69,6 +100,10 @@ struct SenseAssembly { grammar = "" case .example: example = "" + // A decoration can open before any `[mN]` — 479 cards do — and the + // offset it records is 0 either way, because a sense opened later + // starts empty. What matters is that `openSense` no longer throws + // these away, which is what lost the markup on those cards. case .bold: styleStarts[.bold] = length case .italic: @@ -102,6 +137,12 @@ struct SenseAssembly { // MARK: - Senses private mutating func openSense(level: Int) { + // Whatever is still open belongs to the sense being left, so it is + // settled against that sense before the new one exists. Discarding it + // here instead is how 241 sense spans lost their emphasis and how an + // unclosed `[p]` could go on swallowing the *next* sense's prose. + settleOpenSpans() + let parent = lastByLevel .filter { $0.key < level } .max { $0.key < $1.key }? @@ -116,19 +157,44 @@ struct SenseAssembly { } lastByLevel[level] = index current = index - styleStarts.removeAll() - referenceStart = nil } /// `[/m]` ends the span, not the level. /// - /// It closes any emphasis left open inside the sense and nothing more. + /// It settles any emphasis left open inside the sense and nothing more. /// The level hierarchy outlives it — see `lastByLevel` — and text /// between a close and the next open stays with the sense it followed /// rather than being dropped. private mutating func closeDeepestSense() { - styleStarts.removeAll() - referenceStart = nil + settleOpenSpans() + } + + /// Closes everything still open against the sense it was opened in. + /// + /// Markup in this set is not reliably balanced — 241 sense spans close + /// `[/m]` with a style tag still open — and the rule the importer follows + /// is that nothing is ever deleted. An unclosed `[i]` therefore yields the + /// italic run it was asking for, up to where the sense ends, rather than + /// text that silently lost its emphasis. + private mutating func settleOpenSpans() { + closeGrammar() + closeExample() + // `allCases` rather than the dictionary's own key order, so the runs + // come out in the same order on every run. + for style in TextStyle.allCases where styleStarts[style] != nil { + closeStyle(style) + } + closeReference() + } + + /// Text outside any `[mN]` still belongs to the card, so a sense is + /// opened for it rather than letting it fall on the floor. Cards whose + /// article carries no markup at all exist, and dropping them would lose + /// the entry entirely. + private mutating func ensureSense() { + if current == nil { + openSense(level: 0) + } } // MARK: - Text @@ -147,13 +213,7 @@ struct SenseAssembly { example? += text return } - // Text outside any `[mN]` still belongs to the card; a sense is - // opened for it rather than letting it fall on the floor. Cards - // whose body carries no markup at all exist, and dropping them - // would lose the entry entirely. - if current == nil { - openSense(level: 0) - } + ensureSense() guard let current else { return } drafts[current].text += text } @@ -167,6 +227,14 @@ struct SenseAssembly { drafts[current].styles.append(StyleRun(style: style, range: start ..< end)) } + /// A cross-reference names another headword, so the span is trimmed to + /// that headword. + /// + /// 21 `[ref]` spans in the set close after a space — `[ref]尽管 [/ref]` — + /// and a headword with a trailing space matches no entry in any + /// dictionary, so the reference would point nowhere. The span is trimmed + /// rather than only the string, so the range and the word it names cannot + /// disagree. private mutating func closeReference() { guard let current, let start = referenceStart else { return } referenceStart = nil @@ -175,11 +243,22 @@ struct SenseAssembly { guard end > start, let range = Range(NSRange(location: start, length: end - start), in: text) else { return } + + let span = text[range] + let leading = span.prefix(while: \.isWhitespace).count + var word = span.dropFirst(leading) + var trailing = 0 + while let last = word.last, last.isWhitespace { + word = word.dropLast() + trailing += 1 + } + guard !word.isEmpty else { return } + drafts[current].references.append(CrossReference( - simplified: String(text[range]), + simplified: String(word), traditional: nil, reading: nil, - range: start ..< end, + range: (start + leading) ..< (end - trailing), )) } @@ -189,13 +268,14 @@ struct SenseAssembly { /// dropped. A field marker this list has not heard of is still /// information the dictionary chose to print. private mutating func closeGrammar() { - defer { grammar = nil } - guard let text = grammar?.trimmingCharacters(in: .whitespaces), !text.isEmpty else { - return - } - if current == nil { - openSense(level: 0) - } + // Cleared before anything else can run, because `ensureSense` below + // leads back into `settleOpenSpans`, and a buffer still set at that + // point is an infinite recursion. + guard let raw = grammar else { return } + grammar = nil + let text = raw.trimmingCharacters(in: .whitespaces) + guard !text.isEmpty else { return } + ensureSense() guard let current else { return } if Self.partsOfSpeech.contains(text) { drafts[current].partOfSpeech.append(text) @@ -205,11 +285,12 @@ struct SenseAssembly { } private mutating func closeExample() { - defer { example = nil } - guard let raw = example?.trimmingCharacters(in: .whitespaces), !raw.isEmpty else { return } - if current == nil { - openSense(level: 0) - } + // Cleared first, for the same reason as `closeGrammar`. + guard let buffered = example else { return } + example = nil + let raw = buffered.trimmingCharacters(in: .whitespaces) + guard !raw.isEmpty else { return } + ensureSense() guard let current else { return } drafts[current].examples.append(Self.splitExample(raw)) } @@ -217,22 +298,31 @@ struct SenseAssembly { // MARK: - Finishing mutating func finish() -> [Sense] { + // A card can end with markup still open — the set has two more `[mN]` + // opens than `[/m]` closes — so the last sense is settled here rather + // than relying on a close that may never arrive. + settleOpenSpans() lastByLevel.removeAll() current = nil return drafts.enumerated().map { index, draft in + let trimmed = Self.trim( + text: draft.text, + styles: draft.styles, + references: draft.references, + ) var draft = draft - let label = Self.liftLabel(from: &draft) + draft.text = trimmed.text return Sense( id: index, kind: Self.kind(of: draft), gloss: Gloss( - text: draft.text.trimmingCharacters(in: .whitespaces), - references: draft.references, - styles: draft.styles, + text: trimmed.text, + references: trimmed.references, + styles: trimmed.styles, ), level: draft.level, parent: draft.parent, - label: label, + label: trimmed.label, partOfSpeech: draft.partOfSpeech, registers: draft.registers, examples: draft.examples, @@ -246,41 +336,102 @@ struct SenseAssembly { /// Pure functions over a sense's text, kept out of the state machine above /// so each can be read — and tested — without the assembly around it. extension SenseAssembly { - /// Takes the dictionary's own numbering out of the gloss. + /// What a draft's text, label and ranges become in the finished sense. + struct Trimmed { + var text: String + var label: String? + var styles: [StyleRun] + var references: [CrossReference] + } + + /// Strips the dictionary's own numbering and the surrounding whitespace, + /// and moves every recorded range with the text. /// - /// Left in place it appears twice — once from the dictionary and once - /// from whatever numbering the reader's list applies — and it stops the - /// gloss being usable as a one-line summary. - private static func liftLabel(from draft: inout Draft) -> String? { - let text = draft.text.trimmingCharacters(in: .whitespaces) - guard let match = labelPrefix(of: text) else { return nil } - - let removed = draft.text.utf16.count - text.utf16.count - + match.utf16.count - draft.text = String(text.dropFirst(match.count)) - .trimmingCharacters(in: .whitespaces) - draft.styles = shift(draft.styles, by: removed, limit: draft.text.utf16.count) - draft.references = draft.references.compactMap { reference in - let lower = reference.range.lowerBound - removed - let upper = reference.range.upperBound - removed - guard lower >= 0, upper <= draft.text.utf16.count, lower < upper else { return nil } - return CrossReference( - simplified: reference.simplified, - traditional: reference.traditional, - reading: reference.reading, - range: lower ..< upper, - ) + /// Numbering is lifted because left in place it appears twice — once from + /// the dictionary and once from whatever numbering the reader's list + /// applies — and because it stops the gloss reading as a one-line summary. + /// + /// **One place, one offset, and that is the point.** Trimming the text in + /// one function while the ranges were adjusted in another is how every + /// range in a numbered sense ended up one character to the right: the + /// space after `1)` was removed from the text by a second trim that the + /// offset arithmetic knew nothing about. Measured over the whole set, + /// 16,957 senses with a style run and 3,974 with a cross-reference were + /// affected, and a reference reaching the end of its gloss was not merely + /// shifted but dropped. A further 85,395 unnumbered senses — anything + /// whose prose follows a `[p]…[/p]`, which leaves a leading space — had + /// the same fault from the final trim alone. + /// + /// So the prefix is measured here, once, as it is removed: leading + /// whitespace, then the label, then the whitespace after the label. Every + /// range moves by that one number and is clamped to the result, because a + /// range is clamped rather than discarded — a run that overshoots by a + /// trimmed space is still the run the dictionary asked for. + static func trim( + text rawText: String, + styles: [StyleRun] = [], + references: [CrossReference] = [], + ) + -> Trimmed + { + var removed = 0 + var body = Substring(rawText) + + let afterLeading = body.drop(while: \.isWhitespace) + removed += body.utf16.count - afterLeading.utf16.count + body = afterLeading + + var label: String? + if let match = labelPrefix(of: String(body)) { + body = body.dropFirst(match.count) + removed += match.utf16.count + let afterLabel = body.drop(while: \.isWhitespace) + removed += body.utf16.count - afterLabel.utf16.count + body = afterLabel + label = match.trimmingCharacters(in: .whitespaces) } - return match.trimmingCharacters(in: .whitespaces) - } - private static func shift(_ styles: [StyleRun], by offset: Int, limit: Int) -> [StyleRun] { - styles.compactMap { run in - let lower = max(0, run.range.lowerBound - offset) - let upper = min(limit, run.range.upperBound - offset) - guard lower < upper else { return nil } - return StyleRun(style: run.style, range: lower ..< upper) + while let last = body.last, last.isWhitespace { + body = body.dropLast() } + + let text = String(body) + let limit = text.utf16.count + return Trimmed( + text: text, + label: label, + styles: styles.compactMap { run in + guard let range = clamp(run.range, by: removed, limit: limit) else { return nil } + return StyleRun(style: run.style, range: range) + }, + references: references.compactMap { reference in + guard let range = clamp(reference.range, by: removed, limit: limit) else { + return nil + } + return CrossReference( + simplified: reference.simplified, + traditional: reference.traditional, + reading: reference.reading, + range: range, + ) + }, + ) + } + + /// Moves a range back by `offset` and clips it to `0.., + by offset: Int, + limit: Int, + ) + -> Range? + { + let lower = max(0, range.lowerBound - offset) + let upper = min(limit, range.upperBound - offset) + guard lower < upper else { return nil } + return lower ..< upper } /// `I`, `II`, `1)`, `а)` — the forms BKRS numbers senses with. diff --git a/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift b/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift index 5563855..4849d92 100644 --- a/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift +++ b/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift @@ -23,6 +23,27 @@ private func build( )) } +/// The text a style run actually selects. +/// +/// Asserting the substring rather than the offsets is the point: an offset +/// that is off by one still reads as a plausible number in a failure message, +/// where the wrong substring says exactly what went wrong. +private func styled(_ sense: Sense, _ index: Int = 0) -> String? { + guard index < sense.gloss.styles.count else { return nil } + return substring(sense.gloss.text, sense.gloss.styles[index].range) +} + +private func referenced(_ sense: Sense, _ index: Int = 0) -> String? { + guard index < sense.gloss.references.count else { return nil } + return substring(sense.gloss.text, sense.gloss.references[index].range) +} + +private func substring(_ text: String, _ range: Range) -> String? { + let span = NSRange(location: range.lowerBound, length: range.count) + guard let converted = Range(span, in: text) else { return nil } + return String(text[converted]) +} + private func senses(_ body: String...) -> [Sense] { DSLCardBuilder(syllableBases: bases).entries(from: DSLCard( headwords: ["爱"], @@ -250,14 +271,31 @@ struct DSLCardBuilderTests { #expect(result.entries.isEmpty) } - @Test("A second headword line is the traditional spelling") - func traditionalHeadword() { + /// Several headword lines are several spellings of one article, so each + /// becomes its own entry over the same senses. Reading the second as the + /// traditional form of the first is an invention the format does not + /// support — and it dropped every spelling after the second outright. + @Test("Every headword line becomes an entry") + func severalHeadwords() { let result = DSLCardBuilder(syllableBases: bases).entries(from: DSLCard( - headwords: ["爱", "愛"], + headwords: ["爱", "愛", "㤅"], pinyin: "ài", body: ["[m1]любить[/m]"], )) - #expect(result.entries.first?.headword.traditional == "愛") + #expect(result.entries.map(\.headword.simplified) == ["爱", "愛", "㤅"]) + #expect(result.entries.allSatisfy { $0.headword.traditional == nil }) + #expect(result.entries.allSatisfy { $0.senses == result.entries[0].senses }) + } + + @Test("Several headwords and several readings multiply out") + func headwordsAndReadings() { + let result = DSLCardBuilder(syllableBases: bases).entries(from: DSLCard( + headwords: ["了", "瞭"], + pinyin: "le, liǎo", + body: ["[m1]текст[/m]"], + )) + #expect(result.entries.count == 4) + #expect(result.entries.map(\.readingDisplay) == ["le", "liǎo", "le", "liǎo"]) } @Test("Diagnostics from the lexer reach the caller") @@ -266,3 +304,215 @@ struct DSLCardBuilderTests { #expect(result.diagnostics.count == 1) } } + +/// Gloss text is trimmed and renumbered after its markup ranges were +/// recorded against the untrimmed text, so every range has to move with it. +/// These are the cases where it did not. +@Suite("DSL gloss ranges") +struct DSLGlossRangeTests { + /// Lifting `1)` out of the gloss also removes the space after it, and + /// every recorded range has to move by the whole of that. It did not: + /// 16,957 senses in the set carry a style run inside a numbered sense, + /// and all of them pointed one character too far right. + @Test("A style range follows the text past a lifted label") + func styleAfterLabel() { + let result = senses("[m1]1) первый ([i]по порядку[/i])[/m]") + #expect(result[0].label == "1)") + #expect(result[0].gloss.text == "первый (по порядку)") + #expect(styled(result[0]) == "по порядку") + } + + /// Same fault, worse consequence: a reference reaching the end of the + /// gloss overshot it and was discarded rather than clipped. 3,974 senses. + @Test("A reference at the end of a numbered sense survives") + func referenceAtEndOfLabelledSense() { + let result = senses("[m1]2) ключ иероглифа ([p]см.[/p] [ref]横[/ref])[/m]") + #expect(result[0].label == "2)") + #expect(referenced(result[0]) == "横") + #expect(result[0].gloss.references.first?.simplified == "横") + } + + /// 395,878 `[p]` tags, nearly all followed by a space — so the prose of + /// 85,395 senses begins with whitespace that the final trim removed + /// without telling the ranges. No label involved. + @Test("A style range follows the text past trimmed whitespace") + func styleAfterLeadingSpace() { + let result = senses("[m1][p]бот.[/p] девясил каспийский ([i]Inula caspica[/i])[/m]") + #expect(result[0].registers == ["бот."]) + #expect(result[0].gloss.text == "девясил каспийский (Inula caspica)") + #expect(styled(result[0]) == "Inula caspica") + } + + @Test("A reference range follows the text past trimmed whitespace") + func referenceAfterLeadingSpace() { + let result = senses("[m1][p]см.[/p] [ref]磁性碰锁[/ref] и далее[/m]") + #expect(referenced(result[0]) == "磁性碰锁") + } + + /// Both trims at once, which is where an offset computed by subtracting + /// string lengths goes most wrong. + @Test("Ranges survive a leading space, a label and a trailing space") + func rangesSurviveEverything() { + let result = senses("[m1] 12) [b]первый[/b] и [i]второй[/i] [/m]") + #expect(result[0].label == "12)") + #expect(result[0].gloss.text == "первый и второй") + #expect(styled(result[0], 0) == "первый") + #expect(styled(result[0], 1) == "второй") + } + + /// The trim is a pure function, so the awkward inputs can be put to it + /// directly. + @Test("Trimming is exact at the edges", arguments: [ + ("текст", "текст", String?.none), + (" текст ", "текст", nil), + ("1)", "", "1)"), + ("1) ", "", "1)"), + (" ", "", nil), + ("", "", nil), + ]) + func trimEdges(input: String, text: String, label: String?) { + let result = SenseAssembly.trim(text: input) + #expect(result.text == text) + #expect(result.label == label) + } + + /// A run reaching into whitespace that the trim removes is clipped to the + /// gloss, not discarded. It is still the run the dictionary asked for, and + /// dropping it would cost the emphasis over the whole phrase to save a + /// space. + @Test("A run overshooting into trimmed whitespace is clipped") + func runClippedToGloss() { + let result = senses("[m1]текст [i]курсив [/i][/m]") + #expect(result[0].gloss.text == "текст курсив") + #expect(styled(result[0]) == "курсив") + } + + @Test("An unclosed run over trailing whitespace is clipped too") + func unclosedRunClipped() { + let result = senses("[m1]текст [i]курсив ") + #expect(result[0].gloss.text == "текст курсив") + #expect(styled(result[0]) == "курсив") + } + + /// 21 `[ref]` spans in the set close after a space. A headword with a + /// trailing space matches no entry in any dictionary, so the reference + /// would point nowhere — and the range has to be trimmed with the string, + /// or the two disagree about which characters the link covers. + @Test( + "A reference span is trimmed to the headword it names", + arguments: ["[ref]磁性碰锁 [/ref]", "[ref] 磁性碰锁[/ref]", "[ref] 磁性碰锁 [/ref]"], + ) + func referenceSpanTrimmed(markup: String) throws { + let result = senses("[m1]см. \(markup) и далее[/m]") + let reference = try #require(result[0].gloss.references.first) + #expect(reference.simplified == "磁性碰锁") + #expect(referenced(result[0]) == "磁性碰锁") + } + + @Test("A reference span of nothing but whitespace records nothing") + func referenceSpanAllWhitespace() { + #expect(senses("[m1]см. [ref] [/ref][/m]")[0].gloss.references.isEmpty) + } + + @Test("A span lying entirely inside a lifted label is dropped") + func spanInsideLabel() { + // `[b]I[/b]` is the division heading itself: once lifted into the + // label there is no gloss left for the run to point at. + let result = senses("[m1][b]I[/b] [p]гл.[/p][/m]") + #expect(result[0].label == "I") + #expect(result[0].gloss.text.isEmpty) + #expect(result[0].gloss.styles.isEmpty) + } +} + +/// DSL markup is not reliably balanced — the set has two more `[mN]` opens +/// than `[/m]` closes and 241 sense spans with a style tag still open — and +/// the rule the importer follows is that nothing is ever deleted. +@Suite("Unbalanced DSL markup") +struct DSLUnbalancedMarkupTests { + /// Nothing is ever deleted. An unclosed `[i]` kept its text but lost the + /// emphasis it was asking for — 241 sense spans in the set close `[/m]` + /// with a style still open. + @Test("An unclosed emphasis still produces its run") + func unclosedStyle() { + let result = senses("[m1]текст [i]курсив[/m][m1]другой[/m]") + #expect(result.count == 2) + #expect(styled(result[0]) == "курсив") + // And it does not leak into the next sense. + #expect(result[1].gloss.styles.isEmpty) + } + + @Test("An unclosed emphasis at the end of a card still produces its run") + func unclosedStyleAtEnd() { + let result = senses("[m1]текст [i]курсив") + #expect(styled(result[0]) == "курсив") + } + + @Test("An unclosed reference still records the word") + func unclosedReference() { + let result = senses("[m1]см. [ref]磁性碰锁") + #expect(referenced(result[0]) == "磁性碰锁") + } + + /// `[p]` and `[ex]` divert text out of the gloss, so an unclosed one took + /// its contents out of the sense entirely. None occur in this set, but + /// losing text is the one thing this importer must never do. + @Test("An unclosed category keeps its text") + func unclosedGrammar() { + let result = senses("[m1]текст [p]гл.") + #expect(result[0].partOfSpeech == ["гл."]) + #expect(result[0].gloss.text == "текст") + } + + @Test("An unclosed example keeps its text") + func unclosedExample() { + let result = senses("[m1]текст [ex]爱儿童 любить ребёнка") + #expect(result[0].examples.first?.raw == "爱儿童 любить ребёнка") + } + + /// The other half of the same fault: a buffer still open when the next + /// sense began went on collecting that sense's prose. + @Test("An unclosed category does not swallow the next sense") + func unclosedGrammarDoesNotLeak() { + let result = senses("[m1][p]гл.[m2]любить[/m]") + #expect(result.count == 2) + #expect(result[0].partOfSpeech == ["гл."]) + #expect(result[1].gloss.text == "любить") + } + + /// 479 cards begin their article with markup and no `[mN]` at all. The + /// fallback sense used to be created by the first *text* token, which + /// cleared the offset the `[i]` had already recorded. + @Test("Emphasis before any sense tag still produces its run") + func styleBeforeAnySense() { + let result = senses("[i]курсив[/i] и дальше") + #expect(result.count == 1) + #expect(result[0].gloss.text == "курсив и дальше") + #expect(styled(result[0]) == "курсив") + } + + @Test("A reference before any sense tag still records the word") + func referenceBeforeAnySense() { + let result = senses("[ref]磁性碰锁[/ref] см.") + #expect(referenced(result[0]) == "磁性碰锁") + } + + // MARK: - A sense broken across body lines + + /// 乐芙兰, the one card in 3,434,224 whose article breaks a line inside + /// an open `[m1]`. Concatenating gives `Ле Блан(чемпион…)`. + @Test("A sense continued on the next line gains a space, not nothing") + func senseAcrossLines() { + let result = senses("[m1]Ле Блан", "(чемпион из Лиги Легенд)[/m]") + #expect(result.count == 1) + #expect(result[0].gloss.text == "Ле Блан (чемпион из Лиги Легенд)") + } + + /// And a card that closes each sense at the end of its line — which is + /// how most DSL dictionaries are written — gains no stray space. + @Test("One sense per line gains no leading space") + func sensePerLine() { + let result = senses("[m1]первый[/m]", "[m2]второй[/m]") + #expect(result.map(\.gloss.text) == ["первый", "второй"]) + } +}