diff --git a/Sources/HanReaderCore/Format/DSLCardBuilder.swift b/Sources/HanReaderCore/Format/DSLCardBuilder.swift new file mode 100644 index 0000000..6aef101 --- /dev/null +++ b/Sources/HanReaderCore/Format/DSLCardBuilder.swift @@ -0,0 +1,135 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation + +/// One card as it appears in the file: a headword, a reading, and a body. +public struct DSLCard: Hashable, Sendable { + /// Consecutive unindented lines. The DSL spec allows several spellings + /// to share one card; BKRS never uses it, but accepting it costs a line + /// and refusing it would be a parser bug on the next dictionary. + public let headwords: [String] + /// The reading line, already stripped of its indent. `_` means the + /// dictionary has no reading for this word — **77% of BKRS cards**. + public let pinyin: String + /// Everything after the reading line. + public let body: [String] + + public init(headwords: [String], pinyin: String, body: [String]) { + self.headwords = headwords + self.pinyin = pinyin + self.body = body + } +} + +/// Turns a card into entries. +/// +/// ### One entry per reading +/// +/// A card with several readings — `le, liǎo, liào` — becomes several +/// entries sharing a sense run, rather than one entry whose reading is the +/// whole comma-separated string. Three reasons, in order of weight: +/// +/// 1. It is what the schema says. An entry is keyed on headword *and* +/// reading, which is the shape CC-CEDICT already needs for 了. +/// 2. It makes the existing reading-selection rule work for BKRS. The +/// alternative puts `le, liǎo, liào` above the word on the page, which is +/// exactly the prototype's behaviour this rebuild exists to replace. +/// 3. The duplication is nothing: 9,069 of 3,434,224 cards have more than +/// one reading. +/// +/// ### And one entry per spelling, for the same reason +/// +/// A card may carry several headword lines. Each becomes its own entry over +/// the same sense run, so every spelling the dictionary lists can be looked +/// up. The alternative — reading the second line as the traditional form of +/// the first — is an invention: the format says consecutive headword lines +/// are alternative headwords for one article, and says nothing about which +/// script they are in. Guessing would write a wrong `traditional` into the +/// cross-dictionary merge key, and it also silently dropped every spelling +/// after the second. +/// +/// No card in this set has more than one headword, so nothing here changes +/// what BKRS imports. It changes what the next dictionary imports. +/// +/// ### The sense stack is indexed by level, not pushed and popped +/// +/// `[m4]` after an `[m3]` must nest under the right `[m2]`, and a card can +/// reopen a shallower level without closing the deeper one. A stack that +/// simply pushes on open and pops on close attaches a sense to whatever +/// happened to be last, which is wrong whenever the levels skip. +/// +/// Markup is also not reliably balanced: across the whole set there are two +/// more `[mN]` opens than `[/m]` closes. Anything left open at the end of a +/// card is closed there. +public struct DSLCardBuilder: Sendable { + /// The toneless syllables this dictionary's readings are built from, + /// used to split a run-together reading like `tǔ'ěrqísītǎn`. + private let syllableBases: Set + + public init(syllableBases: Set) { + self.syllableBases = syllableBases + } + + public func entries(from card: DSLCard) + -> (entries: [DictionaryEntry], diagnostics: [DSLDiagnostic]) + { + var assembly = SenseAssembly() + var diagnostics: [DSLDiagnostic] = [] + + for (index, line) in card.body.enumerated() { + if index > 0 { + assembly.beginLine() + } + let lexed = DSLLexer.tokens(in: line) + diagnostics.append(contentsOf: lexed.diagnostics) + assembly.consume(lexed.tokens) + } + let senses = assembly.finish() + + guard !card.headwords.isEmpty else { return ([], diagnostics) } + let headwords = card.headwords.map { Headword(simplified: $0, traditional: nil) } + let readings = Self.readings(in: card.pinyin).map { tokens(forReading: $0) } + + let entries = headwords.flatMap { headword in + readings.isEmpty + ? [DictionaryEntry(headword: headword, reading: [], senses: senses)] + : readings.map { + DictionaryEntry(headword: headword, reading: $0, senses: senses) + } + } + return (entries, diagnostics) + } + + // MARK: - Readings + + /// Splits the reading line into individual readings. + /// + /// `_` is not a reading. Returning `[""]` for it — which a naive split + /// does — produces an entry claiming to know a pronunciation that is the + /// empty string, and 77% of this dictionary is that case. + static func readings(in pinyin: String) -> [String] { + let trimmed = pinyin.trimmingCharacters(in: .whitespaces) + guard !trimmed.isEmpty, trimmed != "_" else { return [] } + return trimmed + .split(separator: ",") + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty && $0 != "_" } + } + + /// Converts one diacritic reading into tokens. + /// + /// Falls back to a literal when the reading cannot be split into known + /// syllables. That is a degradation, not a failure: the reading still + /// displays exactly as the dictionary wrote it, and only the + /// cross-dictionary merge key is lost — so the word appears as its own + /// group instead of joining CC-CEDICT's. + func tokens(forReading reading: String) -> [PinyinToken] { + guard let syllables = PinyinSyllabifier.syllables( + fromDiacritic: reading, + bases: syllableBases, + ) else { + return [.literal(reading)] + } + return syllables.map { .syllable($0) } + } +} diff --git a/Sources/HanReaderCore/Format/SenseAssembly.swift b/Sources/HanReaderCore/Format/SenseAssembly.swift new file mode 100644 index 0000000..3368023 --- /dev/null +++ b/Sources/HanReaderCore/Format/SenseAssembly.swift @@ -0,0 +1,528 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation + +/// Builds a card's senses from its token stream. +/// +/// The predecessor produced **one sense per card** — 3,434,222 rows over +/// 3,434,222 headwords, exactly one each — because it treated a line as a +/// sense and a card's whole sense run arrives on a single physical line. It +/// then tried to recover the structure downstream by splitting the blob on +/// semicolons, which is why the reader showed one unseparated string where +/// the dictionary had a numbered list. +/// +/// The structure is in the markup. This reads it. +struct SenseAssembly { + private var drafts: [Draft] = [] + /// The most recent sense seen at each level, by index into `drafts`. + /// + /// Not an open/close stack, and that distinction is the whole of the + /// nesting logic. `[mN]` is an **indent level**, not a container: a card + /// writes `[m1]…[/m][m2]…[/m]`, closing each span before opening the + /// next, so by the time the `[m2]` arrives its `[m1]` parent is already + /// closed. Hierarchy comes from the levels in document order, not from + /// which spans happen to be open. + /// + /// Opening a sense at level L therefore takes the most recent sense at + /// the nearest level above it as its parent, and forgets everything + /// deeper than L — which is what makes `[m1]` after an `[m3]` start a + /// new branch rather than attaching to the old one. + private var lastByLevel: [Int: Int] = [:] + private var current: Int? + + /// Where each emphasis began, as a UTF-16 offset into the sense's text. + private var styleStarts: [TextStyle: Int] = [:] + private var referenceStart: Int? + /// Set while inside `[p]…[/p]`, which is a label rather than prose. + private var grammar: String? + /// Set while inside `[ex]…[/ex]`. + private var example: String? + + private struct Draft { + var level: Int + var parent: Int? + var text = "" + var styles: [StyleRun] = [] + var references: [CrossReference] = [] + var partOfSpeech: [String] = [] + var registers: [String] = [] + var examples: [UsageExample] = [] + } + + // MARK: - Consuming tokens + + /// Joins the next body line to the one before it. + /// + /// A card's article is usually one physical line, but a line break can + /// fall *inside* an open `[mN]`. 乐芙兰 is the one card in 3,434,224 + /// where it does — `[m1]Ле Блан` on one line, `(чемпион из Лиги + /// Легенд)[/m]` on the next — and most other DSL dictionaries write one + /// sense per line, where this matters far more often. + /// + /// The break is worth a space. Concatenating gives + /// `Ле Блан(чемпион из Лиги Легенд)`; inserting one unconditionally would + /// put a leading space on every sense that *is* closed at the end of its + /// line, which is the common case. So: a space only where there is + /// already text to separate. + mutating func beginLine() { + guard let tail = openText, !tail.isEmpty, !tail.hasSuffix(" ") else { return } + append(" ") + } + + /// The text `append` would currently add to, mirroring its precedence. + /// Nil when nothing is open, so beginning a line never opens a sense. + private var openText: String? { + if let grammar { + return grammar + } + if let example { + return example + } + guard let current else { return nil } + return drafts[current].text + } + + mutating func consume(_ tokens: [DSLToken]) { + for token in tokens { + switch token { + case let .text(text): append(text) + case let .open(tag): open(tag) + case let .close(kind): close(kind) + } + } + } + + private mutating func open(_ tag: DSLTag) { + switch tag.kind { + case .sense: + openSense(level: max(0, (tag.level ?? 1) - 1)) + case .grammar: + grammar = "" + case .example: + example = "" + // A decoration can open before any `[mN]` — 479 cards do — and the + // offset it records is 0 either way, because a sense opened later + // starts empty. What matters is that `openSense` no longer throws + // these away, which is what lost the markup on those cards. + case .bold: + styleStarts[.bold] = length + case .italic: + styleStarts[.italic] = length + case .colour: + styleStarts[.highlighted] = length + case .reference: + referenceStart = length + case .exampleBlock: + // `[*]` is a wrapper, and 62 of them in the set contain no + // `[ex]` at all. Treating it as transparent means that path + // needs no special case and loses nothing: the text inside + // simply joins the gloss. + break + } + } + + private mutating func close(_ kind: DSLTagKind) { + switch kind { + case .sense: closeDeepestSense() + case .grammar: closeGrammar() + case .example: closeExample() + case .bold: closeStyle(.bold) + case .italic: closeStyle(.italic) + case .colour: closeStyle(.highlighted) + case .reference: closeReference() + case .exampleBlock: break + } + } + + // MARK: - Senses + + private mutating func openSense(level: Int) { + // Whatever is still open belongs to the sense being left, so it is + // settled against that sense before the new one exists. Discarding it + // here instead is how 241 sense spans lost their emphasis and how an + // unclosed `[p]` could go on swallowing the *next* sense's prose. + settleOpenSpans() + + let parent = lastByLevel + .filter { $0.key < level } + .max { $0.key < $1.key }? + .value + + drafts.append(Draft(level: level, parent: parent)) + let index = drafts.count - 1 + + // Everything deeper than this belongs to the branch just left. + for seen in lastByLevel.keys where seen > level { + lastByLevel.removeValue(forKey: seen) + } + lastByLevel[level] = index + current = index + } + + /// `[/m]` ends the span, not the level. + /// + /// It settles any emphasis left open inside the sense and nothing more. + /// The level hierarchy outlives it — see `lastByLevel` — and text + /// between a close and the next open stays with the sense it followed + /// rather than being dropped. + private mutating func closeDeepestSense() { + settleOpenSpans() + } + + /// Closes everything still open against the sense it was opened in. + /// + /// Markup in this set is not reliably balanced — 241 sense spans close + /// `[/m]` with a style tag still open — and the rule the importer follows + /// is that nothing is ever deleted. An unclosed `[i]` therefore yields the + /// italic run it was asking for, up to where the sense ends, rather than + /// text that silently lost its emphasis. + private mutating func settleOpenSpans() { + closeGrammar() + closeExample() + // `allCases` rather than the dictionary's own key order, so the runs + // come out in the same order on every run. + for style in TextStyle.allCases where styleStarts[style] != nil { + closeStyle(style) + } + closeReference() + } + + /// Text outside any `[mN]` still belongs to the card, so a sense is + /// opened for it rather than letting it fall on the floor. Cards whose + /// article carries no markup at all exist, and dropping them would lose + /// the entry entirely. + private mutating func ensureSense() { + if current == nil { + openSense(level: 0) + } + } + + // MARK: - Text + + private var length: Int { + guard let current else { return 0 } + return drafts[current].text.utf16.count + } + + private mutating func append(_ text: String) { + if grammar != nil { + grammar? += text + return + } + if example != nil { + example? += text + return + } + ensureSense() + guard let current else { return } + drafts[current].text += text + } + + // MARK: - Decorations + + private mutating func closeStyle(_ style: TextStyle) { + guard let current, let start = styleStarts.removeValue(forKey: style) else { return } + let end = drafts[current].text.utf16.count + guard end > start else { return } + drafts[current].styles.append(StyleRun(style: style, range: start ..< end)) + } + + /// A cross-reference names another headword, so the span is trimmed to + /// that headword. + /// + /// 21 `[ref]` spans in the set close after a space — `[ref]尽管 [/ref]` — + /// and a headword with a trailing space matches no entry in any + /// dictionary, so the reference would point nowhere. The span is trimmed + /// rather than only the string, so the range and the word it names cannot + /// disagree. + private mutating func closeReference() { + guard let current, let start = referenceStart else { return } + referenceStart = nil + let text = drafts[current].text + let end = text.utf16.count + guard end > start, + let range = Range(NSRange(location: start, length: end - start), in: text) + else { return } + + let span = text[range] + let leading = span.prefix(while: \.isWhitespace).count + var word = span.dropFirst(leading) + var trailing = 0 + while let last = word.last, last.isWhitespace { + word = word.dropLast() + trailing += 1 + } + guard !word.isEmpty else { return } + + drafts[current].references.append(CrossReference( + simplified: String(word), + traditional: nil, + reading: nil, + range: (start + leading) ..< (end - trailing), + )) + } + + /// `[p]гл.[/p]` is a category, not prose. + /// + /// Unrecognised labels go to `registers` verbatim rather than being + /// dropped. A field marker this list has not heard of is still + /// information the dictionary chose to print. + private mutating func closeGrammar() { + // Cleared before anything else can run, because `ensureSense` below + // leads back into `settleOpenSpans`, and a buffer still set at that + // point is an infinite recursion. + guard let raw = grammar else { return } + grammar = nil + let text = raw.trimmingCharacters(in: .whitespaces) + guard !text.isEmpty else { return } + ensureSense() + guard let current else { return } + if Self.partsOfSpeech.contains(text) { + drafts[current].partOfSpeech.append(text) + } else { + drafts[current].registers.append(text) + } + } + + private mutating func closeExample() { + // Cleared first, for the same reason as `closeGrammar`. + guard let buffered = example else { return } + example = nil + let raw = buffered.trimmingCharacters(in: .whitespaces) + guard !raw.isEmpty else { return } + ensureSense() + guard let current else { return } + drafts[current].examples.append(Self.splitExample(raw)) + } + + // MARK: - Finishing + + mutating func finish() -> [Sense] { + // A card can end with markup still open — the set has two more `[mN]` + // opens than `[/m]` closes — so the last sense is settled here rather + // than relying on a close that may never arrive. + settleOpenSpans() + lastByLevel.removeAll() + current = nil + return drafts.enumerated().map { index, draft in + let trimmed = Self.trim( + text: draft.text, + styles: draft.styles, + references: draft.references, + ) + var draft = draft + draft.text = trimmed.text + return Sense( + id: index, + kind: Self.kind(of: draft), + gloss: Gloss( + text: trimmed.text, + references: trimmed.references, + styles: trimmed.styles, + ), + level: draft.level, + parent: draft.parent, + label: trimmed.label, + partOfSpeech: draft.partOfSpeech, + registers: draft.registers, + examples: draft.examples, + ) + } + } +} + +// MARK: - Text helpers + +/// Pure functions over a sense's text, kept out of the state machine above +/// so each can be read — and tested — without the assembly around it. +extension SenseAssembly { + /// What a draft's text, label and ranges become in the finished sense. + struct Trimmed { + var text: String + var label: String? + var styles: [StyleRun] + var references: [CrossReference] + } + + /// Strips the dictionary's own numbering and the surrounding whitespace, + /// and moves every recorded range with the text. + /// + /// Numbering is lifted because left in place it appears twice — once from + /// the dictionary and once from whatever numbering the reader's list + /// applies — and because it stops the gloss reading as a one-line summary. + /// + /// **One place, one offset, and that is the point.** Trimming the text in + /// one function while the ranges were adjusted in another is how every + /// range in a numbered sense ended up one character to the right: the + /// space after `1)` was removed from the text by a second trim that the + /// offset arithmetic knew nothing about. Measured over the whole set, + /// 16,957 senses with a style run and 3,974 with a cross-reference were + /// affected, and a reference reaching the end of its gloss was not merely + /// shifted but dropped. A further 85,395 unnumbered senses — anything + /// whose prose follows a `[p]…[/p]`, which leaves a leading space — had + /// the same fault from the final trim alone. + /// + /// So the prefix is measured here, once, as it is removed: leading + /// whitespace, then the label, then the whitespace after the label. Every + /// range moves by that one number and is clamped to the result, because a + /// range is clamped rather than discarded — a run that overshoots by a + /// trimmed space is still the run the dictionary asked for. + static func trim( + text rawText: String, + styles: [StyleRun] = [], + references: [CrossReference] = [], + ) + -> Trimmed + { + var removed = 0 + var body = Substring(rawText) + + let afterLeading = body.drop(while: \.isWhitespace) + removed += body.utf16.count - afterLeading.utf16.count + body = afterLeading + + var label: String? + if let match = labelPrefix(of: String(body)) { + body = body.dropFirst(match.count) + removed += match.utf16.count + let afterLabel = body.drop(while: \.isWhitespace) + removed += body.utf16.count - afterLabel.utf16.count + body = afterLabel + label = match.trimmingCharacters(in: .whitespaces) + } + + while let last = body.last, last.isWhitespace { + body = body.dropLast() + } + + let text = String(body) + let limit = text.utf16.count + return Trimmed( + text: text, + label: label, + styles: styles.compactMap { run in + guard let range = clamp(run.range, by: removed, limit: limit) else { return nil } + return StyleRun(style: run.style, range: range) + }, + references: references.compactMap { reference in + guard let range = clamp(reference.range, by: removed, limit: limit) else { + return nil + } + return CrossReference( + simplified: reference.simplified, + traditional: reference.traditional, + reading: reference.reading, + range: range, + ) + }, + ) + } + + /// Moves a range back by `offset` and clips it to `0.., + by offset: Int, + limit: Int, + ) + -> Range? + { + let lower = max(0, range.lowerBound - offset) + let upper = min(limit, range.upperBound - offset) + guard lower < upper else { return nil } + return lower ..< upper + } + + /// `I`, `II`, `1)`, `а)` — the forms BKRS numbers senses with. + static func labelPrefix(of text: String) -> String? { + var scalars = Substring(text) + + // A Roman numeral division heading, alone at the front. + let roman = scalars.prefix { $0 == "I" || $0 == "V" || $0 == "X" } + if !roman.isEmpty { + let rest = scalars.dropFirst(roman.count) + if rest.isEmpty || rest.first == " " { + return String(roman) + } + } + + // `12)` or `а)` — a number or a single letter, then a bracket. + let digits = scalars.prefix(while: \.isNumber) + if !digits.isEmpty, scalars.dropFirst(digits.count).first == ")" { + return String(scalars.prefix(digits.count + 1)) + } + if let first = scalars.first, first.isLetter, + scalars.dropFirst().first == ")" + { + scalars = scalars.dropFirst(2) + return String(text.prefix(2)) + } + return nil + } + + /// Splits an example into its Chinese and its translation. + /// + /// At the end of the leading run of Han characters and the punctuation + /// and spacing that belongs with them. A heuristic, which is why `raw` + /// is always kept: an example that splits badly has to stay readable in + /// full rather than be truncated to whichever half the heuristic liked. + static func splitExample(_ raw: String) -> UsageExample { + var end = raw.startIndex + var index = raw.startIndex + while index < raw.endIndex { + let character = raw[index] + let scalars = character.unicodeScalars + let isHan = scalars.allSatisfy(ScalarClass.isHan) + let isCJKPunctuation = scalars.allSatisfy(isChinesePunctuation) + if isHan || isCJKPunctuation { + index = raw.index(after: index) + end = index + continue + } + if character == " " { + index = raw.index(after: index) + continue + } + break + } + + let chinese = String(raw[raw.startIndex ..< end]).trimmingCharacters(in: .whitespaces) + let translation = String(raw[end...]).trimmingCharacters(in: .whitespaces) + return UsageExample(chinese: chinese, translation: translation, raw: raw) + } + + /// Punctuation that belongs with Chinese text. + /// + /// The CJK blocks are not enough: the quotation marks Chinese actually + /// uses — “ ” ‘ ’ — live in General Punctuation alongside the dash and + /// the ellipsis, so a block test alone stops the split at the first + /// opening quote and hands half the sentence to the translation. + private static func isChinesePunctuation(_ scalar: Unicode.Scalar) -> Bool { + switch scalar.value { + case 0x3000 ... 0x303F: true // CJK symbols and punctuation + case 0xFF00 ... 0xFF65: true // fullwidth forms + case 0x2018, 0x2019, 0x201C, 0x201D: true // the quotation marks + case 0x2014, 0x2026, 0x00B7: true // dash, ellipsis, interpunct + default: false + } + } + + private static func kind(of draft: Draft) -> SenseKind { + if !draft.references.isEmpty, draft.text.count <= 24 { + return .reference + } + if draft.text.isEmpty, !draft.partOfSpeech.isEmpty || !draft.registers.isEmpty { + return .label + } + return .definition + } + + /// Grammatical categories, as BKRS abbreviates them. Anything else is + /// treated as a register or field marker. + static let partsOfSpeech: Set = [ + "гл.", "сущ.", "прил.", "нареч.", "мест.", "числ.", "предлог", + "союз", "частица", "межд.", "служ. сл.", "счётн. сл.", "собств.", + "гл. А", "гл. Б", "гл. В", "усл.", "вспом. гл.", "послелог", + ] +} diff --git a/Sources/HanReaderCore/Model/DictionaryEntry.swift b/Sources/HanReaderCore/Model/DictionaryEntry.swift index 58fa6c1..4bd3a08 100644 --- a/Sources/HanReaderCore/Model/DictionaryEntry.swift +++ b/Sources/HanReaderCore/Model/DictionaryEntry.swift @@ -69,10 +69,91 @@ public struct Gloss: Hashable, Sendable, Codable { /// simplified form so it reads as prose. public let text: String public let references: [CrossReference] + /// Emphasis within the text, as offsets rather than inline markup. + /// + /// Kept rather than flattened because BKRS italicises 631,732 spans, + /// nearly all of them parenthetical grammatical notes — "(кого-л.)", + /// "(о человеке)" — which read as part of the definition once the + /// italics are gone. Dropping them is the difference between a gloss + /// and a gloss with its asides folded into it. + public let styles: [StyleRun] - public init(text: String, references: [CrossReference] = []) { + public init(text: String, references: [CrossReference] = [], styles: [StyleRun] = []) { self.text = text self.references = references + self.styles = styles + } + + private enum CodingKeys: String, CodingKey { + case text, references, styles + } + + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + text = try container.decode(String.self, forKey: .text) + references = try container.decodeIfPresent([CrossReference].self, forKey: .references) ?? [] + styles = try container.decodeIfPresent([StyleRun].self, forKey: .styles) ?? [] + } + + /// Empty collections are omitted rather than written as `[]`. + /// + /// Every CC-CEDICT sense has no references and no styles, and writing + /// the empty arrays anyway added 14 MB to a 35 MB bundled dictionary — + /// 40% of the file, to say "nothing here" 107,619 times. + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(text, forKey: .text) + if !references.isEmpty { + try container.encode(references, forKey: .references) + } + if !styles.isEmpty { + try container.encode(styles, forKey: .styles) + } + } +} + +/// How a span of a gloss is emphasised. +public enum TextStyle: String, Hashable, Sendable, Codable, CaseIterable { + case bold + case italic + /// A coloured span. The source names a colour; what the reader sees is + /// the app's own palette, because a dictionary's `brown` is not a + /// colour that works on both a light and a dark page. + case highlighted +} + +/// One emphasised span of a gloss. +public struct StyleRun: Hashable, Sendable, Codable { + public let style: TextStyle + /// Where the span sits in the gloss, as a UTF-16 offset range — the + /// unit `NSRange` and `AttributedString` both use, and the same unit + /// `CrossReference` uses. + public let range: Range + + public init(style: TextStyle, range: Range) { + self.style = style + self.range = range + } +} + +/// A sentence showing a word in use. +public struct UsageExample: Hashable, Sendable, Codable { + /// The Chinese, if it could be separated from the translation. + public let chinese: String + /// The translation, which for BKRS is Russian. + public let translation: String + /// The example exactly as the dictionary wrote it. + /// + /// Always kept. The split between the two scripts is a heuristic over + /// where the Han characters stop, and an example that splits badly must + /// still be readable in full rather than silently truncated to whichever + /// half the heuristic liked. + public let raw: String + + public init(chinese: String, translation: String, raw: String) { + self.chinese = chinese + self.translation = translation + self.raw = raw } } @@ -96,10 +177,96 @@ public struct Sense: Hashable, Sendable, Identifiable, Codable { public let kind: SenseKind public let gloss: Gloss - public init(id: Int, kind: SenseKind, gloss: Gloss) { + /// How deeply this sense is nested, counting from zero. + /// + /// Flat rather than a tree, with `parent` carrying the structure. A flat + /// list with an explicit level is isomorphic to a tree, encodes without + /// recursion, and renders directly as an indented list — which is all + /// the UI needs. + public let level: Int + /// The enclosing sense, or nil at the top. + public let parent: Int? + /// The dictionary's own numbering: `I`, `1)`, `а)`. + /// + /// Lifted out of the gloss rather than left at the front of it, so the + /// reader can renumber or indent without the original marker appearing + /// twice. + public let label: String? + /// Grammatical category: гл., сущ., прил. + public let partOfSpeech: [String] + /// Field or register: бот., ист., уст. + public let registers: [String] + public let examples: [UsageExample] + + public init( + id: Int, + kind: SenseKind, + gloss: Gloss, + level: Int = 0, + parent: Int? = nil, + label: String? = nil, + partOfSpeech: [String] = [], + registers: [String] = [], + examples: [UsageExample] = [], + ) { self.id = id self.kind = kind self.gloss = gloss + self.level = level + self.parent = parent + self.label = label + self.partOfSpeech = partOfSpeech + self.registers = registers + self.examples = examples + } + + private enum CodingKeys: String, CodingKey { + case id, kind, gloss, level, parent, label, partOfSpeech, registers, examples + } + + /// Decoded field by field with defaults, so a container compiled before + /// these fields existed still opens. A synthesised `init(from:)` fails + /// on any missing key, which would turn a model addition into a forced + /// re-import of a 3.4-million-entry dictionary. + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + id = try container.decode(Int.self, forKey: .id) + kind = try container.decode(SenseKind.self, forKey: .kind) + gloss = try container.decode(Gloss.self, forKey: .gloss) + level = try container.decodeIfPresent(Int.self, forKey: .level) ?? 0 + parent = try container.decodeIfPresent(Int.self, forKey: .parent) + label = try container.decodeIfPresent(String.self, forKey: .label) + partOfSpeech = try container.decodeIfPresent([String].self, forKey: .partOfSpeech) ?? [] + registers = try container.decodeIfPresent([String].self, forKey: .registers) ?? [] + examples = try container.decodeIfPresent([UsageExample].self, forKey: .examples) ?? [] + } + + /// Defaults are omitted, for the same reason as `Gloss`: a dictionary + /// with no nesting, no numbering and no examples should not pay for + /// the fields that describe them. + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(id, forKey: .id) + try container.encode(kind, forKey: .kind) + try container.encode(gloss, forKey: .gloss) + if level != 0 { + try container.encode(level, forKey: .level) + } + if let parent { + try container.encode(parent, forKey: .parent) + } + if let label { + try container.encode(label, forKey: .label) + } + if !partOfSpeech.isEmpty { + try container.encode(partOfSpeech, forKey: .partOfSpeech) + } + if !registers.isEmpty { + try container.encode(registers, forKey: .registers) + } + if !examples.isEmpty { + try container.encode(examples, forKey: .examples) + } } } diff --git a/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift b/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift new file mode 100644 index 0000000..4849d92 --- /dev/null +++ b/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift @@ -0,0 +1,518 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import Testing +@testable import HanReaderCore + +/// Enough syllables to split the readings these tests use. +private let bases: Set = [ + "ai", "le", "liao", "yi", "da", "zhong", "guo", "tu", "er", "qi", "si", "tan", +] + +private func build( + headword: String = "爱", + pinyin: String = "ài", + _ body: String..., +) + -> (entries: [DictionaryEntry], diagnostics: [DSLDiagnostic]) +{ + DSLCardBuilder(syllableBases: bases).entries(from: DSLCard( + headwords: [headword], + pinyin: pinyin, + body: body, + )) +} + +/// The text a style run actually selects. +/// +/// Asserting the substring rather than the offsets is the point: an offset +/// that is off by one still reads as a plausible number in a failure message, +/// where the wrong substring says exactly what went wrong. +private func styled(_ sense: Sense, _ index: Int = 0) -> String? { + guard index < sense.gloss.styles.count else { return nil } + return substring(sense.gloss.text, sense.gloss.styles[index].range) +} + +private func referenced(_ sense: Sense, _ index: Int = 0) -> String? { + guard index < sense.gloss.references.count else { return nil } + return substring(sense.gloss.text, sense.gloss.references[index].range) +} + +private func substring(_ text: String, _ range: Range) -> String? { + let span = NSRange(location: range.lowerBound, length: range.count) + guard let converted = Range(span, in: text) else { return nil } + return String(text[converted]) +} + +private func senses(_ body: String...) -> [Sense] { + DSLCardBuilder(syllableBases: bases).entries(from: DSLCard( + headwords: ["爱"], + pinyin: "ài", + body: body, + )).entries.first?.senses ?? [] +} + +@Suite("DSL card builder") +struct DSLCardBuilderTests { + // MARK: - The bug the rebuild exists for + + /// Test number one. The predecessor produced **one sense per card** — + /// 3,434,222 rows over 3,434,222 headwords, exactly one each — because + /// it treated a line as a sense, and a card's whole sense run arrives on + /// a single physical line. The structure is in the markup. + @Test("Two sense spans on one line become two senses") + func twoSensesOneLine() { + let result = senses("[m1]первый[/m][m2]второй[/m]") + #expect(result.count == 2) + #expect(result.map(\.gloss.text) == ["первый", "второй"]) + #expect(result.map(\.level) == [0, 1]) + } + + @Test("A real card's shape comes out as a list, not a blob") + func realisticCard() { + // A division heading, two numbered senses under it, then a second + // division — the shape most multi-sense cards take. + let division = "[m1][b]I[/b] [p]гл.[/p][/m]" + let numbered = "[m2]1) любить[/m][m2]2) ценить[/m]" + let second = "[m1][b]II[/b] [p]сущ.[/p][/m]" + let result = senses(division + numbered + second) + #expect(result.count == 4) + #expect(result.map(\.level) == [0, 1, 1, 0]) + #expect(result.map(\.label) == ["I", "1)", "2)", "II"]) + #expect(result[1].gloss.text == "любить") + #expect(result[0].partOfSpeech == ["гл."]) + } + + // MARK: - Nesting + + /// `[m4]` after an `[m3]` has to find the right `[m2]`. A stack that + /// pushes on open and pops on close attaches it to whatever was last, + /// which is wrong the moment levels skip. + @Test("A deeper sense nests under the right parent") + func nesting() { + let result = senses("[m1]A[/m][m2]B[/m][m3]C[/m][m4]D[/m][m2]E[/m]") + #expect(result.map(\.level) == [0, 1, 2, 3, 1]) + #expect(result.map(\.parent) == [nil, 0, 1, 2, 0]) + } + + @Test("Reopening a shallower level closes the deeper ones") + func reopeningShallower() { + let result = senses("[m1]A[/m][m3]B[/m][m1]C[/m]") + #expect(result.map(\.level) == [0, 2, 0]) + #expect(result.map(\.parent) == [nil, 0, nil]) + } + + /// Across the whole set there are two more `[mN]` opens than `[/m]` + /// closes, so the builder cannot assume balance. + @Test("Unclosed markup still produces its senses") + func unbalancedMarkup() { + let result = senses("[m1]первый[m2]второй") + #expect(result.count == 2) + #expect(result.map(\.gloss.text) == ["первый", "второй"]) + } + + @Test("Body text with no markup at all is still a sense") + func noMarkup() { + let result = senses("просто текст") + #expect(result.map(\.gloss.text) == ["просто текст"]) + } + + // MARK: - Readings + + /// 77% of cards. Returning `[""]` — which a naive split does — produces + /// an entry claiming its pronunciation is the empty string. + @Test("An underscore is not a reading") + func underscoreReading() { + let result = build(pinyin: "_", "[m1]текст[/m]") + #expect(result.entries.count == 1) + #expect(result.entries[0].reading.isEmpty) + #expect(result.entries[0].readingDisplay.isEmpty) + } + + /// One entry per reading, so the reading-selection rule can choose + /// between them. One entry reading `le, liǎo, liào` would put all three + /// above the word, which is the prototype's behaviour. + @Test("A card with several readings becomes several entries") + func severalReadings() { + let result = build(headword: "了", pinyin: "le, liǎo", "[m1]текст[/m]") + #expect(result.entries.count == 2) + #expect(result.entries.map(\.readingDisplay) == ["le", "liǎo"]) + // They share the sense run rather than splitting it. + #expect(result.entries.allSatisfy { $0.senses.count == 1 }) + } + + @Test("A reading that cannot be split still displays") + func unsplittableReading() { + let result = build(pinyin: "zzzqqq", "[m1]текст[/m]") + #expect(result.entries.count == 1) + // Degraded to a literal: the reading shows, only the merge key is + // lost, so the word forms its own group instead of joining one. + #expect(result.entries[0].readingDisplay == "zzzqqq") + } + + @Test("A run-together reading is split into syllables") + func runTogetherReading() { + let result = build(pinyin: "àiguó", "[m1]текст[/m]") + #expect(result.entries.first?.reading.count == 2) + } + + // MARK: - Labels and categories + + @Test("A grammatical category is separated from the prose") + func partOfSpeech() { + let result = senses("[m1][p]гл.[/p]любить[/m]") + #expect(result[0].partOfSpeech == ["гл."]) + #expect(result[0].gloss.text == "любить") + } + + /// Unrecognised markers go to registers verbatim rather than being + /// dropped. A field marker this list has not heard of is still + /// something the dictionary chose to print. + @Test( + "A field marker is kept even when unrecognised", + arguments: ["бот.", "ист.", "уст.", "зззз."], + ) + func registers(marker: String) { + let result = senses("[m1][p]\(marker)[/p]текст[/m]") + #expect(result[0].registers == [marker]) + #expect(result[0].partOfSpeech.isEmpty) + } + + @Test("Numbering is lifted out of the gloss", arguments: [ + ("1) любить", "1)", "любить"), + ("12) ценить", "12)", "ценить"), + ("а) первый", "а)", "первый"), + ("II сущ", "II", "сущ"), + ]) + func labels(body: String, label: String, gloss: String) { + let result = senses("[m1]\(body)[/m]") + #expect(result[0].label == label) + #expect(result[0].gloss.text == gloss) + } + + @Test("Prose that merely starts with a letter is not a label") + func notALabel() { + let result = senses("[m1]Исторический термин[/m]") + #expect(result[0].label == nil) + #expect(result[0].gloss.text == "Исторический термин") + } + + // MARK: - Emphasis and references + + /// 631,732 italic spans, nearly all parenthetical grammatical notes. + /// Flattened, they read as part of the definition. + @Test("Italics are kept as a range, not flattened") + func italics() throws { + let result = senses("[m1]любить ([i]кого-л.[/i])[/m]") + #expect(result[0].gloss.text == "любить (кого-л.)") + // One `#require` per statement: a macro expanded inside another + // macro's argument is a recursive expansion the compiler refuses. + let run = try #require(result[0].gloss.styles.first) + #expect(run.style == .italic) + let text = result[0].gloss.text + let span = NSRange(location: run.range.lowerBound, length: run.range.count) + let range = try #require(Range(span, in: text)) + #expect(String(text[range]) == "кого-л.") + } + + @Test("A cross-reference records the word it points at") + func reference() { + let result = senses("[m1]см. [ref]磁性碰锁[/ref][/m]") + #expect(result[0].gloss.references.first?.simplified == "磁性碰锁") + #expect(result[0].gloss.text == "см. 磁性碰锁") + } + + @Test("An empty emphasis span records nothing") + func emptyStyle() { + #expect(senses("[m1][i][/i]текст[/m]")[0].gloss.styles.isEmpty) + } + + // MARK: - Examples + + @Test("An example is split into its two languages") + func exampleSplit() { + let result = senses("[m1][*][ex]爱儿童 любить ребёнка (детей)[/ex][/*][/m]") + let example = result[0].examples.first + #expect(example?.chinese == "爱儿童") + #expect(example?.translation == "любить ребёнка (детей)") + #expect(example?.raw == "爱儿童 любить ребёнка (детей)") + } + + @Test("An example with no Chinese still keeps everything") + func exampleWithoutChinese() { + let example = SenseAssembly.splitExample("только по-русски") + #expect(example.chinese.isEmpty) + #expect(example.translation == "только по-русски") + #expect(example.raw == "только по-русски") + } + + @Test("CJK punctuation stays with the Chinese") + func examplePunctuation() { + let example = SenseAssembly.splitExample("他说:“好。” он сказал") + #expect(example.chinese == "他说:“好。”") + #expect(example.translation == "он сказал") + } + + /// 62 `[*]` spans in the set contain no `[ex]`. The block is treated as + /// transparent, so that path needs no special case and loses nothing. + @Test("An example block with no example does not lose its text") + func blockWithoutExample() { + let result = senses("[m1][*]просто текст[/*][/m]") + #expect(result[0].examples.isEmpty) + #expect(result[0].gloss.text == "просто текст") + } + + // MARK: - Entry shape + + @Test("An empty card produces no entries") + func emptyCard() { + let result = DSLCardBuilder(syllableBases: bases) + .entries(from: DSLCard(headwords: [], pinyin: "_", body: [])) + #expect(result.entries.isEmpty) + } + + /// Several headword lines are several spellings of one article, so each + /// becomes its own entry over the same senses. Reading the second as the + /// traditional form of the first is an invention the format does not + /// support — and it dropped every spelling after the second outright. + @Test("Every headword line becomes an entry") + func severalHeadwords() { + let result = DSLCardBuilder(syllableBases: bases).entries(from: DSLCard( + headwords: ["爱", "愛", "㤅"], + pinyin: "ài", + body: ["[m1]любить[/m]"], + )) + #expect(result.entries.map(\.headword.simplified) == ["爱", "愛", "㤅"]) + #expect(result.entries.allSatisfy { $0.headword.traditional == nil }) + #expect(result.entries.allSatisfy { $0.senses == result.entries[0].senses }) + } + + @Test("Several headwords and several readings multiply out") + func headwordsAndReadings() { + let result = DSLCardBuilder(syllableBases: bases).entries(from: DSLCard( + headwords: ["了", "瞭"], + pinyin: "le, liǎo", + body: ["[m1]текст[/m]"], + )) + #expect(result.entries.count == 4) + #expect(result.entries.map(\.readingDisplay) == ["le", "liǎo", "le", "liǎo"]) + } + + @Test("Diagnostics from the lexer reach the caller") + func diagnosticsPropagate() { + let result = build("[m1][zzz]текст[/m]") + #expect(result.diagnostics.count == 1) + } +} + +/// Gloss text is trimmed and renumbered after its markup ranges were +/// recorded against the untrimmed text, so every range has to move with it. +/// These are the cases where it did not. +@Suite("DSL gloss ranges") +struct DSLGlossRangeTests { + /// Lifting `1)` out of the gloss also removes the space after it, and + /// every recorded range has to move by the whole of that. It did not: + /// 16,957 senses in the set carry a style run inside a numbered sense, + /// and all of them pointed one character too far right. + @Test("A style range follows the text past a lifted label") + func styleAfterLabel() { + let result = senses("[m1]1) первый ([i]по порядку[/i])[/m]") + #expect(result[0].label == "1)") + #expect(result[0].gloss.text == "первый (по порядку)") + #expect(styled(result[0]) == "по порядку") + } + + /// Same fault, worse consequence: a reference reaching the end of the + /// gloss overshot it and was discarded rather than clipped. 3,974 senses. + @Test("A reference at the end of a numbered sense survives") + func referenceAtEndOfLabelledSense() { + let result = senses("[m1]2) ключ иероглифа ([p]см.[/p] [ref]横[/ref])[/m]") + #expect(result[0].label == "2)") + #expect(referenced(result[0]) == "横") + #expect(result[0].gloss.references.first?.simplified == "横") + } + + /// 395,878 `[p]` tags, nearly all followed by a space — so the prose of + /// 85,395 senses begins with whitespace that the final trim removed + /// without telling the ranges. No label involved. + @Test("A style range follows the text past trimmed whitespace") + func styleAfterLeadingSpace() { + let result = senses("[m1][p]бот.[/p] девясил каспийский ([i]Inula caspica[/i])[/m]") + #expect(result[0].registers == ["бот."]) + #expect(result[0].gloss.text == "девясил каспийский (Inula caspica)") + #expect(styled(result[0]) == "Inula caspica") + } + + @Test("A reference range follows the text past trimmed whitespace") + func referenceAfterLeadingSpace() { + let result = senses("[m1][p]см.[/p] [ref]磁性碰锁[/ref] и далее[/m]") + #expect(referenced(result[0]) == "磁性碰锁") + } + + /// Both trims at once, which is where an offset computed by subtracting + /// string lengths goes most wrong. + @Test("Ranges survive a leading space, a label and a trailing space") + func rangesSurviveEverything() { + let result = senses("[m1] 12) [b]первый[/b] и [i]второй[/i] [/m]") + #expect(result[0].label == "12)") + #expect(result[0].gloss.text == "первый и второй") + #expect(styled(result[0], 0) == "первый") + #expect(styled(result[0], 1) == "второй") + } + + /// The trim is a pure function, so the awkward inputs can be put to it + /// directly. + @Test("Trimming is exact at the edges", arguments: [ + ("текст", "текст", String?.none), + (" текст ", "текст", nil), + ("1)", "", "1)"), + ("1) ", "", "1)"), + (" ", "", nil), + ("", "", nil), + ]) + func trimEdges(input: String, text: String, label: String?) { + let result = SenseAssembly.trim(text: input) + #expect(result.text == text) + #expect(result.label == label) + } + + /// A run reaching into whitespace that the trim removes is clipped to the + /// gloss, not discarded. It is still the run the dictionary asked for, and + /// dropping it would cost the emphasis over the whole phrase to save a + /// space. + @Test("A run overshooting into trimmed whitespace is clipped") + func runClippedToGloss() { + let result = senses("[m1]текст [i]курсив [/i][/m]") + #expect(result[0].gloss.text == "текст курсив") + #expect(styled(result[0]) == "курсив") + } + + @Test("An unclosed run over trailing whitespace is clipped too") + func unclosedRunClipped() { + let result = senses("[m1]текст [i]курсив ") + #expect(result[0].gloss.text == "текст курсив") + #expect(styled(result[0]) == "курсив") + } + + /// 21 `[ref]` spans in the set close after a space. A headword with a + /// trailing space matches no entry in any dictionary, so the reference + /// would point nowhere — and the range has to be trimmed with the string, + /// or the two disagree about which characters the link covers. + @Test( + "A reference span is trimmed to the headword it names", + arguments: ["[ref]磁性碰锁 [/ref]", "[ref] 磁性碰锁[/ref]", "[ref] 磁性碰锁 [/ref]"], + ) + func referenceSpanTrimmed(markup: String) throws { + let result = senses("[m1]см. \(markup) и далее[/m]") + let reference = try #require(result[0].gloss.references.first) + #expect(reference.simplified == "磁性碰锁") + #expect(referenced(result[0]) == "磁性碰锁") + } + + @Test("A reference span of nothing but whitespace records nothing") + func referenceSpanAllWhitespace() { + #expect(senses("[m1]см. [ref] [/ref][/m]")[0].gloss.references.isEmpty) + } + + @Test("A span lying entirely inside a lifted label is dropped") + func spanInsideLabel() { + // `[b]I[/b]` is the division heading itself: once lifted into the + // label there is no gloss left for the run to point at. + let result = senses("[m1][b]I[/b] [p]гл.[/p][/m]") + #expect(result[0].label == "I") + #expect(result[0].gloss.text.isEmpty) + #expect(result[0].gloss.styles.isEmpty) + } +} + +/// DSL markup is not reliably balanced — the set has two more `[mN]` opens +/// than `[/m]` closes and 241 sense spans with a style tag still open — and +/// the rule the importer follows is that nothing is ever deleted. +@Suite("Unbalanced DSL markup") +struct DSLUnbalancedMarkupTests { + /// Nothing is ever deleted. An unclosed `[i]` kept its text but lost the + /// emphasis it was asking for — 241 sense spans in the set close `[/m]` + /// with a style still open. + @Test("An unclosed emphasis still produces its run") + func unclosedStyle() { + let result = senses("[m1]текст [i]курсив[/m][m1]другой[/m]") + #expect(result.count == 2) + #expect(styled(result[0]) == "курсив") + // And it does not leak into the next sense. + #expect(result[1].gloss.styles.isEmpty) + } + + @Test("An unclosed emphasis at the end of a card still produces its run") + func unclosedStyleAtEnd() { + let result = senses("[m1]текст [i]курсив") + #expect(styled(result[0]) == "курсив") + } + + @Test("An unclosed reference still records the word") + func unclosedReference() { + let result = senses("[m1]см. [ref]磁性碰锁") + #expect(referenced(result[0]) == "磁性碰锁") + } + + /// `[p]` and `[ex]` divert text out of the gloss, so an unclosed one took + /// its contents out of the sense entirely. None occur in this set, but + /// losing text is the one thing this importer must never do. + @Test("An unclosed category keeps its text") + func unclosedGrammar() { + let result = senses("[m1]текст [p]гл.") + #expect(result[0].partOfSpeech == ["гл."]) + #expect(result[0].gloss.text == "текст") + } + + @Test("An unclosed example keeps its text") + func unclosedExample() { + let result = senses("[m1]текст [ex]爱儿童 любить ребёнка") + #expect(result[0].examples.first?.raw == "爱儿童 любить ребёнка") + } + + /// The other half of the same fault: a buffer still open when the next + /// sense began went on collecting that sense's prose. + @Test("An unclosed category does not swallow the next sense") + func unclosedGrammarDoesNotLeak() { + let result = senses("[m1][p]гл.[m2]любить[/m]") + #expect(result.count == 2) + #expect(result[0].partOfSpeech == ["гл."]) + #expect(result[1].gloss.text == "любить") + } + + /// 479 cards begin their article with markup and no `[mN]` at all. The + /// fallback sense used to be created by the first *text* token, which + /// cleared the offset the `[i]` had already recorded. + @Test("Emphasis before any sense tag still produces its run") + func styleBeforeAnySense() { + let result = senses("[i]курсив[/i] и дальше") + #expect(result.count == 1) + #expect(result[0].gloss.text == "курсив и дальше") + #expect(styled(result[0]) == "курсив") + } + + @Test("A reference before any sense tag still records the word") + func referenceBeforeAnySense() { + let result = senses("[ref]磁性碰锁[/ref] см.") + #expect(referenced(result[0]) == "磁性碰锁") + } + + // MARK: - A sense broken across body lines + + /// 乐芙兰, the one card in 3,434,224 whose article breaks a line inside + /// an open `[m1]`. Concatenating gives `Ле Блан(чемпион…)`. + @Test("A sense continued on the next line gains a space, not nothing") + func senseAcrossLines() { + let result = senses("[m1]Ле Блан", "(чемпион из Лиги Легенд)[/m]") + #expect(result.count == 1) + #expect(result[0].gloss.text == "Ле Блан (чемпион из Лиги Легенд)") + } + + /// And a card that closes each sense at the end of its line — which is + /// how most DSL dictionaries are written — gains no stray space. + @Test("One sense per line gains no leading space") + func sensePerLine() { + let result = senses("[m1]первый[/m]", "[m2]второй[/m]") + #expect(result.map(\.gloss.text) == ["первый", "второй"]) + } +} diff --git a/Tests/HanReaderCoreTests/SenseCodingTests.swift b/Tests/HanReaderCoreTests/SenseCodingTests.swift new file mode 100644 index 0000000..04c7443 --- /dev/null +++ b/Tests/HanReaderCoreTests/SenseCodingTests.swift @@ -0,0 +1,106 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import Testing +@testable import HanReaderCore + +@Suite("Sense coding") +struct SenseCodingTests { + private enum CodingTestError: Error { case notUTF8 } + + private func encoded(_ sense: Sense) throws -> String { + let data = try JSONEncoder().encode(sense) + // A plain unwrap rather than `#require`: this helper is called from + // inside `#expect`, and a macro expanded within a macro's argument + // is a recursive expansion the compiler refuses. + guard let text = String(bytes: data, encoding: .utf8) else { + throw CodingTestError.notUTF8 + } + return text + } + + /// Writing `"level":0,"partOfSpeech":[],"registers":[],"examples":[]` + /// for every CC-CEDICT sense added 14 MB to a 35 MB dictionary — 40% of + /// the file to say "nothing here" 107,619 times. + @Test("A plain sense encodes none of the fields it does not use") + func defaultsAreOmitted() throws { + let json = try encoded(Sense(id: 0, kind: .definition, gloss: Gloss(text: "to love"))) + for absent in [ + "level", + "parent", + "label", + "partOfSpeech", + "registers", + "examples", + "styles", + "references", + ] { + #expect(!json.contains(absent), "\(absent) should not be written") + } + #expect(json.contains("to love")) + } + + @Test("A sense that uses the fields writes them") + func populatedFieldsAreWritten() throws { + let sense = Sense( + id: 1, + kind: .definition, + gloss: Gloss(text: "любить", styles: [StyleRun(style: .italic, range: 0 ..< 6)]), + level: 1, + parent: 0, + label: "1)", + partOfSpeech: ["гл."], + examples: [UsageExample(chinese: "爱", translation: "любить", raw: "爱 любить")], + ) + let json = try encoded(sense) + for present in ["level", "parent", "label", "partOfSpeech", "examples", "styles"] { + #expect(json.contains(present)) + } + #expect(!json.contains("registers"), "still omitted when empty") + } + + @Test("Everything round-trips") + func roundTrip() throws { + let sense = Sense( + id: 3, + kind: .reference, + gloss: Gloss( + text: "см. 磁性碰锁", + references: [CrossReference( + simplified: "磁性碰锁", traditional: nil, reading: nil, range: 4 ..< 9, + )], + styles: [StyleRun(style: .bold, range: 0 ..< 3)], + ), + level: 2, + parent: 1, + label: "а)", + partOfSpeech: ["сущ."], + registers: ["бот."], + examples: [UsageExample(chinese: "锁", translation: "замок", raw: "锁 замок")], + ) + let data = try JSONEncoder().encode(sense) + #expect(try JSONDecoder().decode(Sense.self, from: data) == sense) + } + + /// A container compiled before these fields existed must still open. A + /// synthesised `init(from:)` fails on any missing key, which would make + /// a model addition force a re-import of a 3.4-million-entry dictionary. + @Test("A payload written before these fields existed still decodes") + func oldPayloadDecodes() throws { + let old = #"{"id":0,"kind":0,"gloss":{"text":"to love","references":[]}}"# + let sense = try JSONDecoder().decode(Sense.self, from: Data(old.utf8)) + #expect(sense.gloss.text == "to love") + #expect(sense.level == 0) + #expect(sense.examples.isEmpty) + #expect(sense.label == nil) + } + + @Test("A gloss written before styles existed still decodes") + func oldGlossDecodes() throws { + let old = #"{"text":"to love"}"# + let gloss = try JSONDecoder().decode(Gloss.self, from: Data(old.utf8)) + #expect(gloss.text == "to love") + #expect(gloss.styles.isEmpty) + #expect(gloss.references.isEmpty) + } +}