diff --git a/Sources/HanReaderCore/Dictionary/LexiconBuilder.swift b/Sources/HanReaderCore/Dictionary/LexiconBuilder.swift index 20e0e95..444b868 100644 --- a/Sources/HanReaderCore/Dictionary/LexiconBuilder.swift +++ b/Sources/HanReaderCore/Dictionary/LexiconBuilder.swift @@ -62,7 +62,27 @@ public enum LexiconBuilder { /// - **Not a bare cross-reference.** An entry whose only sense is /// "variant of X" is a spelling note, and treating it as a word to /// segment towards produces confident nonsense. - public static func lexicon(from entries: [DictionaryEntry]) -> [Lexeme] { + /// Cap for a dictionary that is not word-level. + /// + /// CC-CEDICT's headwords are genuinely words, so the general cap of six + /// suits it: 125,173 entries produce about a hundred thousand lexemes. + /// BKRS is an encyclopaedic dictionary and its headwords are not — at six + /// characters it contributes **2,804,554** of them, a segmentation + /// lexicon in which almost every string of Han characters is a word, which + /// is the condition under which maximum-matching produces confident + /// nonsense. It is also 176 MB of index. + /// + /// Four is where the measured length distribution says to stop for it: + /// 1→33,372, 2→427,842, 3→455,438, 4→959,810, then 5→585,617 and + /// 6→439,262 of increasingly phrase-like entries. + public static let encyclopaedicWordLength = 4 + + public static func lexicon( + from entries: [DictionaryEntry], + maximumLength: Int = Self.maximumWordLength, + ) + -> [Lexeme] + { var best: [String: Double] = [:] for entry in entries { @@ -73,7 +93,7 @@ public enum LexiconBuilder { // would be segmented character by character. for word in entry.headword.bothScripts { let length = word.count - guard length >= 1, length <= maximumWordLength else { continue } + guard length >= 1, length <= maximumLength else { continue } guard word.allSatisfy(\.isHan) else { continue } // More senses suggests a more established word, which is the @@ -126,15 +146,39 @@ public enum LexiconBuilder { } public static func characterReadings(from entries: [DictionaryEntry]) -> [CharacterReading] { - var byCharacter: [Character: [String: Candidate]] = [:] - + var collector = CharacterReadings() for entry in entries { + collector.add(entry) + } + return collector.ranked() + } + + /// Collects character readings one entry at a time. + /// + /// The same ranking as `characterReadings(from:)`, which is a convenience + /// over this — but without the array. Building the table for an imported + /// BKRS set meant holding every single-character entry at once, and those + /// are the densest cards in the dictionary: 38,618 of them carrying 27 MB + /// of encoded senses, which 一's 57 and 打's 57 are typical of rather than + /// exceptional. Decoded together, with the decoder's own intermediates on + /// top, that took peak memory to 857 MB against an import ceiling of 60. + /// + /// What the ranking needs from an entry is five small values. Keeping + /// those and discarding the entry makes the table's cost a function of how + /// many distinct characters a dictionary has, not of how much it has to + /// say about them. + public struct CharacterReadings: Sendable { + private var byCharacter: [Character: [String: Candidate]] = [:] + + public init() {} + + public mutating func add(_ entry: DictionaryEntry) { guard let syllable = entry.reading.compactMap({ token -> PinyinSyllable? in if case let .syllable(syllable) = token { return syllable } return nil - }).first else { continue } + }).first else { return } let candidate = Candidate( numeric: syllable.numeric, @@ -169,20 +213,22 @@ public enum LexiconBuilder { } } - var readings: [CharacterReading] = [] - for (character, candidates) in byCharacter { - let ordered = candidates.values.sorted(by: isBetter) - for (rank, candidate) in ordered.enumerated() { - readings.append(CharacterReading( - character: character, - numeric: candidate.numeric, - display: candidate.display, - rank: rank, - )) + public func ranked() -> [CharacterReading] { + var readings: [CharacterReading] = [] + for (character, candidates) in byCharacter { + let ordered = candidates.values.sorted(by: isBetter) + for (rank, candidate) in ordered.enumerated() { + readings.append(CharacterReading( + character: character, + numeric: candidate.numeric, + display: candidate.display, + rank: rank, + )) + } + } + return readings.sorted { + ($0.character, $0.rank) < ($1.character, $1.rank) } - } - return readings.sorted { - ($0.character, $0.rank) < ($1.character, $1.rank) } } diff --git a/Sources/HanReaderCore/Dictionary/SensePayloadCodec.swift b/Sources/HanReaderCore/Dictionary/SensePayloadCodec.swift new file mode 100644 index 0000000..5eb7177 --- /dev/null +++ b/Sources/HanReaderCore/Dictionary/SensePayloadCodec.swift @@ -0,0 +1,529 @@ +// HanReader — MIT licensed. See LICENSE. + +public import Foundation + +/// How a container's sense payloads are encoded. +/// +/// A seam rather than a single implementation, because the choice is a +/// measured trade and will be made again. A container records which codec +/// wrote it, so adding one does not invalidate the dictionaries already on +/// disk — and so a future block-compressing codec is a drop-in rather than a +/// migration. +public protocol SensePayloadCodec: Sendable { + /// Written into the container's `meta` table, so a reader can pick the + /// codec the writer used rather than assuming the current one. + static var identifier: String { get } + + func encode(_ senses: [Sense]) throws -> Data + func decode(_ data: Data) throws -> [Sense] +} + +/// Something that cannot be read as a sense payload. +public enum SensePayloadError: Error, Hashable, Sendable, CustomStringConvertible { + case truncated(offset: Int) + case unknownVersion(UInt8) + case unknownCodec(String) + + public var description: String { + switch self { + case let .truncated(offset): + "the payload ends part way through a field, at byte \(offset)" + case let .unknownVersion(version): + "payload format version \(version) is newer than this build understands" + case let .unknownCodec(identifier): + "this container's senses were written by the '\(identifier)' codec, " + + "which this build does not have" + } + } +} + +// MARK: - JSON, as the first milestones wrote it + +/// JSON, kept so a container compiled before the compact codec still opens. +/// +/// It is not what new containers use. Measured over the BKRS set, only +/// **40.7%** of a JSON payload is text; the rest is syntax and key names, +/// written out once per sense across 3,691,468 of them. +public struct JSONSenseCodec: SensePayloadCodec { + public static let identifier = "json" + + public init() {} + + public func encode(_ senses: [Sense]) throws -> Data { + try JSONEncoder().encode(senses) + } + + public func decode(_ data: Data) throws -> [Sense] { + try JSONDecoder().decode([Sense].self, from: data) + } +} + +// MARK: - The packed encoding + +/// A packed binary encoding of a sense run. +/// +/// ### Why +/// +/// Measured over the 3,442,623-entry BKRS container: the `entry` table was +/// 625 MB, its sense payloads 327 MB of that, and **only 40.7% of a payload +/// was text**. The rest was `{"gloss":{"text":…},"level":0,"kind":…}` written +/// out once per sense. Payload size is also most of the import's wall-clock, +/// since every byte is one SQLite has to write. +/// +/// ### How +/// +/// Three ideas, each worth its complexity: +/// +/// - **A presence byte.** The overwhelmingly common sense is a gloss, a level +/// and nothing else — 3,318,399 of 3,434,224 cards carry a single sense — so +/// every field it lacks costs one bit instead of a key and a literal. +/// - **Varints.** Levels are 0–3, parents and offsets are small, counts are +/// usually one. A varint spends a byte on each where JSON spends a decimal +/// rendering plus punctuation. +/// - **An example is stored once.** `UsageExample` keeps `chinese`, +/// `translation` *and* `raw`, where `raw` is the example as written and the +/// other two are a heuristic split of it; measured, the halves cost as much +/// again as `raw` itself. Here `raw` is stored with the two boundaries +/// inside it, reproducing both halves exactly — including a split that +/// trimmed whitespace — from one copy of the text. +/// +/// `Sense.id` is not stored at all: it is the index in the run. +/// +/// ### Compatibility +/// +/// The leading version byte is checked, and a payload claiming a newer version +/// is refused rather than misread. Adding a field means a new presence bit and +/// a version bump; `meta` says which codec wrote a container, so an older one +/// stays readable. +public struct CompactSenseCodec: SensePayloadCodec { + public static let identifier = "compact-v1" + + /// Bumped when the layout changes. A payload is refused rather than + /// guessed at, because a misread sense is worse than an error. + static let version: UInt8 = 1 + + enum Field: UInt8 { + case parent = 0x01 + case label = 0x02 + case styles = 0x04 + case references = 0x08 + case partOfSpeech = 0x10 + case registers = 0x20 + case examples = 0x40 + /// Set when the kind is not `.definition`, which almost every sense is. + case kind = 0x80 + } + + public init() {} + + public func encode(_ senses: [Sense]) throws -> Data { + var bytes: [UInt8] = [Self.version] + // A rough guess at the common case, to avoid a handful of growths per + // entry across three and a half million of them. + bytes.reserveCapacity(64 * senses.count + 8) + Self.appendVarint(senses.count, to: &bytes) + for sense in senses { + Self.append(sense, to: &bytes) + } + return Data(bytes) + } + + public func decode(_ data: Data) throws -> [Sense] { + var reader = Reader(data) + let version = try reader.byte() + guard version == Self.version else { throw SensePayloadError.unknownVersion(version) } + + let count = try reader.varint() + var senses: [Sense] = [] + senses.reserveCapacity(count) + for index in 0 ..< count { + try senses.append(Self.decodeSense(at: index, from: &reader)) + } + return senses + } +} + +// MARK: - Writing + +extension CompactSenseCodec { + static func append(_ sense: Sense, to bytes: inout [UInt8]) { + bytes.append(presence(of: sense)) + + if sense.kind != .definition { + bytes.append(sense.kind.rawValue) + } + appendVarint(sense.level, to: &bytes) + if let parent = sense.parent { + appendVarint(parent, to: &bytes) + } + if let label = sense.label { + appendText(label, to: &bytes) + } + appendText(sense.gloss.text, to: &bytes) + + if !sense.gloss.styles.isEmpty { + append(styles: sense.gloss.styles, to: &bytes) + } + if !sense.gloss.references.isEmpty { + append(references: sense.gloss.references, to: &bytes) + } + if !sense.partOfSpeech.isEmpty { + appendTexts(sense.partOfSpeech, to: &bytes) + } + if !sense.registers.isEmpty { + appendTexts(sense.registers, to: &bytes) + } + if !sense.examples.isEmpty { + appendVarint(sense.examples.count, to: &bytes) + for example in sense.examples { + appendExample(example, to: &bytes) + } + } + } + + private static func append(styles: [StyleRun], to bytes: inout [UInt8]) { + appendVarint(styles.count, to: &bytes) + for run in styles { + bytes.append(run.style.encoded) + appendVarint(run.range.lowerBound, to: &bytes) + appendVarint(run.range.count, to: &bytes) + } + } + + private static func append(references: [CrossReference], to bytes: inout [UInt8]) { + appendVarint(references.count, to: &bytes) + for reference in references { + appendText(reference.simplified, to: &bytes) + appendOptionalText(reference.traditional, to: &bytes) + appendOptionalText(reference.reading, to: &bytes) + appendVarint(reference.range.lowerBound, to: &bytes) + appendVarint(reference.range.count, to: &bytes) + } + } + + /// One bit per optional field. + static func presence(of sense: Sense) -> UInt8 { + var flags: UInt8 = 0 + if sense.parent != nil { + flags |= Field.parent.rawValue + } + if sense.label != nil { + flags |= Field.label.rawValue + } + if !sense.gloss.styles.isEmpty { + flags |= Field.styles.rawValue + } + if !sense.gloss.references.isEmpty { + flags |= Field.references.rawValue + } + if !sense.partOfSpeech.isEmpty { + flags |= Field.partOfSpeech.rawValue + } + if !sense.registers.isEmpty { + flags |= Field.registers.rawValue + } + if !sense.examples.isEmpty { + flags |= Field.examples.rawValue + } + if sense.kind != .definition { + flags |= Field.kind.rawValue + } + return flags + } + + /// Writes an example as its raw text plus where the two halves sit in it. + /// + /// The split is a heuristic over where the Han characters stop, and the + /// translation side is trimmed, so neither half is a plain suffix of the + /// raw text. Two offsets reproduce both exactly from one copy. + /// + /// When a half is not a slice of the raw text at all — which a different + /// splitter could produce — it is written out in full and marked, so this + /// stays an optimisation rather than an invariant the model must keep. + static func appendExample(_ example: UsageExample, to bytes: inout [UInt8]) { + let raw = Array(example.raw.utf8) + let chinese = Array(example.chinese.utf8) + let translation = Array(example.translation.utf8) + + // One-based, so zero can mean "this half is empty" — which is how an + // example with no Chinese at all is told from one whose Chinese starts + // at offset zero, the ordinary case. + let chineseStart = chinese.isEmpty + ? 0 + : (raw.firstRange(of: chinese)?.lowerBound).map { $0 + 1 } + let translationStart = translation.isEmpty + ? 0 + : (raw.lastRange(of: translation)?.lowerBound).map { $0 + 1 } + + guard let chineseStart, let translationStart else { + bytes.append(1) + appendText(example.raw, to: &bytes) + appendText(example.chinese, to: &bytes) + appendText(example.translation, to: &bytes) + return + } + bytes.append(0) + appendText(example.raw, to: &bytes) + appendVarint(chineseStart, to: &bytes) + appendVarint(chinese.count, to: &bytes) + appendVarint(translationStart, to: &bytes) + appendVarint(translation.count, to: &bytes) + } + + /// LEB128, unsigned. Every integer in this format is small and + /// non-negative — levels, parents, offsets, counts — so a fixed width + /// would spend seven bytes saying nothing. + static func appendVarint(_ value: Int, to bytes: inout [UInt8]) { + var remaining = UInt(bitPattern: value) + while true { + let byte = UInt8(remaining & 0x7F) + remaining >>= 7 + if remaining == 0 { + bytes.append(byte) + return + } + bytes.append(byte | 0x80) + } + } + + static func appendText(_ text: String, to bytes: inout [UInt8]) { + let utf8 = text.utf8 + appendVarint(utf8.count, to: &bytes) + bytes.append(contentsOf: utf8) + } + + /// A length of zero means the empty string; nil is one less than that, so + /// an absent traditional spelling costs one byte and an empty one is still + /// distinguishable from it. + static func appendOptionalText(_ text: String?, to bytes: inout [UInt8]) { + guard let text else { + appendVarint(0, to: &bytes) + return + } + let utf8 = text.utf8 + appendVarint(utf8.count + 1, to: &bytes) + bytes.append(contentsOf: utf8) + } + + static func appendTexts(_ texts: [String], to bytes: inout [UInt8]) { + appendVarint(texts.count, to: &bytes) + for text in texts { + appendText(text, to: &bytes) + } + } +} + +// MARK: - Reading + +extension CompactSenseCodec { + static func decodeSense(at index: Int, from reader: inout Reader) throws -> Sense { + let flags = try reader.byte() + let kind = try flags & Field.kind.rawValue != 0 + ? SenseKind(rawValue: reader.byte()) ?? .definition + : .definition + let level = try reader.varint() + let parent = flags & Field.parent.rawValue != 0 ? try reader.varint() : nil + let label = flags & Field.label.rawValue != 0 ? try reader.text() : nil + let text = try reader.text() + let styles = flags & Field.styles.rawValue != 0 ? try decodeStyles(from: &reader) : [] + let references = flags & Field.references.rawValue != 0 + ? try decodeReferences(from: &reader) + : [] + let partOfSpeech = flags & Field.partOfSpeech.rawValue != 0 ? try reader.texts() : [] + let registers = flags & Field.registers.rawValue != 0 ? try reader.texts() : [] + var examples: [UsageExample] = [] + if flags & Field.examples.rawValue != 0 { + for _ in try 0 ..< (reader.varint()) { + try examples.append(decodeExample(from: &reader)) + } + } + + return Sense( + id: index, + kind: kind, + gloss: Gloss(text: text, references: references, styles: styles), + level: level, + parent: parent, + label: label, + partOfSpeech: partOfSpeech, + registers: registers, + examples: examples, + ) + } + + private static func decodeStyles(from reader: inout Reader) throws -> [StyleRun] { + var styles: [StyleRun] = [] + for _ in try 0 ..< (reader.varint()) { + let style = try TextStyle(encoded: reader.byte()) + let start = try reader.varint() + let length = try reader.varint() + styles.append(StyleRun(style: style, range: start ..< (start + length))) + } + return styles + } + + private static func decodeReferences(from reader: inout Reader) throws -> [CrossReference] { + var references: [CrossReference] = [] + for _ in try 0 ..< (reader.varint()) { + let simplified = try reader.text() + let traditional = try reader.optionalText() + let reading = try reader.optionalText() + let start = try reader.varint() + let length = try reader.varint() + references.append(CrossReference( + simplified: simplified, + traditional: traditional, + reading: reading, + range: start ..< (start + length), + )) + } + return references + } + + private static func decodeExample(from reader: inout Reader) throws -> UsageExample { + let form = try reader.byte() + let raw = try reader.text() + guard form == 0 else { + return try UsageExample( + chinese: reader.text(), + translation: reader.text(), + raw: raw, + ) + } + let chineseStart = try reader.varint() + let chineseLength = try reader.varint() + let translationStart = try reader.varint() + let translationLength = try reader.varint() + let bytes = Array(raw.utf8) + return UsageExample( + chinese: slice(bytes, start: chineseStart, length: chineseLength), + translation: slice(bytes, start: translationStart, length: translationLength), + raw: raw, + ) + } + + // A payload is derived data in a file on disk. Where it is damaged, a + // replacement character costs the characters it replaces and leaves the + // entry readable, which beats an error costing the whole entry — and + // structural damage is already an error, from the bounds checks below. + // The same bargain the byte reader makes with a malformed source. + // swiftlint:disable optional_data_string_conversion + + /// A half of an example, by its one-based start. Zero means empty, which + /// is how "no Chinese at all" is told from "the Chinese starts at zero". + private static func slice(_ bytes: [UInt8], start: Int, length: Int) -> String { + guard start > 0, length > 0 else { return "" } + let from = start - 1 + guard from + length <= bytes.count else { return "" } + return String(decoding: bytes[from ..< (from + length)], as: UTF8.self) + } + + /// A cursor over the payload that refuses to read past the end. + /// + /// Every read is bounds-checked and throws rather than trapping: a payload + /// is data from a file on disk, and a corrupt one should cost that entry, + /// not the process. + struct Reader { + private let bytes: [UInt8] + private var offset = 0 + + init(_ data: Data) { + bytes = [UInt8](data) + } + + mutating func byte() throws -> UInt8 { + guard offset < bytes.count else { throw SensePayloadError.truncated(offset: offset) } + defer { offset += 1 } + return bytes[offset] + } + + mutating func varint() throws -> Int { + var result = 0 + var shift = 0 + while true { + let byte = try byte() + result |= Int(byte & 0x7F) << shift + if byte & 0x80 == 0 { + return result + } + shift += 7 + guard shift < 64 else { throw SensePayloadError.truncated(offset: offset) } + } + } + + mutating func text() throws -> String { + try take(count: varint()) + } + + mutating func optionalText() throws -> String? { + let length = try varint() + guard length > 0 else { return nil } + return try take(count: length - 1) + } + + mutating func texts() throws -> [String] { + let count = try varint() + var result: [String] = [] + result.reserveCapacity(count) + for _ in 0 ..< count { + try result.append(text()) + } + return result + } + + private mutating func take(count: Int) throws -> String { + guard count >= 0, offset + count <= bytes.count else { + throw SensePayloadError.truncated(offset: offset) + } + defer { offset += count } + return String(decoding: bytes[offset ..< (offset + count)], as: UTF8.self) + } + } + + // swiftlint:enable optional_data_string_conversion +} + +// MARK: - Enumerations on the wire + +extension TextStyle { + /// A fixed byte per case, chosen here rather than taken from the case + /// order: reordering the enum must not change what is already on disk. + var encoded: UInt8 { + switch self { + case .bold: 1 + case .italic: 2 + case .highlighted: 3 + } + } + + init(encoded: UInt8) { + switch encoded { + case 1: self = .bold + case 3: self = .highlighted + default: self = .italic + } + } +} + +// MARK: - Choosing a codec + +/// Which codec reads or writes a container. +public enum SensePayloadCodecs { + /// What a writer should use now. + public static let current: any SensePayloadCodec = CompactSenseCodec() + + /// Resolves a recorded identifier. + /// + /// An unrecognised one is an error rather than a fallback to the current + /// codec: misreading a payload produces plausible nonsense, and a + /// container written by a newer build should say so plainly. + public static func named(_ identifier: String?) throws -> any SensePayloadCodec { + switch identifier { + // A container from before the codec was recorded is JSON, because that + // is what the only writer there was then used. + case nil, JSONSenseCodec.identifier: JSONSenseCodec() + case CompactSenseCodec.identifier: CompactSenseCodec() + case let other: throw SensePayloadError.unknownCodec(other ?? "") + } + } +} diff --git a/Sources/HanReaderDictionaryImport/DSLImporter.swift b/Sources/HanReaderDictionaryImport/DSLImporter.swift new file mode 100644 index 0000000..839f568 --- /dev/null +++ b/Sources/HanReaderDictionaryImport/DSLImporter.swift @@ -0,0 +1,596 @@ +// HanReader — MIT licensed. See LICENSE. + +public import Foundation +public import HanReaderCore +import HanReaderPersistence + +/// How far an import has got. +public struct DSLImportProgress: Hashable, Sendable { + public let fileIndex: Int + public let fileCount: Int + public let bytesRead: Int + public let totalBytes: Int + public let cardsRead: Int + public let entriesWritten: Int + + /// 0 to 1 across the whole set, not the current file — a reader watching + /// a three-volume import does not want the bar to reach the end and start + /// again twice. + public var fraction: Double { + guard totalBytes > 0 else { return 0 } + return min(1, Double(bytesRead) / Double(totalBytes)) + } +} + +/// What an import produced. +public struct DSLImportSummary: Hashable, Sendable { + public let cardsRead: Int + public let entriesWritten: Int + /// Problems inside the files — unknown markup, an article with no + /// headword. + public let diagnostics: Int + /// Problems with the *set*: a volume an `#INCLUDE` names but that is not + /// there, or a set longer than discovery will follow. Separate from the + /// count above, because a reader has to be told about these whatever the + /// import otherwise achieved — a dictionary quietly missing a third of + /// itself is the failure they would never notice. + public let discovery: [DSLDiagnostic] + public let resumed: Bool + /// False when the import stopped early because it was asked to. + /// + /// An unfinished container keeps its job row, so the next run picks up + /// where this one left off. It is not a dictionary yet, and nothing has + /// built its lookup indexes. + public let completed: Bool + public let metadata: DictionaryMetadata + + /// Everything that went wrong, as one headline number. + public var totalDiagnostics: Int { + diagnostics + discovery.count + } +} + +/// Imports an ABBYY DSL dictionary into a container. +/// +/// The pieces it drives all live in `HanReaderCore` and are tested there: +/// `DSLFileSet` finds the volumes, `DSLCardReader` turns bytes into cards, and +/// `DSLCardBuilder` turns a card into entries. This owns only the I/O around +/// them — batching, checkpointing, throttled progress and cancellation — +/// which is the part that cannot be a pure function. +/// +/// ### Memory is bounded by the batch, not by the dictionary +/// +/// 3,442,623 entries over 3,691,468 senses do not fit in the 60 MB the import +/// is allowed. Nothing is accumulated: cards stream in, a batch of entries is +/// written, and the batch is released. Measured peak over the real 350 MB set +/// is a few tens of megabytes, almost all of it SQLite's page cache. +/// +/// ### Progress is throttled, because the alternative was measurable +/// +/// The predecessor fired its progress handler every 128 KB and hopped to the +/// main actor each time — about 2,700 hops for this set, every one of them +/// contending with the thread doing CJK layout. Here progress is reported at +/// most ten times a second, and always once at the end so a bar cannot stop +/// short of full. +public struct DSLImporter: Sendable { + /// Bumped when a parser change alters what gets written. + /// + /// A container built by an older parser is rebuilt rather than migrated, + /// because no schema migration can repair data that was parsed wrongly. + public static let parserVersion = 1 + + /// Entries per transaction. A row at a time takes minutes; batched it + /// takes seconds, and 5,000 entries is a few megabytes. + public static let defaultBatchSize = 5000 + + /// Ten a second. Any faster is invisible to a reader and costs the + /// importer an actor hop. + static let progressInterval: TimeInterval = 0.1 + + private let fileSet: DSLFileSet + private let destination: URL + private let syllableBases: Set + private let batchSize: Int + + /// - Parameters: + /// - fileSet: the volumes to read, from `DSLFileSet.discover(from:)`. + /// - destination: the container to write, created or resumed. + /// - syllableBases: toneless syllables for splitting a run-together + /// reading such as `tǔ'ěrqísītǎn`. These come from CC-CEDICT, which is + /// the phonetic backbone for both language modes; without them a + /// reading still displays exactly as the dictionary wrote it, and only + /// the cross-dictionary merge key is lost. + public init( + fileSet: DSLFileSet, + destination: URL, + syllableBases: Set, + batchSize: Int = Self.defaultBatchSize, + ) { + self.fileSet = fileSet + self.destination = destination + self.syllableBases = syllableBases + self.batchSize = max(1, batchSize) + } + + // MARK: - Running + + /// Runs the import, resuming an interrupted one. + /// + /// Two ways to stop it, for two different needs. `shouldContinue` + /// returning false stops at the next card boundary and returns a summary + /// with `completed == false`, which is what a Cancel button wants: the + /// work so far is committed, the job row stays, and nothing has thrown. + /// Cancelling the task instead throws `CancellationError`, for a caller + /// that is tearing down rather than pausing. + /// + /// Either way the container is left in the state a `kill -9` leaves — + /// everything up to the last committed batch — and resumes the same way. + @discardableResult + public func run( + progress report: (@Sendable (DSLImportProgress) -> Void)? = nil, + shouldContinue: (@Sendable () -> Bool)? = nil, + ) throws + -> DSLImportSummary + { + let sources = try fileSet.files.map { try FileDescription(url: $0) } + guard let first = sources.first else { + throw DSLImportError.noFiles + } + let totalBytes = sources.reduce(0) { $0 + $1.size } + + let (writer, resume) = try DictionaryWriter.open( + at: destination, + source: DictionaryWriter.Source( + name: first.name, + fingerprint: Self.fingerprint(of: sources, bases: syllableBases), + fileCount: sources.count, + totalBytes: totalBytes, + parserVersion: Self.parserVersion, + ), + // A dictionary whose headwords run to phrases must not supply the + // segmentation lexicon as if they were words: at the general cap + // of six, BKRS contributes 2,804,554 lexemes and almost every run + // of Han characters becomes a word to match towards. + lexiconMaximumLength: LexiconBuilder.encyclopaedicWordLength, + ) + + let run = Run( + sources: sources, + resume: resume, + writer: writer, + builder: DSLCardBuilder(syllableBases: syllableBases), + batchSize: batchSize, + report: report, + shouldContinue: shouldContinue, + ) + for index in run.state.checkpoint.fileIndex ..< sources.count { + try run.importFile(at: index) + if run.stopped { + break + } + } + + return try summary(for: run, writer: writer, firstFile: first.name, resumed: resume != nil) + } + + /// Finishes the container unless the run was asked to stop, and describes + /// what it produced either way. + private func summary( + for run: Run, + writer: DictionaryWriter, + firstFile: String, + resumed: Bool, + ) throws + -> DSLImportSummary + { + let metadata = Self.metadata( + for: fileSet, + entryCount: writer.count, + fileName: firstFile, + ) + if !run.stopped { + try writer.finish(metadata: metadata) + run.report?(run.state.progress(written: writer.count, completed: true)) + } + return DSLImportSummary( + cardsRead: run.state.checkpoint.cardsRead, + entriesWritten: writer.count, + diagnostics: run.state.checkpoint.diagnostics, + // Discovery's own complaints are not in the checkpoint, because + // they are not something a resume re-reads. Reported here so a + // dictionary missing a volume cannot look finished. + discovery: fileSet.diagnostics, + resumed: resumed, + completed: !run.stopped, + metadata: metadata, + ) + } + + /// The same import as a stream of progress values, for a view to observe. + /// + /// Runs on a detached task so the caller's actor is never the one doing + /// the parsing, and finishes by throwing whatever the import threw. + public func events() -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + let task = Task.detached(priority: .utility) { + do { + try run { continuation.yield($0) } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + } +} + +// MARK: - One file at a time + +extension DSLImporter { + /// Everything the output depends on, as one comparable string. + /// + /// Spelled out rather than hashed, for two reasons. `Hasher` is seeded per + /// process, so `hashValue` is not comparable across runs at all — the bug + /// would be invisible and intermittent. And at a couple of kilobytes for a + /// three-volume set this costs one row, where a mismatch that wrongly + /// restarts a 200-second import costs rather more. + /// + /// Every volume, not just the one the reader picked: a changed third + /// volume with an unchanged first one would otherwise resume onto entries + /// from the old version. And the syllable inventory, because it decides + /// how a reading is split and so what the cross-dictionary merge key is — + /// re-running with a different `--bases` would mix readings parsed two + /// ways. + static func fingerprint(of sources: [FileDescription], bases: Set) -> String { + let files = sources.map { + "\($0.name):\($0.size):\(Int($0.modified.timeIntervalSince1970))" + } + return (files + ["bases"] + bases.sorted()).joined(separator: "|") + } +} + +extension DSLImporter { + /// The mutable half of a run. + /// + /// A single object rather than a handful of `inout` parameters threaded + /// through a function, so the checkpoint cannot drift out of step with the + /// batch it describes — they are only ever advanced together, here. + final class Run { + let sources: [FileDescription] + let resume: DictionaryWriter.Checkpoint? + let writer: DictionaryWriter + let builder: DSLCardBuilder + let batchSize: Int + let report: (@Sendable (DSLImportProgress) -> Void)? + let shouldContinue: (@Sendable () -> Bool)? + + var state: Checkpointing + /// Set when `shouldContinue` said to stop, so `run` knows not to + /// finish the container. + var stopped = false + var batch: [DictionaryWriter.SourcedEntry] = [] + var lastReport = Date.distantPast + + init( + sources: [FileDescription], + resume: DictionaryWriter.Checkpoint?, + writer: DictionaryWriter, + builder: DSLCardBuilder, + batchSize: Int, + report: (@Sendable (DSLImportProgress) -> Void)?, + shouldContinue: (@Sendable () -> Bool)? = nil, + ) { + self.sources = sources + self.resume = resume + self.writer = writer + self.builder = builder + self.batchSize = batchSize + self.report = report + self.shouldContinue = shouldContinue + state = Checkpointing( + checkpoint: resume ?? DictionaryWriter.Checkpoint(), + sources: sources, + totalBytes: sources.reduce(0) { $0 + $1.size }, + ) + batch.reserveCapacity(batchSize) + } + + /// Commits the pending batch, checkpointed at the card that filled + /// it. + /// + /// The checkpoint is that card's own offset, so it is re-read on + /// resume rather than skipped: it is already in the batch being + /// committed, and the resume path deletes everything at or past the + /// offset first. + /// + /// A card with no offset was assembled across a file seam, and the + /// only point that restarts it is in the *previous* file. Leaving the + /// checkpoint where it is re-reads from there, which rebuilds the seam + /// correctly; moving it here would resume past the carried headword + /// and lose the card. The carried headwords stay for the same reason — + /// cleared before this commit, a checkpoint whose frontier is still + /// the seam card restarts the next file without that headword, so the + /// card's entries are deleted and its article cannot rebuild them. + private func commitBatch( + frontier: DSLCardRecord, + settled: (cards: Int, diagnostics: Int), + ) throws { + var commit = state.checkpoint + commit.cardsRead = settled.cards + commit.diagnostics = settled.diagnostics + if let offset = frontier.offset { + state.checkpoint.byteOffset = offset + state.checkpoint.leadingHeadwords = [] + commit.byteOffset = offset + commit.leadingHeadwords = [] + } + try writer.append(batch, checkpoint: commit) + batch.removeAll(keepingCapacity: true) + lastReport = state.reportIfDue( + after: lastReport, + written: writer.count, + to: report, + ) + } + + /// Reads one file of the set, committing a batch whenever one fills. + func importFile(at index: Int) throws { + // Only the file the checkpoint names starts part way in, and only + // when it names a real card offset. Zero is not one: it is where + // the byte-order mark is, and handing it back as a resume point + // reads the mark as the first character of the first headword. An + // end-of-file checkpoint stores zero precisely to mean "from the + // beginning", which is what nil says to the reader. + let storedOffset = index == resume?.fileIndex ? (resume?.byteOffset ?? 0) : 0 + var reader = try DSLCardReader( + source: DSLFileBytes(url: sources[index].url), + resumingAt: storedOffset > 0 ? storedOffset : nil, + leadingHeadwords: index == state.checkpoint.fileIndex + ? state.checkpoint.leadingHeadwords + : [], + ) + state.checkpoint.fileIndex = index + var readerDiagnostics = 0 + + while let record = try reader.next() { + try Task.checkCancellation() + if shouldContinue?() == false { + stopped = true + return + } + // The reader's own complaints are counted first, and they + // count as *settled*. They describe lines the reader has + // already gone past — an article with no headword above it + // sits before the card that follows it — so a resume starting + // at that card will never see them again, and leaving them out + // of its checkpoint loses them for good. Taken per card rather + // than at end of file for the same reason. + state.checkpoint.diagnostics += reader.diagnostics.count - readerDiagnostics + readerDiagnostics = reader.diagnostics.count + + // This card's own totals are not settled, because a resume + // re-reads it: counting it on both sides inflates the totals + // by one card and its diagnostics per interruption. + let settled = ( + cards: state.checkpoint.cardsRead, + diagnostics: state.checkpoint.diagnostics, + ) + state.checkpoint.cardsRead += 1 + let built = builder.entries(from: record.card) + state.checkpoint.diagnostics += built.diagnostics.count + // Each entry carries its own card's offset. A card assembled + // across a file seam has none of its own, and its entries take + // the checkpoint's — which still points into the previous + // file, so a resume from there deletes and rebuilds them. + batch.append(contentsOf: built.entries.map { + DictionaryWriter.SourcedEntry( + entry: $0, + sourceFile: index, + sourceOffset: record.offset ?? 0, + ) + }) + + guard batch.count >= batchSize else { continue } + try commitBatch(frontier: record, settled: settled) + } + + // End of file: the next card is in the next file, so the + // checkpoint moves past this one in the same commit as its last + // batch. + state.checkpoint.diagnostics += reader.diagnostics.count - readerDiagnostics + state.checkpoint.fileIndex = index + 1 + state.checkpoint.byteOffset = 0 + state.checkpoint.leadingHeadwords = reader.trailingHeadwords + if batch.isEmpty { + try writer.checkpoint(state.checkpoint) + } else { + try writer.append(batch, checkpoint: state.checkpoint) + batch.removeAll(keepingCapacity: true) + } + } + } +} + +// MARK: - Progress bookkeeping + +extension DSLImporter { + /// One source file, measured once. + struct FileDescription: Sendable { + let url: URL + let name: String + let size: Int + let modified: Date + + init(url: URL) throws { + self.url = url + name = url.lastPathComponent + let values = try url.resourceValues(forKeys: [ + .fileSizeKey, + .contentModificationDateKey, + ]) + size = values.fileSize ?? 0 + modified = values.contentModificationDate ?? .distantPast + } + } + + /// The mutable half of a run, kept together so `run` reads as a loop over + /// cards rather than as bookkeeping with a loop in it. + struct Checkpointing { + var checkpoint: DictionaryWriter.Checkpoint + let sources: [FileDescription] + let totalBytes: Int + + /// Bytes finished, counting whole files before the current one. + var bytesRead: Int { + let whole = sources.prefix(checkpoint.fileIndex).reduce(0) { $0 + $1.size } + return min(totalBytes, whole + checkpoint.byteOffset) + } + + func progress(written: Int, completed: Bool = false) -> DSLImportProgress { + DSLImportProgress( + fileIndex: min(checkpoint.fileIndex, max(0, sources.count - 1)), + fileCount: sources.count, + bytesRead: completed ? totalBytes : bytesRead, + totalBytes: totalBytes, + cardsRead: checkpoint.cardsRead, + entriesWritten: written, + ) + } + + /// Reports at most ten times a second, returning when it last did. + func reportIfDue( + after last: Date, + written: Int, + to report: (@Sendable (DSLImportProgress) -> Void)?, + ) + -> Date + { + guard let report else { return last } + let now = Date() + guard now.timeIntervalSince(last) >= DSLImporter.progressInterval else { return last } + report(progress(written: written)) + return now + } + } +} + +// MARK: - Metadata + +/// Something that stops an import before it starts. +public enum DSLImportError: Error, Hashable, Sendable, CustomStringConvertible { + case noFiles + + public var description: String { + switch self { + case .noFiles: "the dictionary has no files to read" + } + } +} + +extension DSLImporter { + /// Builds the container's metadata from what the file says about itself. + /// + /// Nothing here is hardcoded. The predecessor took the dictionary's name + /// from a constant in its source, so importing any other DSL dictionary + /// mislabelled it. + /// + /// **No licence is claimed.** 大БКРС states no terms, and inventing one + /// would be worse than admitting it: the Acknowledgements screen renders + /// this verbatim, so whatever goes here is what the reader is told. The + /// honest statement is that the file came from them, HanReader does not + /// redistribute it, and its terms are the source dictionary's own. + static func metadata( + for fileSet: DSLFileSet, + entryCount: Int, + fileName: String, + ) + -> DictionaryMetadata + { + let declared = fileSet.header.name?.trimmingCharacters(in: .whitespaces) ?? "" + let name = volumeSuffix(strippedFrom: declared) + let display = name.isEmpty ? fileName : name + return DictionaryMetadata( + slug: slug(from: display, fallback: fileName), + displayName: display, + format: "abbyy-dsl", + indexLanguage: languageTag(fileSet.header.indexLanguage) ?? "zh-Hans", + glossLanguage: languageTag(fileSet.header.contentsLanguage) ?? "und", + licence: "Not stated by the source", + attribution: """ + \(display) — imported from a copy supplied by the reader. HanReader \ + ships an importer, not this dictionary's data, and does not \ + redistribute it. Its terms are whatever the source dictionary \ + states. Changes made: parsed from the distributed DSL format into a \ + SQLite database for lookup; sense structure, usage examples and \ + grammatical labels were read from the markup, and a per-character \ + reading table and segmentation word list derived from the headword \ + set. No entry content was altered, removed, or added. + """, + sourceVersion: version(in: declared), + parserVersion: Self.parserVersion, + entryCount: entryCount, + ) + } + + /// `大БКРС - 250920 ( 1 / 3 )` → `大БКРС - 250920`. + /// + /// The volume count belongs to the file, not to the dictionary, and the + /// three volumes compile into one container. + static func volumeSuffix(strippedFrom name: String) -> String { + guard let open = name.lastIndex(of: "("), name.hasSuffix(")") else { return name } + let inside = name[name.index(after: open) ..< name.index(before: name.endIndex)] + guard inside.contains("/"), + inside.allSatisfy({ $0.isNumber || $0.isWhitespace || $0 == "/" }) + else { return name } + return String(name[name.startIndex ..< open]).trimmingCharacters( + in: .whitespaces.union(CharacterSet(charactersIn: "-–—")), + ) + } + + /// The date-like run in a declared name, as a version. + static func version(in name: String) -> String? { + let runs = name.split(whereSeparator: { !$0.isNumber }) + return runs.first { $0.count >= 6 }.map(String.init) + } + + /// A stable identifier: lowercase, ASCII-safe, hyphen-separated. + /// + /// Transliterated rather than stripped, which is why `大БКРС - 250920` + /// becomes `dabkrs-250920` and a wholly Han name like `大辞典` becomes + /// `da-ci-dian` instead of nothing. The file name is the fallback for a + /// declared name with nothing transliterable in it at all, because a slug + /// has to work as a file name and as a settings key. + static func slug(from name: String, fallback: String) -> String { + let folded = name.lowercased() + .applyingTransform(.toLatin, reverse: false)? + .applyingTransform(.stripDiacritics, reverse: false) ?? name.lowercased() + let parts = folded.split { !$0.isASCII || !($0.isLetter || $0.isNumber) } + let slug = parts.joined(separator: "-") + guard slug.count >= 3 else { + let stem = fallback.split(separator: ".").first.map(String.init) ?? fallback + // Through the same splitter, so the two paths cannot disagree + // about what a slug looks like. + return stem.lowercased() + .split { !$0.isASCII || !($0.isLetter || $0.isNumber) } + .joined(separator: "-") + } + return slug + } + + /// Maps a DSL language name to a BCP-47 tag. + /// + /// Only the names this format actually writes, and anything else is kept + /// verbatim rather than guessed at: an unrecognised language should be + /// reported as the dictionary stated it, not relabelled as one we know. + static func languageTag(_ declared: String?) -> String? { + guard let declared, !declared.isEmpty else { return nil } + switch declared.lowercased() { + case "chinese", "chinesesimplified", "chinese simplified": return "zh-Hans" + case "chinesetraditional", "chinese traditional": return "zh-Hant" + case "russian": return "ru" + case "english": return "en" + default: return declared + } + } +} diff --git a/Sources/HanReaderPersistence/DictionaryContainer.swift b/Sources/HanReaderPersistence/DictionaryContainer.swift index b16e5ee..2d8655d 100644 --- a/Sources/HanReaderPersistence/DictionaryContainer.swift +++ b/Sources/HanReaderPersistence/DictionaryContainer.swift @@ -58,7 +58,7 @@ public struct DictionaryContainer: Sendable { let queue = try DatabaseQueue(path: url.path, configuration: configuration) try DictionarySchema.migrator().migrate(queue) - let encoder = JSONEncoder() + let codec = SensePayloadCodecs.current var written = 0 for chunk in entries.chunked(into: batchSize) { @@ -69,15 +69,14 @@ public struct DictionaryContainer: Sendable { try db.execute(sql: """ INSERT INTO entry (simplified, traditional, readingNumeric, - readingDisplay, senses, summary) - VALUES (?, ?, ?, ?, ?, ?) + readingDisplay, senses) + VALUES (?, ?, ?, ?, ?) """, arguments: [ entry.headword.simplified, entry.headword.traditional, entry.readingKey, entry.readingDisplay, - encoder.encode(entry.senses), - entry.summarySense?.gloss.text ?? "", + codec.encode(entry.senses), ]) } } @@ -134,12 +133,15 @@ public struct DictionaryContainer: Sendable { } } - private static func writeMetadata( + static func writeMetadata( _ metadata: DictionaryMetadata, to queue: DatabaseQueue, ) throws { let values: [String: String?] = [ "formatVersion": String(DictionarySchema.formatVersion), + // Which codec wrote the sense payloads, so a reader picks the one + // that can read them rather than assuming the current one. + "sensePayloadCodec": type(of: SensePayloadCodecs.current).identifier, "slug": metadata.slug, "displayName": metadata.displayName, "format": metadata.format, @@ -166,6 +168,22 @@ public struct DictionaryContainer: Sendable { // MARK: - Reading + /// The codec this container's sense payloads were written with. + /// + /// Read from the file rather than assumed, which is what lets a new codec + /// land without invalidating the dictionaries already on disk — and what + /// makes a container written by a *newer* build say so plainly instead of + /// decoding into plausible nonsense. + public func sensePayloadCodec() throws -> any SensePayloadCodec { + let identifier = try dbQueue.read { db in + try String.fetchOne( + db, + sql: "SELECT value FROM meta WHERE key = 'sensePayloadCodec'", + ) + } + return try SensePayloadCodecs.named(identifier) + } + public func metadata() throws -> DictionaryMetadata? { let values = try dbQueue.read { db in try Row.fetchAll(db, sql: "SELECT key, value FROM meta") @@ -196,7 +214,7 @@ public struct DictionaryContainer: Sendable { /// readings; the prototype's `LIMIT 1` is the bug this signature exists to /// make impossible. public func entries(for headword: String) throws -> [DictionaryEntry] { - let decoder = JSONDecoder() + let codec = try sensePayloadCodec() return try dbQueue.read { db in // Matched against BOTH scripts. Storing the traditional form and // then only ever querying the simplified one means a reader of @@ -213,7 +231,7 @@ public struct DictionaryContainer: Sendable { traditional: row["traditional"], ), reading: Pinyin.parse(numeric: row["readingNumeric"]), - senses: decoder.decode([Sense].self, from: row["senses"]), + senses: codec.decode(row["senses"]), ) } } @@ -252,6 +270,47 @@ public struct DictionaryContainer: Sendable { } } + /// How many words the segmentation lexicon holds. + /// + /// Separate from `lexicon()`, which returns all of them. A BKRS container + /// has 1,806,893, and fetching them to print one number took 450 MB — in + /// a `--verbose` summary, where it looked for a while like the import + /// itself had blown its memory ceiling. + public func lexemeCount() throws -> Int { + try dbQueue.read { db in + try Int.fetchOne(db, sql: "SELECT count(*) FROM lexeme") ?? 0 + } + } + + /// Where each entry came from, in row order. + /// + /// For verifying that a resumable import stamps each entry with its own + /// card rather than with the one that happened to fill its batch. + public func sourceOffsets() throws -> [Int] { + try dbQueue.read { db in + try Int.fetchAll(db, sql: "SELECT sourceOffset FROM entry ORDER BY id") + } + } + + /// The longest word in the segmentation lexicon, in characters. + /// + /// Asked of the database rather than by fetching the lexicon: a BKRS + /// container holds 1,806,893 lexemes, and loading them to take a maximum + /// costs hundreds of megabytes. + public func longestLexeme() throws -> Int { + try dbQueue.read { db in + try Int.fetchOne(db, sql: "SELECT max(charLength) FROM lexeme") ?? 0 + } + } + + /// Every headword in the container, for tests that need to assert over + /// all of them rather than look one up. + public func headwords() throws -> [String] { + try dbQueue.read { db in + try String.fetchAll(db, sql: "SELECT simplified FROM entry ORDER BY id") + } + } + public func entryCount() throws -> Int { try dbQueue.read { db in try Int.fetchOne(db, sql: "SELECT count(*) FROM entry") ?? 0 @@ -259,6 +318,75 @@ public struct DictionaryContainer: Sendable { } } +// MARK: - Introspection + +/// Reading the shape of a container rather than its content. +/// +/// Used by the tests that assert an import leaves the right indexes behind +/// and that the retired `summary` column is gone — both things a reader +/// should never need to ask, and both worth asserting once. +extension DictionaryContainer { + /// The indexes the file actually carries. + /// + /// For asserting that an import leaves behind the ones a reader needs and + /// not the one only the importer wanted. + public func indexNames() throws -> Set { + try dbQueue.read { db in + try Set(String.fetchAll( + db, + sql: "SELECT name FROM sqlite_schema WHERE type = 'index' AND name IS NOT NULL", + )) + } + } + + /// The columns a table carries, for asserting that one is gone. + public func columnNames(of table: String) throws -> Set { + try dbQueue.read { db in + try Set(db.columns(in: table).map(\.name)) + } + } + + /// Writes a one-entry container with a chosen codec. + /// + /// Only so a test can build the kind of container an earlier build + /// produced, and prove it still opens — which is the whole point of + /// recording the codec rather than assuming it. + static func writeForTesting( + entry: DictionaryEntry, + codec: any SensePayloadCodec, + metadata: DictionaryMetadata, + to url: URL, + ) throws { + try? FileManager.default.removeItem(at: url) + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true, + ) + let queue = try DatabaseQueue(path: url.path) + try DictionarySchema.migrator().migrate(queue) + try queue.write { db in + try db.execute(sql: """ + INSERT INTO entry + (simplified, traditional, readingNumeric, readingDisplay, senses) + VALUES (?, ?, ?, ?, ?) + """, arguments: [ + entry.headword.simplified, + entry.headword.traditional, + entry.readingKey, + entry.readingDisplay, + codec.encode(entry.senses), + ]) + } + try writeMetadata(metadata, to: queue) + try queue.write { db in + try db.execute( + sql: "INSERT OR REPLACE INTO meta (key, value) VALUES ('sensePayloadCodec', ?)", + arguments: [type(of: codec).identifier], + ) + } + } +} + extension Array { /// Splits into fixed-size chunks, so a batch can be one transaction. func chunked(into size: Int) -> [ArraySlice] { diff --git a/Sources/HanReaderPersistence/DictionarySchema.swift b/Sources/HanReaderPersistence/DictionarySchema.swift index 59ecbb9..b871b31 100644 --- a/Sources/HanReaderPersistence/DictionarySchema.swift +++ b/Sources/HanReaderPersistence/DictionarySchema.swift @@ -20,7 +20,7 @@ enum DictionarySchema { /// Distinct from a parser version: no schema migration can repair data /// that was parsed wrongly, so a parser fix invalidates a container /// through `meta.parserVersion` and triggers a rebuild instead. - static let formatVersion = 1 + static let formatVersion = 3 static func migrator() -> DatabaseMigrator { var migrator = DatabaseMigrator() @@ -33,6 +33,30 @@ enum DictionarySchema { try createSyllableBaseTable(db) } + // A separate migration rather than an edit to v1, even though nothing + // has been released yet. A container is a file on a reader's disk the + // moment they import a dictionary, and the habit of editing the + // migration that built it is the one that eventually destroys + // somebody's data. + migrator.registerMigration("v2_resumable_import") { db in + try addSourceAnchors(db) + try createImportJobTable(db) + } + + // The denormalised first gloss, written since v1 and read by nothing. + // On a BKRS container it is **160 MB** — 3,442,623 rows of Cyrillic + // prose that is already inside the sense payload beside it. It was + // there so a result list could render without decoding a payload per + // row; with the compact codec that decode is microseconds for the ten + // rows a list shows, and 160 MB is not a reasonable price for it. If + // a list ever does need it, it can come back as a short prefix rather + // than a whole copy. + migrator.registerMigration("v3_drop_summary") { db in + try db.alter(table: "entry") { table in + table.drop(column: "summary") + } + } + return migrator } @@ -125,4 +149,76 @@ enum DictionarySchema { table.column("base", .text).notNull().primaryKey() } } + + // MARK: - v2: resumable import + + /// Where each entry came from in the source. + /// + /// Only an import that can be interrupted needs these, which is why they + /// are not in v1. A resumed import deletes everything at or past its + /// checkpoint before writing again, so a batch that was partly committed + /// cannot leave duplicates behind — the belt to the braces of writing the + /// rows and the checkpoint in one transaction. + private static func addSourceAnchors(_ db: Database) throws { + try db.alter(table: "entry") { table in + table.add(column: "sourceFile", .integer).notNull().defaults(to: 0) + table.add(column: "sourceOffset", .integer).notNull().defaults(to: 0) + } + try db.create( + index: "entry_on_source", + on: "entry", + columns: ["sourceFile", "sourceOffset"], + ) + } + + /// The one in-progress import, if there is one. + /// + /// Resumability is a requirement rather than a refinement: a BKRS set is + /// ~350 MB of UTF-16, and on iOS the app will be suspended or killed part + /// way through. One row, because a container holds one dictionary. + /// + /// The source is identified by a fingerprint rather than by a path. A + /// path is the wrong anchor on iOS, where the container directory's UUID + /// changes across installs and restores — the same mistake the + /// predecessor made storing absolute audio paths. + /// + /// The fingerprint covers **everything that affects the output**, which + /// is more than the file the reader picked: the name, size and + /// modification time of *every* volume, and the syllable inventory the + /// readings were split with. Checking only the first volume lets a + /// changed third volume resume onto entries from the old one, and + /// re-running with a different `--bases` mixes readings parsed two ways. + /// Both produce a dictionary that is half one thing and half another, and + /// report success. + private static func createImportJobTable(_ db: Database) throws { + try db.create(table: "importJob") { table in + // A single row, enforced rather than assumed. + table.column("id", .integer).notNull().primaryKey().check { $0 == 1 } + + table.column("fileIndex", .integer).notNull() + table.column("byteOffset", .integer).notNull() + table.column("fileCount", .integer).notNull() + + // The first volume's name, for messages only. + table.column("sourceName", .text).notNull() + table.column("sourceFingerprint", .text).notNull() + table.column("totalBytes", .integer).notNull() + + // A parser fix invalidates what was already written, so a resume + // across versions starts again rather than stitching output from + // two parsers together. + table.column("parserVersion", .integer).notNull() + + table.column("entriesWritten", .integer).notNull() + table.column("cardsRead", .integer).notNull() + table.column("diagnostics", .integer).notNull() + + // A headword at the end of a file whose article is at the start of + // the next one. Zero for a card-aligned set; carried across the + // seam for a set someone split on a byte boundary. + table.column("leadingHeadwords", .text).notNull() + + table.column("updatedAt", .double).notNull() + } + } } diff --git a/Sources/HanReaderPersistence/DictionaryWriter.swift b/Sources/HanReaderPersistence/DictionaryWriter.swift new file mode 100644 index 0000000..a91add5 --- /dev/null +++ b/Sources/HanReaderPersistence/DictionaryWriter.swift @@ -0,0 +1,556 @@ +// HanReader — MIT licensed. See LICENSE. + +public import Foundation +import GRDB +public import HanReaderCore + +/// A container being filled a batch at a time, able to pick up where it left +/// off. +/// +/// `DictionaryContainer.write(entries:…)` takes the whole dictionary as an +/// array, which is right for CC-CEDICT: 107,619 entries parsed in under three +/// seconds at build time, and holding them costs a few tens of megabytes. It +/// is not right for a 350 MB BKRS set — 3,442,623 entries over 3,691,468 +/// senses would not fit inside the import's 60 MB ceiling, and the import has +/// to survive being killed half way through. +/// +/// So this writer holds no entries at all. Batches arrive, are written, and are +/// forgotten; the only state it keeps between batches is a few counters. +/// +/// ### Crash safety is the whole design, and it costs a pragma +/// +/// Compiling CC-CEDICT runs under `journal_mode = OFF, synchronous = OFF`, +/// which is safe there because the output is a build artefact regenerated from +/// its source in seconds — on failure you delete it and run again. +/// +/// A reader's own BKRS import is not that. It takes a minute or more, and on +/// iOS the app *will* be suspended or killed part way through, so "delete it +/// and run again" is the behaviour this exists to avoid. Under +/// `journal_mode = OFF` a process killed mid-transaction can leave the file +/// corrupt, and a corrupt file cannot be resumed — the two requirements are +/// mutually exclusive, and resumability is the one that was asked for. WAL +/// with `synchronous = NORMAL` makes every committed batch survive the process +/// dying, which is exactly the guarantee `kill -9` mid-import needs. (It does +/// not promise survival of a power cut, which would need `FULL`; a dictionary +/// import is not worth an fsync per batch, and the job simply resumes from the +/// last batch that made it to disk.) +/// +/// ### Rows and the checkpoint go in one transaction +/// +/// Written separately, a crash between them leaves the checkpoint behind the +/// rows and the next run writes those rows again. Together, the file can only +/// ever be in a state where the checkpoint describes exactly what is in it. +/// The resume path still deletes anything at or past the checkpoint, because a +/// belt costs one indexed `DELETE` and braces alone are an argument. +public final class DictionaryWriter { + /// When the lookup indexes get built. + /// + /// They are needed only once an import is finished, and the choice of + /// *when* to build them trades the import's two budgets against each + /// other. Measured over the full 350 MB BKRS set, same machine, same + /// output: + /// + /// | | time | peak memory | size | + /// |---|---|---|---| + /// | `.atTheEnd` | **75 s** | 134 MB | **493 MB** | + /// | `.whileWriting` | 188 s | **53 MB** | 556 MB | + /// + /// Building at the end is the ordinary bulk-load shape: one pass over + /// rows that are all already there, instead of a random-order insert into + /// an 84 MB B-tree three and a half million times. It is also 63 MB + /// smaller on disk, because an index built in one pass does not carry the + /// page splits that random insertion leaves behind. + /// + /// Its cost is memory: `CREATE INDEX` is a single atomic statement, so the + /// whole new index is dirty at once — about 92 MB of it, which no + /// `cache_size`, `temp_store`, `threads`, `soft_heap_limit` or + /// `hard_heap_limit` pragma reduces. Measured, not assumed; all five were + /// tried. + /// + /// `.atTheEnd` is the default because compiling a DSL set is a desktop + /// operation by design — a Mac compiles it once and a phone imports the + /// compiled container — so the import's memory ceiling, which exists for + /// iOS, is not the binding constraint on this path. `.whileWriting` is + /// there for the day something does run an import where it is. + public enum Indexing: Sendable { + case atTheEnd + case whileWriting + } + + /// Where the reader got to, recorded at a card boundary. + /// + /// A card boundary, not a line or a batch boundary, because it is the only + /// place an import can restart without either losing a card or writing one + /// twice. + public struct Checkpoint: Hashable, Sendable { + /// Index into the file set. + public var fileIndex: Int + /// Byte offset of the next unread card in that file. + public var byteOffset: Int + /// A headword whose article is in the next file. Empty for a + /// card-aligned set. + public var leadingHeadwords: [String] + public var cardsRead: Int + public var diagnostics: Int + + public init( + fileIndex: Int = 0, + byteOffset: Int = 0, + leadingHeadwords: [String] = [], + cardsRead: Int = 0, + diagnostics: Int = 0, + ) { + self.fileIndex = fileIndex + self.byteOffset = byteOffset + self.leadingHeadwords = leadingHeadwords + self.cardsRead = cardsRead + self.diagnostics = diagnostics + } + } + + /// An entry together with the card it came from. + /// + /// Per entry, not per batch, and that distinction is the resume. Stamping + /// a whole batch with the offset of the card that happened to fill it + /// means a resume deletes every row of that batch and re-reads only its + /// last card: measured over three `kill -9`s, 14,997 entries of 574,709 + /// simply vanished, and the import reported success. + public struct SourcedEntry: Sendable { + public let entry: DictionaryEntry + public let sourceFile: Int + public let sourceOffset: Int + + public init(entry: DictionaryEntry, sourceFile: Int, sourceOffset: Int) { + self.entry = entry + self.sourceFile = sourceFile + self.sourceOffset = sourceOffset + } + } + + /// What the job is reading, as identity rather than as a location. + /// + /// A fingerprint rather than a path: on iOS the container directory's UUID + /// changes across installs and restores, so a stored path is stale by the + /// time it matters — the same mistake the predecessor made with absolute + /// audio paths. + /// + /// The fingerprint has to cover **everything that affects the output**, + /// which is more than the file the reader picked. Checking only the first + /// volume lets a changed third volume resume onto entries from the old + /// one; ignoring the syllable inventory lets a re-run with different + /// `--bases` mix readings parsed two ways. Either produces a dictionary + /// that is half one thing and half another, and reports success. + public struct Source: Hashable, Sendable { + /// The first volume's name, for messages. + public var name: String + /// Identity of every input: all volumes, and whatever else the parse + /// depends on. Compared exactly. + public var fingerprint: String + public var fileCount: Int + /// Across the whole set, for progress. + public var totalBytes: Int + public var parserVersion: Int + + public init( + name: String, + fingerprint: String, + fileCount: Int, + totalBytes: Int, + parserVersion: Int, + ) { + self.name = name + self.fingerprint = fingerprint + self.fileCount = fileCount + self.totalBytes = totalBytes + self.parserVersion = parserVersion + } + } + + private let dbQueue: DatabaseQueue + private let source: Source + private let lexiconMaximumLength: Int + private let indexing: Indexing + private let codec = SensePayloadCodecs.current + private var entriesWritten: Int + + /// Opens a container at `url`, resuming the job in it when there is one + /// for this exact source. + /// + /// Returns nil for `resume` when the container is new, or when what is in + /// it was built from a different file or by a different parser version. In + /// that case everything already written is discarded first: no schema + /// migration can reconcile output from two parsers, and half a dictionary + /// from each is worse than either. + /// - Parameter lexiconMaximumLength: longest headword admitted to the + /// segmentation lexicon. The default suits a word-level dictionary; + /// `LexiconBuilder.encyclopaedicWordLength` suits one whose headwords + /// run to phrases, which is most of BKRS. + public static func open( + at url: URL, + source: Source, + lexiconMaximumLength: Int = LexiconBuilder.maximumWordLength, + indexing: Indexing = .atTheEnd, + ) throws + -> (writer: DictionaryWriter, resume: Checkpoint?) + { + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true, + ) + + var configuration = Configuration() + configuration.prepareDatabase { db in + try db.execute(sql: "PRAGMA journal_mode = WAL") + try db.execute(sql: "PRAGMA synchronous = NORMAL") + } + let queue = try DatabaseQueue(path: url.path, configuration: configuration) + try DictionarySchema.migrator().migrate(queue) + + try prepareIndexes(queue, indexing: indexing) + + let stored = try queue.read { db in + try Row.fetchOne(db, sql: "SELECT * FROM importJob WHERE id = 1") + } + let resume = stored.flatMap { row -> Checkpoint? in + guard row["sourceFingerprint"] as String == source.fingerprint, + row["parserVersion"] as Int == source.parserVersion + else { return nil } + return Checkpoint( + fileIndex: row["fileIndex"], + byteOffset: row["byteOffset"], + leadingHeadwords: Self.decodeHeadwords(row["leadingHeadwords"]), + cardsRead: row["cardsRead"], + diagnostics: row["diagnostics"], + ) + } + + let written = try prepare(queue, resuming: resume, hadJob: stored != nil) + + let writer = DictionaryWriter( + dbQueue: queue, + source: source, + lexiconMaximumLength: lexiconMaximumLength, + indexing: indexing, + entriesWritten: written, + ) + return (writer, resume) + } + + /// Sets up the indexes an import wants, and takes down the ones it does + /// not. + /// + /// A lookup index is maintained per row while it exists, and + /// `entry_on_simplified` means a random-order insert into an 84 MB B-tree + /// three and a half million times. Nothing queries it until the import is + /// over, so it comes down for the duration and is built in one pass at the + /// end — the ordinary bulk-load shape. + /// + /// The source index is the mirror image: wanted only *during* an import, + /// for the delete a resume does, and dropped when one finishes. + /// + /// A container interrupted part way therefore has no lookup indexes. That + /// is correct rather than unfortunate — it is not a dictionary yet, and + /// the `importJob` row is what says so. + private static func prepareIndexes(_ queue: DatabaseQueue, indexing: Indexing) throws { + try queue.write { db in + try db.execute(sql: """ + CREATE INDEX IF NOT EXISTS entry_on_source ON entry (sourceFile, sourceOffset) + """) + guard indexing == .atTheEnd else { return } + try db.execute(sql: "DROP INDEX IF EXISTS entry_on_simplified") + try db.execute(sql: "DROP INDEX IF EXISTS entry_on_traditional") + } + } + + /// Makes the container ready to be written into, and says how many + /// entries it already holds. + private static func prepare( + _ queue: DatabaseQueue, + resuming resume: Checkpoint?, + hadJob: Bool, + ) throws + -> Int + { + let existing = try queue.read { db in + try Int.fetchOne(db, sql: "SELECT count(*) FROM entry") ?? 0 + } + guard let resume else { + // No job to resume, so anything already here was written by a + // different file or a different parser version. No schema + // migration can reconcile output from two parsers, and half a + // dictionary from each is worse than either. + guard hadJob || existing > 0 else { return 0 } + try queue.write { db in + for table in ["entry", "lexeme", "charReading", "syllableBase", "meta"] { + try db.execute(sql: "DELETE FROM \(table)") + } + try db.execute(sql: "DELETE FROM importJob") + } + return 0 + } + // Anything at or past the checkpoint is from a batch that did not + // commit, or from a run that got further and was discarded. Either way + // it is about to be written again. + try queue.write { db in + try db.execute(sql: """ + DELETE FROM entry + WHERE sourceFile > ? OR (sourceFile = ? AND sourceOffset >= ?) + """, arguments: [resume.fileIndex, resume.fileIndex, resume.byteOffset]) + } + return try queue.read { db in + try Int.fetchOne(db, sql: "SELECT count(*) FROM entry") ?? 0 + } + } + + private init( + dbQueue: DatabaseQueue, + source: Source, + lexiconMaximumLength: Int, + indexing: Indexing, + entriesWritten: Int, + ) { + self.dbQueue = dbQueue + self.source = source + self.lexiconMaximumLength = lexiconMaximumLength + self.indexing = indexing + self.entriesWritten = entriesWritten + } + + /// How many entries the container holds. + public var count: Int { + entriesWritten + } + + // MARK: - Appending + + /// Writes a batch and the checkpoint that describes it, in one + /// transaction. + /// + /// The lexicon is upserted here rather than derived at the end, because + /// deriving it would mean decoding three and a half million sense payloads + /// back out of the file. A word's weight depends only on its own entry, so + /// taking the best weight per word batch by batch gives the same table as + /// computing it over the whole dictionary at once — and it survives the + /// process dying, since it lands in the same transaction as the rows. + public func append(_ entries: [SourcedEntry], checkpoint: Checkpoint) throws { + try dbQueue.write { db in + for sourced in entries { + let entry = sourced.entry + try db.execute(sql: """ + INSERT INTO entry + (simplified, traditional, readingNumeric, readingDisplay, + senses, sourceFile, sourceOffset) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, arguments: [ + entry.headword.simplified, + entry.headword.traditional, + entry.readingKey, + entry.readingDisplay, + codec.encode(entry.senses), + sourced.sourceFile, + sourced.sourceOffset, + ]) + } + + for lexeme in LexiconBuilder.lexicon( + from: entries.map(\.entry), + maximumLength: lexiconMaximumLength, + ) { + try db.execute(sql: """ + INSERT INTO lexeme (word, charLength, weight) VALUES (?, ?, ?) + ON CONFLICT(word) DO UPDATE SET weight = max(weight, excluded.weight) + """, arguments: [lexeme.word, lexeme.characterLength, lexeme.weight]) + } + + try self.writeJob(checkpoint, to: db) + } + entriesWritten += entries.count + } + + /// Records a checkpoint with no entries behind it. + /// + /// A run of cards can produce no entries at all — a file whose remaining + /// cards are all headwords with no article — and without this the job + /// would appear to have made no progress and re-read them on resume. + public func checkpoint(_ checkpoint: Checkpoint) throws { + try dbQueue.write { db in + try self.writeJob(checkpoint, to: db) + } + } + + private func writeJob(_ checkpoint: Checkpoint, to db: Database) throws { + try db.execute(sql: """ + INSERT INTO importJob + (id, fileIndex, byteOffset, fileCount, sourceName, + sourceFingerprint, totalBytes, parserVersion, entriesWritten, + cardsRead, diagnostics, leadingHeadwords, updatedAt) + VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + fileIndex = excluded.fileIndex, + byteOffset = excluded.byteOffset, + entriesWritten = excluded.entriesWritten, + cardsRead = excluded.cardsRead, + diagnostics = excluded.diagnostics, + leadingHeadwords = excluded.leadingHeadwords, + updatedAt = excluded.updatedAt + """, arguments: [ + checkpoint.fileIndex, + checkpoint.byteOffset, + source.fileCount, + source.name, + source.fingerprint, + source.totalBytes, + source.parserVersion, + entriesWritten, + checkpoint.cardsRead, + checkpoint.diagnostics, + Self.encodeHeadwords(checkpoint.leadingHeadwords), + Date().timeIntervalSince1970, + ]) + } + + // MARK: - Finishing + + /// Builds the derived tables, writes the metadata, and clears the job. + /// + /// The character readings and the syllable inventory are read back out of + /// the `entry` table rather than accumulated while writing. Both need to + /// see the whole dictionary — a character's readings are *ranked* against + /// each other — and accumulating them in memory would be state that a + /// crash destroys and a resume cannot rebuild. Derived at the end from + /// what is on disk, they are correct however many times the import was + /// interrupted. It costs one indexed pass; the single-character headwords + /// it decodes number 33,372 in BKRS, not three million. + public func finish(metadata: DictionaryMetadata) throws { + try writeCharacterReadings() + try writeSyllableBases() + try writeMetadata(metadata) + + try dbQueue.write { db in + try db.execute(sql: "DELETE FROM importJob") + // The source index exists only so a resume can delete the rows + // from a batch that did not commit. Once the import is finished + // nothing queries it ever again, and on a BKRS-sized dictionary it + // is 53 MB the reader would carry for nothing. The columns stay, + // so a re-import can rebuild the index and resume as before. + try db.execute(sql: "DROP INDEX IF EXISTS entry_on_source") + + // And now the ones a reader needs, built in one pass over rows + // that are all already there rather than maintained as they + // arrived. Lookup matches BOTH scripts: storing the traditional + // form and only ever querying the simplified one means a reader of + // traditional text finds nothing at all. + try db.execute(sql: """ + CREATE INDEX IF NOT EXISTS entry_on_simplified ON entry (simplified) + """) + try db.execute(sql: """ + CREATE INDEX IF NOT EXISTS entry_on_traditional ON entry (traditional) + WHERE traditional IS NOT NULL + """) + try db.execute(sql: "ANALYZE") + } + // Folding the write-ahead log back in leaves one file rather than + // three, which is what the reader copies around and what the size + // reported to them should mean. + try dbQueue.writeWithoutTransaction { db in + try db.execute(sql: "PRAGMA wal_checkpoint(TRUNCATE)") + } + } +} + +// MARK: - Derived tables + +extension DictionaryWriter { + // Reads the single-character entries through a cursor, one at a time. + + /// + /// Fetching them into an array first is what took peak memory to 857 MB + /// against a 60 MB ceiling: single-character headwords are the densest + /// cards in a dictionary — 38,618 of them in BKRS carrying 27 MB of + /// encoded senses, 一 and 打 with 57 apiece — and decoding all of them at + /// once, with the decoder's own intermediates on top, costs vastly more + /// than the 27 MB suggests. The collector keeps five small values per + /// candidate reading and the entry is released immediately. + private func writeCharacterReadings() throws { + var collector = LexiconBuilder.CharacterReadings() + try dbQueue.read { db in + let cursor = try Row.fetchCursor(db, sql: """ + SELECT simplified, traditional, readingNumeric, senses + FROM entry WHERE length(simplified) = 1 ORDER BY id + """) + while let row = try cursor.next() { + try collector.add(DictionaryEntry( + headword: Headword( + simplified: row["simplified"], + traditional: row["traditional"], + ), + reading: Pinyin.parse(numeric: row["readingNumeric"]), + senses: self.codec.decode(row["senses"]), + )) + } + } + try dbQueue.write { db in + try db.execute(sql: "DELETE FROM charReading") + for reading in collector.ranked() { + try db.execute(sql: """ + INSERT INTO charReading (character, readingNumeric, readingDisplay, rank) + VALUES (?, ?, ?, ?) + """, arguments: [ + String(reading.character), reading.numeric, reading.display, reading.rank, + ]) + } + } + } + + /// The toneless syllables this dictionary actually uses. + /// + /// Read as a cursor over the reading column alone, so three and a half + /// million rows cost one small set rather than one large array. + private func writeSyllableBases() throws { + var bases: Set = [] + try dbQueue.read { db in + let cursor = try String.fetchCursor( + db, + sql: "SELECT readingNumeric FROM entry WHERE readingNumeric <> ''", + ) + while let numeric = try cursor.next() { + for token in Pinyin.parse(numeric: numeric) { + if case let .syllable(syllable) = token { + // Lowercased, as `LexiconBuilder.syllableBases` does. + // The syllabifier looks up lowercase candidates, so an + // uppercase-only base stored here — `Bei` from a + // capitalised reading — is one a later import using + // this inventory could never match. + bases.insert(syllable.base.lowercased()) + } + } + } + } + try dbQueue.write { db in + try db.execute(sql: "DELETE FROM syllableBase") + for base in bases.sorted() { + try db.execute(sql: "INSERT INTO syllableBase (base) VALUES (?)", arguments: [base]) + } + } + } + + private func writeMetadata(_ metadata: DictionaryMetadata) throws { + try dbQueue.write { db in + try db.execute(sql: "DELETE FROM meta") + } + try DictionaryContainer.writeMetadata(metadata, to: dbQueue) + } + + // MARK: - Headword carry-over + + /// Tab-separated, because a headword cannot contain a tab and this is one + /// field read by one function. JSON here would be a dependency on a + /// decoder for a list that is empty 100% of the time on real input. + private static func encodeHeadwords(_ headwords: [String]) -> String { + headwords.joined(separator: "\t") + } + + private static func decodeHeadwords(_ encoded: String) -> [String] { + encoded.isEmpty ? [] : encoded.components(separatedBy: "\t") + } +} diff --git a/Sources/hanreader-dictgen/CEDICTCommand.swift b/Sources/hanreader-dictgen/CEDICTCommand.swift index 360b9aa..7fd84fb 100644 --- a/Sources/hanreader-dictgen/CEDICTCommand.swift +++ b/Sources/hanreader-dictgen/CEDICTCommand.swift @@ -91,7 +91,7 @@ struct CompileCEDICT: ParsableCommand { .attributesOfItem(atPath: outputURL.path)[.size] as? Int) ?? 0 try print(""" entries: \(parsed.entries.count) - lexemes: \(container.lexicon().count) + lexemes: \(container.lexemeCount()) characters: \(container.syllableBases().count) syllable bases size: \(size / 1_048_576) MB parse: \(String(format: "%.2f", parseTime))s diff --git a/Sources/hanreader-dictgen/DSLCommand.swift b/Sources/hanreader-dictgen/DSLCommand.swift new file mode 100644 index 0000000..d0c6b32 --- /dev/null +++ b/Sources/hanreader-dictgen/DSLCommand.swift @@ -0,0 +1,165 @@ +// HanReader — MIT licensed. See LICENSE. + +import ArgumentParser +import Foundation +import HanReaderCore +import HanReaderDictionaryImport +import HanReaderPersistence + +/// Compiles an ABBYY DSL dictionary into a `.hanreaderdict` container. +/// +/// Unlike `cedict`, this is not a build step: the source is the reader's own +/// file and is never in the repository. It exists so the import can be run and +/// measured outside an app — which is how the resume behaviour is exercised, +/// since `kill -9` is easier to arrange against a command than against a +/// window. +/// +/// Interrupting it is expected and safe. Run the same command again and it +/// picks up from the last committed batch. +struct CompileDSL: ParsableCommand { + static let configuration = CommandConfiguration( + commandName: "dsl", + abstract: "Compile an ABBYY DSL dictionary into a HanReader dictionary.", + discussion: """ + Point it at any file of the set; the others are found through its \ + #INCLUDE directives, or failing that through numbered siblings beside \ + it. Interrupting the command is safe — running it again resumes from \ + the last committed batch rather than starting over. + + A dictionary supplies readings as diacritics, which have to be split \ + into syllables to be comparable with CC-CEDICT's numeric form. Pass \ + --bases pointing at a compiled CC-CEDICT container to enable that; \ + without it a reading still displays exactly as written and only the \ + cross-dictionary merge key is lost. + """, + ) + + @Argument(help: "Path to any file of the DSL set.") var input: String + + @Option(name: .shortAndLong, help: "Where to write the .hanreaderdict file.") var output: String + + @Option( + help: "A compiled dictionary to take syllable bases from, normally CC-CEDICT.", + ) var bases: String? + + @Option(help: "Entries per transaction.") var batchSize = DSLImporter.defaultBatchSize + + @Option(help: "Fail if fewer than this many entries are produced.") var minimumEntries = 1 + + @Flag(name: .shortAndLong, help: "Print progress and statistics.") var verbose = false + + func run() throws { + let outputURL = URL(fileURLWithPath: output) + let fileSet = try DSLFileSet.discover(from: URL(fileURLWithPath: input)) + + var syllableBases: Set = [] + if let bases { + syllableBases = try DictionaryContainer(contentsOf: URL(fileURLWithPath: bases)) + .syllableBases() + } + + if verbose { + print(""" + discovery: \(fileSet.discovery) + files: \(fileSet.files.map(\.lastPathComponent).joined(separator: ", ")) + name: \(fileSet.header.name ?? "not declared") + languages: \(fileSet.header.indexLanguage ?? "?") \ + to \(fileSet.header.contentsLanguage ?? "?") + bases: \(syllableBases.count) + """) + for diagnostic in fileSet.diagnostics { + print(" \(diagnostic.kind): \(diagnostic.text)") + } + } + + let started = Date() + let importer = DSLImporter( + fileSet: fileSet, + destination: outputURL, + syllableBases: syllableBases, + batchSize: batchSize, + ) + let reporter = ProgressReporter(enabled: verbose) + let summary = try importer.run(progress: { reporter.report($0) }) + reporter.finish() + let elapsed = Date().timeIntervalSince(started) + + // The failure this guards against is a parser change that silently + // halves the dictionary: everything still builds, the app still runs, + // and words simply stop being found. + guard summary.entriesWritten >= minimumEntries else { + throw CompileDSLError.tooFewEntries( + got: summary.entriesWritten, + expected: minimumEntries, + ) + } + + try reportResult(summary, at: outputURL, elapsed: elapsed) + } + + private func reportResult( + _ summary: DSLImportSummary, + at outputURL: URL, + elapsed: TimeInterval, + ) throws { + guard verbose else { + print("Wrote \(summary.entriesWritten) entries to \(outputURL.lastPathComponent)") + return + } + let container = try DictionaryContainer(contentsOf: outputURL) + let size = (try? FileManager.default + .attributesOfItem(atPath: outputURL.path)[.size] as? Int) ?? 0 + try print(""" + resumed: \(summary.resumed) + cards: \(summary.cardsRead) + entries: \(summary.entriesWritten) + lexemes: \(container.lexemeCount()) + characters: \(container.syllableBases().count) syllable bases + diagnostics:\(summary.diagnostics) + slug: \(summary.metadata.slug) + name: \(summary.metadata.displayName) + size: \(size / 1_048_576) MB + elapsed: \(String(format: "%.1f", elapsed))s + """) + } + + /// Prints a single rewritten line, so a long import is legible in a + /// terminal without scrolling it off the top. + /// + /// Stateless, which is what lets it be `Sendable` and go straight into the + /// importer's `@Sendable` progress callback. The importer reports on its + /// first batch and again at the end, so there is always a line to + /// terminate. + private struct ProgressReporter: Sendable { + let enabled: Bool + + func report(_ progress: DSLImportProgress) { + guard enabled else { return } + let percent = Int(progress.fraction * 100) + FileHandle.standardError.write(Data(""" + \r file \(progress.fileIndex + 1)/\(progress.fileCount) \ + \(percent)% \(progress.cardsRead) cards \ + \(progress.entriesWritten) entries + """.utf8)) + } + + func finish() { + guard enabled else { return } + FileHandle.standardError.write(Data("\n".utf8)) + } + } + + enum CompileDSLError: Error, CustomStringConvertible { + case tooFewEntries(got: Int, expected: Int) + + var description: String { + switch self { + case let .tooFewEntries(got, expected): + """ + only \(got) entries were produced, expected at least \(expected). \ + A parser change has probably started dropping entries silently. + """ + } + } + } +} diff --git a/Sources/hanreader-dictgen/DictgenCommand.swift b/Sources/hanreader-dictgen/DictgenCommand.swift index 0a40151..2be5a97 100644 --- a/Sources/hanreader-dictgen/DictgenCommand.swift +++ b/Sources/hanreader-dictgen/DictgenCommand.swift @@ -33,11 +33,13 @@ struct Dictgen: AsyncParsableCommand { repository keeps the reviewable .u8 text and the app ships a database \ it only has to open. - The dsl subcommand, for ABBYY DSL dictionaries such as BKRS, arrives \ - in milestone M6. + The dsl subcommand compiles an ABBYY DSL dictionary such as 大БКРС. \ + That source is the reader's own file and is never in this repository, \ + so it is not a build step — and interrupting it is safe, since the \ + import resumes from the last committed batch. """, version: HanReaderCore.version, - subcommands: [CompileCEDICT.self, VerifyDictionary.self], + subcommands: [CompileCEDICT.self, CompileDSL.self, VerifyDictionary.self], ) func run() async throws { diff --git a/Sources/hanreader-dictgen/VerifyCommand.swift b/Sources/hanreader-dictgen/VerifyCommand.swift index ca289a1..a117711 100644 --- a/Sources/hanreader-dictgen/VerifyCommand.swift +++ b/Sources/hanreader-dictgen/VerifyCommand.swift @@ -19,6 +19,12 @@ struct VerifyDictionary: ParsableCommand { @Argument(help: "Path to the .hanreaderdict file.") var input: String + /// Defaults sized for the bundled dictionary, which is what this command + /// was written to guard: the failure it catches is a parser refactor that + /// silently halves it, where everything still builds and words simply + /// stop being found. They are applied only to the format they describe — + /// a container compiled from someone's own DSL dictionary may legitimately + /// hold a hundred entries — and are options so a caller can state its own. @Option(help: "Fail if fewer than this many entries are present.") var minimumEntries = 100_000 @Option(help: "Fail if fewer than this many lexemes are present.") var minimumLexemes = 80000 @@ -37,21 +43,17 @@ struct VerifyDictionary: ParsableCommand { } let entryCount = try container.entryCount() - let lexemes = try container.lexicon() + let lexemeCount = try container.lexemeCount() + let longestLexeme = try container.longestLexeme() let bases = try container.syllableBases() let size = (try? FileManager.default.attributesOfItem(atPath: url.path)[.size] as? Int) ?? 0 - print(""" - \(metadata.displayName) (\(metadata.slug)) - licence: \(metadata.licence) - source: \(metadata.sourceVersion ?? "unspecified") - parser: v\(metadata.parserVersion) - entries: \(entryCount) - lexemes: \(lexemes.count) - syllables: \(bases.count) bases - size: \(size / 1_048_576) MB - open: \(String(format: "%.1f", openTime * 1000)) ms - """) + report(metadata: metadata, openTime: openTime, counts: Counts( + entries: entryCount, + lexemes: lexemeCount, + bases: bases.count, + bytes: size, + )) // Attribution is a licence obligation, not a nicety, so its absence is // a hard failure rather than a warning. @@ -61,21 +63,32 @@ struct VerifyDictionary: ParsableCommand { guard entryCount == metadata.entryCount else { throw VerifyError.entryCountMismatch(recorded: metadata.entryCount, actual: entryCount) } - guard entryCount >= minimumEntries else { + let isBundled = metadata.format == "cc-cedict" + guard !isBundled || entryCount >= minimumEntries else { throw VerifyError.tooFew(what: "entries", got: entryCount, expected: minimumEntries) } - guard lexemes.count >= minimumLexemes else { - throw VerifyError.tooFew(what: "lexemes", got: lexemes.count, expected: minimumLexemes) + guard !isBundled || lexemeCount >= minimumLexemes else { + throw VerifyError.tooFew(what: "lexemes", got: lexemeCount, expected: minimumLexemes) + } + guard entryCount > 0 else { + throw VerifyError.tooFew(what: "entries", got: 0, expected: 1) } // A lexeme longer than the cap means maximum-matching could swallow a // whole clause, which is the specific failure the cap prevents. - if let longest = lexemes.map(\.characterLength).max(), - longest > LexiconBuilder.maximumWordLength - { - throw VerifyError.lexemeTooLong(longest) + if longestLexeme > LexiconBuilder.maximumWordLength { + throw VerifyError.lexemeTooLong(longestLexeme) } - try checkKnownWords(container) + // The spot checks below are CC-CEDICT's own words, and every check + // above this line is structural. A container compiled from someone's + // DSL dictionary is a valid container that does not contain 中国 with + // the reading `zhong1 guo2`, and failing it for that would make + // `verify` useless for the format this milestone added. + if isBundled { + try checkKnownWords(container) + } else { + print(" (content checks are CC-CEDICT's; skipped for \(metadata.format))") + } if benchmark { try measureLookups(container) @@ -83,6 +96,32 @@ struct VerifyDictionary: ParsableCommand { print(" OK") } + /// What the container turned out to hold. + private struct Counts { + let entries: Int + let lexemes: Int + let bases: Int + let bytes: Int + } + + private func report( + metadata: DictionaryMetadata, + openTime: TimeInterval, + counts: Counts, + ) { + print(""" + \(metadata.displayName) (\(metadata.slug)) + licence: \(metadata.licence) + source: \(metadata.sourceVersion ?? "unspecified") + parser: v\(metadata.parserVersion) + entries: \(counts.entries) + lexemes: \(counts.lexemes) + syllables: \(counts.bases) bases + size: \(counts.bytes / 1_048_576) MB + open: \(String(format: "%.1f", openTime * 1000)) ms + """) + } + /// Words that must be findable, with the reading they must have. /// /// A spot check rather than a sample: each is here because it would catch diff --git a/Tests/HanReaderCoreTests/SensePayloadCodecTests.swift b/Tests/HanReaderCoreTests/SensePayloadCodecTests.swift new file mode 100644 index 0000000..88b19d6 --- /dev/null +++ b/Tests/HanReaderCoreTests/SensePayloadCodecTests.swift @@ -0,0 +1,275 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import Testing +@testable import HanReaderCore + +// MARK: - Fixtures + +/// Every sense in the committed slice of real 大БКРС data. +/// +/// Round-tripping invented senses tests the cases someone thought of. +/// Round-tripping 523 real ones tests the cases nobody did — the sense whose +/// gloss is empty because its text was all label, the example that splits at +/// a quotation mark, the reference that ends where the gloss does. +private enum Corpus { + static let senses: [[Sense]] = { + guard let url = Bundle.module.url( + forResource: "Fixtures/bkrs-slice.dsl", + withExtension: "txt", + ), + let data = try? Data(contentsOf: url), + let text = String(data: data, encoding: .utf8) + else { + Issue.record("could not load the BKRS fixture") + return [] + } + var bytes: [UInt8] = [0xFF, 0xFE] + for unit in Array(text.utf16) { + bytes += [UInt8(unit & 0xFF), UInt8(unit >> 8)] + } + let builder = DSLCardBuilder(syllableBases: []) + var runs: [[Sense]] = [] + do { + var reader = try DSLCardReader(source: DSLMemoryBytes(bytes)) + while let record = try reader.next() { + if let senses = builder.entries(from: record.card).entries.first?.senses { + runs.append(senses) + } + } + } catch { + Issue.record("could not read the BKRS fixture: \(error)") + } + return runs + }() +} + +private func sense( + id: Int = 0, + kind: SenseKind = .definition, + text: String = "значение", + level: Int = 0, + parent: Int? = nil, + label: String? = nil, + styles: [StyleRun] = [], + references: [CrossReference] = [], + partOfSpeech: [String] = [], + registers: [String] = [], + examples: [UsageExample] = [], +) + -> Sense +{ + Sense( + id: id, + kind: kind, + gloss: Gloss(text: text, references: references, styles: styles), + level: level, + parent: parent, + label: label, + partOfSpeech: partOfSpeech, + registers: registers, + examples: examples, + ) +} + +@Suite("Compact sense payloads") +struct SensePayloadCodecTests { + private let codec = CompactSenseCodec() + + // MARK: - Round trips + + /// The whole committed slice, sense for sense. Equality is on `Sense` + /// itself, so every field has to come back — a codec that silently drops + /// `registers` would pass a test that only checked gloss text. + @Test("Every sense in the real slice survives a round trip") + func realCorpus() throws { + #expect(!Corpus.senses.isEmpty) + var total = 0 + for run in Corpus.senses { + let decoded = try codec.decode(codec.encode(run)) + #expect(decoded == run) + total += run.count + } + #expect(total == 523) + } + + @Test("An empty run round-trips") + func emptyRun() throws { + #expect(try codec.decode(codec.encode([])).isEmpty) + } + + @Test("Each field survives on its own", arguments: [ + sense(), + sense(kind: .reference, text: "см. другое"), + sense(kind: .label, text: "фамилия"), + sense(text: "", level: 3, parent: 1, label: "I"), + sense(level: 9, parent: 300), + sense(styles: [ + StyleRun(style: .bold, range: 0 ..< 2), + StyleRun(style: .italic, range: 3 ..< 8), + StyleRun(style: .highlighted, range: 0 ..< 1), + ]), + sense(references: [CrossReference( + simplified: "磁性碰锁", + traditional: "磁性碰鎖", + reading: "cíxìng pèngsuǒ", + range: 2 ..< 6, + )]), + sense(references: [CrossReference( + simplified: "横", + traditional: nil, + reading: nil, + range: 0 ..< 1, + )]), + sense(partOfSpeech: ["гл.", "сущ."], registers: ["бот.", "уст."]), + sense(examples: [UsageExample( + chinese: "爱儿童", + translation: "любить ребёнка", + raw: "爱儿童 любить ребёнка", + )]), + ]) + func singleFields(input: Sense) throws { + #expect(try codec.decode(codec.encode([input])) == [input]) + } + + /// `id` is the index in the run and is not stored, so a run decodes with + /// its positions renumbered — which is what every reader of it assumes. + @Test("Identifiers come back as positions") + func identifiersArePositions() throws { + let run = (0 ..< 5).map { sense(id: 97 + $0, text: "s\($0)") } + let decoded = try codec.decode(codec.encode(run)) + #expect(decoded.map(\.id) == [0, 1, 2, 3, 4]) + #expect(decoded.map(\.gloss.text) == run.map(\.gloss.text)) + } + + // MARK: - Examples, which are stored once + + /// `raw` is the example as written and the two halves are a heuristic + /// split of it, so storing all three copies the text twice. Measured over + /// the set, the halves cost as much again as `raw`. + @Test("An example is stored once and split on the way out", arguments: [ + UsageExample(chinese: "爱儿童", translation: "любить ребёнка", raw: "爱儿童 любить ребёнка"), + UsageExample(chinese: "", translation: "только по-русски", raw: "только по-русски"), + UsageExample(chinese: "他说：“好。”", translation: "он сказал", raw: "他说：“好。” он сказал"), + UsageExample(chinese: "一", translation: "", raw: "一"), + UsageExample(chinese: "", translation: "", raw: ""), + ]) + func examples(input: UsageExample) throws { + let decoded = try codec.decode(codec.encode([sense(examples: [input])])) + #expect(decoded.first?.examples == [input]) + } + + /// A split that is not a slice of the raw text — which a different + /// splitter could produce — is written out in full rather than silently + /// reconstructed wrongly. The compact form is an optimisation, not an + /// invariant the model has to keep. + @Test("An example whose halves are not slices of its raw text still round-trips") + func examplesThatAreNotSlices() throws { + let odd = UsageExample(chinese: "完全", translation: "иное", raw: "ничего общего") + let decoded = try codec.decode(codec.encode([sense(examples: [odd])])) + #expect(decoded.first?.examples == [odd]) + } + + /// The same text twice over, where a naive "find the half in the raw" + /// would pick the wrong occurrence. + @Test("A repeated half picks the right occurrence") + func repeatedHalf() throws { + let repeated = UsageExample(chinese: "好", translation: "好", raw: "好 好") + let decoded = try codec.decode(codec.encode([sense(examples: [repeated])])) + #expect(decoded.first?.examples == [repeated]) + } + + // MARK: - Size + + /// The reason the codec exists. Measured over the real slice rather than + /// asserted as a ratio someone liked: JSON spends most of a payload on + /// syntax and key names, written out once per sense across three and a + /// half million of them. + @Test("The compact form is substantially smaller than JSON") + func size() throws { + let json = JSONSenseCodec() + var compactBytes = 0 + var jsonBytes = 0 + for run in Corpus.senses { + compactBytes += try codec.encode(run).count + jsonBytes += try json.encode(run).count + } + #expect(compactBytes * 2 < jsonBytes, "compact \(compactBytes), json \(jsonBytes)") + } + + // MARK: - Malformed payloads + + /// A payload is data from a file on disk. A corrupt one costs that entry, + /// not the process, so every read is bounds-checked rather than trapping. + @Test("A truncated payload throws rather than trapping") + func truncated() throws { + let full = try codec.encode([sense( + text: "значение", + label: "1)", + examples: [UsageExample(chinese: "一", translation: "один", raw: "一 один")], + )]) + for length in 0 ..< full.count { + #expect(throws: (any Error).self) { + _ = try codec.decode(full.prefix(length)) + } + } + } + + @Test("A payload from a newer format is refused, not guessed at") + func newerVersion() throws { + var bytes = try [UInt8](codec.encode([sense()])) + bytes[0] = CompactSenseCodec.version + 1 + #expect(throws: SensePayloadError.unknownVersion(CompactSenseCodec.version + 1)) { + _ = try codec.decode(Data(bytes)) + } + } + + @Test("An empty payload is refused") + func emptyPayload() { + #expect(throws: (any Error).self) { + _ = try codec.decode(Data()) + } + } + + // MARK: - Codec selection + + /// A container records which codec wrote it, so adding one does not + /// invalidate the dictionaries already on disk. + @Test("A container's recorded codec is what reads it") + func selection() throws { + #expect(try SensePayloadCodecs.named("compact-v1") is CompactSenseCodec) + #expect(try SensePayloadCodecs.named("json") is JSONSenseCodec) + // Written before the codec was recorded, which means JSON. + #expect(try SensePayloadCodecs.named(nil) is JSONSenseCodec) + #expect(throws: SensePayloadError.unknownCodec("compact-v99")) { + _ = try SensePayloadCodecs.named("compact-v99") + } + } + + /// JSON still reads, so a dictionary compiled by an earlier build opens. + @Test("JSON payloads still round-trip") + func jsonStillWorks() throws { + let json = JSONSenseCodec() + let run = try #require(Corpus.senses.first { $0.count > 2 }) + #expect(try json.decode(json.encode(run)) == run) + } + + // MARK: - Wire format + + /// The byte for a style is fixed here rather than taken from the case + /// order, so reordering the enum cannot change what is already on disk. + @Test("Style bytes are stable", arguments: TextStyle.allCases) + func styleBytes(style: TextStyle) { + #expect(TextStyle(encoded: style.encoded) == style) + } + + @Test("Varints round-trip at the boundaries", arguments: [ + 0, 1, 127, 128, 255, 256, 16383, 16384, 1 << 20, Int(Int32.max), + ]) + func varints(value: Int) throws { + var bytes: [UInt8] = [] + CompactSenseCodec.appendVarint(value, to: &bytes) + var reader = CompactSenseCodec.Reader(Data(bytes)) + #expect(try reader.varint() == value) + } +} diff --git a/Tests/HanReaderDictionaryImportTests/DSLImporterTests.swift b/Tests/HanReaderDictionaryImportTests/DSLImporterTests.swift new file mode 100644 index 0000000..c3e99a5 --- /dev/null +++ b/Tests/HanReaderDictionaryImportTests/DSLImporterTests.swift @@ -0,0 +1,546 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import HanReaderCore +import HanReaderPersistence +import Testing +@testable import HanReaderDictionaryImport + +// MARK: - Fixtures + +/// A throwaway directory holding a DSL set and the container built from it. +private struct Workspace: ~Copyable { + let root: URL + + init() throws { + root = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("hanreader-import-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + } + + deinit { + try? FileManager.default.removeItem(at: root) + } + + var destination: URL { + root.appendingPathComponent("out.hanreaderdict") + } + + /// Writes a DSL file as the real ones are written: UTF-16LE with a BOM. + @discardableResult + func write(_ name: String, _ text: String) throws -> URL { + let url = root.appendingPathComponent(name) + var bytes: [UInt8] = [0xFF, 0xFE] + for unit in Array(text.utf16) { + bytes += [UInt8(unit & 0xFF), UInt8(unit >> 8)] + } + try Data(bytes).write(to: url) + return url + } + + var bases: Set { + ["ai", "le", "liao", "yi", "da", "guo"] + } + + /// Plants the job row an interruption would have left. + /// + /// Needed because a *completed* import clears the job, so a test that + /// runs one to the end and then runs it again is testing a fresh import + /// and nothing else — which is how three of these tests first passed + /// against bugs they were written for. + func plantJob( + _ checkpoint: DictionaryWriter.Checkpoint, + files: [String], + bases: Set? = nil, + ) throws { + let sources = try files.map { + try DSLImporter.FileDescription(url: root.appendingPathComponent($0)) + } + let (writer, _) = try DictionaryWriter.open( + at: destination, + source: DictionaryWriter.Source( + name: files[0], + fingerprint: DSLImporter.fingerprint(of: sources, bases: bases ?? self.bases), + fileCount: files.count, + totalBytes: sources.reduce(0) { $0 + $1.size }, + parserVersion: DSLImporter.parserVersion, + ), + ) + try writer.checkpoint(checkpoint) + } + + func importer( + from name: String, + batchSize: Int = DSLImporter.defaultBatchSize, + ) throws + -> DSLImporter + { + try DSLImporter( + fileSet: DSLFileSet.discover(from: root.appendingPathComponent(name)), + destination: destination, + syllableBases: bases, + batchSize: batchSize, + ) + } +} + +/// A card run long enough to span several batches. +private func cards(_ count: Int, from offset: Int = 0) -> String { + (0 ..< count).map { index in + let word = String(index + offset, radix: 36) + return "w\(word)\n ài\n [m1]значение \(index)[/m][m2]второе[/m]\n" + }.joined() +} + +private let header = """ +#NAME "大БКРС - 250920 ( 1 / 3 )"\r +#INDEX_LANGUAGE "Chinese" +#CONTENTS_LANGUAGE "Russian" + +""" + +/// A flag two closures can share. +private final class Flag: @unchecked Sendable { + private let lock = NSLock() + private var value = false + + var isSet: Bool { + lock.withLock { value } + } + + func set() { + lock.withLock { value = true } + } +} + +/// Runs an import and stops it after its first committed batch. +/// +/// Deterministic and synchronous: progress is reported when a batch commits, +/// and `shouldContinue` is asked at the next card, so the importer stops with +/// exactly one batch written and the rest of the file unread. +/// +/// Two earlier attempts at this were wrong in instructive ways. Sleeping and +/// then cancelling raced — a twenty-card import finishes before an async test +/// can react. Signalling the test and blocking the callback until it cancelled +/// was deterministic but **blocked a thread of the cooperative pool**, and +/// with Swift Testing running these tests in parallel the pool deadlocked: CI +/// sat in that suite for twenty minutes. Asking the importer to stop needs no +/// concurrency at all. +@discardableResult +private func stopAfterFirstBatch(_ importer: DSLImporter) throws -> DSLImportSummary { + let committed = Flag() + return try importer.run( + progress: { _ in committed.set() }, + shouldContinue: { !committed.isSet }, + ) +} + +@Suite("DSL importer") +struct DSLImporterTests { + // MARK: - A plain import + + @Test("A set becomes a container") + func wholeImport() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + cards(40)) + + let summary = try workspace.importer(from: "a.dsl").run() + #expect(summary.cardsRead == 40) + #expect(summary.entriesWritten == 40) + #expect(summary.diagnostics == 0) + #expect(!summary.resumed) + + let container = try DictionaryContainer(contentsOf: workspace.destination) + #expect(try container.entryCount() == 40) + let entries = try container.entries(for: "w0") + #expect(entries.count == 1) + #expect(entries[0].senses.count == 2) + #expect(entries[0].senses[0].gloss.text == "значение 0") + } + + /// The job row is cleared on success, so a finished container does not + /// look like one that was interrupted. + @Test("A finished import leaves no job behind") + func noJobAfterFinishing() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + cards(10)) + try workspace.importer(from: "a.dsl").run() + + // Opening for writing again finds nothing to resume, which is what + // makes the second run a fresh import rather than a continuation. + let reopened = try DictionaryWriter.open( + at: workspace.destination, + source: DictionaryWriter.Source( + name: "a.dsl", + fingerprint: "anything", + fileCount: 1, + totalBytes: 1, + parserVersion: DSLImporter.parserVersion, + ), + ) + #expect(reopened.resume == nil) + } + + @Test("Several files are imported in order") + func severalFiles() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + "#INCLUDE \"b.dsl\"\n\n" + cards(5)) + try workspace.write("b.dsl", "#NAME \"B\"\n\n" + cards(5, from: 100)) + + let summary = try workspace.importer(from: "a.dsl").run() + #expect(summary.cardsRead == 10) + let container = try DictionaryContainer(contentsOf: workspace.destination) + #expect(try container.entryCount() == 10) + } + + // MARK: - Resuming + + /// The gate this exists for. Interrupting leaves the same state a + /// `kill -9` leaves — everything up to the last committed batch — and the + /// resumed result has to equal the uninterrupted one exactly. + /// + /// Interrupted deterministically rather than after a sleep. Progress is + /// reported once the first batch commits and then at most ten times a + /// second, so cancelling on the first report stops the import after + /// exactly one batch however fast the machine is. A sleep raced: on a + /// quiet machine the whole 400-card import finished inside 40 ms. + @Test("An interrupted import resumes to the same result") + func resumeMatchesUninterrupted() throws { + let reference = try Workspace() + try reference.write("a.dsl", header + cards(400)) + let expected = try reference.importer(from: "a.dsl", batchSize: 25).run() + + let interrupted = try Workspace() + try interrupted.write("a.dsl", header + cards(400)) + let importer = try interrupted.importer(from: "a.dsl", batchSize: 25) + + // The first progress value means the first batch has committed, so + // waiting for it and then cancelling interrupts the import at a batch + // boundary with work done and work left — no sleep, no race. + let partialRun = try stopAfterFirstBatch(importer) + #expect(!partialRun.completed) + + let partial = try DictionaryContainer(contentsOf: interrupted.destination).entryCount() + #expect(partial > 0, "nothing was committed before the interruption") + #expect(partial < 400, "the import was not actually interrupted") + // A whole number of batches, because a batch and its checkpoint are + // one transaction: a half-written batch cannot survive. + #expect(partial.isMultiple(of: 25)) + + let resumed = try interrupted.importer(from: "a.dsl", batchSize: 25).run() + #expect(resumed.resumed) + #expect(resumed.entriesWritten == expected.entriesWritten) + #expect(resumed.cardsRead == expected.cardsRead) + + // Not just the counts: the same rows, and the same senses in them. + let left = try DictionaryContainer(contentsOf: reference.destination) + let right = try DictionaryContainer(contentsOf: interrupted.destination) + #expect(try left.entryCount() == right.entryCount()) + #expect(try left.sourceOffsets() == right.sourceOffsets()) + for word in ["w0", "wa", "wrr", "wb3"] { + let before = try left.entries(for: word).map(\.senses) + let after = try right.entries(for: word).map(\.senses) + #expect(before == after, "\(word) differs") + } + } + + /// An end-of-file checkpoint stores byte offset zero to mean "from the + /// beginning of the next file". Handing that straight back to the byte + /// reader starts it *at* the byte-order mark, which then arrives as the + /// first character of the first headword — so every multi-file resume at a + /// file boundary corrupted one headword, silently. + /// + /// The job row is planted rather than reached by interrupting, because a + /// batch big enough to span a volume reports no progress inside the card + /// loop and a batch small enough to report never stops *on* the boundary. + /// It is exactly the row a real interruption leaves. + @Test("Resuming at a file boundary does not read the byte-order mark") + func resumeAtFileBoundary() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + "#INCLUDE \"b.dsl\"\n\n" + cards(5)) + try workspace.write("b.dsl", "#NAME \"B\"\n\n" + cards(5, from: 500)) + try workspace.plantJob( + DictionaryWriter.Checkpoint(fileIndex: 1, byteOffset: 0), + files: ["a.dsl", "b.dsl"], + ) + + let summary = try workspace.importer(from: "a.dsl").run() + #expect(summary.resumed) + + let container = try DictionaryContainer(contentsOf: workspace.destination) + // The headword that follows the mark, intact. Read as content, the + // mark makes it "\u{FEFF}wdw" and nothing finds it again. + #expect(try !container.entries(for: "w" + String(500, radix: 36)).isEmpty) + for word in try container.headwords() { + #expect(!word.hasPrefix("\u{FEFF}"), "\(word.debugDescription) kept a byte-order mark") + } + } + + /// A card whose headword is in one volume and whose article is in the next + /// has no resume point in the file it was completed in. Clearing the + /// carried headword before that card's batch commits leaves a checkpoint + /// that restarts the next file without it, so the card's entries are + /// deleted on resume and its article cannot rebuild them. + /// + /// Interrupted at exactly that card: the first volume holds nothing but + /// the dangling headword, so the first batch to commit is the seam card's, + /// and the first progress report is therefore the one that follows it. + @Test("A seam card survives being the frontier of a checkpoint") + func seamCardSurvivesResume() throws { + let seamFiles = { (workspace: borrowing Workspace) throws in + try workspace.write("a.dsl", header + "#INCLUDE \"b.dsl\"\n\nwseam\n") + try workspace.write( + "b.dsl", + "#NAME \"B\"\n\n ài\n [m1]разрезанное[/m]\n" + cards(4, from: 900), + ) + } + + let reference = try Workspace() + try seamFiles(reference) + let expected = try reference.importer(from: "a.dsl", batchSize: 1).run() + #expect(expected.cardsRead == 5) + + let workspace = try Workspace() + try seamFiles(workspace) + let importer = try workspace.importer(from: "a.dsl", batchSize: 1) + let partialRun = try stopAfterFirstBatch(importer) + #expect(!partialRun.completed) + + let resumed = try workspace.importer(from: "a.dsl", batchSize: 1).run() + #expect(resumed.resumed) + #expect(resumed.cardsRead == expected.cardsRead) + + let seam = try DictionaryContainer(contentsOf: workspace.destination) + .entries(for: "wseam") + #expect(seam.count == 1, "the seam card was lost") + #expect(seam.first?.senses.first?.gloss.text == "разрезанное") + } + + // MARK: - Progress + + @Test("Progress is throttled and ends at the end") + func progress() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + cards(600)) + + nonisolated(unsafe) var values: [DSLImportProgress] = [] + try workspace.importer(from: "a.dsl", batchSize: 5).run { values.append($0) } + + // 120 batches, but never more than ten reports a second. + #expect(values.count < 120) + #expect(values.last?.fraction == 1) + #expect(values.last?.bytesRead == values.last?.totalBytes) + #expect(values.map(\.cardsRead) == values.map(\.cardsRead).sorted()) + } + + // MARK: - Metadata + + /// Nothing is hardcoded: the predecessor took the dictionary's name from a + /// constant in its source, so importing any other DSL dictionary + /// mislabelled it. + @Test("Metadata comes out of the file") + func metadata() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + cards(3)) + let summary = try workspace.importer(from: "a.dsl").run() + + #expect(summary.metadata.displayName == "大БКРС - 250920") + #expect(summary.metadata.slug == "dabkrs-250920") + #expect(summary.metadata.format == "abbyy-dsl") + #expect(summary.metadata.indexLanguage == "zh-Hans") + #expect(summary.metadata.glossLanguage == "ru") + #expect(summary.metadata.sourceVersion == "250920") + #expect(summary.metadata.entryCount == 3) + } + + /// The volume count belongs to the file, not to the dictionary — the three + /// volumes compile into one container. + @Test("The volume suffix is stripped from the name", arguments: [ + ("大БКРС - 250920 ( 1 / 3 )", "大БКРС - 250920"), + ("Example (2/2)", "Example"), + ("Example (revised)", "Example (revised)"), + ("Example", "Example"), + ]) + func volumeSuffix(declared: String, expected: String) { + #expect(DSLImporter.volumeSuffix(strippedFrom: declared) == expected) + } + + /// BKRS states no licence, and inventing one would be worse than + /// admitting it: the Acknowledgements screen renders this verbatim. + @Test("No licence is claimed for a source that states none") + func noLicenceClaimed() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + cards(1)) + let summary = try workspace.importer(from: "a.dsl").run() + #expect(summary.metadata.licence == "Not stated by the source") + #expect(summary.metadata.attribution.contains("does not")) + #expect(summary.metadata.attribution.contains("Changes made:")) + } + + @Test("A language the format does not name is kept verbatim") + func unknownLanguage() { + #expect(DSLImporter.languageTag("Chinese") == "zh-Hans") + #expect(DSLImporter.languageTag("Russian") == "ru") + #expect(DSLImporter.languageTag("Klingon") == "Klingon") + #expect(DSLImporter.languageTag(nil) == nil) + } + + /// A slug has to work as a file name and a settings key, and `大БКРС` + /// has nothing ASCII in it at all. Transliterating rather than stripping + /// is what gives it one. + @Test("A name is transliterated into its slug", arguments: [ + ("大БКРС - 250920", "dabkrs-250920"), + ("大辞典", "da-ci-dian"), + ("Example Dictionary", "example-dictionary"), + ]) + func slugFromName(declared: String, expected: String) { + #expect(DSLImporter.slug(from: declared, fallback: "ignored.dsl") == expected) + } + + @Test("A name with nothing transliterable falls back to the file name") + func slugFallback() { + #expect(DSLImporter.slug(from: "", fallback: "my_dict.dsl") == "my-dict") + #expect(DSLImporter.slug(from: "—", fallback: "vol_1.dsl") == "vol-1") + } +} + +/// What an import is allowed to resume onto. +/// +/// A job is only continuable if everything the output depends on is unchanged +/// — every volume, and the syllable inventory the readings were split with. +/// Half a dictionary from each of two versions is worse than either, and +/// reports success. +@Suite("DSL import identity") +struct DSLImportIdentityTests { + /// Checking only the first volume lets a changed third volume resume onto + /// entries from the old one — a dictionary half from each version, and it + /// reports success. + @Test("A changed later volume starts the import again") + func changedLaterVolumeRestarts() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + "#INCLUDE \"b.dsl\"\n\n" + cards(5)) + try workspace.write("b.dsl", "#NAME \"B\"\n\n" + cards(5, from: 500)) + try workspace.plantJob( + DictionaryWriter.Checkpoint(fileIndex: 1, byteOffset: 0, cardsRead: 5), + files: ["a.dsl", "b.dsl"], + ) + + // The first volume is untouched; only the second changes. + try workspace.write("b.dsl", "#NAME \"B\"\n\n" + cards(2, from: 500)) + let again = try workspace.importer(from: "a.dsl").run() + #expect(!again.resumed) + #expect(again.cardsRead == 7) + #expect(again.entriesWritten == 7) + } + + /// The syllable inventory decides how a reading is split and therefore + /// what the cross-dictionary merge key is, so changing it between runs + /// would leave readings parsed two ways in one container. + @Test("Changed syllable bases start the import again") + func changedBasesRestart() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + cards(6)) + try workspace.plantJob( + DictionaryWriter.Checkpoint(fileIndex: 0, byteOffset: 200, cardsRead: 2), + files: ["a.dsl"], + bases: ["ai"], + ) + + let again = try DSLImporter( + fileSet: DSLFileSet.discover(from: workspace.root.appendingPathComponent("a.dsl")), + destination: workspace.destination, + syllableBases: ["ai", "guo"], + ).run() + #expect(!again.resumed) + #expect(again.cardsRead == 6) + } + + /// The same job with the same inputs *does* resume, so the checks above + /// are not passing because nothing ever resumes. + @Test("An unchanged source resumes") + func unchangedSourceResumes() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + cards(6)) + try workspace.plantJob( + DictionaryWriter.Checkpoint(fileIndex: 0, byteOffset: 0, cardsRead: 0), + files: ["a.dsl"], + ) + #expect(try workspace.importer(from: "a.dsl").run().resumed) + } + + /// A volume an `#INCLUDE` names but that is not there means a dictionary + /// missing part of itself, which is the failure a reader would never + /// notice on their own. + @Test("A missing volume is reported, not finished over") + func missingVolumeReported() throws { + let workspace = try Workspace() + try workspace.write("a.dsl", header + "#INCLUDE \"gone.dsl\"\n\n" + cards(4)) + + let summary = try workspace.importer(from: "a.dsl").run() + #expect(summary.entriesWritten == 4) + #expect(summary.diagnostics == 0) + #expect(summary.discovery.map(\.kind) == [.missingInclude("gone.dsl")]) + #expect(summary.totalDiagnostics == 1) + } + + /// A card counted on both sides of an interruption is counted twice, and + /// so are its diagnostics. Reader diagnostics have the opposite problem: + /// added only at end of file, a resume seeking past them loses them. + @Test("Diagnostic counts survive an interruption") + func diagnosticCountsSurviveResume() throws { + // `[zzz]` is one lexer diagnostic per card; the dangling headword at + // the end is one from the reader. + let body = (0 ..< 20).map { "wz\($0)\n ài\n [m1][zzz]текст[/m]\n" }.joined() + let make = { (workspace: borrowing Workspace) throws in + _ = try workspace.write("a.dsl", header + body + "wdangling\n") + } + + let reference = try Workspace() + try make(reference) + let expected = try reference.importer(from: "a.dsl", batchSize: 1).run() + #expect(expected.diagnostics == 21) + + let workspace = try Workspace() + try make(workspace) + let importer = try workspace.importer(from: "a.dsl", batchSize: 1) + let partialRun = try stopAfterFirstBatch(importer) + #expect(!partialRun.completed) + + let resumed = try workspace.importer(from: "a.dsl", batchSize: 1).run() + #expect(resumed.resumed) + #expect(resumed.cardsRead == expected.cardsRead) + #expect(resumed.diagnostics == expected.diagnostics) + } + + /// The other half: a reader diagnostic that happens *before* a checkpoint + /// has to be in that checkpoint. Added only at end of file, it is lost + /// whenever a resume seeks past the lines that caused it — and the + /// end-of-file catch-up hides the fault in any run that reaches the end, + /// which is why this case needs an interruption to show at all. + @Test("A diagnostic before the checkpoint is carried across a resume") + func readerDiagnosticsBeforeCheckpoint() throws { + // An article line with no headword above it, before any card. + let make = { (workspace: borrowing Workspace) throws in + _ = try workspace.write("a.dsl", header + " [m1]сирота[/m]\n" + cards(20)) + } + + let reference = try Workspace() + try make(reference) + let expected = try reference.importer(from: "a.dsl", batchSize: 1).run() + #expect(expected.diagnostics == 1) + + let workspace = try Workspace() + try make(workspace) + let importer = try workspace.importer(from: "a.dsl", batchSize: 1) + let partialRun = try stopAfterFirstBatch(importer) + #expect(!partialRun.completed) + + let partial = try DictionaryContainer(contentsOf: workspace.destination).entryCount() + let resumed = try workspace.importer(from: "a.dsl", batchSize: 1).run() + #expect(resumed.resumed) + #expect( + resumed.diagnostics == expected.diagnostics, + "resumed \(resumed.diagnostics), reference \(expected.diagnostics), partial \(partial)", + ) + } +} diff --git a/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift b/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift new file mode 100644 index 0000000..eb1fe45 --- /dev/null +++ b/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift @@ -0,0 +1,208 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import HanReaderCore +import Testing +@testable import HanReaderPersistence + +// MARK: - Fixtures + +private func entry( + _ simplified: String, + traditional: String? = nil, + reading: String = "ai4", + senses: [String] = ["значение"], +) + -> DictionaryEntry +{ + DictionaryEntry( + headword: Headword(simplified: simplified, traditional: traditional), + reading: Pinyin.parse(numeric: reading), + senses: senses.enumerated().map { index, text in + Sense( + id: index, + kind: .definition, + gloss: Gloss(text: text, references: [], styles: []), + level: 0, + parent: nil, + label: nil, + partOfSpeech: [], + registers: [], + examples: [], + ) + }, + ) +} + +private func metadata(entryCount: Int) -> DictionaryMetadata { + DictionaryMetadata( + slug: "test", + displayName: "Test", + format: "abbyy-dsl", + indexLanguage: "zh-Hans", + glossLanguage: "ru", + licence: "none", + attribution: "none", + parserVersion: 1, + entryCount: entryCount, + ) +} + +private func temporaryURL() -> URL { + URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("hanreader-storage-\(UUID().uuidString)") + .appendingPathComponent("out.hanreaderdict") +} + +private func source(fingerprint: String = "f") -> DictionaryWriter.Source { + DictionaryWriter.Source( + name: "a.dsl", + fingerprint: fingerprint, + fileCount: 1, + totalBytes: 1000, + parserVersion: 1, + ) +} + +@Suite("Dictionary storage") +struct DictionaryStorageTests { + /// Whatever the indexing strategy, a finished container has to answer a + /// lookup — the whole point of building the indexes at the end is that it + /// is invisible from the outside. + @Test("A finished container looks words up", arguments: [ + DictionaryWriter.Indexing.atTheEnd, + DictionaryWriter.Indexing.whileWriting, + ]) + func lookupAfterEitherIndexing(indexing: DictionaryWriter.Indexing) throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let (writer, resume) = try DictionaryWriter.open( + at: url, + source: source(), + indexing: indexing, + ) + #expect(resume == nil) + try writer.append( + [ + DictionaryWriter.SourcedEntry( + entry: entry("爱", traditional: "愛"), + sourceFile: 0, + sourceOffset: 10, + ), + DictionaryWriter.SourcedEntry( + entry: entry("中国", reading: "zhong1 guo2"), + sourceFile: 0, + sourceOffset: 20, + ), + ], + checkpoint: DictionaryWriter.Checkpoint(fileIndex: 0, byteOffset: 20, cardsRead: 2), + ) + try writer.finish(metadata: metadata(entryCount: 2)) + + let container = try DictionaryContainer(contentsOf: url) + #expect(try container.entryCount() == 2) + #expect(try container.entries(for: "爱").count == 1) + // Matched against both scripts, or a reader of traditional text finds + // nothing at all. + #expect(try container.entries(for: "愛").count == 1) + #expect(try container.entries(for: "中国").first?.senses.first?.gloss.text == "значение") + } + + /// The index the resume path uses is wanted only during an import, and the + /// lookup indexes only after one. A finished container should carry the + /// second pair and not the first. + @Test("A finished container carries the reader's indexes and not the importer's") + func indexesAfterFinishing() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let (writer, _) = try DictionaryWriter.open(at: url, source: source()) + try writer.append( + [DictionaryWriter.SourcedEntry(entry: entry("爱"), sourceFile: 0, sourceOffset: 1)], + checkpoint: DictionaryWriter.Checkpoint(byteOffset: 1, cardsRead: 1), + ) + try writer.finish(metadata: metadata(entryCount: 1)) + + let indexes = try DictionaryContainer(contentsOf: url).indexNames() + #expect(indexes.contains("entry_on_simplified")) + #expect(indexes.contains("entry_on_traditional")) + #expect(!indexes.contains("entry_on_source")) + } + + /// A re-import needs the source index back, since a finished import drops + /// it and the resume path's delete depends on it. + @Test("Reopening a finished container restores the importer's index") + func reopeningRestoresSourceIndex() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let (first, _) = try DictionaryWriter.open(at: url, source: source()) + try first.append( + [DictionaryWriter.SourcedEntry(entry: entry("爱"), sourceFile: 0, sourceOffset: 1)], + checkpoint: DictionaryWriter.Checkpoint(byteOffset: 1, cardsRead: 1), + ) + try first.finish(metadata: metadata(entryCount: 1)) + + _ = try DictionaryWriter.open(at: url, source: source(fingerprint: "changed")) + #expect(try DictionaryContainer(contentsOf: url).indexNames() + .contains("entry_on_source")) + } + + // MARK: - Sense payloads + + /// A container records which codec wrote it, so a new codec does not + /// invalidate the dictionaries already on disk. + @Test("A container says which codec wrote its senses") + func recordedCodec() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + try DictionaryContainer.write( + entries: [entry("爱", senses: ["первое", "второе"])], + metadata: metadata(entryCount: 1), + to: url, + ) + let container = try DictionaryContainer(contentsOf: url) + #expect(try type(of: container.sensePayloadCodec()).identifier + == CompactSenseCodec.identifier) + #expect(try container.entries(for: "爱").first?.senses.count == 2) + } + + /// The denormalised first gloss, written since v1 and read by nothing. On + /// a BKRS container it was 160 MB of prose already sitting in the payload + /// beside it. + @Test("The summary column is gone") + func summaryDropped() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + try DictionaryContainer.write( + entries: [entry("爱")], + metadata: metadata(entryCount: 1), + to: url, + ) + #expect(try !DictionaryContainer(contentsOf: url).columnNames(of: "entry") + .contains("summary")) + } + + /// A container built before the compact codec existed still opens, which + /// is what the recorded identifier is for. + @Test("A JSON container still reads") + func jsonContainerStillReads() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let senses = entry("爱", senses: ["первое", "второе"]).senses + try DictionaryContainer.writeForTesting( + entry: entry("爱", senses: ["первое", "второе"]), + codec: JSONSenseCodec(), + metadata: metadata(entryCount: 1), + to: url, + ) + let container = try DictionaryContainer(contentsOf: url) + #expect(try type(of: container.sensePayloadCodec()).identifier + == JSONSenseCodec.identifier) + #expect(try container.entries(for: "爱").first?.senses == senses) + } +}