diff --git a/Sources/HanReaderCore/Dictionary/SensePayloadCodec.swift b/Sources/HanReaderCore/Dictionary/SensePayloadCodec.swift new file mode 100644 index 0000000..5eb7177 --- /dev/null +++ b/Sources/HanReaderCore/Dictionary/SensePayloadCodec.swift @@ -0,0 +1,529 @@ +// HanReader — MIT licensed. See LICENSE. + +public import Foundation + +/// How a container's sense payloads are encoded. +/// +/// A seam rather than a single implementation, because the choice is a +/// measured trade and will be made again. A container records which codec +/// wrote it, so adding one does not invalidate the dictionaries already on +/// disk — and so a future block-compressing codec is a drop-in rather than a +/// migration. +public protocol SensePayloadCodec: Sendable { + /// Written into the container's `meta` table, so a reader can pick the + /// codec the writer used rather than assuming the current one. + static var identifier: String { get } + + func encode(_ senses: [Sense]) throws -> Data + func decode(_ data: Data) throws -> [Sense] +} + +/// Something that cannot be read as a sense payload. +public enum SensePayloadError: Error, Hashable, Sendable, CustomStringConvertible { + case truncated(offset: Int) + case unknownVersion(UInt8) + case unknownCodec(String) + + public var description: String { + switch self { + case let .truncated(offset): + "the payload ends part way through a field, at byte \(offset)" + case let .unknownVersion(version): + "payload format version \(version) is newer than this build understands" + case let .unknownCodec(identifier): + "this container's senses were written by the '\(identifier)' codec, " + + "which this build does not have" + } + } +} + +// MARK: - JSON, as the first milestones wrote it + +/// JSON, kept so a container compiled before the compact codec still opens. +/// +/// It is not what new containers use. Measured over the BKRS set, only +/// **40.7%** of a JSON payload is text; the rest is syntax and key names, +/// written out once per sense across 3,691,468 of them. +public struct JSONSenseCodec: SensePayloadCodec { + public static let identifier = "json" + + public init() {} + + public func encode(_ senses: [Sense]) throws -> Data { + try JSONEncoder().encode(senses) + } + + public func decode(_ data: Data) throws -> [Sense] { + try JSONDecoder().decode([Sense].self, from: data) + } +} + +// MARK: - The packed encoding + +/// A packed binary encoding of a sense run. +/// +/// ### Why +/// +/// Measured over the 3,442,623-entry BKRS container: the `entry` table was +/// 625 MB, its sense payloads 327 MB of that, and **only 40.7% of a payload +/// was text**. The rest was `{"gloss":{"text":…},"level":0,"kind":…}` written +/// out once per sense. Payload size is also most of the import's wall-clock, +/// since every byte is one SQLite has to write. +/// +/// ### How +/// +/// Three ideas, each worth its complexity: +/// +/// - **A presence byte.** The overwhelmingly common sense is a gloss, a level +/// and nothing else — 3,318,399 of 3,434,224 cards carry a single sense — so +/// every field it lacks costs one bit instead of a key and a literal. +/// - **Varints.** Levels are 0–3, parents and offsets are small, counts are +/// usually one. A varint spends a byte on each where JSON spends a decimal +/// rendering plus punctuation. +/// - **An example is stored once.** `UsageExample` keeps `chinese`, +/// `translation` *and* `raw`, where `raw` is the example as written and the +/// other two are a heuristic split of it; measured, the halves cost as much +/// again as `raw` itself. Here `raw` is stored with the two boundaries +/// inside it, reproducing both halves exactly — including a split that +/// trimmed whitespace — from one copy of the text. +/// +/// `Sense.id` is not stored at all: it is the index in the run. +/// +/// ### Compatibility +/// +/// The leading version byte is checked, and a payload claiming a newer version +/// is refused rather than misread. Adding a field means a new presence bit and +/// a version bump; `meta` says which codec wrote a container, so an older one +/// stays readable. +public struct CompactSenseCodec: SensePayloadCodec { + public static let identifier = "compact-v1" + + /// Bumped when the layout changes. A payload is refused rather than + /// guessed at, because a misread sense is worse than an error. + static let version: UInt8 = 1 + + enum Field: UInt8 { + case parent = 0x01 + case label = 0x02 + case styles = 0x04 + case references = 0x08 + case partOfSpeech = 0x10 + case registers = 0x20 + case examples = 0x40 + /// Set when the kind is not `.definition`, which almost every sense is. + case kind = 0x80 + } + + public init() {} + + public func encode(_ senses: [Sense]) throws -> Data { + var bytes: [UInt8] = [Self.version] + // A rough guess at the common case, to avoid a handful of growths per + // entry across three and a half million of them. + bytes.reserveCapacity(64 * senses.count + 8) + Self.appendVarint(senses.count, to: &bytes) + for sense in senses { + Self.append(sense, to: &bytes) + } + return Data(bytes) + } + + public func decode(_ data: Data) throws -> [Sense] { + var reader = Reader(data) + let version = try reader.byte() + guard version == Self.version else { throw SensePayloadError.unknownVersion(version) } + + let count = try reader.varint() + var senses: [Sense] = [] + senses.reserveCapacity(count) + for index in 0 ..< count { + try senses.append(Self.decodeSense(at: index, from: &reader)) + } + return senses + } +} + +// MARK: - Writing + +extension CompactSenseCodec { + static func append(_ sense: Sense, to bytes: inout [UInt8]) { + bytes.append(presence(of: sense)) + + if sense.kind != .definition { + bytes.append(sense.kind.rawValue) + } + appendVarint(sense.level, to: &bytes) + if let parent = sense.parent { + appendVarint(parent, to: &bytes) + } + if let label = sense.label { + appendText(label, to: &bytes) + } + appendText(sense.gloss.text, to: &bytes) + + if !sense.gloss.styles.isEmpty { + append(styles: sense.gloss.styles, to: &bytes) + } + if !sense.gloss.references.isEmpty { + append(references: sense.gloss.references, to: &bytes) + } + if !sense.partOfSpeech.isEmpty { + appendTexts(sense.partOfSpeech, to: &bytes) + } + if !sense.registers.isEmpty { + appendTexts(sense.registers, to: &bytes) + } + if !sense.examples.isEmpty { + appendVarint(sense.examples.count, to: &bytes) + for example in sense.examples { + appendExample(example, to: &bytes) + } + } + } + + private static func append(styles: [StyleRun], to bytes: inout [UInt8]) { + appendVarint(styles.count, to: &bytes) + for run in styles { + bytes.append(run.style.encoded) + appendVarint(run.range.lowerBound, to: &bytes) + appendVarint(run.range.count, to: &bytes) + } + } + + private static func append(references: [CrossReference], to bytes: inout [UInt8]) { + appendVarint(references.count, to: &bytes) + for reference in references { + appendText(reference.simplified, to: &bytes) + appendOptionalText(reference.traditional, to: &bytes) + appendOptionalText(reference.reading, to: &bytes) + appendVarint(reference.range.lowerBound, to: &bytes) + appendVarint(reference.range.count, to: &bytes) + } + } + + /// One bit per optional field. + static func presence(of sense: Sense) -> UInt8 { + var flags: UInt8 = 0 + if sense.parent != nil { + flags |= Field.parent.rawValue + } + if sense.label != nil { + flags |= Field.label.rawValue + } + if !sense.gloss.styles.isEmpty { + flags |= Field.styles.rawValue + } + if !sense.gloss.references.isEmpty { + flags |= Field.references.rawValue + } + if !sense.partOfSpeech.isEmpty { + flags |= Field.partOfSpeech.rawValue + } + if !sense.registers.isEmpty { + flags |= Field.registers.rawValue + } + if !sense.examples.isEmpty { + flags |= Field.examples.rawValue + } + if sense.kind != .definition { + flags |= Field.kind.rawValue + } + return flags + } + + /// Writes an example as its raw text plus where the two halves sit in it. + /// + /// The split is a heuristic over where the Han characters stop, and the + /// translation side is trimmed, so neither half is a plain suffix of the + /// raw text. Two offsets reproduce both exactly from one copy. + /// + /// When a half is not a slice of the raw text at all — which a different + /// splitter could produce — it is written out in full and marked, so this + /// stays an optimisation rather than an invariant the model must keep. + static func appendExample(_ example: UsageExample, to bytes: inout [UInt8]) { + let raw = Array(example.raw.utf8) + let chinese = Array(example.chinese.utf8) + let translation = Array(example.translation.utf8) + + // One-based, so zero can mean "this half is empty" — which is how an + // example with no Chinese at all is told from one whose Chinese starts + // at offset zero, the ordinary case. + let chineseStart = chinese.isEmpty + ? 0 + : (raw.firstRange(of: chinese)?.lowerBound).map { $0 + 1 } + let translationStart = translation.isEmpty + ? 0 + : (raw.lastRange(of: translation)?.lowerBound).map { $0 + 1 } + + guard let chineseStart, let translationStart else { + bytes.append(1) + appendText(example.raw, to: &bytes) + appendText(example.chinese, to: &bytes) + appendText(example.translation, to: &bytes) + return + } + bytes.append(0) + appendText(example.raw, to: &bytes) + appendVarint(chineseStart, to: &bytes) + appendVarint(chinese.count, to: &bytes) + appendVarint(translationStart, to: &bytes) + appendVarint(translation.count, to: &bytes) + } + + /// LEB128, unsigned. Every integer in this format is small and + /// non-negative — levels, parents, offsets, counts — so a fixed width + /// would spend seven bytes saying nothing. + static func appendVarint(_ value: Int, to bytes: inout [UInt8]) { + var remaining = UInt(bitPattern: value) + while true { + let byte = UInt8(remaining & 0x7F) + remaining >>= 7 + if remaining == 0 { + bytes.append(byte) + return + } + bytes.append(byte | 0x80) + } + } + + static func appendText(_ text: String, to bytes: inout [UInt8]) { + let utf8 = text.utf8 + appendVarint(utf8.count, to: &bytes) + bytes.append(contentsOf: utf8) + } + + /// A length of zero means the empty string; nil is one less than that, so + /// an absent traditional spelling costs one byte and an empty one is still + /// distinguishable from it. + static func appendOptionalText(_ text: String?, to bytes: inout [UInt8]) { + guard let text else { + appendVarint(0, to: &bytes) + return + } + let utf8 = text.utf8 + appendVarint(utf8.count + 1, to: &bytes) + bytes.append(contentsOf: utf8) + } + + static func appendTexts(_ texts: [String], to bytes: inout [UInt8]) { + appendVarint(texts.count, to: &bytes) + for text in texts { + appendText(text, to: &bytes) + } + } +} + +// MARK: - Reading + +extension CompactSenseCodec { + static func decodeSense(at index: Int, from reader: inout Reader) throws -> Sense { + let flags = try reader.byte() + let kind = try flags & Field.kind.rawValue != 0 + ? SenseKind(rawValue: reader.byte()) ?? .definition + : .definition + let level = try reader.varint() + let parent = flags & Field.parent.rawValue != 0 ? try reader.varint() : nil + let label = flags & Field.label.rawValue != 0 ? try reader.text() : nil + let text = try reader.text() + let styles = flags & Field.styles.rawValue != 0 ? try decodeStyles(from: &reader) : [] + let references = flags & Field.references.rawValue != 0 + ? try decodeReferences(from: &reader) + : [] + let partOfSpeech = flags & Field.partOfSpeech.rawValue != 0 ? try reader.texts() : [] + let registers = flags & Field.registers.rawValue != 0 ? try reader.texts() : [] + var examples: [UsageExample] = [] + if flags & Field.examples.rawValue != 0 { + for _ in try 0 ..< (reader.varint()) { + try examples.append(decodeExample(from: &reader)) + } + } + + return Sense( + id: index, + kind: kind, + gloss: Gloss(text: text, references: references, styles: styles), + level: level, + parent: parent, + label: label, + partOfSpeech: partOfSpeech, + registers: registers, + examples: examples, + ) + } + + private static func decodeStyles(from reader: inout Reader) throws -> [StyleRun] { + var styles: [StyleRun] = [] + for _ in try 0 ..< (reader.varint()) { + let style = try TextStyle(encoded: reader.byte()) + let start = try reader.varint() + let length = try reader.varint() + styles.append(StyleRun(style: style, range: start ..< (start + length))) + } + return styles + } + + private static func decodeReferences(from reader: inout Reader) throws -> [CrossReference] { + var references: [CrossReference] = [] + for _ in try 0 ..< (reader.varint()) { + let simplified = try reader.text() + let traditional = try reader.optionalText() + let reading = try reader.optionalText() + let start = try reader.varint() + let length = try reader.varint() + references.append(CrossReference( + simplified: simplified, + traditional: traditional, + reading: reading, + range: start ..< (start + length), + )) + } + return references + } + + private static func decodeExample(from reader: inout Reader) throws -> UsageExample { + let form = try reader.byte() + let raw = try reader.text() + guard form == 0 else { + return try UsageExample( + chinese: reader.text(), + translation: reader.text(), + raw: raw, + ) + } + let chineseStart = try reader.varint() + let chineseLength = try reader.varint() + let translationStart = try reader.varint() + let translationLength = try reader.varint() + let bytes = Array(raw.utf8) + return UsageExample( + chinese: slice(bytes, start: chineseStart, length: chineseLength), + translation: slice(bytes, start: translationStart, length: translationLength), + raw: raw, + ) + } + + // A payload is derived data in a file on disk. Where it is damaged, a + // replacement character costs the characters it replaces and leaves the + // entry readable, which beats an error costing the whole entry — and + // structural damage is already an error, from the bounds checks below. + // The same bargain the byte reader makes with a malformed source. + // swiftlint:disable optional_data_string_conversion + + /// A half of an example, by its one-based start. Zero means empty, which + /// is how "no Chinese at all" is told from "the Chinese starts at zero". + private static func slice(_ bytes: [UInt8], start: Int, length: Int) -> String { + guard start > 0, length > 0 else { return "" } + let from = start - 1 + guard from + length <= bytes.count else { return "" } + return String(decoding: bytes[from ..< (from + length)], as: UTF8.self) + } + + /// A cursor over the payload that refuses to read past the end. + /// + /// Every read is bounds-checked and throws rather than trapping: a payload + /// is data from a file on disk, and a corrupt one should cost that entry, + /// not the process. + struct Reader { + private let bytes: [UInt8] + private var offset = 0 + + init(_ data: Data) { + bytes = [UInt8](data) + } + + mutating func byte() throws -> UInt8 { + guard offset < bytes.count else { throw SensePayloadError.truncated(offset: offset) } + defer { offset += 1 } + return bytes[offset] + } + + mutating func varint() throws -> Int { + var result = 0 + var shift = 0 + while true { + let byte = try byte() + result |= Int(byte & 0x7F) << shift + if byte & 0x80 == 0 { + return result + } + shift += 7 + guard shift < 64 else { throw SensePayloadError.truncated(offset: offset) } + } + } + + mutating func text() throws -> String { + try take(count: varint()) + } + + mutating func optionalText() throws -> String? { + let length = try varint() + guard length > 0 else { return nil } + return try take(count: length - 1) + } + + mutating func texts() throws -> [String] { + let count = try varint() + var result: [String] = [] + result.reserveCapacity(count) + for _ in 0 ..< count { + try result.append(text()) + } + return result + } + + private mutating func take(count: Int) throws -> String { + guard count >= 0, offset + count <= bytes.count else { + throw SensePayloadError.truncated(offset: offset) + } + defer { offset += count } + return String(decoding: bytes[offset ..< (offset + count)], as: UTF8.self) + } + } + + // swiftlint:enable optional_data_string_conversion +} + +// MARK: - Enumerations on the wire + +extension TextStyle { + /// A fixed byte per case, chosen here rather than taken from the case + /// order: reordering the enum must not change what is already on disk. + var encoded: UInt8 { + switch self { + case .bold: 1 + case .italic: 2 + case .highlighted: 3 + } + } + + init(encoded: UInt8) { + switch encoded { + case 1: self = .bold + case 3: self = .highlighted + default: self = .italic + } + } +} + +// MARK: - Choosing a codec + +/// Which codec reads or writes a container. +public enum SensePayloadCodecs { + /// What a writer should use now. + public static let current: any SensePayloadCodec = CompactSenseCodec() + + /// Resolves a recorded identifier. + /// + /// An unrecognised one is an error rather than a fallback to the current + /// codec: misreading a payload produces plausible nonsense, and a + /// container written by a newer build should say so plainly. + public static func named(_ identifier: String?) throws -> any SensePayloadCodec { + switch identifier { + // A container from before the codec was recorded is JSON, because that + // is what the only writer there was then used. + case nil, JSONSenseCodec.identifier: JSONSenseCodec() + case CompactSenseCodec.identifier: CompactSenseCodec() + case let other: throw SensePayloadError.unknownCodec(other ?? "") + } + } +} diff --git a/Sources/HanReaderPersistence/DictionaryContainer.swift b/Sources/HanReaderPersistence/DictionaryContainer.swift index 46a71b5..2d8655d 100644 --- a/Sources/HanReaderPersistence/DictionaryContainer.swift +++ b/Sources/HanReaderPersistence/DictionaryContainer.swift @@ -58,7 +58,7 @@ public struct DictionaryContainer: Sendable { let queue = try DatabaseQueue(path: url.path, configuration: configuration) try DictionarySchema.migrator().migrate(queue) - let encoder = JSONEncoder() + let codec = SensePayloadCodecs.current var written = 0 for chunk in entries.chunked(into: batchSize) { @@ -69,15 +69,14 @@ public struct DictionaryContainer: Sendable { try db.execute(sql: """ INSERT INTO entry (simplified, traditional, readingNumeric, - readingDisplay, senses, summary) - VALUES (?, ?, ?, ?, ?, ?) + readingDisplay, senses) + VALUES (?, ?, ?, ?, ?) """, arguments: [ entry.headword.simplified, entry.headword.traditional, entry.readingKey, entry.readingDisplay, - encoder.encode(entry.senses), - entry.summarySense?.gloss.text ?? "", + codec.encode(entry.senses), ]) } } @@ -140,6 +139,9 @@ public struct DictionaryContainer: Sendable { ) throws { let values: [String: String?] = [ "formatVersion": String(DictionarySchema.formatVersion), + // Which codec wrote the sense payloads, so a reader picks the one + // that can read them rather than assuming the current one. + "sensePayloadCodec": type(of: SensePayloadCodecs.current).identifier, "slug": metadata.slug, "displayName": metadata.displayName, "format": metadata.format, @@ -166,6 +168,22 @@ public struct DictionaryContainer: Sendable { // MARK: - Reading + /// The codec this container's sense payloads were written with. + /// + /// Read from the file rather than assumed, which is what lets a new codec + /// land without invalidating the dictionaries already on disk — and what + /// makes a container written by a *newer* build say so plainly instead of + /// decoding into plausible nonsense. + public func sensePayloadCodec() throws -> any SensePayloadCodec { + let identifier = try dbQueue.read { db in + try String.fetchOne( + db, + sql: "SELECT value FROM meta WHERE key = 'sensePayloadCodec'", + ) + } + return try SensePayloadCodecs.named(identifier) + } + public func metadata() throws -> DictionaryMetadata? { let values = try dbQueue.read { db in try Row.fetchAll(db, sql: "SELECT key, value FROM meta") @@ -196,7 +214,7 @@ public struct DictionaryContainer: Sendable { /// readings; the prototype's `LIMIT 1` is the bug this signature exists to /// make impossible. public func entries(for headword: String) throws -> [DictionaryEntry] { - let decoder = JSONDecoder() + let codec = try sensePayloadCodec() return try dbQueue.read { db in // Matched against BOTH scripts. Storing the traditional form and // then only ever querying the simplified one means a reader of @@ -213,7 +231,7 @@ public struct DictionaryContainer: Sendable { traditional: row["traditional"], ), reading: Pinyin.parse(numeric: row["readingNumeric"]), - senses: decoder.decode([Sense].self, from: row["senses"]), + senses: codec.decode(row["senses"]), ) } } @@ -300,6 +318,75 @@ public struct DictionaryContainer: Sendable { } } +// MARK: - Introspection + +/// Reading the shape of a container rather than its content. +/// +/// Used by the tests that assert an import leaves the right indexes behind +/// and that the retired `summary` column is gone — both things a reader +/// should never need to ask, and both worth asserting once. +extension DictionaryContainer { + /// The indexes the file actually carries. + /// + /// For asserting that an import leaves behind the ones a reader needs and + /// not the one only the importer wanted. + public func indexNames() throws -> Set { + try dbQueue.read { db in + try Set(String.fetchAll( + db, + sql: "SELECT name FROM sqlite_schema WHERE type = 'index' AND name IS NOT NULL", + )) + } + } + + /// The columns a table carries, for asserting that one is gone. + public func columnNames(of table: String) throws -> Set { + try dbQueue.read { db in + try Set(db.columns(in: table).map(\.name)) + } + } + + /// Writes a one-entry container with a chosen codec. + /// + /// Only so a test can build the kind of container an earlier build + /// produced, and prove it still opens — which is the whole point of + /// recording the codec rather than assuming it. + static func writeForTesting( + entry: DictionaryEntry, + codec: any SensePayloadCodec, + metadata: DictionaryMetadata, + to url: URL, + ) throws { + try? FileManager.default.removeItem(at: url) + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true, + ) + let queue = try DatabaseQueue(path: url.path) + try DictionarySchema.migrator().migrate(queue) + try queue.write { db in + try db.execute(sql: """ + INSERT INTO entry + (simplified, traditional, readingNumeric, readingDisplay, senses) + VALUES (?, ?, ?, ?, ?) + """, arguments: [ + entry.headword.simplified, + entry.headword.traditional, + entry.readingKey, + entry.readingDisplay, + codec.encode(entry.senses), + ]) + } + try writeMetadata(metadata, to: queue) + try queue.write { db in + try db.execute( + sql: "INSERT OR REPLACE INTO meta (key, value) VALUES ('sensePayloadCodec', ?)", + arguments: [type(of: codec).identifier], + ) + } + } +} + extension Array { /// Splits into fixed-size chunks, so a batch can be one transaction. func chunked(into size: Int) -> [ArraySlice] { diff --git a/Sources/HanReaderPersistence/DictionarySchema.swift b/Sources/HanReaderPersistence/DictionarySchema.swift index 99c8871..b871b31 100644 --- a/Sources/HanReaderPersistence/DictionarySchema.swift +++ b/Sources/HanReaderPersistence/DictionarySchema.swift @@ -20,7 +20,7 @@ enum DictionarySchema { /// Distinct from a parser version: no schema migration can repair data /// that was parsed wrongly, so a parser fix invalidates a container /// through `meta.parserVersion` and triggers a rebuild instead. - static let formatVersion = 2 + static let formatVersion = 3 static func migrator() -> DatabaseMigrator { var migrator = DatabaseMigrator() @@ -43,6 +43,20 @@ enum DictionarySchema { try createImportJobTable(db) } + // The denormalised first gloss, written since v1 and read by nothing. + // On a BKRS container it is **160 MB** — 3,442,623 rows of Cyrillic + // prose that is already inside the sense payload beside it. It was + // there so a result list could render without decoding a payload per + // row; with the compact codec that decode is microseconds for the ten + // rows a list shows, and 160 MB is not a reasonable price for it. If + // a list ever does need it, it can come back as a short prefix rather + // than a whole copy. + migrator.registerMigration("v3_drop_summary") { db in + try db.alter(table: "entry") { table in + table.drop(column: "summary") + } + } + return migrator } diff --git a/Sources/HanReaderPersistence/DictionaryWriter.swift b/Sources/HanReaderPersistence/DictionaryWriter.swift index 2fdccc7..a91add5 100644 --- a/Sources/HanReaderPersistence/DictionaryWriter.swift +++ b/Sources/HanReaderPersistence/DictionaryWriter.swift @@ -43,6 +43,40 @@ public import HanReaderCore /// The resume path still deletes anything at or past the checkpoint, because a /// belt costs one indexed `DELETE` and braces alone are an argument. public final class DictionaryWriter { + /// When the lookup indexes get built. + /// + /// They are needed only once an import is finished, and the choice of + /// *when* to build them trades the import's two budgets against each + /// other. Measured over the full 350 MB BKRS set, same machine, same + /// output: + /// + /// | | time | peak memory | size | + /// |---|---|---|---| + /// | `.atTheEnd` | **75 s** | 134 MB | **493 MB** | + /// | `.whileWriting` | 188 s | **53 MB** | 556 MB | + /// + /// Building at the end is the ordinary bulk-load shape: one pass over + /// rows that are all already there, instead of a random-order insert into + /// an 84 MB B-tree three and a half million times. It is also 63 MB + /// smaller on disk, because an index built in one pass does not carry the + /// page splits that random insertion leaves behind. + /// + /// Its cost is memory: `CREATE INDEX` is a single atomic statement, so the + /// whole new index is dirty at once — about 92 MB of it, which no + /// `cache_size`, `temp_store`, `threads`, `soft_heap_limit` or + /// `hard_heap_limit` pragma reduces. Measured, not assumed; all five were + /// tried. + /// + /// `.atTheEnd` is the default because compiling a DSL set is a desktop + /// operation by design — a Mac compiles it once and a phone imports the + /// compiled container — so the import's memory ceiling, which exists for + /// iOS, is not the binding constraint on this path. `.whileWriting` is + /// there for the day something does run an import where it is. + public enum Indexing: Sendable { + case atTheEnd + case whileWriting + } + /// Where the reader got to, recorded at a card boundary. /// /// A card boundary, not a line or a batch boundary, because it is the only @@ -135,7 +169,8 @@ public final class DictionaryWriter { private let dbQueue: DatabaseQueue private let source: Source private let lexiconMaximumLength: Int - private let encoder = JSONEncoder() + private let indexing: Indexing + private let codec = SensePayloadCodecs.current private var entriesWritten: Int /// Opens a container at `url`, resuming the job in it when there is one @@ -154,6 +189,7 @@ public final class DictionaryWriter { at url: URL, source: Source, lexiconMaximumLength: Int = LexiconBuilder.maximumWordLength, + indexing: Indexing = .atTheEnd, ) throws -> (writer: DictionaryWriter, resume: Checkpoint?) { @@ -170,14 +206,7 @@ public final class DictionaryWriter { let queue = try DatabaseQueue(path: url.path, configuration: configuration) try DictionarySchema.migrator().migrate(queue) - // A finished import drops this index, so a re-import needs it back. - // Here rather than in the migration, because the migration has already - // run by the time it is dropped. - try queue.write { db in - try db.execute(sql: """ - CREATE INDEX IF NOT EXISTS entry_on_source ON entry (sourceFile, sourceOffset) - """) - } + try prepareIndexes(queue, indexing: indexing) let stored = try queue.read { db in try Row.fetchOne(db, sql: "SELECT * FROM importJob WHERE id = 1") @@ -201,11 +230,38 @@ public final class DictionaryWriter { dbQueue: queue, source: source, lexiconMaximumLength: lexiconMaximumLength, + indexing: indexing, entriesWritten: written, ) return (writer, resume) } + /// Sets up the indexes an import wants, and takes down the ones it does + /// not. + /// + /// A lookup index is maintained per row while it exists, and + /// `entry_on_simplified` means a random-order insert into an 84 MB B-tree + /// three and a half million times. Nothing queries it until the import is + /// over, so it comes down for the duration and is built in one pass at the + /// end — the ordinary bulk-load shape. + /// + /// The source index is the mirror image: wanted only *during* an import, + /// for the delete a resume does, and dropped when one finishes. + /// + /// A container interrupted part way therefore has no lookup indexes. That + /// is correct rather than unfortunate — it is not a dictionary yet, and + /// the `importJob` row is what says so. + private static func prepareIndexes(_ queue: DatabaseQueue, indexing: Indexing) throws { + try queue.write { db in + try db.execute(sql: """ + CREATE INDEX IF NOT EXISTS entry_on_source ON entry (sourceFile, sourceOffset) + """) + guard indexing == .atTheEnd else { return } + try db.execute(sql: "DROP INDEX IF EXISTS entry_on_simplified") + try db.execute(sql: "DROP INDEX IF EXISTS entry_on_traditional") + } + } + /// Makes the container ready to be written into, and says how many /// entries it already holds. private static func prepare( @@ -250,11 +306,13 @@ public final class DictionaryWriter { dbQueue: DatabaseQueue, source: Source, lexiconMaximumLength: Int, + indexing: Indexing, entriesWritten: Int, ) { self.dbQueue = dbQueue self.source = source self.lexiconMaximumLength = lexiconMaximumLength + self.indexing = indexing self.entriesWritten = entriesWritten } @@ -281,15 +339,14 @@ public final class DictionaryWriter { try db.execute(sql: """ INSERT INTO entry (simplified, traditional, readingNumeric, readingDisplay, - senses, summary, sourceFile, sourceOffset) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + senses, sourceFile, sourceOffset) + VALUES (?, ?, ?, ?, ?, ?, ?) """, arguments: [ entry.headword.simplified, entry.headword.traditional, entry.readingKey, entry.readingDisplay, - encoder.encode(entry.senses), - entry.summarySense?.gloss.text ?? "", + codec.encode(entry.senses), sourced.sourceFile, sourced.sourceOffset, ]) @@ -377,6 +434,19 @@ public final class DictionaryWriter { // is 53 MB the reader would carry for nothing. The columns stay, // so a re-import can rebuild the index and resume as before. try db.execute(sql: "DROP INDEX IF EXISTS entry_on_source") + + // And now the ones a reader needs, built in one pass over rows + // that are all already there rather than maintained as they + // arrived. Lookup matches BOTH scripts: storing the traditional + // form and only ever querying the simplified one means a reader of + // traditional text finds nothing at all. + try db.execute(sql: """ + CREATE INDEX IF NOT EXISTS entry_on_simplified ON entry (simplified) + """) + try db.execute(sql: """ + CREATE INDEX IF NOT EXISTS entry_on_traditional ON entry (traditional) + WHERE traditional IS NOT NULL + """) try db.execute(sql: "ANALYZE") } // Folding the write-ahead log back in leaves one file rather than @@ -402,7 +472,6 @@ extension DictionaryWriter { /// than the 27 MB suggests. The collector keeps five small values per /// candidate reading and the entry is released immediately. private func writeCharacterReadings() throws { - let decoder = JSONDecoder() var collector = LexiconBuilder.CharacterReadings() try dbQueue.read { db in let cursor = try Row.fetchCursor(db, sql: """ @@ -416,7 +485,7 @@ extension DictionaryWriter { traditional: row["traditional"], ), reading: Pinyin.parse(numeric: row["readingNumeric"]), - senses: decoder.decode([Sense].self, from: row["senses"]), + senses: self.codec.decode(row["senses"]), )) } } diff --git a/Tests/HanReaderCoreTests/SensePayloadCodecTests.swift b/Tests/HanReaderCoreTests/SensePayloadCodecTests.swift new file mode 100644 index 0000000..88b19d6 --- /dev/null +++ b/Tests/HanReaderCoreTests/SensePayloadCodecTests.swift @@ -0,0 +1,275 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import Testing +@testable import HanReaderCore + +// MARK: - Fixtures + +/// Every sense in the committed slice of real 大БКРС data. +/// +/// Round-tripping invented senses tests the cases someone thought of. +/// Round-tripping 523 real ones tests the cases nobody did — the sense whose +/// gloss is empty because its text was all label, the example that splits at +/// a quotation mark, the reference that ends where the gloss does. +private enum Corpus { + static let senses: [[Sense]] = { + guard let url = Bundle.module.url( + forResource: "Fixtures/bkrs-slice.dsl", + withExtension: "txt", + ), + let data = try? Data(contentsOf: url), + let text = String(data: data, encoding: .utf8) + else { + Issue.record("could not load the BKRS fixture") + return [] + } + var bytes: [UInt8] = [0xFF, 0xFE] + for unit in Array(text.utf16) { + bytes += [UInt8(unit & 0xFF), UInt8(unit >> 8)] + } + let builder = DSLCardBuilder(syllableBases: []) + var runs: [[Sense]] = [] + do { + var reader = try DSLCardReader(source: DSLMemoryBytes(bytes)) + while let record = try reader.next() { + if let senses = builder.entries(from: record.card).entries.first?.senses { + runs.append(senses) + } + } + } catch { + Issue.record("could not read the BKRS fixture: \(error)") + } + return runs + }() +} + +private func sense( + id: Int = 0, + kind: SenseKind = .definition, + text: String = "значение", + level: Int = 0, + parent: Int? = nil, + label: String? = nil, + styles: [StyleRun] = [], + references: [CrossReference] = [], + partOfSpeech: [String] = [], + registers: [String] = [], + examples: [UsageExample] = [], +) + -> Sense +{ + Sense( + id: id, + kind: kind, + gloss: Gloss(text: text, references: references, styles: styles), + level: level, + parent: parent, + label: label, + partOfSpeech: partOfSpeech, + registers: registers, + examples: examples, + ) +} + +@Suite("Compact sense payloads") +struct SensePayloadCodecTests { + private let codec = CompactSenseCodec() + + // MARK: - Round trips + + /// The whole committed slice, sense for sense. Equality is on `Sense` + /// itself, so every field has to come back — a codec that silently drops + /// `registers` would pass a test that only checked gloss text. + @Test("Every sense in the real slice survives a round trip") + func realCorpus() throws { + #expect(!Corpus.senses.isEmpty) + var total = 0 + for run in Corpus.senses { + let decoded = try codec.decode(codec.encode(run)) + #expect(decoded == run) + total += run.count + } + #expect(total == 523) + } + + @Test("An empty run round-trips") + func emptyRun() throws { + #expect(try codec.decode(codec.encode([])).isEmpty) + } + + @Test("Each field survives on its own", arguments: [ + sense(), + sense(kind: .reference, text: "см. другое"), + sense(kind: .label, text: "фамилия"), + sense(text: "", level: 3, parent: 1, label: "I"), + sense(level: 9, parent: 300), + sense(styles: [ + StyleRun(style: .bold, range: 0 ..< 2), + StyleRun(style: .italic, range: 3 ..< 8), + StyleRun(style: .highlighted, range: 0 ..< 1), + ]), + sense(references: [CrossReference( + simplified: "磁性碰锁", + traditional: "磁性碰鎖", + reading: "cíxìng pèngsuǒ", + range: 2 ..< 6, + )]), + sense(references: [CrossReference( + simplified: "横", + traditional: nil, + reading: nil, + range: 0 ..< 1, + )]), + sense(partOfSpeech: ["гл.", "сущ."], registers: ["бот.", "уст."]), + sense(examples: [UsageExample( + chinese: "爱儿童", + translation: "любить ребёнка", + raw: "爱儿童 любить ребёнка", + )]), + ]) + func singleFields(input: Sense) throws { + #expect(try codec.decode(codec.encode([input])) == [input]) + } + + /// `id` is the index in the run and is not stored, so a run decodes with + /// its positions renumbered — which is what every reader of it assumes. + @Test("Identifiers come back as positions") + func identifiersArePositions() throws { + let run = (0 ..< 5).map { sense(id: 97 + $0, text: "s\($0)") } + let decoded = try codec.decode(codec.encode(run)) + #expect(decoded.map(\.id) == [0, 1, 2, 3, 4]) + #expect(decoded.map(\.gloss.text) == run.map(\.gloss.text)) + } + + // MARK: - Examples, which are stored once + + /// `raw` is the example as written and the two halves are a heuristic + /// split of it, so storing all three copies the text twice. Measured over + /// the set, the halves cost as much again as `raw`. + @Test("An example is stored once and split on the way out", arguments: [ + UsageExample(chinese: "爱儿童", translation: "любить ребёнка", raw: "爱儿童 любить ребёнка"), + UsageExample(chinese: "", translation: "только по-русски", raw: "только по-русски"), + UsageExample(chinese: "他说：“好。”", translation: "он сказал", raw: "他说：“好。” он сказал"), + UsageExample(chinese: "一", translation: "", raw: "一"), + UsageExample(chinese: "", translation: "", raw: ""), + ]) + func examples(input: UsageExample) throws { + let decoded = try codec.decode(codec.encode([sense(examples: [input])])) + #expect(decoded.first?.examples == [input]) + } + + /// A split that is not a slice of the raw text — which a different + /// splitter could produce — is written out in full rather than silently + /// reconstructed wrongly. The compact form is an optimisation, not an + /// invariant the model has to keep. + @Test("An example whose halves are not slices of its raw text still round-trips") + func examplesThatAreNotSlices() throws { + let odd = UsageExample(chinese: "完全", translation: "иное", raw: "ничего общего") + let decoded = try codec.decode(codec.encode([sense(examples: [odd])])) + #expect(decoded.first?.examples == [odd]) + } + + /// The same text twice over, where a naive "find the half in the raw" + /// would pick the wrong occurrence. + @Test("A repeated half picks the right occurrence") + func repeatedHalf() throws { + let repeated = UsageExample(chinese: "好", translation: "好", raw: "好 好") + let decoded = try codec.decode(codec.encode([sense(examples: [repeated])])) + #expect(decoded.first?.examples == [repeated]) + } + + // MARK: - Size + + /// The reason the codec exists. Measured over the real slice rather than + /// asserted as a ratio someone liked: JSON spends most of a payload on + /// syntax and key names, written out once per sense across three and a + /// half million of them. + @Test("The compact form is substantially smaller than JSON") + func size() throws { + let json = JSONSenseCodec() + var compactBytes = 0 + var jsonBytes = 0 + for run in Corpus.senses { + compactBytes += try codec.encode(run).count + jsonBytes += try json.encode(run).count + } + #expect(compactBytes * 2 < jsonBytes, "compact \(compactBytes), json \(jsonBytes)") + } + + // MARK: - Malformed payloads + + /// A payload is data from a file on disk. A corrupt one costs that entry, + /// not the process, so every read is bounds-checked rather than trapping. + @Test("A truncated payload throws rather than trapping") + func truncated() throws { + let full = try codec.encode([sense( + text: "значение", + label: "1)", + examples: [UsageExample(chinese: "一", translation: "один", raw: "一 один")], + )]) + for length in 0 ..< full.count { + #expect(throws: (any Error).self) { + _ = try codec.decode(full.prefix(length)) + } + } + } + + @Test("A payload from a newer format is refused, not guessed at") + func newerVersion() throws { + var bytes = try [UInt8](codec.encode([sense()])) + bytes[0] = CompactSenseCodec.version + 1 + #expect(throws: SensePayloadError.unknownVersion(CompactSenseCodec.version + 1)) { + _ = try codec.decode(Data(bytes)) + } + } + + @Test("An empty payload is refused") + func emptyPayload() { + #expect(throws: (any Error).self) { + _ = try codec.decode(Data()) + } + } + + // MARK: - Codec selection + + /// A container records which codec wrote it, so adding one does not + /// invalidate the dictionaries already on disk. + @Test("A container's recorded codec is what reads it") + func selection() throws { + #expect(try SensePayloadCodecs.named("compact-v1") is CompactSenseCodec) + #expect(try SensePayloadCodecs.named("json") is JSONSenseCodec) + // Written before the codec was recorded, which means JSON. + #expect(try SensePayloadCodecs.named(nil) is JSONSenseCodec) + #expect(throws: SensePayloadError.unknownCodec("compact-v99")) { + _ = try SensePayloadCodecs.named("compact-v99") + } + } + + /// JSON still reads, so a dictionary compiled by an earlier build opens. + @Test("JSON payloads still round-trip") + func jsonStillWorks() throws { + let json = JSONSenseCodec() + let run = try #require(Corpus.senses.first { $0.count > 2 }) + #expect(try json.decode(json.encode(run)) == run) + } + + // MARK: - Wire format + + /// The byte for a style is fixed here rather than taken from the case + /// order, so reordering the enum cannot change what is already on disk. + @Test("Style bytes are stable", arguments: TextStyle.allCases) + func styleBytes(style: TextStyle) { + #expect(TextStyle(encoded: style.encoded) == style) + } + + @Test("Varints round-trip at the boundaries", arguments: [ + 0, 1, 127, 128, 255, 256, 16383, 16384, 1 << 20, Int(Int32.max), + ]) + func varints(value: Int) throws { + var bytes: [UInt8] = [] + CompactSenseCodec.appendVarint(value, to: &bytes) + var reader = CompactSenseCodec.Reader(Data(bytes)) + #expect(try reader.varint() == value) + } +} diff --git a/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift b/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift new file mode 100644 index 0000000..eb1fe45 --- /dev/null +++ b/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift @@ -0,0 +1,208 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import HanReaderCore +import Testing +@testable import HanReaderPersistence + +// MARK: - Fixtures + +private func entry( + _ simplified: String, + traditional: String? = nil, + reading: String = "ai4", + senses: [String] = ["значение"], +) + -> DictionaryEntry +{ + DictionaryEntry( + headword: Headword(simplified: simplified, traditional: traditional), + reading: Pinyin.parse(numeric: reading), + senses: senses.enumerated().map { index, text in + Sense( + id: index, + kind: .definition, + gloss: Gloss(text: text, references: [], styles: []), + level: 0, + parent: nil, + label: nil, + partOfSpeech: [], + registers: [], + examples: [], + ) + }, + ) +} + +private func metadata(entryCount: Int) -> DictionaryMetadata { + DictionaryMetadata( + slug: "test", + displayName: "Test", + format: "abbyy-dsl", + indexLanguage: "zh-Hans", + glossLanguage: "ru", + licence: "none", + attribution: "none", + parserVersion: 1, + entryCount: entryCount, + ) +} + +private func temporaryURL() -> URL { + URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("hanreader-storage-\(UUID().uuidString)") + .appendingPathComponent("out.hanreaderdict") +} + +private func source(fingerprint: String = "f") -> DictionaryWriter.Source { + DictionaryWriter.Source( + name: "a.dsl", + fingerprint: fingerprint, + fileCount: 1, + totalBytes: 1000, + parserVersion: 1, + ) +} + +@Suite("Dictionary storage") +struct DictionaryStorageTests { + /// Whatever the indexing strategy, a finished container has to answer a + /// lookup — the whole point of building the indexes at the end is that it + /// is invisible from the outside. + @Test("A finished container looks words up", arguments: [ + DictionaryWriter.Indexing.atTheEnd, + DictionaryWriter.Indexing.whileWriting, + ]) + func lookupAfterEitherIndexing(indexing: DictionaryWriter.Indexing) throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let (writer, resume) = try DictionaryWriter.open( + at: url, + source: source(), + indexing: indexing, + ) + #expect(resume == nil) + try writer.append( + [ + DictionaryWriter.SourcedEntry( + entry: entry("爱", traditional: "愛"), + sourceFile: 0, + sourceOffset: 10, + ), + DictionaryWriter.SourcedEntry( + entry: entry("中国", reading: "zhong1 guo2"), + sourceFile: 0, + sourceOffset: 20, + ), + ], + checkpoint: DictionaryWriter.Checkpoint(fileIndex: 0, byteOffset: 20, cardsRead: 2), + ) + try writer.finish(metadata: metadata(entryCount: 2)) + + let container = try DictionaryContainer(contentsOf: url) + #expect(try container.entryCount() == 2) + #expect(try container.entries(for: "爱").count == 1) + // Matched against both scripts, or a reader of traditional text finds + // nothing at all. + #expect(try container.entries(for: "愛").count == 1) + #expect(try container.entries(for: "中国").first?.senses.first?.gloss.text == "значение") + } + + /// The index the resume path uses is wanted only during an import, and the + /// lookup indexes only after one. A finished container should carry the + /// second pair and not the first. + @Test("A finished container carries the reader's indexes and not the importer's") + func indexesAfterFinishing() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let (writer, _) = try DictionaryWriter.open(at: url, source: source()) + try writer.append( + [DictionaryWriter.SourcedEntry(entry: entry("爱"), sourceFile: 0, sourceOffset: 1)], + checkpoint: DictionaryWriter.Checkpoint(byteOffset: 1, cardsRead: 1), + ) + try writer.finish(metadata: metadata(entryCount: 1)) + + let indexes = try DictionaryContainer(contentsOf: url).indexNames() + #expect(indexes.contains("entry_on_simplified")) + #expect(indexes.contains("entry_on_traditional")) + #expect(!indexes.contains("entry_on_source")) + } + + /// A re-import needs the source index back, since a finished import drops + /// it and the resume path's delete depends on it. + @Test("Reopening a finished container restores the importer's index") + func reopeningRestoresSourceIndex() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let (first, _) = try DictionaryWriter.open(at: url, source: source()) + try first.append( + [DictionaryWriter.SourcedEntry(entry: entry("爱"), sourceFile: 0, sourceOffset: 1)], + checkpoint: DictionaryWriter.Checkpoint(byteOffset: 1, cardsRead: 1), + ) + try first.finish(metadata: metadata(entryCount: 1)) + + _ = try DictionaryWriter.open(at: url, source: source(fingerprint: "changed")) + #expect(try DictionaryContainer(contentsOf: url).indexNames() + .contains("entry_on_source")) + } + + // MARK: - Sense payloads + + /// A container records which codec wrote it, so a new codec does not + /// invalidate the dictionaries already on disk. + @Test("A container says which codec wrote its senses") + func recordedCodec() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + try DictionaryContainer.write( + entries: [entry("爱", senses: ["первое", "второе"])], + metadata: metadata(entryCount: 1), + to: url, + ) + let container = try DictionaryContainer(contentsOf: url) + #expect(try type(of: container.sensePayloadCodec()).identifier + == CompactSenseCodec.identifier) + #expect(try container.entries(for: "爱").first?.senses.count == 2) + } + + /// The denormalised first gloss, written since v1 and read by nothing. On + /// a BKRS container it was 160 MB of prose already sitting in the payload + /// beside it. + @Test("The summary column is gone") + func summaryDropped() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + try DictionaryContainer.write( + entries: [entry("爱")], + metadata: metadata(entryCount: 1), + to: url, + ) + #expect(try !DictionaryContainer(contentsOf: url).columnNames(of: "entry") + .contains("summary")) + } + + /// A container built before the compact codec existed still opens, which + /// is what the recorded identifier is for. + @Test("A JSON container still reads") + func jsonContainerStillReads() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let senses = entry("爱", senses: ["первое", "второе"]).senses + try DictionaryContainer.writeForTesting( + entry: entry("爱", senses: ["первое", "второе"]), + codec: JSONSenseCodec(), + metadata: metadata(entryCount: 1), + to: url, + ) + let container = try DictionaryContainer(contentsOf: url) + #expect(try type(of: container.sensePayloadCodec()).identifier + == JSONSenseCodec.identifier) + #expect(try container.entries(for: "爱").first?.senses == senses) + } +}