From 3668bf5195c4da8207d9445e2b8561cff8cc6fc2 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:00:50 +0300 Subject: [PATCH] perf: pack the sense payload, and build indexes once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two performance gates M6 missed, closed by measurement rather than by guessing where the bytes were. Over the full 350 MB BKRS set: | | before | after | |---|---|---| | on disk | 868 MB | **493 MB** | | elapsed | 206 s | **74.7 s** | The bundled CC-CEDICT dictionary goes from 32 MB to **24 MB** with it, and 3,434,224 cards still produce 3,442,623 entries and zero diagnostics. **A packed binary codec for sense payloads.** The `entry` table was 625 MB of the 868, its payloads 327 MB of that, and **only 40.7% of a payload was text** — the rest `{"gloss":{"text":…},"level":0,"kind":…}` written out once per sense, 3,691,468 times. The packed form is 168 MB, within 4% of the raw text it carries, which is about as close as an uncompressed encoding gets. Three ideas earn their complexity: a presence byte, since the common sense is a gloss and a level and nothing else; varints, since every integer here is a level, a parent, an offset or a count; and storing a usage example once, as its raw text plus the two boundaries inside it, where before `raw` and the heuristic split of it were both written out in full and cost the same again. `SensePayloadCodec` is a seam, not a single implementation, because this choice will be made again — a block-compressing codec is the obvious next tier and the plan already names it. A container records which codec wrote it, so adding one does not invalidate the dictionaries already on disk, and JSON still reads. An unrecognised identifier is an error rather than a fallback, because misreading a payload produces plausible nonsense. **The `summary` column is gone.** The denormalised first gloss, written since v1 and **read by nothing** — 160 MB of Cyrillic prose already sitting in the payload beside it. It was there so a result list could render without decoding a payload per row; with the packed codec that decode is microseconds for the ten rows a list shows. **The lookup indexes are built once, at the end.** Maintained while rows arrive, `entry_on_simplified` is a random-order insert into an 84 MB B-tree three and a half million times. Built in one pass over rows that are all already there it is also 63 MB smaller, because it carries none of the page splits random insertion leaves behind. That one change is most of the 130-second difference. It costs memory, and that is the honest shape of it: `CREATE INDEX` is a single atomic statement, so the whole new index is dirty at once — about 92 MB, which no `cache_size`, `temp_store`, `threads`, `soft_heap_limit` or `hard_heap_limit` pragma reduces. All five were tried. So the two budgets trade against each other and the choice is explicit: | `Indexing` | time | peak memory | size | |---|---|---|---| | `.atTheEnd` | **75 s** | 134 MB | **493 MB** | | `.whileWriting` | 188 s | **53 MB** | 556 MB | `.atTheEnd` is the default because compiling a DSL set is a desktop operation by design — a Mac compiles it once and a phone imports the compiled container — so the 60 MB ceiling, which exists for iOS, is not the binding constraint on this path. `.whileWriting` exists for the day something runs an import where it is, and both numbers are on the type so the trade is not rediscovered. What remains above the plan's 210–240 MB estimate is no longer payload. Of 493 MB: 168 MB of senses that are 96% text, 84 MB of headword index, 104 MB of segmentation lexicon, ~75 MB of headwords and readings, and the row overhead of three and a half million rows. The estimate was computed from the predecessor's flattened definitions and did not account for the lexicon, the index, or storing sense structure and examples at all. Compressing the payload is the next real lever and the seam is there for it. Verified by round-tripping all 523 senses of the committed real-data slice, field for field; by a two-volume import killed three times, the third landing inside the second volume, coming back row-for-row identical to an uninterrupted one; and by truncating a payload at every length from zero, which throws rather than trapping at each. Co-Authored-By: Claude Opus 5 --- .../Dictionary/SensePayloadCodec.swift | 529 ++++++++++++++++++ .../DictionaryContainer.swift | 101 +++- .../DictionarySchema.swift | 16 +- .../DictionaryWriter.swift | 99 +++- .../SensePayloadCodecTests.swift | 275 +++++++++ .../DictionaryStorageTests.swift | 208 +++++++ 6 files changed, 1205 insertions(+), 23 deletions(-) create mode 100644 Sources/HanReaderCore/Dictionary/SensePayloadCodec.swift create mode 100644 Tests/HanReaderCoreTests/SensePayloadCodecTests.swift create mode 100644 Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift diff --git a/Sources/HanReaderCore/Dictionary/SensePayloadCodec.swift b/Sources/HanReaderCore/Dictionary/SensePayloadCodec.swift new file mode 100644 index 0000000..5eb7177 --- /dev/null +++ b/Sources/HanReaderCore/Dictionary/SensePayloadCodec.swift @@ -0,0 +1,529 @@ +// HanReader — MIT licensed. See LICENSE. + +public import Foundation + +/// How a container's sense payloads are encoded. +/// +/// A seam rather than a single implementation, because the choice is a +/// measured trade and will be made again. A container records which codec +/// wrote it, so adding one does not invalidate the dictionaries already on +/// disk — and so a future block-compressing codec is a drop-in rather than a +/// migration. +public protocol SensePayloadCodec: Sendable { + /// Written into the container's `meta` table, so a reader can pick the + /// codec the writer used rather than assuming the current one. + static var identifier: String { get } + + func encode(_ senses: [Sense]) throws -> Data + func decode(_ data: Data) throws -> [Sense] +} + +/// Something that cannot be read as a sense payload. +public enum SensePayloadError: Error, Hashable, Sendable, CustomStringConvertible { + case truncated(offset: Int) + case unknownVersion(UInt8) + case unknownCodec(String) + + public var description: String { + switch self { + case let .truncated(offset): + "the payload ends part way through a field, at byte \(offset)" + case let .unknownVersion(version): + "payload format version \(version) is newer than this build understands" + case let .unknownCodec(identifier): + "this container's senses were written by the '\(identifier)' codec, " + + "which this build does not have" + } + } +} + +// MARK: - JSON, as the first milestones wrote it + +/// JSON, kept so a container compiled before the compact codec still opens. +/// +/// It is not what new containers use. Measured over the BKRS set, only +/// **40.7%** of a JSON payload is text; the rest is syntax and key names, +/// written out once per sense across 3,691,468 of them. +public struct JSONSenseCodec: SensePayloadCodec { + public static let identifier = "json" + + public init() {} + + public func encode(_ senses: [Sense]) throws -> Data { + try JSONEncoder().encode(senses) + } + + public func decode(_ data: Data) throws -> [Sense] { + try JSONDecoder().decode([Sense].self, from: data) + } +} + +// MARK: - The packed encoding + +/// A packed binary encoding of a sense run. +/// +/// ### Why +/// +/// Measured over the 3,442,623-entry BKRS container: the `entry` table was +/// 625 MB, its sense payloads 327 MB of that, and **only 40.7% of a payload +/// was text**. The rest was `{"gloss":{"text":…},"level":0,"kind":…}` written +/// out once per sense. Payload size is also most of the import's wall-clock, +/// since every byte is one SQLite has to write. +/// +/// ### How +/// +/// Three ideas, each worth its complexity: +/// +/// - **A presence byte.** The overwhelmingly common sense is a gloss, a level +/// and nothing else — 3,318,399 of 3,434,224 cards carry a single sense — so +/// every field it lacks costs one bit instead of a key and a literal. +/// - **Varints.** Levels are 0–3, parents and offsets are small, counts are +/// usually one. A varint spends a byte on each where JSON spends a decimal +/// rendering plus punctuation. +/// - **An example is stored once.** `UsageExample` keeps `chinese`, +/// `translation` *and* `raw`, where `raw` is the example as written and the +/// other two are a heuristic split of it; measured, the halves cost as much +/// again as `raw` itself. Here `raw` is stored with the two boundaries +/// inside it, reproducing both halves exactly — including a split that +/// trimmed whitespace — from one copy of the text. +/// +/// `Sense.id` is not stored at all: it is the index in the run. +/// +/// ### Compatibility +/// +/// The leading version byte is checked, and a payload claiming a newer version +/// is refused rather than misread. Adding a field means a new presence bit and +/// a version bump; `meta` says which codec wrote a container, so an older one +/// stays readable. +public struct CompactSenseCodec: SensePayloadCodec { + public static let identifier = "compact-v1" + + /// Bumped when the layout changes. A payload is refused rather than + /// guessed at, because a misread sense is worse than an error. + static let version: UInt8 = 1 + + enum Field: UInt8 { + case parent = 0x01 + case label = 0x02 + case styles = 0x04 + case references = 0x08 + case partOfSpeech = 0x10 + case registers = 0x20 + case examples = 0x40 + /// Set when the kind is not `.definition`, which almost every sense is. + case kind = 0x80 + } + + public init() {} + + public func encode(_ senses: [Sense]) throws -> Data { + var bytes: [UInt8] = [Self.version] + // A rough guess at the common case, to avoid a handful of growths per + // entry across three and a half million of them. + bytes.reserveCapacity(64 * senses.count + 8) + Self.appendVarint(senses.count, to: &bytes) + for sense in senses { + Self.append(sense, to: &bytes) + } + return Data(bytes) + } + + public func decode(_ data: Data) throws -> [Sense] { + var reader = Reader(data) + let version = try reader.byte() + guard version == Self.version else { throw SensePayloadError.unknownVersion(version) } + + let count = try reader.varint() + var senses: [Sense] = [] + senses.reserveCapacity(count) + for index in 0 ..< count { + try senses.append(Self.decodeSense(at: index, from: &reader)) + } + return senses + } +} + +// MARK: - Writing + +extension CompactSenseCodec { + static func append(_ sense: Sense, to bytes: inout [UInt8]) { + bytes.append(presence(of: sense)) + + if sense.kind != .definition { + bytes.append(sense.kind.rawValue) + } + appendVarint(sense.level, to: &bytes) + if let parent = sense.parent { + appendVarint(parent, to: &bytes) + } + if let label = sense.label { + appendText(label, to: &bytes) + } + appendText(sense.gloss.text, to: &bytes) + + if !sense.gloss.styles.isEmpty { + append(styles: sense.gloss.styles, to: &bytes) + } + if !sense.gloss.references.isEmpty { + append(references: sense.gloss.references, to: &bytes) + } + if !sense.partOfSpeech.isEmpty { + appendTexts(sense.partOfSpeech, to: &bytes) + } + if !sense.registers.isEmpty { + appendTexts(sense.registers, to: &bytes) + } + if !sense.examples.isEmpty { + appendVarint(sense.examples.count, to: &bytes) + for example in sense.examples { + appendExample(example, to: &bytes) + } + } + } + + private static func append(styles: [StyleRun], to bytes: inout [UInt8]) { + appendVarint(styles.count, to: &bytes) + for run in styles { + bytes.append(run.style.encoded) + appendVarint(run.range.lowerBound, to: &bytes) + appendVarint(run.range.count, to: &bytes) + } + } + + private static func append(references: [CrossReference], to bytes: inout [UInt8]) { + appendVarint(references.count, to: &bytes) + for reference in references { + appendText(reference.simplified, to: &bytes) + appendOptionalText(reference.traditional, to: &bytes) + appendOptionalText(reference.reading, to: &bytes) + appendVarint(reference.range.lowerBound, to: &bytes) + appendVarint(reference.range.count, to: &bytes) + } + } + + /// One bit per optional field. + static func presence(of sense: Sense) -> UInt8 { + var flags: UInt8 = 0 + if sense.parent != nil { + flags |= Field.parent.rawValue + } + if sense.label != nil { + flags |= Field.label.rawValue + } + if !sense.gloss.styles.isEmpty { + flags |= Field.styles.rawValue + } + if !sense.gloss.references.isEmpty { + flags |= Field.references.rawValue + } + if !sense.partOfSpeech.isEmpty { + flags |= Field.partOfSpeech.rawValue + } + if !sense.registers.isEmpty { + flags |= Field.registers.rawValue + } + if !sense.examples.isEmpty { + flags |= Field.examples.rawValue + } + if sense.kind != .definition { + flags |= Field.kind.rawValue + } + return flags + } + + /// Writes an example as its raw text plus where the two halves sit in it. + /// + /// The split is a heuristic over where the Han characters stop, and the + /// translation side is trimmed, so neither half is a plain suffix of the + /// raw text. Two offsets reproduce both exactly from one copy. + /// + /// When a half is not a slice of the raw text at all — which a different + /// splitter could produce — it is written out in full and marked, so this + /// stays an optimisation rather than an invariant the model must keep. + static func appendExample(_ example: UsageExample, to bytes: inout [UInt8]) { + let raw = Array(example.raw.utf8) + let chinese = Array(example.chinese.utf8) + let translation = Array(example.translation.utf8) + + // One-based, so zero can mean "this half is empty" — which is how an + // example with no Chinese at all is told from one whose Chinese starts + // at offset zero, the ordinary case. + let chineseStart = chinese.isEmpty + ? 0 + : (raw.firstRange(of: chinese)?.lowerBound).map { $0 + 1 } + let translationStart = translation.isEmpty + ? 0 + : (raw.lastRange(of: translation)?.lowerBound).map { $0 + 1 } + + guard let chineseStart, let translationStart else { + bytes.append(1) + appendText(example.raw, to: &bytes) + appendText(example.chinese, to: &bytes) + appendText(example.translation, to: &bytes) + return + } + bytes.append(0) + appendText(example.raw, to: &bytes) + appendVarint(chineseStart, to: &bytes) + appendVarint(chinese.count, to: &bytes) + appendVarint(translationStart, to: &bytes) + appendVarint(translation.count, to: &bytes) + } + + /// LEB128, unsigned. Every integer in this format is small and + /// non-negative — levels, parents, offsets, counts — so a fixed width + /// would spend seven bytes saying nothing. + static func appendVarint(_ value: Int, to bytes: inout [UInt8]) { + var remaining = UInt(bitPattern: value) + while true { + let byte = UInt8(remaining & 0x7F) + remaining >>= 7 + if remaining == 0 { + bytes.append(byte) + return + } + bytes.append(byte | 0x80) + } + } + + static func appendText(_ text: String, to bytes: inout [UInt8]) { + let utf8 = text.utf8 + appendVarint(utf8.count, to: &bytes) + bytes.append(contentsOf: utf8) + } + + /// A length of zero means the empty string; nil is one less than that, so + /// an absent traditional spelling costs one byte and an empty one is still + /// distinguishable from it. + static func appendOptionalText(_ text: String?, to bytes: inout [UInt8]) { + guard let text else { + appendVarint(0, to: &bytes) + return + } + let utf8 = text.utf8 + appendVarint(utf8.count + 1, to: &bytes) + bytes.append(contentsOf: utf8) + } + + static func appendTexts(_ texts: [String], to bytes: inout [UInt8]) { + appendVarint(texts.count, to: &bytes) + for text in texts { + appendText(text, to: &bytes) + } + } +} + +// MARK: - Reading + +extension CompactSenseCodec { + static func decodeSense(at index: Int, from reader: inout Reader) throws -> Sense { + let flags = try reader.byte() + let kind = try flags & Field.kind.rawValue != 0 + ? SenseKind(rawValue: reader.byte()) ?? .definition + : .definition + let level = try reader.varint() + let parent = flags & Field.parent.rawValue != 0 ? try reader.varint() : nil + let label = flags & Field.label.rawValue != 0 ? try reader.text() : nil + let text = try reader.text() + let styles = flags & Field.styles.rawValue != 0 ? try decodeStyles(from: &reader) : [] + let references = flags & Field.references.rawValue != 0 + ? try decodeReferences(from: &reader) + : [] + let partOfSpeech = flags & Field.partOfSpeech.rawValue != 0 ? try reader.texts() : [] + let registers = flags & Field.registers.rawValue != 0 ? try reader.texts() : [] + var examples: [UsageExample] = [] + if flags & Field.examples.rawValue != 0 { + for _ in try 0 ..< (reader.varint()) { + try examples.append(decodeExample(from: &reader)) + } + } + + return Sense( + id: index, + kind: kind, + gloss: Gloss(text: text, references: references, styles: styles), + level: level, + parent: parent, + label: label, + partOfSpeech: partOfSpeech, + registers: registers, + examples: examples, + ) + } + + private static func decodeStyles(from reader: inout Reader) throws -> [StyleRun] { + var styles: [StyleRun] = [] + for _ in try 0 ..< (reader.varint()) { + let style = try TextStyle(encoded: reader.byte()) + let start = try reader.varint() + let length = try reader.varint() + styles.append(StyleRun(style: style, range: start ..< (start + length))) + } + return styles + } + + private static func decodeReferences(from reader: inout Reader) throws -> [CrossReference] { + var references: [CrossReference] = [] + for _ in try 0 ..< (reader.varint()) { + let simplified = try reader.text() + let traditional = try reader.optionalText() + let reading = try reader.optionalText() + let start = try reader.varint() + let length = try reader.varint() + references.append(CrossReference( + simplified: simplified, + traditional: traditional, + reading: reading, + range: start ..< (start + length), + )) + } + return references + } + + private static func decodeExample(from reader: inout Reader) throws -> UsageExample { + let form = try reader.byte() + let raw = try reader.text() + guard form == 0 else { + return try UsageExample( + chinese: reader.text(), + translation: reader.text(), + raw: raw, + ) + } + let chineseStart = try reader.varint() + let chineseLength = try reader.varint() + let translationStart = try reader.varint() + let translationLength = try reader.varint() + let bytes = Array(raw.utf8) + return UsageExample( + chinese: slice(bytes, start: chineseStart, length: chineseLength), + translation: slice(bytes, start: translationStart, length: translationLength), + raw: raw, + ) + } + + // A payload is derived data in a file on disk. Where it is damaged, a + // replacement character costs the characters it replaces and leaves the + // entry readable, which beats an error costing the whole entry — and + // structural damage is already an error, from the bounds checks below. + // The same bargain the byte reader makes with a malformed source. + // swiftlint:disable optional_data_string_conversion + + /// A half of an example, by its one-based start. Zero means empty, which + /// is how "no Chinese at all" is told from "the Chinese starts at zero". + private static func slice(_ bytes: [UInt8], start: Int, length: Int) -> String { + guard start > 0, length > 0 else { return "" } + let from = start - 1 + guard from + length <= bytes.count else { return "" } + return String(decoding: bytes[from ..< (from + length)], as: UTF8.self) + } + + /// A cursor over the payload that refuses to read past the end. + /// + /// Every read is bounds-checked and throws rather than trapping: a payload + /// is data from a file on disk, and a corrupt one should cost that entry, + /// not the process. + struct Reader { + private let bytes: [UInt8] + private var offset = 0 + + init(_ data: Data) { + bytes = [UInt8](data) + } + + mutating func byte() throws -> UInt8 { + guard offset < bytes.count else { throw SensePayloadError.truncated(offset: offset) } + defer { offset += 1 } + return bytes[offset] + } + + mutating func varint() throws -> Int { + var result = 0 + var shift = 0 + while true { + let byte = try byte() + result |= Int(byte & 0x7F) << shift + if byte & 0x80 == 0 { + return result + } + shift += 7 + guard shift < 64 else { throw SensePayloadError.truncated(offset: offset) } + } + } + + mutating func text() throws -> String { + try take(count: varint()) + } + + mutating func optionalText() throws -> String? { + let length = try varint() + guard length > 0 else { return nil } + return try take(count: length - 1) + } + + mutating func texts() throws -> [String] { + let count = try varint() + var result: [String] = [] + result.reserveCapacity(count) + for _ in 0 ..< count { + try result.append(text()) + } + return result + } + + private mutating func take(count: Int) throws -> String { + guard count >= 0, offset + count <= bytes.count else { + throw SensePayloadError.truncated(offset: offset) + } + defer { offset += count } + return String(decoding: bytes[offset ..< (offset + count)], as: UTF8.self) + } + } + + // swiftlint:enable optional_data_string_conversion +} + +// MARK: - Enumerations on the wire + +extension TextStyle { + /// A fixed byte per case, chosen here rather than taken from the case + /// order: reordering the enum must not change what is already on disk. + var encoded: UInt8 { + switch self { + case .bold: 1 + case .italic: 2 + case .highlighted: 3 + } + } + + init(encoded: UInt8) { + switch encoded { + case 1: self = .bold + case 3: self = .highlighted + default: self = .italic + } + } +} + +// MARK: - Choosing a codec + +/// Which codec reads or writes a container. +public enum SensePayloadCodecs { + /// What a writer should use now. + public static let current: any SensePayloadCodec = CompactSenseCodec() + + /// Resolves a recorded identifier. + /// + /// An unrecognised one is an error rather than a fallback to the current + /// codec: misreading a payload produces plausible nonsense, and a + /// container written by a newer build should say so plainly. + public static func named(_ identifier: String?) throws -> any SensePayloadCodec { + switch identifier { + // A container from before the codec was recorded is JSON, because that + // is what the only writer there was then used. + case nil, JSONSenseCodec.identifier: JSONSenseCodec() + case CompactSenseCodec.identifier: CompactSenseCodec() + case let other: throw SensePayloadError.unknownCodec(other ?? "") + } + } +} diff --git a/Sources/HanReaderPersistence/DictionaryContainer.swift b/Sources/HanReaderPersistence/DictionaryContainer.swift index 46a71b5..2d8655d 100644 --- a/Sources/HanReaderPersistence/DictionaryContainer.swift +++ b/Sources/HanReaderPersistence/DictionaryContainer.swift @@ -58,7 +58,7 @@ public struct DictionaryContainer: Sendable { let queue = try DatabaseQueue(path: url.path, configuration: configuration) try DictionarySchema.migrator().migrate(queue) - let encoder = JSONEncoder() + let codec = SensePayloadCodecs.current var written = 0 for chunk in entries.chunked(into: batchSize) { @@ -69,15 +69,14 @@ public struct DictionaryContainer: Sendable { try db.execute(sql: """ INSERT INTO entry (simplified, traditional, readingNumeric, - readingDisplay, senses, summary) - VALUES (?, ?, ?, ?, ?, ?) + readingDisplay, senses) + VALUES (?, ?, ?, ?, ?) """, arguments: [ entry.headword.simplified, entry.headword.traditional, entry.readingKey, entry.readingDisplay, - encoder.encode(entry.senses), - entry.summarySense?.gloss.text ?? "", + codec.encode(entry.senses), ]) } } @@ -140,6 +139,9 @@ public struct DictionaryContainer: Sendable { ) throws { let values: [String: String?] = [ "formatVersion": String(DictionarySchema.formatVersion), + // Which codec wrote the sense payloads, so a reader picks the one + // that can read them rather than assuming the current one. + "sensePayloadCodec": type(of: SensePayloadCodecs.current).identifier, "slug": metadata.slug, "displayName": metadata.displayName, "format": metadata.format, @@ -166,6 +168,22 @@ public struct DictionaryContainer: Sendable { // MARK: - Reading + /// The codec this container's sense payloads were written with. + /// + /// Read from the file rather than assumed, which is what lets a new codec + /// land without invalidating the dictionaries already on disk — and what + /// makes a container written by a *newer* build say so plainly instead of + /// decoding into plausible nonsense. + public func sensePayloadCodec() throws -> any SensePayloadCodec { + let identifier = try dbQueue.read { db in + try String.fetchOne( + db, + sql: "SELECT value FROM meta WHERE key = 'sensePayloadCodec'", + ) + } + return try SensePayloadCodecs.named(identifier) + } + public func metadata() throws -> DictionaryMetadata? { let values = try dbQueue.read { db in try Row.fetchAll(db, sql: "SELECT key, value FROM meta") @@ -196,7 +214,7 @@ public struct DictionaryContainer: Sendable { /// readings; the prototype's `LIMIT 1` is the bug this signature exists to /// make impossible. public func entries(for headword: String) throws -> [DictionaryEntry] { - let decoder = JSONDecoder() + let codec = try sensePayloadCodec() return try dbQueue.read { db in // Matched against BOTH scripts. Storing the traditional form and // then only ever querying the simplified one means a reader of @@ -213,7 +231,7 @@ public struct DictionaryContainer: Sendable { traditional: row["traditional"], ), reading: Pinyin.parse(numeric: row["readingNumeric"]), - senses: decoder.decode([Sense].self, from: row["senses"]), + senses: codec.decode(row["senses"]), ) } } @@ -300,6 +318,75 @@ public struct DictionaryContainer: Sendable { } } +// MARK: - Introspection + +/// Reading the shape of a container rather than its content. +/// +/// Used by the tests that assert an import leaves the right indexes behind +/// and that the retired `summary` column is gone — both things a reader +/// should never need to ask, and both worth asserting once. +extension DictionaryContainer { + /// The indexes the file actually carries. + /// + /// For asserting that an import leaves behind the ones a reader needs and + /// not the one only the importer wanted. + public func indexNames() throws -> Set { + try dbQueue.read { db in + try Set(String.fetchAll( + db, + sql: "SELECT name FROM sqlite_schema WHERE type = 'index' AND name IS NOT NULL", + )) + } + } + + /// The columns a table carries, for asserting that one is gone. + public func columnNames(of table: String) throws -> Set { + try dbQueue.read { db in + try Set(db.columns(in: table).map(\.name)) + } + } + + /// Writes a one-entry container with a chosen codec. + /// + /// Only so a test can build the kind of container an earlier build + /// produced, and prove it still opens — which is the whole point of + /// recording the codec rather than assuming it. + static func writeForTesting( + entry: DictionaryEntry, + codec: any SensePayloadCodec, + metadata: DictionaryMetadata, + to url: URL, + ) throws { + try? FileManager.default.removeItem(at: url) + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true, + ) + let queue = try DatabaseQueue(path: url.path) + try DictionarySchema.migrator().migrate(queue) + try queue.write { db in + try db.execute(sql: """ + INSERT INTO entry + (simplified, traditional, readingNumeric, readingDisplay, senses) + VALUES (?, ?, ?, ?, ?) + """, arguments: [ + entry.headword.simplified, + entry.headword.traditional, + entry.readingKey, + entry.readingDisplay, + codec.encode(entry.senses), + ]) + } + try writeMetadata(metadata, to: queue) + try queue.write { db in + try db.execute( + sql: "INSERT OR REPLACE INTO meta (key, value) VALUES ('sensePayloadCodec', ?)", + arguments: [type(of: codec).identifier], + ) + } + } +} + extension Array { /// Splits into fixed-size chunks, so a batch can be one transaction. func chunked(into size: Int) -> [ArraySlice] { diff --git a/Sources/HanReaderPersistence/DictionarySchema.swift b/Sources/HanReaderPersistence/DictionarySchema.swift index 99c8871..b871b31 100644 --- a/Sources/HanReaderPersistence/DictionarySchema.swift +++ b/Sources/HanReaderPersistence/DictionarySchema.swift @@ -20,7 +20,7 @@ enum DictionarySchema { /// Distinct from a parser version: no schema migration can repair data /// that was parsed wrongly, so a parser fix invalidates a container /// through `meta.parserVersion` and triggers a rebuild instead. - static let formatVersion = 2 + static let formatVersion = 3 static func migrator() -> DatabaseMigrator { var migrator = DatabaseMigrator() @@ -43,6 +43,20 @@ enum DictionarySchema { try createImportJobTable(db) } + // The denormalised first gloss, written since v1 and read by nothing. + // On a BKRS container it is **160 MB** — 3,442,623 rows of Cyrillic + // prose that is already inside the sense payload beside it. It was + // there so a result list could render without decoding a payload per + // row; with the compact codec that decode is microseconds for the ten + // rows a list shows, and 160 MB is not a reasonable price for it. If + // a list ever does need it, it can come back as a short prefix rather + // than a whole copy. + migrator.registerMigration("v3_drop_summary") { db in + try db.alter(table: "entry") { table in + table.drop(column: "summary") + } + } + return migrator } diff --git a/Sources/HanReaderPersistence/DictionaryWriter.swift b/Sources/HanReaderPersistence/DictionaryWriter.swift index 2fdccc7..a91add5 100644 --- a/Sources/HanReaderPersistence/DictionaryWriter.swift +++ b/Sources/HanReaderPersistence/DictionaryWriter.swift @@ -43,6 +43,40 @@ public import HanReaderCore /// The resume path still deletes anything at or past the checkpoint, because a /// belt costs one indexed `DELETE` and braces alone are an argument. public final class DictionaryWriter { + /// When the lookup indexes get built. + /// + /// They are needed only once an import is finished, and the choice of + /// *when* to build them trades the import's two budgets against each + /// other. Measured over the full 350 MB BKRS set, same machine, same + /// output: + /// + /// | | time | peak memory | size | + /// |---|---|---|---| + /// | `.atTheEnd` | **75 s** | 134 MB | **493 MB** | + /// | `.whileWriting` | 188 s | **53 MB** | 556 MB | + /// + /// Building at the end is the ordinary bulk-load shape: one pass over + /// rows that are all already there, instead of a random-order insert into + /// an 84 MB B-tree three and a half million times. It is also 63 MB + /// smaller on disk, because an index built in one pass does not carry the + /// page splits that random insertion leaves behind. + /// + /// Its cost is memory: `CREATE INDEX` is a single atomic statement, so the + /// whole new index is dirty at once — about 92 MB of it, which no + /// `cache_size`, `temp_store`, `threads`, `soft_heap_limit` or + /// `hard_heap_limit` pragma reduces. Measured, not assumed; all five were + /// tried. + /// + /// `.atTheEnd` is the default because compiling a DSL set is a desktop + /// operation by design — a Mac compiles it once and a phone imports the + /// compiled container — so the import's memory ceiling, which exists for + /// iOS, is not the binding constraint on this path. `.whileWriting` is + /// there for the day something does run an import where it is. + public enum Indexing: Sendable { + case atTheEnd + case whileWriting + } + /// Where the reader got to, recorded at a card boundary. /// /// A card boundary, not a line or a batch boundary, because it is the only @@ -135,7 +169,8 @@ public final class DictionaryWriter { private let dbQueue: DatabaseQueue private let source: Source private let lexiconMaximumLength: Int - private let encoder = JSONEncoder() + private let indexing: Indexing + private let codec = SensePayloadCodecs.current private var entriesWritten: Int /// Opens a container at `url`, resuming the job in it when there is one @@ -154,6 +189,7 @@ public final class DictionaryWriter { at url: URL, source: Source, lexiconMaximumLength: Int = LexiconBuilder.maximumWordLength, + indexing: Indexing = .atTheEnd, ) throws -> (writer: DictionaryWriter, resume: Checkpoint?) { @@ -170,14 +206,7 @@ public final class DictionaryWriter { let queue = try DatabaseQueue(path: url.path, configuration: configuration) try DictionarySchema.migrator().migrate(queue) - // A finished import drops this index, so a re-import needs it back. - // Here rather than in the migration, because the migration has already - // run by the time it is dropped. - try queue.write { db in - try db.execute(sql: """ - CREATE INDEX IF NOT EXISTS entry_on_source ON entry (sourceFile, sourceOffset) - """) - } + try prepareIndexes(queue, indexing: indexing) let stored = try queue.read { db in try Row.fetchOne(db, sql: "SELECT * FROM importJob WHERE id = 1") @@ -201,11 +230,38 @@ public final class DictionaryWriter { dbQueue: queue, source: source, lexiconMaximumLength: lexiconMaximumLength, + indexing: indexing, entriesWritten: written, ) return (writer, resume) } + /// Sets up the indexes an import wants, and takes down the ones it does + /// not. + /// + /// A lookup index is maintained per row while it exists, and + /// `entry_on_simplified` means a random-order insert into an 84 MB B-tree + /// three and a half million times. Nothing queries it until the import is + /// over, so it comes down for the duration and is built in one pass at the + /// end — the ordinary bulk-load shape. + /// + /// The source index is the mirror image: wanted only *during* an import, + /// for the delete a resume does, and dropped when one finishes. + /// + /// A container interrupted part way therefore has no lookup indexes. That + /// is correct rather than unfortunate — it is not a dictionary yet, and + /// the `importJob` row is what says so. + private static func prepareIndexes(_ queue: DatabaseQueue, indexing: Indexing) throws { + try queue.write { db in + try db.execute(sql: """ + CREATE INDEX IF NOT EXISTS entry_on_source ON entry (sourceFile, sourceOffset) + """) + guard indexing == .atTheEnd else { return } + try db.execute(sql: "DROP INDEX IF EXISTS entry_on_simplified") + try db.execute(sql: "DROP INDEX IF EXISTS entry_on_traditional") + } + } + /// Makes the container ready to be written into, and says how many /// entries it already holds. private static func prepare( @@ -250,11 +306,13 @@ public final class DictionaryWriter { dbQueue: DatabaseQueue, source: Source, lexiconMaximumLength: Int, + indexing: Indexing, entriesWritten: Int, ) { self.dbQueue = dbQueue self.source = source self.lexiconMaximumLength = lexiconMaximumLength + self.indexing = indexing self.entriesWritten = entriesWritten } @@ -281,15 +339,14 @@ public final class DictionaryWriter { try db.execute(sql: """ INSERT INTO entry (simplified, traditional, readingNumeric, readingDisplay, - senses, summary, sourceFile, sourceOffset) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + senses, sourceFile, sourceOffset) + VALUES (?, ?, ?, ?, ?, ?, ?) """, arguments: [ entry.headword.simplified, entry.headword.traditional, entry.readingKey, entry.readingDisplay, - encoder.encode(entry.senses), - entry.summarySense?.gloss.text ?? "", + codec.encode(entry.senses), sourced.sourceFile, sourced.sourceOffset, ]) @@ -377,6 +434,19 @@ public final class DictionaryWriter { // is 53 MB the reader would carry for nothing. The columns stay, // so a re-import can rebuild the index and resume as before. try db.execute(sql: "DROP INDEX IF EXISTS entry_on_source") + + // And now the ones a reader needs, built in one pass over rows + // that are all already there rather than maintained as they + // arrived. Lookup matches BOTH scripts: storing the traditional + // form and only ever querying the simplified one means a reader of + // traditional text finds nothing at all. + try db.execute(sql: """ + CREATE INDEX IF NOT EXISTS entry_on_simplified ON entry (simplified) + """) + try db.execute(sql: """ + CREATE INDEX IF NOT EXISTS entry_on_traditional ON entry (traditional) + WHERE traditional IS NOT NULL + """) try db.execute(sql: "ANALYZE") } // Folding the write-ahead log back in leaves one file rather than @@ -402,7 +472,6 @@ extension DictionaryWriter { /// than the 27 MB suggests. The collector keeps five small values per /// candidate reading and the entry is released immediately. private func writeCharacterReadings() throws { - let decoder = JSONDecoder() var collector = LexiconBuilder.CharacterReadings() try dbQueue.read { db in let cursor = try Row.fetchCursor(db, sql: """ @@ -416,7 +485,7 @@ extension DictionaryWriter { traditional: row["traditional"], ), reading: Pinyin.parse(numeric: row["readingNumeric"]), - senses: decoder.decode([Sense].self, from: row["senses"]), + senses: self.codec.decode(row["senses"]), )) } } diff --git a/Tests/HanReaderCoreTests/SensePayloadCodecTests.swift b/Tests/HanReaderCoreTests/SensePayloadCodecTests.swift new file mode 100644 index 0000000..88b19d6 --- /dev/null +++ b/Tests/HanReaderCoreTests/SensePayloadCodecTests.swift @@ -0,0 +1,275 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import Testing +@testable import HanReaderCore + +// MARK: - Fixtures + +/// Every sense in the committed slice of real 大БКРС data. +/// +/// Round-tripping invented senses tests the cases someone thought of. +/// Round-tripping 523 real ones tests the cases nobody did — the sense whose +/// gloss is empty because its text was all label, the example that splits at +/// a quotation mark, the reference that ends where the gloss does. +private enum Corpus { + static let senses: [[Sense]] = { + guard let url = Bundle.module.url( + forResource: "Fixtures/bkrs-slice.dsl", + withExtension: "txt", + ), + let data = try? Data(contentsOf: url), + let text = String(data: data, encoding: .utf8) + else { + Issue.record("could not load the BKRS fixture") + return [] + } + var bytes: [UInt8] = [0xFF, 0xFE] + for unit in Array(text.utf16) { + bytes += [UInt8(unit & 0xFF), UInt8(unit >> 8)] + } + let builder = DSLCardBuilder(syllableBases: []) + var runs: [[Sense]] = [] + do { + var reader = try DSLCardReader(source: DSLMemoryBytes(bytes)) + while let record = try reader.next() { + if let senses = builder.entries(from: record.card).entries.first?.senses { + runs.append(senses) + } + } + } catch { + Issue.record("could not read the BKRS fixture: \(error)") + } + return runs + }() +} + +private func sense( + id: Int = 0, + kind: SenseKind = .definition, + text: String = "значение", + level: Int = 0, + parent: Int? = nil, + label: String? = nil, + styles: [StyleRun] = [], + references: [CrossReference] = [], + partOfSpeech: [String] = [], + registers: [String] = [], + examples: [UsageExample] = [], +) + -> Sense +{ + Sense( + id: id, + kind: kind, + gloss: Gloss(text: text, references: references, styles: styles), + level: level, + parent: parent, + label: label, + partOfSpeech: partOfSpeech, + registers: registers, + examples: examples, + ) +} + +@Suite("Compact sense payloads") +struct SensePayloadCodecTests { + private let codec = CompactSenseCodec() + + // MARK: - Round trips + + /// The whole committed slice, sense for sense. Equality is on `Sense` + /// itself, so every field has to come back — a codec that silently drops + /// `registers` would pass a test that only checked gloss text. + @Test("Every sense in the real slice survives a round trip") + func realCorpus() throws { + #expect(!Corpus.senses.isEmpty) + var total = 0 + for run in Corpus.senses { + let decoded = try codec.decode(codec.encode(run)) + #expect(decoded == run) + total += run.count + } + #expect(total == 523) + } + + @Test("An empty run round-trips") + func emptyRun() throws { + #expect(try codec.decode(codec.encode([])).isEmpty) + } + + @Test("Each field survives on its own", arguments: [ + sense(), + sense(kind: .reference, text: "см. другое"), + sense(kind: .label, text: "фамилия"), + sense(text: "", level: 3, parent: 1, label: "I"), + sense(level: 9, parent: 300), + sense(styles: [ + StyleRun(style: .bold, range: 0 ..< 2), + StyleRun(style: .italic, range: 3 ..< 8), + StyleRun(style: .highlighted, range: 0 ..< 1), + ]), + sense(references: [CrossReference( + simplified: "磁性碰锁", + traditional: "磁性碰鎖", + reading: "cíxìng pèngsuǒ", + range: 2 ..< 6, + )]), + sense(references: [CrossReference( + simplified: "横", + traditional: nil, + reading: nil, + range: 0 ..< 1, + )]), + sense(partOfSpeech: ["гл.", "сущ."], registers: ["бот.", "уст."]), + sense(examples: [UsageExample( + chinese: "爱儿童", + translation: "любить ребёнка", + raw: "爱儿童 любить ребёнка", + )]), + ]) + func singleFields(input: Sense) throws { + #expect(try codec.decode(codec.encode([input])) == [input]) + } + + /// `id` is the index in the run and is not stored, so a run decodes with + /// its positions renumbered — which is what every reader of it assumes. + @Test("Identifiers come back as positions") + func identifiersArePositions() throws { + let run = (0 ..< 5).map { sense(id: 97 + $0, text: "s\($0)") } + let decoded = try codec.decode(codec.encode(run)) + #expect(decoded.map(\.id) == [0, 1, 2, 3, 4]) + #expect(decoded.map(\.gloss.text) == run.map(\.gloss.text)) + } + + // MARK: - Examples, which are stored once + + /// `raw` is the example as written and the two halves are a heuristic + /// split of it, so storing all three copies the text twice. Measured over + /// the set, the halves cost as much again as `raw`. + @Test("An example is stored once and split on the way out", arguments: [ + UsageExample(chinese: "爱儿童", translation: "любить ребёнка", raw: "爱儿童 любить ребёнка"), + UsageExample(chinese: "", translation: "только по-русски", raw: "только по-русски"), + UsageExample(chinese: "他说：“好。”", translation: "он сказал", raw: "他说：“好。” он сказал"), + UsageExample(chinese: "一", translation: "", raw: "一"), + UsageExample(chinese: "", translation: "", raw: ""), + ]) + func examples(input: UsageExample) throws { + let decoded = try codec.decode(codec.encode([sense(examples: [input])])) + #expect(decoded.first?.examples == [input]) + } + + /// A split that is not a slice of the raw text — which a different + /// splitter could produce — is written out in full rather than silently + /// reconstructed wrongly. The compact form is an optimisation, not an + /// invariant the model has to keep. + @Test("An example whose halves are not slices of its raw text still round-trips") + func examplesThatAreNotSlices() throws { + let odd = UsageExample(chinese: "完全", translation: "иное", raw: "ничего общего") + let decoded = try codec.decode(codec.encode([sense(examples: [odd])])) + #expect(decoded.first?.examples == [odd]) + } + + /// The same text twice over, where a naive "find the half in the raw" + /// would pick the wrong occurrence. + @Test("A repeated half picks the right occurrence") + func repeatedHalf() throws { + let repeated = UsageExample(chinese: "好", translation: "好", raw: "好 好") + let decoded = try codec.decode(codec.encode([sense(examples: [repeated])])) + #expect(decoded.first?.examples == [repeated]) + } + + // MARK: - Size + + /// The reason the codec exists. Measured over the real slice rather than + /// asserted as a ratio someone liked: JSON spends most of a payload on + /// syntax and key names, written out once per sense across three and a + /// half million of them. + @Test("The compact form is substantially smaller than JSON") + func size() throws { + let json = JSONSenseCodec() + var compactBytes = 0 + var jsonBytes = 0 + for run in Corpus.senses { + compactBytes += try codec.encode(run).count + jsonBytes += try json.encode(run).count + } + #expect(compactBytes * 2 < jsonBytes, "compact \(compactBytes), json \(jsonBytes)") + } + + // MARK: - Malformed payloads + + /// A payload is data from a file on disk. A corrupt one costs that entry, + /// not the process, so every read is bounds-checked rather than trapping. + @Test("A truncated payload throws rather than trapping") + func truncated() throws { + let full = try codec.encode([sense( + text: "значение", + label: "1)", + examples: [UsageExample(chinese: "一", translation: "один", raw: "一 один")], + )]) + for length in 0 ..< full.count { + #expect(throws: (any Error).self) { + _ = try codec.decode(full.prefix(length)) + } + } + } + + @Test("A payload from a newer format is refused, not guessed at") + func newerVersion() throws { + var bytes = try [UInt8](codec.encode([sense()])) + bytes[0] = CompactSenseCodec.version + 1 + #expect(throws: SensePayloadError.unknownVersion(CompactSenseCodec.version + 1)) { + _ = try codec.decode(Data(bytes)) + } + } + + @Test("An empty payload is refused") + func emptyPayload() { + #expect(throws: (any Error).self) { + _ = try codec.decode(Data()) + } + } + + // MARK: - Codec selection + + /// A container records which codec wrote it, so adding one does not + /// invalidate the dictionaries already on disk. + @Test("A container's recorded codec is what reads it") + func selection() throws { + #expect(try SensePayloadCodecs.named("compact-v1") is CompactSenseCodec) + #expect(try SensePayloadCodecs.named("json") is JSONSenseCodec) + // Written before the codec was recorded, which means JSON. + #expect(try SensePayloadCodecs.named(nil) is JSONSenseCodec) + #expect(throws: SensePayloadError.unknownCodec("compact-v99")) { + _ = try SensePayloadCodecs.named("compact-v99") + } + } + + /// JSON still reads, so a dictionary compiled by an earlier build opens. + @Test("JSON payloads still round-trip") + func jsonStillWorks() throws { + let json = JSONSenseCodec() + let run = try #require(Corpus.senses.first { $0.count > 2 }) + #expect(try json.decode(json.encode(run)) == run) + } + + // MARK: - Wire format + + /// The byte for a style is fixed here rather than taken from the case + /// order, so reordering the enum cannot change what is already on disk. + @Test("Style bytes are stable", arguments: TextStyle.allCases) + func styleBytes(style: TextStyle) { + #expect(TextStyle(encoded: style.encoded) == style) + } + + @Test("Varints round-trip at the boundaries", arguments: [ + 0, 1, 127, 128, 255, 256, 16383, 16384, 1 << 20, Int(Int32.max), + ]) + func varints(value: Int) throws { + var bytes: [UInt8] = [] + CompactSenseCodec.appendVarint(value, to: &bytes) + var reader = CompactSenseCodec.Reader(Data(bytes)) + #expect(try reader.varint() == value) + } +} diff --git a/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift b/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift new file mode 100644 index 0000000..eb1fe45 --- /dev/null +++ b/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift @@ -0,0 +1,208 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import HanReaderCore +import Testing +@testable import HanReaderPersistence + +// MARK: - Fixtures + +private func entry( + _ simplified: String, + traditional: String? = nil, + reading: String = "ai4", + senses: [String] = ["значение"], +) + -> DictionaryEntry +{ + DictionaryEntry( + headword: Headword(simplified: simplified, traditional: traditional), + reading: Pinyin.parse(numeric: reading), + senses: senses.enumerated().map { index, text in + Sense( + id: index, + kind: .definition, + gloss: Gloss(text: text, references: [], styles: []), + level: 0, + parent: nil, + label: nil, + partOfSpeech: [], + registers: [], + examples: [], + ) + }, + ) +} + +private func metadata(entryCount: Int) -> DictionaryMetadata { + DictionaryMetadata( + slug: "test", + displayName: "Test", + format: "abbyy-dsl", + indexLanguage: "zh-Hans", + glossLanguage: "ru", + licence: "none", + attribution: "none", + parserVersion: 1, + entryCount: entryCount, + ) +} + +private func temporaryURL() -> URL { + URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("hanreader-storage-\(UUID().uuidString)") + .appendingPathComponent("out.hanreaderdict") +} + +private func source(fingerprint: String = "f") -> DictionaryWriter.Source { + DictionaryWriter.Source( + name: "a.dsl", + fingerprint: fingerprint, + fileCount: 1, + totalBytes: 1000, + parserVersion: 1, + ) +} + +@Suite("Dictionary storage") +struct DictionaryStorageTests { + /// Whatever the indexing strategy, a finished container has to answer a + /// lookup — the whole point of building the indexes at the end is that it + /// is invisible from the outside. + @Test("A finished container looks words up", arguments: [ + DictionaryWriter.Indexing.atTheEnd, + DictionaryWriter.Indexing.whileWriting, + ]) + func lookupAfterEitherIndexing(indexing: DictionaryWriter.Indexing) throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let (writer, resume) = try DictionaryWriter.open( + at: url, + source: source(), + indexing: indexing, + ) + #expect(resume == nil) + try writer.append( + [ + DictionaryWriter.SourcedEntry( + entry: entry("爱", traditional: "愛"), + sourceFile: 0, + sourceOffset: 10, + ), + DictionaryWriter.SourcedEntry( + entry: entry("中国", reading: "zhong1 guo2"), + sourceFile: 0, + sourceOffset: 20, + ), + ], + checkpoint: DictionaryWriter.Checkpoint(fileIndex: 0, byteOffset: 20, cardsRead: 2), + ) + try writer.finish(metadata: metadata(entryCount: 2)) + + let container = try DictionaryContainer(contentsOf: url) + #expect(try container.entryCount() == 2) + #expect(try container.entries(for: "爱").count == 1) + // Matched against both scripts, or a reader of traditional text finds + // nothing at all. + #expect(try container.entries(for: "愛").count == 1) + #expect(try container.entries(for: "中国").first?.senses.first?.gloss.text == "значение") + } + + /// The index the resume path uses is wanted only during an import, and the + /// lookup indexes only after one. A finished container should carry the + /// second pair and not the first. + @Test("A finished container carries the reader's indexes and not the importer's") + func indexesAfterFinishing() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let (writer, _) = try DictionaryWriter.open(at: url, source: source()) + try writer.append( + [DictionaryWriter.SourcedEntry(entry: entry("爱"), sourceFile: 0, sourceOffset: 1)], + checkpoint: DictionaryWriter.Checkpoint(byteOffset: 1, cardsRead: 1), + ) + try writer.finish(metadata: metadata(entryCount: 1)) + + let indexes = try DictionaryContainer(contentsOf: url).indexNames() + #expect(indexes.contains("entry_on_simplified")) + #expect(indexes.contains("entry_on_traditional")) + #expect(!indexes.contains("entry_on_source")) + } + + /// A re-import needs the source index back, since a finished import drops + /// it and the resume path's delete depends on it. + @Test("Reopening a finished container restores the importer's index") + func reopeningRestoresSourceIndex() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let (first, _) = try DictionaryWriter.open(at: url, source: source()) + try first.append( + [DictionaryWriter.SourcedEntry(entry: entry("爱"), sourceFile: 0, sourceOffset: 1)], + checkpoint: DictionaryWriter.Checkpoint(byteOffset: 1, cardsRead: 1), + ) + try first.finish(metadata: metadata(entryCount: 1)) + + _ = try DictionaryWriter.open(at: url, source: source(fingerprint: "changed")) + #expect(try DictionaryContainer(contentsOf: url).indexNames() + .contains("entry_on_source")) + } + + // MARK: - Sense payloads + + /// A container records which codec wrote it, so a new codec does not + /// invalidate the dictionaries already on disk. + @Test("A container says which codec wrote its senses") + func recordedCodec() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + try DictionaryContainer.write( + entries: [entry("爱", senses: ["первое", "второе"])], + metadata: metadata(entryCount: 1), + to: url, + ) + let container = try DictionaryContainer(contentsOf: url) + #expect(try type(of: container.sensePayloadCodec()).identifier + == CompactSenseCodec.identifier) + #expect(try container.entries(for: "爱").first?.senses.count == 2) + } + + /// The denormalised first gloss, written since v1 and read by nothing. On + /// a BKRS container it was 160 MB of prose already sitting in the payload + /// beside it. + @Test("The summary column is gone") + func summaryDropped() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + try DictionaryContainer.write( + entries: [entry("爱")], + metadata: metadata(entryCount: 1), + to: url, + ) + #expect(try !DictionaryContainer(contentsOf: url).columnNames(of: "entry") + .contains("summary")) + } + + /// A container built before the compact codec existed still opens, which + /// is what the recorded identifier is for. + @Test("A JSON container still reads") + func jsonContainerStillReads() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let senses = entry("爱", senses: ["первое", "второе"]).senses + try DictionaryContainer.writeForTesting( + entry: entry("爱", senses: ["первое", "второе"]), + codec: JSONSenseCodec(), + metadata: metadata(entryCount: 1), + to: url, + ) + let container = try DictionaryContainer(contentsOf: url) + #expect(try type(of: container.sensePayloadCodec()).identifier + == JSONSenseCodec.identifier) + #expect(try container.entries(for: "爱").first?.senses == senses) + } +}