From 9ccf38ec0690041091ed6ac2135db16b20f1a4c3 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:04:16 +0300 Subject: [PATCH] feat: let the reader use more than one dictionary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now the app opened exactly one file, the bundled CC-CEDICT, so an imported BKRS container was a 493 MB file nothing read. This is the catalogue, the merge, and the three bugs that only appeared once the app actually had two dictionaries open. **Found by scanning, not by remembering where things were.** Every `.hanreaderdict` in the dictionaries directory is installed, plus the bundled one. No stored path, so no path to go stale — which on iOS it would, since the container directory's UUID changes on every install and restore. And a dictionary describes itself: name, languages, licence and attribution come from its own `meta` each time it is listed, so the acknowledgements cannot show a licence the file contradicts. The reader's own choices — order and permissions — live in the *library* database, because that is the half that cannot be regenerated and the half that belongs in a backup. **Answers are grouped by reading and never deduplicated.** 了 is `le` in one dictionary and `liǎo` in another; without grouping a reader sees four entries with no indication which are the same word said the same way. Two dictionaries defining one word are two answers, not one with a duplicate: merging them would mean choosing whose wording survives, which is a licensing problem as much as a usability one. Each answer is labelled with where it came from, because CC BY-SA attribution has to reach the reader where they read the gloss. **Three permissions, not one switch.** A reader can turn off CC-CEDICT's English definitions and still need it: 77% of BKRS entries carry no reading, so its single-character headwords are what put pinyin on the page in both language modes. One "enabled" switch would let them turn off the thing their ruby depends on. The third permission is the segmentation word list, opt-in and off by default for a dictionary whose headwords are not words — BKRS contributes 1,806,893 lexemes, which is a quarter of a gigabyte held for the life of the process and a word list in which almost every run of Han characters is a word. A dictionary nobody has ordered sorts by its gloss language against the reader's own, so a Russian-locale reader sees БКРС first without touching a setting. ### Three bugs that only running it could find **A finished container could not be opened.** It was left in write-ahead logging mode, and a database whose header says WAL needs a shared-memory file that a *read-only* connection cannot create. So an imported dictionary failed with "unable to open database file" and simply did not appear in the reader's list, while every writer-side test passed — because a writer opens for writing. Journal mode is an import-time choice for crash safety; a finished dictionary is read-only data. **Adding BKRS changed 打's reading from `dǎ` to `dá` on screen.** The composer ranks a reading partly by how many senses carry it, which is sound within one dictionary and meaningless across two — and especially for BKRS, where one entry per reading shares a single sense run by design, so all of 打's readings have 57 senses and the tie fell to alphabetical order. A dictionary with more senses per entry would simply outvote a better answer from the dictionary the reader put first. Readings now come from the first reading-providing dictionary that has the word, so the reader's ordering decides and each dictionary's ranking applies to its own entries. **`许谎` displayed as `xǔ; hǔang`.** 1,750 reading lines separate their readings with a semicolon rather than a comma, and 287 carry a register note between two readings — `zhǔ; [i]в сочет. также[/i] zhù` — of which some have no separator at all. A reading is now a maximal run of Latin-alphabet words, which takes `nāi … lāi` as two readings, leaves the Russian between them out, and needs no list of this dictionary's abbreviations. It recovers 4,114 readings from those 1,825 awkward lines where splitting on separators alone recovers 3,810 and shows `в сочет. также zhù` above a word for the rest. The whole set goes from 3,442,623 entries to 3,445,226. Also: a sense with nothing in it is no longer chosen as an entry's one-line summary. A BKRS card opens with a division heading whose whole text is a Roman numeral and a part of speech, both lifted out of the gloss, so 爱's summary rendered as a blank line with 32 senses behind it. Verified against the real 496 MB container installed where the app looks for it: both dictionaries open and merge, 爱 gets CC-CEDICT's 4 senses alongside BKRS's 32, 了 gains the third reading `liào` that CC-CEDICT does not have, and 一 and 打 get 57 each. The two reading defects were found by looking at the running app and are fixed there. Co-Authored-By: Claude Opus 5 --- .../Dictionary/DictionarySet.swift | 200 +++++++++++ .../HanReaderCore/Format/DSLCardBuilder.swift | 77 ++++- .../HanReaderCore/Model/DictionaryEntry.swift | 10 +- .../HanReaderPersistence/AppDatabase.swift | 13 + .../DictionaryCatalogue.swift | 250 ++++++++++++++ .../DictionaryWriter.swift | 11 + .../LibraryRepository.swift | 83 +++++ .../HanReaderPersistence/LibrarySchema.swift | 46 +++ Sources/HanReaderUI/HanReaderRootView.swift | 23 +- Sources/HanReaderUI/Models/ReaderModel.swift | 22 +- .../HanReaderUI/Screens/ReaderScreen.swift | 1 + .../Screens/WordDetailContent.swift | 93 ++++- .../HanReaderUI/Services/AppServices.swift | 124 ++++--- .../Services/DictionaryService.swift | 40 +++ .../CEDICTParserTests.swift | 29 ++ .../DSLCardBuilderTests.swift | 32 ++ .../DictionarySetTests.swift | 315 +++++++++++++++++ .../DictionaryCatalogueTests.swift | 317 ++++++++++++++++++ .../DictionaryStorageTests.swift | 60 ++++ .../MigrationTests.swift | 6 +- 20 files changed, 1659 insertions(+), 93 deletions(-) create mode 100644 Sources/HanReaderCore/Dictionary/DictionarySet.swift create mode 100644 Sources/HanReaderPersistence/DictionaryCatalogue.swift create mode 100644 Tests/HanReaderCoreTests/DictionarySetTests.swift create mode 100644 Tests/HanReaderPersistenceTests/DictionaryCatalogueTests.swift diff --git a/Sources/HanReaderCore/Dictionary/DictionarySet.swift b/Sources/HanReaderCore/Dictionary/DictionarySet.swift new file mode 100644 index 0000000..0b1f149 --- /dev/null +++ b/Sources/HanReaderCore/Dictionary/DictionarySet.swift @@ -0,0 +1,200 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation + +/// An entry together with the dictionary it came from. +/// +/// The source travels with the entry rather than being looked up later, +/// because it is not decoration: CC-CEDICT is CC BY-SA, and attribution has +/// to reach the reader at the point they read the gloss. It is also the only +/// way to render two dictionaries' answers for one word without implying they +/// are one dictionary's. +public struct AttributedEntry: Hashable, Sendable { + /// The dictionary's slug, as its container records it. + public let dictionary: String + public let entry: DictionaryEntry + + public init(dictionary: String, entry: DictionaryEntry) { + self.dictionary = dictionary + self.entry = entry + } +} + +/// Every dictionary's entries for one word that share a reading. +/// +/// Grouped by reading, and that is the crux of consulting several +/// dictionaries at once. 了 is `le` in one and `liǎo` in another; without +/// grouping, a reader tapping it sees four entries and no indication that two +/// of them are the same word said the same way. +public struct ReadingGroup: Hashable, Sendable { + /// Numeric pinyin — the cross-dictionary merge key. BKRS writes + /// diacritics and CC-CEDICT numerals, and both normalise to this. + /// + /// Empty for the group of entries that carry no reading at all, which is + /// 77% of BKRS. + public let readingKey: String + public let entries: [AttributedEntry] + + public init(readingKey: String, entries: [AttributedEntry]) { + self.readingKey = readingKey + self.entries = entries + } + + /// The reading as the reader should see it, taken from the first entry + /// that has one. + /// + /// From an entry rather than converted back from the key, so the + /// dictionary's own spelling is what shows — including the apostrophe in + /// `tǔ'ěrqísītǎn`, which a round trip through the key would lose. + public var readingDisplay: String { + entries.first { !$0.entry.reading.isEmpty }?.entry.readingDisplay ?? "" + } +} + +/// The dictionaries a lookup consults, in the reader's order. +/// +/// ### Nothing is ever deduplicated +/// +/// Two dictionaries defining the same word are two answers, not one with a +/// duplicate. Merging them would mean choosing whose wording survives, which +/// is both a licensing problem — CC BY-SA attribution is per work — and a +/// usability one, since the reason to install a second dictionary is to read +/// what it says. +/// +/// ### Glosses, readings and the word list are separate permissions +/// +/// A reader can turn off CC-CEDICT's English glosses and still need it: 77% +/// of BKRS entries carry no reading, so CC-CEDICT's ten thousand +/// single-character headwords are what put pinyin on the page in *both* +/// language modes. Treating "enabled" as one switch would let a Russian +/// reader turn off the thing their ruby depends on. So the three uses are +/// three flags, and `providesReadings` stays on for a dictionary disabled as +/// a gloss source. +public struct DictionarySet: Sendable { + /// One installed dictionary and what it is allowed to do. + public struct Member: Sendable { + public let slug: String + public let lookup: any DictionaryLookup + /// Whether its definitions are shown to the reader. + public let providesGlosses: Bool + /// Whether its readings feed the ruby above the text. + public let providesReadings: Bool + + public init( + slug: String, + lookup: any DictionaryLookup, + providesGlosses: Bool = true, + providesReadings: Bool = true, + ) { + self.slug = slug + self.lookup = lookup + self.providesGlosses = providesGlosses + self.providesReadings = providesReadings + } + } + + /// In the order the reader wants them, which decides which dictionary a + /// detail panel shows first. + public let members: [Member] + + public init(members: [Member]) { + self.members = members + } + + public var isEmpty: Bool { + members.isEmpty + } + + // MARK: - Looking up + + /// Every gloss-providing dictionary's entries for a word, grouped by + /// reading. + /// + /// Groups come out in the order their reading was first seen, walking the + /// dictionaries in the reader's order — so the first group is the first + /// reading of the dictionary they put first, which is the one a detail + /// panel should open on. Entries within a group keep that order too. + /// + /// A dictionary that throws is skipped rather than failing the lookup: + /// one unreadable container should cost its own answers, not the answers + /// of every other dictionary installed. + public func groups(for headword: String) -> (groups: [ReadingGroup], failures: [String]) { + var order: [String] = [] + var byKey: [String: [AttributedEntry]] = [:] + var failures: [String] = [] + + for member in members where member.providesGlosses { + let entries: [DictionaryEntry] + do { + entries = try member.lookup.entries(for: headword) + } catch { + failures.append(member.slug) + continue + } + for entry in entries { + let key = entry.readingKey + if byKey[key] == nil { + order.append(key) + } + byKey[key, default: []].append( + AttributedEntry(dictionary: member.slug, entry: entry), + ) + } + } + return ( + order.map { ReadingGroup(readingKey: $0, entries: byKey[$0] ?? []) }, + failures, + ) + } +} + +// MARK: - Reading the set as one dictionary + +/// So the reading composer, which only wants to know what a word sounds like, +/// can consult the whole set without knowing there is a set. +/// +/// This path uses `providesReadings`, not `providesGlosses` — the distinction +/// the two flags exist for. +extension DictionarySet: DictionaryLookup { + /// The entries a reading should be chosen from: the **first** + /// reading-providing dictionary that has any, not every dictionary's + /// concatenated. + /// + /// Concatenating them broke reading selection the moment a second + /// dictionary was installed, and visibly: 遭打 went from `zāo dǎ` to + /// `zāo dá`. The composer ranks candidates partly by how many senses + /// carry them, which is a sound signal *within* one dictionary and + /// meaningless across two — and especially so for BKRS, where one entry + /// per reading shares a single sense run by design, so all of 打's + /// readings have 57 senses and the tie falls to alphabetical order. A + /// dictionary with more senses per entry would simply outvote a better + /// answer from a dictionary the reader put first. + /// + /// So the reader's ordering decides, which is what they set it for, and + /// each dictionary's own ranking is applied to its own entries. + public func entries(for headword: String) throws -> [DictionaryEntry] { + for member in members where member.providesReadings { + if let entries = try? member.lookup.entries(for: headword), !entries.isEmpty { + return entries + } + } + return [] + } + + /// The first reading-providing dictionary that knows the character. + /// + /// First rather than merged: the ranking within a container is computed + /// over that container's whole headword set, and interleaving two + /// rankings produces an order neither dictionary would endorse. The + /// reader's ordering decides, which is what they set it for. + public func readings(forCharacter character: Character) throws -> [CharacterReading] { + for member in members where member.providesReadings { + if let readings = try? member.lookup.readings(forCharacter: character), + !readings.isEmpty + { + return readings + } + } + return [] + } +} diff --git a/Sources/HanReaderCore/Format/DSLCardBuilder.swift b/Sources/HanReaderCore/Format/DSLCardBuilder.swift index 6aef101..35a87a5 100644 --- a/Sources/HanReaderCore/Format/DSLCardBuilder.swift +++ b/Sources/HanReaderCore/Format/DSLCardBuilder.swift @@ -107,15 +107,80 @@ public struct DSLCardBuilder: Sendable { /// `_` is not a reading. Returning `[""]` for it — which a naive split /// does — produces an entry claiming to know a pronunciation that is the /// empty string, and 77% of this dictionary is that case. + /// + /// Three things the line can do that a split on `,` does not survive, + /// each measured over the 784,004 cards that have a reading at all: + /// + /// - **1,750 lines separate readings with `;`** rather than a comma. + /// - **287 carry markup**, a register note sitting between two readings: + /// `zhǔ; [i]в сочет. также[/i] zhù`. Lexing it away leaves the note + /// behind as text. + /// - and some of those have **no separator at all** — + /// `nāi [i]гуанчжоуск. диал. также[/i] lāi` — so splitting cannot find + /// the second reading however many separators it knows. + /// + /// So a reading is a maximal run of Latin-alphabet words. That takes + /// `nāi … lāi` as two readings and leaves the Russian between them out, + /// and it needs no list of the abbreviations this dictionary uses. It + /// recovers 4,114 readings from those 1,825 awkward lines where splitting + /// on separators alone recovers 3,810 and shows `в сочет. также zhù` + /// above a word for the rest. static func readings(in pinyin: String) -> [String] { - let trimmed = pinyin.trimmingCharacters(in: .whitespaces) - guard !trimmed.isEmpty, trimmed != "_" else { return [] } - return trimmed - .split(separator: ",") - .map { $0.trimmingCharacters(in: .whitespaces) } - .filter { !$0.isEmpty && $0 != "_" } + let text = DSLLexer.tokens(in: pinyin).tokens.reduce(into: "") { result, token in + if case let .text(text) = token { + result += text + } + } + + var readings: [String] = [] + // Two levels, and they are not the same thing. A comma or semicolon + // separates readings; a space inside one joins its syllables, as in + // `yībānxìng tǎolùn`. + for candidate in text.split(whereSeparator: { $0 == "," || $0 == ";" }) { + var run: [Substring] = [] + func endRun() { + guard !run.isEmpty else { return } + readings.append(run.joined(separator: " ")) + run = [] + } + for word in candidate.split(separator: " ") { + if isLatinScript(word) { + run.append(word) + } else { + endRun() + } + } + endRun() + } + return readings + } + + /// Whether a word is written in the alphabet pinyin is written in. + /// + /// `_` has no letters at all and so is not a reading either, which is how + /// the 77% of cards that carry no reading come out as none rather than as + /// one empty one. + static func isLatinScript(_ word: Substring) -> Bool { + var sawLetter = false + for scalar in word.unicodeScalars where scalar.properties.isAlphabetic { + sawLetter = true + guard latinRanges.contains(where: { $0.contains(scalar.value) }) else { + return false + } + } + return sawLetter } + /// The blocks pinyin is written in: ASCII, Latin-1 Supplement and Latin + /// Extended-A/B for `ǎ`, `ǔ` and `ǖ`, and Latin Extended Additional for + /// the `ḿ` that CC-CEDICT spells `m2`. + private static let latinRanges: [ClosedRange] = [ + 0x41 ... 0x5A, + 0x61 ... 0x7A, + 0xC0 ... 0x24F, + 0x1E00 ... 0x1EFF, + ] + /// Converts one diacritic reading into tokens. /// /// Falls back to a literal when the reading cannot be split into known diff --git a/Sources/HanReaderCore/Model/DictionaryEntry.swift b/Sources/HanReaderCore/Model/DictionaryEntry.swift index 4bd3a08..3a68f52 100644 --- a/Sources/HanReaderCore/Model/DictionaryEntry.swift +++ b/Sources/HanReaderCore/Model/DictionaryEntry.swift @@ -307,6 +307,14 @@ public struct DictionaryEntry: Hashable, Sendable { /// first of any kind. Used for one-line summaries, where "variant of 妳" /// is close to useless. public var summarySense: Sense? { - senses.first { $0.kind == .definition } ?? senses.first + // Nor one that says nothing. A BKRS card opens with a division + // heading — `[m1][b]I[/b] [p]гл.[/p][/m]` — whose whole text is the + // Roman numeral and the part of speech, both of which are lifted out + // of the gloss. Taken as the summary it renders as an empty line, so + // 爱's one-line answer would be blank while 32 senses sat behind it. + senses.first { $0.kind == .definition && !$0.gloss.text.isEmpty } + ?? senses.first { !$0.gloss.text.isEmpty } + ?? senses.first { $0.kind == .definition } + ?? senses.first } } diff --git a/Sources/HanReaderPersistence/AppDatabase.swift b/Sources/HanReaderPersistence/AppDatabase.swift index 4b7b9b2..24383da 100644 --- a/Sources/HanReaderPersistence/AppDatabase.swift +++ b/Sources/HanReaderPersistence/AppDatabase.swift @@ -47,6 +47,19 @@ public enum AppDatabase { self.audio = audio } + /// Where imported dictionary containers live. + /// + /// A directory rather than a list of paths, because scanning it is + /// what lets a container be found after the app's container UUID has + /// changed — which on iOS it does on every install and restore. + /// Derived from `dictionaries` rather than stored, so the two cannot + /// be configured inconsistently. + public var dictionaryDirectory: URL { + dictionaries + .deletingLastPathComponent() + .appendingPathComponent("dictionaries", isDirectory: true) + } + /// The standard locations inside the application's support directory. /// /// Identical code on both platforms: `FileManager` already returns the diff --git a/Sources/HanReaderPersistence/DictionaryCatalogue.swift b/Sources/HanReaderPersistence/DictionaryCatalogue.swift new file mode 100644 index 0000000..1386cd6 --- /dev/null +++ b/Sources/HanReaderPersistence/DictionaryCatalogue.swift @@ -0,0 +1,250 @@ +// HanReader — MIT licensed. See LICENSE. + +public import Foundation +import GRDB +public import HanReaderCore + +/// What the reader has decided about one dictionary. +/// +/// Three permissions rather than one switch, because the three uses are +/// genuinely separate: a reader can turn off CC-CEDICT's English definitions +/// and still need its readings, since 77% of BKRS entries carry none. +public struct DictionaryPreference: Hashable, Sendable { + public let slug: String + public let providesGlosses: Bool + public let providesReadings: Bool + public let providesLexicon: Bool + public let sortOrder: Int + + public init( + slug: String, + providesGlosses: Bool = true, + providesReadings: Bool = true, + providesLexicon: Bool = false, + sortOrder: Int = 0, + ) { + self.slug = slug + self.providesGlosses = providesGlosses + self.providesReadings = providesReadings + self.providesLexicon = providesLexicon + self.sortOrder = sortOrder + } +} + +/// An installed dictionary, as the reader's list shows it. +public struct InstalledDictionary: Hashable, Sendable { + /// From the container's own `meta`, never from a copy. + public let metadata: DictionaryMetadata + public let file: URL + /// Shipped with the app, so it cannot be removed. + public let isBuiltIn: Bool + + public let providesGlosses: Bool + public let providesReadings: Bool + public let providesLexicon: Bool + public let sortOrder: Int + + public var slug: String { + metadata.slug + } +} + +/// A container that is present but cannot be read. +/// +/// Listed rather than hidden: a reader whose dictionary stopped working +/// should be told which one, and offered the chance to remove or re-import +/// it. Silently showing a shorter list is how "my Russian definitions +/// disappeared" becomes unanswerable. +public struct UnreadableDictionary: Hashable, Sendable { + public let file: URL + public let reason: String +} + +/// What is installed, in the order the reader wants it. +/// +/// ### Found by scanning, not by remembering where things were +/// +/// Every `.hanreaderdict` in the dictionaries directory is installed, plus +/// the one in the app bundle. There is no stored path, so there is no path to +/// go stale — which on iOS it would, since the container directory's UUID +/// changes across installs and restores. The predecessor stored absolute +/// audio paths and broke exactly this way. +/// +/// ### A dictionary describes itself +/// +/// Name, languages, licence and attribution come from the container's own +/// `meta` table every time it is listed. Copying them into a registry would +/// let the Acknowledgements screen show a licence the file contradicts, and +/// attribution reaching the reader intact is a CC BY-SA obligation rather +/// than a nicety. +public struct DictionaryCatalogue: Sendable { + public let installed: [InstalledDictionary] + public let unreadable: [UnreadableDictionary] + + public init(installed: [InstalledDictionary], unreadable: [UnreadableDictionary] = []) { + self.installed = installed + self.unreadable = unreadable + } + + /// The extension a compiled dictionary has. + public static let fileExtension = "hanreaderdict" + + public var isEmpty: Bool { + installed.isEmpty + } +} + +// MARK: - Building the set to look words up in + +extension DictionaryCatalogue { + /// Opens every installed dictionary that is allowed to answer something. + /// + /// A container that will not open is reported rather than dropped, for + /// the same reason the catalogue lists an unreadable file. + public func makeSet() -> (set: DictionarySet, unreadable: [UnreadableDictionary]) { + var members: [DictionarySet.Member] = [] + var failures: [UnreadableDictionary] = [] + + for entry in installed.sorted(by: { $0.sortOrder < $1.sortOrder }) { + guard entry.providesGlosses || entry.providesReadings else { continue } + do { + try members.append(DictionarySet.Member( + slug: entry.slug, + lookup: DictionaryContainer(contentsOf: entry.file), + providesGlosses: entry.providesGlosses, + providesReadings: entry.providesReadings, + )) + } catch { + failures.append(UnreadableDictionary( + file: entry.file, + reason: String(describing: error), + )) + } + } + return (DictionarySet(members: members), failures) + } + + /// The segmentation word list, from the dictionaries allowed to supply + /// one. + /// + /// Opt-in per dictionary, and off by default for one whose headwords are + /// not words. BKRS contributes **1,806,893** lexemes: a quarter of a + /// gigabyte held for the life of the process, and a word list in which + /// almost every run of Han characters is a word — which is the condition + /// under which maximum-matching produces confident nonsense. CC-CEDICT's + /// 163,746 are genuinely word-level and cost a few megabytes. + public func lexicon() -> Lexicon { + var lexemes: [Lexeme] = [] + for entry in installed.sorted(by: { $0.sortOrder < $1.sortOrder }) + where entry.providesLexicon + { + lexemes += (try? DictionaryContainer(contentsOf: entry.file).lexicon()) ?? [] + } + return Lexicon(lexemes) + } +} + +// MARK: - Reading the catalogue + +extension AppDatabase { + /// Lists what is installed, applying the reader's preferences. + /// + /// - Parameters: + /// - directory: where imported dictionaries live. Scanned; absent is + /// not an error, it is a reader who has imported nothing. + /// - bundled: the container shipped with the app, if there is one. + /// - preferredLanguages: the reader's own languages, used to order a + /// dictionary they have never had an opinion about — so a + /// Russian-locale reader sees a Russian dictionary first with no + /// configuration at all. + public static func catalogue( + in directory: URL, + bundled: URL?, + library: LibraryRepository, + preferredLanguages: [String] = Locale.preferredLanguages, + fileManager: FileManager = .default, + ) async throws + -> DictionaryCatalogue + { + var files: [(url: URL, isBuiltIn: Bool)] = bundled.map { [($0, true)] } ?? [] + let contents = (try? fileManager.contentsOfDirectory( + at: directory, + includingPropertiesForKeys: nil, + )) ?? [] + files += contents + .filter { $0.pathExtension == DictionaryCatalogue.fileExtension } + // Sorted so the list is the same on every launch, whatever order + // the file system happens to answer in. + .sorted { $0.lastPathComponent < $1.lastPathComponent } + .map { ($0, false) } + + let preferences = try await library.dictionaryPreferences() + var installed: [InstalledDictionary] = [] + var unreadable: [UnreadableDictionary] = [] + + for (url, isBuiltIn) in files { + do { + guard let metadata = try DictionaryContainer(contentsOf: url).metadata() else { + unreadable.append(UnreadableDictionary( + file: url, + reason: "the file has no dictionary metadata", + )) + continue + } + let stored = preferences[metadata.slug] + installed.append(InstalledDictionary( + metadata: metadata, + file: url, + isBuiltIn: isBuiltIn, + providesGlosses: stored?.providesGlosses ?? true, + providesReadings: stored?.providesReadings ?? true, + // A word list is opt-in, so a dictionary nobody has had + // an opinion about does not silently cost a quarter of a + // gigabyte. The bundled one is the exception: it is + // word-level, and it is the lexicon the reader has until + // they say otherwise. + providesLexicon: stored?.providesLexicon ?? isBuiltIn, + sortOrder: stored?.sortOrder ?? defaultOrder( + for: metadata, + preferredLanguages: preferredLanguages, + ), + )) + } catch { + unreadable.append(UnreadableDictionary( + file: url, + reason: String(describing: error), + )) + } + } + return DictionaryCatalogue(installed: installed, unreadable: unreadable) + } + + /// Where a dictionary the reader has not ordered should sit. + /// + /// By how well its gloss language matches theirs, most-preferred first. + /// A Russian-locale reader who imports БКРС should see Russian first + /// without touching a setting — and an English-locale reader importing + /// the same file should not. + /// + /// Ties keep their discovery order, which puts the bundled dictionary + /// first among equals because it is scanned first. + static func defaultOrder( + for metadata: DictionaryMetadata, + preferredLanguages: [String], + ) + -> Int + { + let gloss = metadata.glossLanguage.lowercased() + for (rank, language) in preferredLanguages.enumerated() { + let code = language.lowercased() + // `ru` matches `ru-RU`, and `zh-Hans` matches `zh-Hans-CN`, + // either way round — a reader whose system language is `ru-RU` + // wants a dictionary that says `ru`. + if code == gloss || code.hasPrefix(gloss + "-") || gloss.hasPrefix(code + "-") { + return rank + } + } + // Below everything the reader reads, but in a stable place. + return preferredLanguages.count + 1 + } +} diff --git a/Sources/HanReaderPersistence/DictionaryWriter.swift b/Sources/HanReaderPersistence/DictionaryWriter.swift index a91add5..15de644 100644 --- a/Sources/HanReaderPersistence/DictionaryWriter.swift +++ b/Sources/HanReaderPersistence/DictionaryWriter.swift @@ -452,8 +452,19 @@ public final class DictionaryWriter { // Folding the write-ahead log back in leaves one file rather than // three, which is what the reader copies around and what the size // reported to them should mean. + // + // Then out of write-ahead logging altogether, and that part is not + // cosmetic. A database whose header says WAL needs a shared-memory + // file, and a **read-only** connection cannot create one — so a + // finished container opened the way the app opens it failed with + // "unable to open database file", and the reader's imported + // dictionary simply did not appear. Write-ahead logging is an + // import-time choice, made for crash safety while the file is being + // built; a finished dictionary is read-only data and has no use for + // it. try dbQueue.writeWithoutTransaction { db in try db.execute(sql: "PRAGMA wal_checkpoint(TRUNCATE)") + try db.execute(sql: "PRAGMA journal_mode = DELETE") } } } diff --git a/Sources/HanReaderPersistence/LibraryRepository.swift b/Sources/HanReaderPersistence/LibraryRepository.swift index 5280dca..19f4a33 100644 --- a/Sources/HanReaderPersistence/LibraryRepository.swift +++ b/Sources/HanReaderPersistence/LibraryRepository.swift @@ -347,3 +347,86 @@ public enum LibraryError: Error, Sendable, Equatable, LocalizedError { } } } + +// MARK: - Dictionary preferences + +/// What the reader has decided about each dictionary. +/// +/// In the library database rather than the dictionary one: `dictionaries` is +/// derived data that can be deleted and rebuilt, while an ordering somebody +/// dragged into place cannot, and it is the half that belongs in a backup. +extension LibraryRepository { + /// What the reader has decided about each dictionary, by slug. + /// + /// A dictionary with no row is not disabled — it is one they have not had + /// an opinion about, and the catalogue gives it defaults from its own + /// gloss language against theirs. + public func dictionaryPreferences() async throws -> [String: DictionaryPreference] { + try await dbQueue.read { db in + try Row.fetchAll(db, sql: """ + SELECT slug, providesGlosses, providesReadings, providesLexicon, sortOrder + FROM dictionaryPreference + """).reduce(into: [:]) { result, row in + result[row["slug"] as String] = DictionaryPreference( + slug: row["slug"], + providesGlosses: row["providesGlosses"], + providesReadings: row["providesReadings"], + providesLexicon: row["providesLexicon"], + sortOrder: row["sortOrder"], + ) + } + } + } + + /// Records what the reader decided about one dictionary. + public func save(_ preference: DictionaryPreference) async throws { + try await dbQueue.write { db in + try db.execute(sql: """ + INSERT INTO dictionaryPreference + (slug, providesGlosses, providesReadings, providesLexicon, sortOrder) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(slug) DO UPDATE SET + providesGlosses = excluded.providesGlosses, + providesReadings = excluded.providesReadings, + providesLexicon = excluded.providesLexicon, + sortOrder = excluded.sortOrder + """, arguments: [ + preference.slug, + preference.providesGlosses, + preference.providesReadings, + preference.providesLexicon, + preference.sortOrder, + ]) + } + } + + /// Writes an order the reader dragged into place. + /// + /// Written for every slug in one transaction, so a list that was + /// reordered is never half reordered — the state a reader would see as + /// two dictionaries swapping places at random. + public func setDictionaryOrder(_ slugs: [String]) async throws { + try await dbQueue.write { db in + for (position, slug) in slugs.enumerated() { + try db.execute(sql: """ + INSERT INTO dictionaryPreference (slug, sortOrder) VALUES (?, ?) + ON CONFLICT(slug) DO UPDATE SET sortOrder = excluded.sortOrder + """, arguments: [slug, position]) + } + } + } + + /// Forgets what the reader decided about a dictionary they removed. + /// + /// Separate from deleting the file, because the two can disagree: a + /// reader who deletes a container and re-imports it should get their + /// ordering back, so this is called only when they remove it on purpose. + public func forgetDictionary(_ slug: String) async throws { + try await dbQueue.write { db in + try db.execute( + sql: "DELETE FROM dictionaryPreference WHERE slug = ?", + arguments: [slug], + ) + } + } +} diff --git a/Sources/HanReaderPersistence/LibrarySchema.swift b/Sources/HanReaderPersistence/LibrarySchema.swift index 46e49fd..68bf9e4 100644 --- a/Sources/HanReaderPersistence/LibrarySchema.swift +++ b/Sources/HanReaderPersistence/LibrarySchema.swift @@ -34,9 +34,55 @@ enum LibrarySchema { try createAppStateTable(db) } + migrator.registerMigration("v2_dictionary_preferences") { db in + try createDictionaryPreferenceTable(db) + } + return migrator } + // MARK: - v2: what the reader has decided about each dictionary + + /// The reader's choices about an installed dictionary, and nothing else. + /// + /// **In the library database, not the dictionary one.** `dictionaries` is + /// derived data that can be deleted and rebuilt; an ordering someone + /// dragged into place cannot. It is also the half that belongs in a + /// backup. + /// + /// **No path, and no copy of the dictionary's own metadata.** A container + /// is found by scanning the dictionaries directory, so there is no stored + /// path to go stale — which on iOS it would, since the container + /// directory's UUID changes across installs and restores. And its name, + /// licence and attribution are read from the container's own `meta` every + /// time, so the Acknowledgements screen cannot show a licence the file + /// contradicts. A row here is the reader's opinion; the file is the fact. + /// + /// A dictionary with no row is not disabled — it is one the reader has + /// not had an opinion about yet, and gets sensible defaults from its own + /// gloss language against theirs. + private static func createDictionaryPreferenceTable(_ db: Database) throws { + try db.create(table: "dictionaryPreference") { table in + table.column("slug", .text).notNull().primaryKey() + + // Three permissions rather than one switch. A reader can turn off + // CC-CEDICT's English glosses and still need it: 77% of BKRS + // entries carry no reading, so its single-character headwords are + // what put pinyin on the page in both language modes. + table.column("providesGlosses", .boolean).notNull().defaults(to: true) + table.column("providesReadings", .boolean).notNull().defaults(to: true) + // Whether its headwords feed the segmenter. Off by default for a + // dictionary whose headwords are not words: BKRS contributes + // 1,806,893 lexemes, which is both a quarter of a gigabyte in + // memory and a word list in which almost every run of Han + // characters is a word — the condition under which + // maximum-matching produces confident nonsense. + table.column("providesLexicon", .boolean).notNull().defaults(to: false) + + table.column("sortOrder", .integer).notNull() + } + } + // MARK: - v1 tables private static func createTextTable(_ db: Database) throws { diff --git a/Sources/HanReaderUI/HanReaderRootView.swift b/Sources/HanReaderUI/HanReaderRootView.swift index 54c24f2..f044198 100644 --- a/Sources/HanReaderUI/HanReaderRootView.swift +++ b/Sources/HanReaderUI/HanReaderRootView.swift @@ -26,6 +26,9 @@ public final class HanReaderLaunch { /// detail of this module. enum State { case loading + /// Claimed by the first window to call `start`, so a second one does + /// not open the library again while the first is still awaiting. + case opening case ready(AppServices, Settings, SpeechModel) case failed(String) } @@ -36,10 +39,18 @@ public final class HanReaderLaunch { /// Opens the databases. Safe to call repeatedly; only the first does /// anything, because every window's root view calls it on appear. - func start() { + /// + /// Asynchronous now that the dictionaries are a catalogue rather than one + /// bundled file: it scans a directory and reads each container's own + /// metadata, which is work that should not run on the main actor even + /// though it is only a few milliseconds. + func start() async { guard case .loading = state else { return } + // Claimed before awaiting, or two windows appearing together both see + // `.loading` and both open the library. + state = .opening do { - state = try .ready(AppServices.launch(), Settings(), SpeechModel()) + state = try await .ready(AppServices.launch(), Settings(), SpeechModel()) } catch { // Only a failure to open the *library* reaches here. A missing // or unreadable dictionary is handled inside `AppServices` and @@ -62,11 +73,11 @@ public struct HanReaderRootView: View { public var body: some View { switch launch.state { - case .loading: - // Opening a prepared dictionary container takes about a - // millisecond, so there is nothing worth showing a spinner for. + case .loading, .opening: + // Opening prepared dictionary containers takes a few + // milliseconds, so there is nothing worth showing a spinner for. Color.clear - .task { launch.start() } + .task { await launch.start() } case let .failed(message): ContentUnavailableView( "HanReader could not start", diff --git a/Sources/HanReaderUI/Models/ReaderModel.swift b/Sources/HanReaderUI/Models/ReaderModel.swift index 355809a..8d12af0 100644 --- a/Sources/HanReaderUI/Models/ReaderModel.swift +++ b/Sources/HanReaderUI/Models/ReaderModel.swift @@ -15,9 +15,16 @@ enum ReaderPhase: Equatable, Sendable { struct WordDetail: Equatable, Sendable { let word: String let reading: TokenReading? - let entries: [DictionaryEntry] + /// Every dictionary's answer, grouped by reading. + let groups: [ReadingGroup] - /// Whether the dictionary defines this word. + /// Flattened, for the places that want senses without caring which + /// dictionary or reading they belong to. + var entries: [DictionaryEntry] { + groups.flatMap { $0.entries.map(\.entry) } + } + + /// Whether any dictionary defines this word. /// /// A word can have a reading and no entry — composed from its /// characters — which is why "not found" is a property of the result @@ -25,7 +32,7 @@ struct WordDetail: Equatable, Sendable { /// reading on the floor: tapping a word the dictionary does not define /// removed the pinyin that was already visible above it. var isDefined: Bool { - !entries.isEmpty + groups.contains { !$0.entries.isEmpty } } } @@ -265,7 +272,7 @@ final class ReaderModel { detailTask?.cancel() detail = .loading(word) detailTask = Task { - let entries = await services.dictionary.entries(for: word) + let groups = await services.dictionary.groups(for: word) // Both checks are needed. Cancellation covers the task being // torn down; the state check covers a result arriving for a word // that is no longer the one being asked about. @@ -276,8 +283,11 @@ final class ReaderModel { // entry is the one composed from its characters. Falling // back to the entries covers a word tapped before its // reading was fetched. - reading: readings[word] ?? ReadingComposer.reading(from: entries), - entries: entries, + reading: readings[word] + ?? ReadingComposer.reading( + from: groups.flatMap { $0.entries.map(\.entry) }, + ), + groups: groups, )) } } diff --git a/Sources/HanReaderUI/Screens/ReaderScreen.swift b/Sources/HanReaderUI/Screens/ReaderScreen.swift index 33400c4..c50f14d 100644 --- a/Sources/HanReaderUI/Screens/ReaderScreen.swift +++ b/Sources/HanReaderUI/Screens/ReaderScreen.swift @@ -155,6 +155,7 @@ struct ReaderScreen: View { detail: detail, style: style, onSpeak: { speech.speak($0, suppressed: voiceOverEnabled) }, + dictionaryNames: services.dictionaryNames, ) .inspectorColumnWidth(min: 260, ideal: 320, max: 480) } else { diff --git a/Sources/HanReaderUI/Screens/WordDetailContent.swift b/Sources/HanReaderUI/Screens/WordDetailContent.swift index 523ce07..0584be2 100644 --- a/Sources/HanReaderUI/Screens/WordDetailContent.swift +++ b/Sources/HanReaderUI/Screens/WordDetailContent.swift @@ -13,14 +13,18 @@ struct WordDetailContent: View { let detail: WordDetail let style: ReaderStyle let onSpeak: (String) -> Void + /// Slug to display name, so an answer can say which dictionary it came + /// from. Attribution is a licence obligation for CC BY-SA data, not a + /// nicety, and it has to reach the reader where they read the gloss. + var dictionaryNames: [String: String] = [:] var body: some View { ScrollView { VStack(alignment: .leading, spacing: 20) { header if detail.isDefined { - ForEach(Array(detail.entries.enumerated()), id: \.offset) { _, entry in - entryView(entry) + ForEach(Array(detail.groups.enumerated()), id: \.offset) { _, group in + groupView(group) } } else { Text("No entry in the dictionary for this word.") @@ -66,33 +70,86 @@ struct WordDetailContent: View { } } - private func entryView(_ entry: DictionaryEntry) -> some View { - VStack(alignment: .leading, spacing: 6) { + /// One reading, with every dictionary's answer for it. + /// + /// The reading heads the group rather than repeating on each answer: + /// 了's two dictionaries both say `le`, and printing it twice implies + /// they are two readings. + private func groupView(_ group: ReadingGroup) -> some View { + VStack(alignment: .leading, spacing: 10) { HStack(spacing: 8) { - Text(verbatim: entry.readingDisplay) + Text(verbatim: group.readingDisplay) .font(.headline) - if entry.headword.scriptsDiffer { - // Shown only when the two forms actually differ, which - // is true for 63% of CC-CEDICT. Printing the same - // characters twice for the rest would be noise. - Text(verbatim: entry.headword.traditionalOrSimplified) + if let traditional = Self.traditional(in: group) { + // Shown only when the two forms actually differ, which is + // true for 63% of CC-CEDICT. Printing the same characters + // twice for the rest would be noise. + Text(verbatim: traditional) .font(.subheadline) .foregroundStyle(.secondary) .accessibilityLabel(Text("Traditional form")) } } - ForEach(entry.senses) { sense in - HStack(alignment: .firstTextBaseline, spacing: 8) { - Text(verbatim: "\(sense.id + 1).") - .font(.callout.monospacedDigit()) - .foregroundStyle(.tertiary) - Text(verbatim: sense.gloss.text) - .font(.callout) + ForEach(Array(group.entries.enumerated()), id: \.offset) { _, attributed in + answerView(attributed) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + } + + private func answerView(_ attributed: AttributedEntry) -> some View { + VStack(alignment: .leading, spacing: 6) { + if let name = dictionaryNames[attributed.dictionary] { + Text(verbatim: name) + .font(.caption.weight(.medium)) + .foregroundStyle(.secondary) + .accessibilityLabel(Text("From \(name)")) + } + ForEach(attributed.entry.senses) { sense in + senseView(sense) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + } + + /// One sense, indented by how deeply the dictionary nested it. + /// + /// The indent is the whole reason `Sense` carries a level: BKRS writes + /// Roman-numeral divisions with numbered senses beneath them and examples + /// beneath those, and flattening that is what made the predecessor's + /// definitions unreadable. + private func senseView(_ sense: Sense) -> some View { + HStack(alignment: .firstTextBaseline, spacing: 8) { + Text(verbatim: sense.label ?? "\(sense.id + 1).") + .font(.callout.monospacedDigit()) + .foregroundStyle(.tertiary) + VStack(alignment: .leading, spacing: 2) { + if !sense.partOfSpeech.isEmpty || !sense.registers.isEmpty { + Text(verbatim: (sense.partOfSpeech + sense.registers) + .joined(separator: " ")) + .font(.caption.italic()) + .foregroundStyle(.secondary) + } + Text(verbatim: sense.gloss.text) + .font(.callout) + .textSelection(.enabled) + ForEach(Array(sense.examples.enumerated()), id: \.offset) { _, example in + Text(verbatim: example.raw) + .font(.caption) + .foregroundStyle(.secondary) .textSelection(.enabled) } } } - .frame(maxWidth: .infinity, alignment: .leading) + .padding(.leading, Double(sense.level) * 14) + } + + /// The traditional spelling, if any dictionary in the group gives one + /// that differs. + private static func traditional(in group: ReadingGroup) -> String? { + group.entries + .first { $0.entry.headword.scriptsDiffer }? + .entry.headword.traditionalOrSimplified } /// Optional rather than an empty string: a reading the dictionary diff --git a/Sources/HanReaderUI/Services/AppServices.swift b/Sources/HanReaderUI/Services/AppServices.swift index a520ec6..d37d925 100644 --- a/Sources/HanReaderUI/Services/AppServices.swift +++ b/Sources/HanReaderUI/Services/AppServices.swift @@ -20,42 +20,95 @@ final class AppServices: Sendable { let dictionary: DictionaryService let segmenter: DocumentSegmenter let locations: AppDatabase.Locations - /// Nil when no dictionary could be opened, which is not fatal: the reader - /// still segments and still displays text, just with no annotations. - let dictionaryMetadata: DictionaryMetadata? + /// Every dictionary the reader has, in their order, with what each is + /// allowed to do and whether it opened. + /// + /// Empty is not fatal: the reader still segments and still displays text, + /// just with no annotations. + let catalogue: DictionaryCatalogue + + /// The first gloss-providing dictionary, for the places that show one + /// dictionary's name — which is a stopgap until the acknowledgements + /// screen lists them all. + var dictionaryMetadata: DictionaryMetadata? { + catalogue.installed + .sorted { $0.sortOrder < $1.sortOrder } + .first { $0.providesGlosses }? + .metadata + } + + /// Slug to display name, for labelling which dictionary an answer came + /// from. Attribution is a licence obligation for CC BY-SA data, so the + /// name has to be available wherever a gloss is shown. + var dictionaryNames: [String: String] { + catalogue.installed.reduce(into: [:]) { names, entry in + names[entry.slug] = entry.metadata.displayName + } + } private init( library: LibraryRepository, dictionary: DictionaryService, segmenter: DocumentSegmenter, locations: AppDatabase.Locations, - dictionaryMetadata: DictionaryMetadata?, + catalogue: DictionaryCatalogue, ) { self.library = library self.dictionary = dictionary self.segmenter = segmenter self.locations = locations - self.dictionaryMetadata = dictionaryMetadata + self.catalogue = catalogue } - /// Opens the real library and the bundled dictionary. + /// Opens the real library and every dictionary the reader has. /// - /// The dictionary is compiled at build time and shipped in the bundle, so - /// this is a file open rather than an import — around a millisecond - /// against the 2.6 seconds parsing the source text would take. That is - /// the whole reason `hanreader-dictgen` exists. - static func launch(bundle: Bundle = .main) throws -> AppServices { + /// The bundled one is compiled at build time and shipped in the app, so + /// that one is a file open rather than an import — around a millisecond + /// against the 2.6 seconds parsing its source would take, which is the + /// whole reason `hanreader-dictgen` exists. Anything else was imported by + /// the reader and is found by scanning, so there is no stored path to go + /// stale. + static func launch(bundle: Bundle = .main) async throws -> AppServices { let locations = try AppDatabase.Locations.standard() let library = try LibraryRepository(locations: locations) - let container = Self.openBundledDictionary(in: bundle) - let lookup: any DictionaryLookup = container ?? EmptyDictionary() + let catalogue = try await AppDatabase.catalogue( + in: locations.dictionaryDirectory, + bundled: bundle.url( + forResource: bundledDictionaryName, + withExtension: DictionaryCatalogue.fileExtension, + ), + library: library, + ) + for failure in catalogue.unreadable { + Log.error("services", "\(failure.file.lastPathComponent): \(failure.reason)") + } + + let (set, unopened) = catalogue.makeSet() + for failure in unopened { + Log.error("services", "\(failure.file.lastPathComponent): \(failure.reason)") + } + if set.isEmpty { + // Not fatal. Without a dictionary the reader still opens texts, + // segments them and displays them; it just cannot annotate or + // define anything. Refusing to launch would turn a degraded + // reader into no reader. + Log.error("services", "no dictionary could be opened") + } return AppServices( library: library, - dictionary: DictionaryService(lookup: lookup), - segmenter: Self.makeSegmenter(container: container), + dictionary: DictionaryService(lookup: set), + // The dictionary's own word list, so a word the segmenter finds + // is a word the reader can look up. That removes the prototype's + // silent mismatch, where the tokenizer produced a token no + // dictionary defined and the panel reported "not found" for what + // was really a segmentation error. + segmenter: DocumentSegmenter( + tokenizer: TextSegmenter.system(), + lexicon: catalogue.lexicon(), + ), locations: locations, - dictionaryMetadata: container.flatMap { try? $0.metadata() }, + catalogue: catalogue, ) } @@ -84,50 +137,13 @@ final class AppServices: Sendable { dictionaries: root.appendingPathComponent("dictionaries.sqlite"), audio: root, ), - dictionaryMetadata: nil, + catalogue: DictionaryCatalogue(installed: []), ) } // MARK: - Construction - private static func openBundledDictionary(in bundle: Bundle) -> DictionaryContainer? { - guard let url = bundle.url( - forResource: bundledDictionaryName, - withExtension: bundledDictionaryExtension, - ) else { - Log.error("services", "the bundled dictionary is missing from the app bundle") - return nil - } - do { - return try DictionaryContainer(contentsOf: url) - } catch { - // Not fatal. Without a dictionary the reader still opens texts, - // segments them character by character and displays them; it just - // cannot annotate or define anything. Refusing to launch would - // turn a degraded reader into no reader. - Log.error("services", "could not open the bundled dictionary: \(error)") - return nil - } - } - - private static func makeSegmenter(container: DictionaryContainer?) -> DocumentSegmenter { - // The segmentation lexicon is the dictionary's own, so a word the - // segmenter finds is a word the reader can look up. That removes the - // prototype's silent mismatch, where the tokenizer produced a token - // no dictionary defined and the panel reported "not found" for what - // was really a segmentation error. - var lexemes: [Lexeme] = [] - if let container { - lexemes = (try? container.lexicon()) ?? [] - } - return DocumentSegmenter( - tokenizer: TextSegmenter.system(), - lexicon: Lexicon(lexemes), - ) - } - static let bundledDictionaryName = "cedict" - static let bundledDictionaryExtension = "hanreaderdict" } /// A dictionary that knows nothing. diff --git a/Sources/HanReaderUI/Services/DictionaryService.swift b/Sources/HanReaderUI/Services/DictionaryService.swift index 188b164..7d299e4 100644 --- a/Sources/HanReaderUI/Services/DictionaryService.swift +++ b/Sources/HanReaderUI/Services/DictionaryService.swift @@ -25,6 +25,13 @@ import HanReaderCore /// a screenful is one actor hop rather than four hundred. actor DictionaryService { private let lookup: any DictionaryLookup + /// The dictionaries a definition may come from, in the reader's order. + /// + /// Separate from `lookup`, which is the *reading* path: a reader can turn + /// off CC-CEDICT's English definitions and still need its readings, since + /// 77% of BKRS entries carry none. Nil when the service was built from a + /// single lookup, as the tests and previews do. + private let set: DictionarySet? private var readingCache: BoundedCache private var entryCache: BoundedCache @@ -47,10 +54,43 @@ actor DictionaryService { /// annotated and tapping it does not query again. init(lookup: some DictionaryLookup, capacity: Int = 4096, entryCapacity: Int = 1024) { self.lookup = lookup + set = lookup as? DictionarySet readingCache = BoundedCache(capacity: capacity) entryCache = BoundedCache(capacity: entryCapacity) } + /// Every gloss-providing dictionary's answer for a word, grouped by + /// reading and labelled with where it came from. + /// + /// Grouped because 了 is `le` in one dictionary and `liǎo` in another, + /// and a reader tapping it should see two readings with two answers each + /// rather than four entries with no indication which are the same word. + /// Labelled because CC-CEDICT is CC BY-SA and attribution has to reach + /// the reader at the point they read the gloss. + func groups(for word: String) -> [ReadingGroup] { + guard let set else { + // Built from a single lookup, so there is nothing to attribute + // and nothing to merge; one unnamed group keeps the caller's + // shape the same either way. + let entries = entries(for: word) + guard !entries.isEmpty else { return [] } + return Dictionary(grouping: entries, by: \.readingKey) + .sorted { $0.key < $1.key } + .map { key, entries in + ReadingGroup( + readingKey: key, + entries: entries.map { AttributedEntry(dictionary: "", entry: $0) }, + ) + } + } + let result = set.groups(for: word) + for slug in result.failures where !hasReportedFailure { + hasReportedFailure = true + Log.error("dictionary", "\(slug) could not be read") + } + return result.groups + } + // MARK: - Readings /// Readings for a window of words, as one batch. diff --git a/Tests/HanReaderCoreTests/CEDICTParserTests.swift b/Tests/HanReaderCoreTests/CEDICTParserTests.swift index 50eff29..4384b61 100644 --- a/Tests/HanReaderCoreTests/CEDICTParserTests.swift +++ b/Tests/HanReaderCoreTests/CEDICTParserTests.swift @@ -212,6 +212,35 @@ struct CEDICTSenseTests { #expect(entry.summarySense?.gloss.text == "an actual meaning") } + /// Nor one that says nothing. A BKRS card opens with a division heading + /// whose whole text is a Roman numeral and a part of speech, both lifted + /// out of the gloss — so taken as the summary it renders as a blank line + /// while thirty-two senses sit behind it. + @Test("Summary skips a sense with nothing in it") + func summarySkipsEmptyGloss() { + let entry = DictionaryEntry( + headword: Headword(simplified: "爱", traditional: nil), + reading: Pinyin.parse(numeric: "ai4"), + senses: [ + Sense(id: 0, kind: .definition, gloss: Gloss(text: ""), label: "I"), + Sense(id: 1, kind: .definition, gloss: Gloss(text: "любить"), level: 1), + ], + ) + #expect(entry.summarySense?.gloss.text == "любить") + } + + /// An entry whose senses are all empty still summarises as one of them, + /// rather than claiming the word is undefined. + @Test("Summary falls back to an empty sense rather than to nothing") + func summaryFallsBack() { + let entry = DictionaryEntry( + headword: Headword(simplified: "x", traditional: nil), + reading: [], + senses: [Sense(id: 0, kind: .definition, gloss: Gloss(text: ""), label: "I")], + ) + #expect(entry.summarySense != nil) + } + /// Senses are not split on `;`. Only about 2% of entries contain one, and /// splitting mangles text like `Down's syndrome; trisomy 21`. @Test("Semicolons do not split a sense") diff --git a/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift b/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift index 4849d92..d64145c 100644 --- a/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift +++ b/Tests/HanReaderCoreTests/DSLCardBuilderTests.swift @@ -141,6 +141,38 @@ struct DSLCardBuilderTests { #expect(result.entries.allSatisfy { $0.senses.count == 1 }) } + /// 1,750 reading lines separate their readings with a semicolon rather + /// than a comma. + @Test("A semicolon separates readings too") + func semicolonSeparatesReadings() { + let result = build(headword: "獭", pinyin: "chán; tán; dàn", "[m1]текст[/m]") + #expect(result.entries.map(\.readingDisplay) == ["chán", "tán", "dàn"]) + } + + /// 287 reading lines carry a register note between two readings. Lexed, + /// the markup goes but the Russian stays, and without a script test + /// `в сочет. также zhù` ends up above the word on the page — which is + /// exactly what the app showed when BKRS first fed the ruby. + @Test("A register note between two readings is not a reading", arguments: [ + ("zhǔ; [i]в сочет. также[/i] zhù", ["zhǔ", "zhù"]), + ("lùzhóu,[c][i] разг.[/c] [c][/i][/c]liùzhóu", ["lùzhóu", "liùzhóu"]), + // No separator at all, so splitting alone could never find the + // second reading however many separators it knew. + ("nāi [c][i]гуанчжоуск. диал. также[/c] [c][/i][/c]lāi", ["nāi", "lāi"]), + ]) + func registerNotesAreNotReadings(line: String, expected: [String]) { + #expect(DSLCardBuilder.readings(in: line) == expected) + } + + /// A space inside one reading joins its syllables; a comma between two + /// separates them. Conflating the two made `le, liǎo` a single reading + /// called `le liǎo`. + @Test("A space inside a reading is not a separator") + func spacesJoinSyllables() { + #expect(DSLCardBuilder.readings(in: "yībānxìng tǎolùn") == ["yībānxìng tǎolùn"]) + #expect(DSLCardBuilder.readings(in: "le, liǎo") == ["le", "liǎo"]) + } + @Test("A reading that cannot be split still displays") func unsplittableReading() { let result = build(pinyin: "zzzqqq", "[m1]текст[/m]") diff --git a/Tests/HanReaderCoreTests/DictionarySetTests.swift b/Tests/HanReaderCoreTests/DictionarySetTests.swift new file mode 100644 index 0000000..9e3506b --- /dev/null +++ b/Tests/HanReaderCoreTests/DictionarySetTests.swift @@ -0,0 +1,315 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import Testing +@testable import HanReaderCore + +// MARK: - Fixtures + +/// A dictionary that answers from a table, so the merge can be tested without +/// a database. +private struct FakeDictionary: DictionaryLookup { + var entries: [String: [DictionaryEntry]] = [:] + var characters: [Character: [CharacterReading]] = [:] + /// Set to make every lookup fail, as an unreadable container would. + var broken = false + + func entries(for headword: String) throws -> [DictionaryEntry] { + if broken { + throw FakeFailure() + } + return entries[headword] ?? [] + } + + func readings(forCharacter character: Character) throws -> [CharacterReading] { + if broken { + throw FakeFailure() + } + return characters[character] ?? [] + } +} + +private struct FakeFailure: Error {} + +private func entry( + _ simplified: String, + reading: String, + gloss: String, +) + -> DictionaryEntry +{ + DictionaryEntry( + headword: Headword(simplified: simplified, traditional: nil), + reading: Pinyin.parse(numeric: reading), + senses: [Sense( + id: 0, + kind: .definition, + gloss: Gloss(text: gloss, references: [], styles: []), + level: 0, + parent: nil, + label: nil, + partOfSpeech: [], + registers: [], + examples: [], + )], + ) +} + +private func member( + _ slug: String, + _ entries: [String: [DictionaryEntry]], + characters: [Character: [CharacterReading]] = [:], + glosses: Bool = true, + readings: Bool = true, + broken: Bool = false, +) + -> DictionarySet.Member +{ + DictionarySet.Member( + slug: slug, + lookup: FakeDictionary(entries: entries, characters: characters, broken: broken), + providesGlosses: glosses, + providesReadings: readings, + ) +} + +@Suite("Dictionary set") +struct DictionarySetTests { + // MARK: - Grouping by reading + + /// The crux of consulting two dictionaries at once. BKRS writes `liǎo` + /// and CC-CEDICT `liao3`; both normalise to the same key, so a reader + /// tapping 了 sees two readings with two answers each, not four entries + /// with no indication which are the same word. + @Test("Two dictionaries' entries group by reading, not by dictionary") + func groupsByReading() { + let set = DictionarySet(members: [ + member("cc-cedict", ["了": [ + entry("了", reading: "le5", gloss: "completed action marker"), + entry("了", reading: "liao3", gloss: "to understand"), + ]]), + member("dabkrs", ["了": [ + entry("了", reading: "le5", gloss: "частица"), + entry("了", reading: "liao3", gloss: "понимать"), + ]]), + ]) + + let result = set.groups(for: "了") + #expect(result.failures.isEmpty) + #expect(result.groups.count == 2) + #expect(result.groups.map(\.readingKey) == ["le5", "liao3"]) + #expect(result.groups[0].entries.map(\.dictionary) == ["cc-cedict", "dabkrs"]) + #expect(result.groups[0].entries.map(\.entry.senses[0].gloss.text) + == ["completed action marker", "частица"]) + } + + /// Two dictionaries defining the same word are two answers. Merging them + /// would mean choosing whose wording survives, which is a licensing + /// problem as well as a usability one — the reason to install a second + /// dictionary is to read what it says. + @Test("Identical entries from two dictionaries are both kept") + func neverDeduplicated() { + let same = entry("爱", reading: "ai4", gloss: "to love") + let set = DictionarySet(members: [ + member("one", ["爱": [same]]), + member("two", ["爱": [same]]), + ]) + let groups = set.groups(for: "爱").groups + #expect(groups.count == 1) + #expect(groups[0].entries.count == 2) + #expect(groups[0].entries.map(\.dictionary) == ["one", "two"]) + } + + /// 77% of BKRS entries carry no reading at all, so they group together + /// rather than each becoming a group of one. + @Test("Entries with no reading form one group") + func entriesWithoutReadings() { + let set = DictionarySet(members: [ + member("dabkrs", ["碟仙": [ + DictionaryEntry( + headword: Headword(simplified: "碟仙", traditional: nil), + reading: [], + senses: entry("碟仙", reading: "", gloss: "спиритическая доска").senses, + ), + DictionaryEntry( + headword: Headword(simplified: "碟仙", traditional: nil), + reading: [], + senses: entry("碟仙", reading: "", gloss: "второе").senses, + ), + ]]), + ]) + let groups = set.groups(for: "碟仙").groups + #expect(groups.count == 1) + #expect(groups[0].readingKey.isEmpty) + #expect(groups[0].entries.count == 2) + #expect(groups[0].readingDisplay.isEmpty) + } + + /// The reader's order decides which answer a detail panel opens on, so a + /// Russian-preferring reader who puts БКРС first sees Russian first. + @Test("The reader's order decides which group comes first") + func readerOrderDecides() { + let entries = ["爱": [entry("爱", reading: "ai4", gloss: "to love")]] + let russian = ["爱": [entry("爱", reading: "ai4", gloss: "любить")]] + + let englishFirst = DictionarySet(members: [ + member("cc-cedict", entries), member("dabkrs", russian), + ]) + let russianFirst = DictionarySet(members: [ + member("dabkrs", russian), member("cc-cedict", entries), + ]) + #expect(englishFirst.groups(for: "爱").groups[0].entries[0].dictionary == "cc-cedict") + #expect(russianFirst.groups(for: "爱").groups[0].entries[0].dictionary == "dabkrs") + } + + /// The display spelling comes from an entry, not from converting the key + /// back — a round trip would lose the apostrophe the dictionary wrote. + @Test("The displayed reading is the dictionary's own spelling") + func readingDisplayKeepsItsSpelling() { + let set = DictionarySet(members: [ + member("dabkrs", ["西安": [entry("西安", reading: "Xi1 an1", gloss: "Сиань")]]), + ]) + #expect(set.groups(for: "西安").groups[0].readingDisplay == "Xī'ān") + } + + @Test("A word nothing defines produces no groups") + func noGroups() { + let set = DictionarySet(members: [member("one", [:])]) + #expect(set.groups(for: "爱").groups.isEmpty) + } + + // MARK: - One unreadable container + + /// One bad container should cost its own answers, not everyone else's. + @Test("A dictionary that fails is skipped and reported") + func brokenDictionaryIsSkipped() { + let set = DictionarySet(members: [ + member("broken", ["爱": [entry("爱", reading: "ai4", gloss: "x")]], broken: true), + member("good", ["爱": [entry("爱", reading: "ai4", gloss: "любить")]]), + ]) + let result = set.groups(for: "爱") + #expect(result.failures == ["broken"]) + #expect(result.groups.count == 1) + #expect(result.groups[0].entries.map(\.dictionary) == ["good"]) + } + + // MARK: - Three permissions, not one switch + + /// A reader can turn off CC-CEDICT's English glosses and still need it: + /// 77% of BKRS entries carry no reading, so its single-character + /// headwords are what put pinyin on the page. One "enabled" switch would + /// let them turn off the thing their ruby depends on. + @Test("A dictionary disabled for glosses still supplies readings") + func glossesOffReadingsOn() throws { + let set = DictionarySet(members: [ + member("dabkrs", ["爱": [entry("爱", reading: "ai4", gloss: "любить")]]), + member( + "cc-cedict", + ["爱": [entry("爱", reading: "ai4", gloss: "to love")]], + characters: ["爱": [CharacterReading( + character: "爱", + numeric: "ai4", + display: "ài", + rank: 0, + )]], + glosses: false, + ), + ]) + + // Not in the reader's definitions… + let groups = set.groups(for: "爱").groups + #expect(groups.count == 1) + #expect(groups[0].entries.map(\.dictionary) == ["dabkrs"]) + + // …but still above the word. One dictionary's entries, not both + // concatenated: the first that has any, which here is БКРС. + #expect(try set.readings(forCharacter: "爱").map(\.display) == ["ài"]) + #expect(try set.entries(for: "爱").count == 1) + } + + @Test("A dictionary disabled for readings still supplies glosses") + func readingsOffGlossesOn() throws { + let set = DictionarySet(members: [ + member( + "dabkrs", + ["爱": [entry("爱", reading: "ai4", gloss: "любить")]], + characters: ["爱": [CharacterReading( + character: "爱", + numeric: "ai4", + display: "ài", + rank: 0, + )]], + readings: false, + ), + ]) + #expect(set.groups(for: "爱").groups.count == 1) + #expect(try set.readings(forCharacter: "爱").isEmpty) + #expect(try set.entries(for: "爱").isEmpty) + } + + /// Interleaving two containers' rankings produces an order neither + /// dictionary would endorse, so the reader's ordering decides instead. + @Test("Character readings come from the first dictionary that has any") + func characterReadingsAreNotInterleaved() throws { + let first = CharacterReading(character: "行", numeric: "xing2", display: "xíng", rank: 0) + let second = CharacterReading(character: "行", numeric: "hang2", display: "háng", rank: 0) + let set = DictionarySet(members: [ + member("empty", [:]), + member("one", [:], characters: ["行": [first]]), + member("two", [:], characters: ["行": [second]]), + ]) + #expect(try set.readings(forCharacter: "行") == [first]) + } + + /// Ranking a reading partly by how many senses carry it is sound within + /// one dictionary and meaningless across two. BKRS gives every reading of + /// a card the same sense run by design, so all of 打's readings have 57 + /// senses and the tie falls to alphabetical order — which is how adding a + /// second dictionary turned 遭打 from `zāo dǎ` into `zāo dá` on screen. + @Test("A reading comes from one dictionary, not from both at once") + func readingsComeFromOneDictionary() throws { + let cedict = ["打": [ + entry("打", reading: "da2", gloss: "dozen"), + entry("打", reading: "da3", gloss: "to beat"), + ]] + // Same two readings, equal sense counts — the shape that outvoted + // CC-CEDICT's ranking when the two were concatenated. + let bkrs = ["打": [ + entry("打", reading: "da2", gloss: "дюжина"), + entry("打", reading: "da3", gloss: "бить"), + ]] + + let cedictFirst = DictionarySet(members: [ + member("cc-cedict", cedict), member("dabkrs", bkrs), + ]) + #expect(try cedictFirst.entries(for: "打").map(\.senses[0].gloss.text) + == ["dozen", "to beat"]) + + let bkrsFirst = DictionarySet(members: [ + member("dabkrs", bkrs), member("cc-cedict", cedict), + ]) + #expect(try bkrsFirst.entries(for: "打").map(\.senses[0].gloss.text) + == ["дюжина", "бить"]) + } + + /// A dictionary that does not have the word is skipped rather than + /// ending the search, or the first dictionary would be the only one that + /// ever supplied a reading. + @Test("The first dictionary that has the word supplies its reading") + func firstWithTheWord() throws { + let set = DictionarySet(members: [ + member("sparse", [:]), + member("full", ["爱": [entry("爱", reading: "ai4", gloss: "любить")]]), + ]) + #expect(try set.entries(for: "爱").count == 1) + } + + @Test("An empty set answers nothing rather than failing") + func emptySet() throws { + let set = DictionarySet(members: []) + #expect(set.isEmpty) + #expect(set.groups(for: "爱").groups.isEmpty) + #expect(try set.entries(for: "爱").isEmpty) + #expect(try set.readings(forCharacter: "爱").isEmpty) + } +} diff --git a/Tests/HanReaderPersistenceTests/DictionaryCatalogueTests.swift b/Tests/HanReaderPersistenceTests/DictionaryCatalogueTests.swift new file mode 100644 index 0000000..59c0ea4 --- /dev/null +++ b/Tests/HanReaderPersistenceTests/DictionaryCatalogueTests.swift @@ -0,0 +1,317 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation +import HanReaderCore +import Testing +@testable import HanReaderPersistence + +// MARK: - Fixtures + +/// A throwaway directory with real compiled containers in it. +private struct Workspace: ~Copyable { + let root: URL + let library: LibraryRepository + + init() throws { + root = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("hanreader-catalogue-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + library = try LibraryRepository.inMemory() + } + + deinit { + try? FileManager.default.removeItem(at: root) + } + + var directory: URL { + root.appendingPathComponent("dictionaries", isDirectory: true) + } + + /// Compiles a one-word dictionary, so the catalogue has something real to + /// open and read metadata from. + @discardableResult + func install( + slug: String, + glossLanguage: String, + word: String = "爱", + gloss: String = "to love", + in folder: URL? = nil, + ) throws + -> URL + { + let url = (folder ?? directory) + .appendingPathComponent("\(slug).\(DictionaryCatalogue.fileExtension)") + try DictionaryContainer.write( + entries: [DictionaryEntry( + headword: Headword(simplified: word, traditional: nil), + reading: Pinyin.parse(numeric: "ai4"), + senses: [Sense( + id: 0, + kind: .definition, + gloss: Gloss(text: gloss, references: [], styles: []), + level: 0, + parent: nil, + label: nil, + partOfSpeech: [], + registers: [], + examples: [], + )], + )], + metadata: DictionaryMetadata( + slug: slug, + displayName: slug.uppercased(), + format: "test", + indexLanguage: "zh-Hans", + glossLanguage: glossLanguage, + licence: "none", + attribution: "none", + parserVersion: 1, + entryCount: 1, + ), + to: url, + ) + return url + } + + func catalogue(preferredLanguages: [String] = ["en"]) async throws -> DictionaryCatalogue { + try await AppDatabase.catalogue( + in: directory, + bundled: nil, + library: library, + preferredLanguages: preferredLanguages, + ) + } +} + +@Suite("Dictionary catalogue") +struct DictionaryCatalogueTests { + // MARK: - Discovery + + /// Found by scanning, not by remembering where things were. There is no + /// stored path, so there is no path to go stale — which on iOS it would, + /// since the container directory's UUID changes across installs. + @Test("Every container in the directory is installed") + func discovery() async throws { + let workspace = try Workspace() + try workspace.install(slug: "cc-cedict", glossLanguage: "en") + try workspace.install(slug: "dabkrs", glossLanguage: "ru") + + let catalogue = try await workspace.catalogue() + #expect(catalogue.installed.map(\.slug).sorted() == ["cc-cedict", "dabkrs"]) + #expect(catalogue.unreadable.isEmpty) + } + + /// A dictionary describes itself. Copying its name and licence into a + /// registry would let the Acknowledgements screen show a licence the file + /// contradicts, and attribution reaching the reader intact is a CC BY-SA + /// obligation. + @Test("A dictionary's own metadata is what the list shows") + func metadataComesFromTheFile() async throws { + let workspace = try Workspace() + try workspace.install(slug: "dabkrs", glossLanguage: "ru") + + let entry = try #require(try await workspace.catalogue().installed.first) + #expect(entry.metadata.displayName == "DABKRS") + #expect(entry.metadata.glossLanguage == "ru") + #expect(!entry.isBuiltIn) + } + + @Test("A reader who has imported nothing gets an empty catalogue") + func nothingInstalled() async throws { + let workspace = try Workspace() + let catalogue = try await workspace.catalogue() + #expect(catalogue.isEmpty) + #expect(catalogue.unreadable.isEmpty) + } + + /// Listed rather than hidden: a reader whose dictionary stopped working + /// should be told which one. Showing a shorter list is how "my Russian + /// definitions disappeared" becomes unanswerable. + @Test("A file that will not open is listed, not dropped") + func unreadableIsListed() async throws { + let workspace = try Workspace() + try workspace.install(slug: "good", glossLanguage: "en") + try FileManager.default.createDirectory( + at: workspace.directory, + withIntermediateDirectories: true, + ) + try Data("not a database".utf8).write( + to: workspace.directory + .appendingPathComponent("broken.\(DictionaryCatalogue.fileExtension)"), + ) + + let catalogue = try await workspace.catalogue() + #expect(catalogue.installed.map(\.slug) == ["good"]) + #expect(catalogue.unreadable.count == 1) + #expect(catalogue.unreadable[0].file.lastPathComponent + == "broken.\(DictionaryCatalogue.fileExtension)") + } + + @Test("A file with another extension is not a dictionary") + func otherFilesIgnored() async throws { + let workspace = try Workspace() + try workspace.install(slug: "good", glossLanguage: "en") + try Data("notes".utf8).write( + to: workspace.directory.appendingPathComponent("notes.txt"), + ) + let catalogue = try await workspace.catalogue() + #expect(catalogue.installed.count == 1) + #expect(catalogue.unreadable.isEmpty) + } + + // MARK: - Order without configuration + + /// A Russian-locale reader who imports БКРС should see Russian first + /// without touching a setting, and an English-locale reader importing the + /// same file should not. + @Test("A dictionary nobody has ordered sorts by the reader's own languages") + func defaultOrderFollowsLocale() async throws { + let workspace = try Workspace() + try workspace.install(slug: "cc-cedict", glossLanguage: "en") + try workspace.install(slug: "dabkrs", glossLanguage: "ru") + + let forRussian = try await workspace.catalogue(preferredLanguages: ["ru-RU", "en-GB"]) + #expect(forRussian.installed.sorted { $0.sortOrder < $1.sortOrder } + .map(\.slug) == ["dabkrs", "cc-cedict"]) + + let forEnglish = try await workspace.catalogue(preferredLanguages: ["en-GB", "ru-RU"]) + #expect(forEnglish.installed.sorted { $0.sortOrder < $1.sortOrder } + .map(\.slug) == ["cc-cedict", "dabkrs"]) + } + + @Test("A language tag matches either way round", arguments: [ + (["ru-RU"], "ru", 0), + (["ru"], "ru-RU", 0), + (["en-GB", "ru"], "ru", 1), + (["en"], "ru", 2), + ]) + func languageMatching(preferred: [String], gloss: String, expected: Int) { + let metadata = DictionaryMetadata( + slug: "x", + displayName: "X", + format: "test", + indexLanguage: "zh-Hans", + glossLanguage: gloss, + licence: "", + attribution: "", + parserVersion: 1, + entryCount: 0, + ) + #expect(AppDatabase.defaultOrder( + for: metadata, + preferredLanguages: preferred, + ) == expected) + } + + /// Once the reader has dragged a list into place, that is the order. + @Test("A stored order overrides the locale default") + func storedOrderWins() async throws { + let workspace = try Workspace() + try workspace.install(slug: "cc-cedict", glossLanguage: "en") + try workspace.install(slug: "dabkrs", glossLanguage: "ru") + try await workspace.library.setDictionaryOrder(["dabkrs", "cc-cedict"]) + + let catalogue = try await workspace.catalogue(preferredLanguages: ["en"]) + #expect(catalogue.installed.sorted { $0.sortOrder < $1.sortOrder } + .map(\.slug) == ["dabkrs", "cc-cedict"]) + } + + // MARK: - The three permissions + + @Test("Permissions default to on for glosses and readings") + func defaultPermissions() async throws { + let workspace = try Workspace() + try workspace.install(slug: "dabkrs", glossLanguage: "ru") + let entry = try #require(try await workspace.catalogue().installed.first) + #expect(entry.providesGlosses) + #expect(entry.providesReadings) + // A word list is opt-in, so an imported dictionary does not silently + // cost a quarter of a gigabyte at launch. + #expect(!entry.providesLexicon) + } + + @Test("A stored permission is applied") + func storedPermissions() async throws { + let workspace = try Workspace() + try workspace.install(slug: "cc-cedict", glossLanguage: "en") + try await workspace.library.save(DictionaryPreference( + slug: "cc-cedict", + providesGlosses: false, + providesReadings: true, + providesLexicon: true, + sortOrder: 0, + )) + + let entry = try #require(try await workspace.catalogue().installed.first) + #expect(!entry.providesGlosses) + #expect(entry.providesReadings) + #expect(entry.providesLexicon) + } + + @Test("Forgetting a dictionary restores its defaults") + func forgetting() async throws { + let workspace = try Workspace() + try workspace.install(slug: "cc-cedict", glossLanguage: "en") + try await workspace.library.save(DictionaryPreference( + slug: "cc-cedict", + providesGlosses: false, + )) + try await workspace.library.forgetDictionary("cc-cedict") + #expect(try await workspace.catalogue().installed.first?.providesGlosses == true) + } + + // MARK: - Opening the set + + @Test("The set is built in the reader's order") + func setOrder() async throws { + let workspace = try Workspace() + try workspace.install(slug: "cc-cedict", glossLanguage: "en", gloss: "to love") + try workspace.install(slug: "dabkrs", glossLanguage: "ru", gloss: "любить") + try await workspace.library.setDictionaryOrder(["dabkrs", "cc-cedict"]) + + let (set, unreadable) = try await workspace.catalogue().makeSet() + #expect(unreadable.isEmpty) + #expect(set.members.map(\.slug) == ["dabkrs", "cc-cedict"]) + + let groups = set.groups(for: "爱").groups + #expect(groups.count == 1) + #expect(groups[0].entries.map(\.dictionary) == ["dabkrs", "cc-cedict"]) + #expect(groups[0].entries.map(\.entry.senses[0].gloss.text) == ["любить", "to love"]) + } + + /// A dictionary allowed to do nothing is not opened at all, so turning + /// everything off for a 500 MB container costs nothing to have installed. + @Test("A dictionary with no permissions is not opened") + func fullyDisabledIsNotOpened() async throws { + let workspace = try Workspace() + try workspace.install(slug: "dabkrs", glossLanguage: "ru") + try await workspace.library.save(DictionaryPreference( + slug: "dabkrs", + providesGlosses: false, + providesReadings: false, + )) + let (set, _) = try await workspace.catalogue().makeSet() + #expect(set.isEmpty) + } + + /// Opt-in, and off by default for a dictionary whose headwords are not + /// words: BKRS contributes 1,806,893 lexemes, which is both a quarter of + /// a gigabyte and a word list in which almost every run of Han characters + /// is a word. + @Test("Only dictionaries allowed to supply a word list contribute to it") + func lexiconIsOptIn() async throws { + let workspace = try Workspace() + try workspace.install(slug: "cc-cedict", glossLanguage: "en", word: "中国") + try workspace.install(slug: "dabkrs", glossLanguage: "ru", word: "一家人") + + #expect(try await workspace.catalogue().lexicon().isEmpty) + + try await workspace.library.save(DictionaryPreference( + slug: "cc-cedict", + providesLexicon: true, + )) + let lexicon = try await workspace.catalogue().lexicon() + #expect(lexicon.contains("中国")) + #expect(!lexicon.contains("一家人")) + } +} diff --git a/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift b/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift index eb1fe45..44b808c 100644 --- a/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift +++ b/Tests/HanReaderPersistenceTests/DictionaryStorageTests.swift @@ -149,6 +149,66 @@ struct DictionaryStorageTests { .contains("entry_on_source")) } + /// The way the app opens one: read-only, which is where this broke. + /// + /// A database whose header says WAL needs a shared-memory file, and a + /// read-only connection cannot create one — so an imported dictionary + /// opened the way the reader's app opens it failed with "unable to open + /// database file" and simply did not appear in their list. Nothing in the + /// writer's own tests caught it, because the writer opens for writing. + @Test("A finished container opens read-only") + func opensReadOnly() throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } + + let (writer, _) = try DictionaryWriter.open(at: url, source: source()) + try writer.append( + [DictionaryWriter.SourcedEntry(entry: entry("爱"), sourceFile: 0, sourceOffset: 1)], + checkpoint: DictionaryWriter.Checkpoint(byteOffset: 1, cardsRead: 1), + ) + try writer.finish(metadata: metadata(entryCount: 1)) + + // No write-ahead log beside it. (The shared-memory file lingers + // until the writer's own connection closes, which is why the test + // below copies the container instead of inspecting the directory: + // what matters is that a reader needs nothing but the one file.) + let siblings = try FileManager.default.contentsOfDirectory( + atPath: url.deletingLastPathComponent().path, + ) + #expect(!siblings.contains { $0.hasSuffix("-wal") }, "left behind \(siblings)") + + let container = try DictionaryContainer(contentsOf: url) + #expect(try container.metadata()?.slug == "test") + #expect(try container.entries(for: "爱").count == 1) + } + + /// And the same file copied somewhere else, which is what installing a + /// dictionary is: a container that needed its sidecar files would open + /// where it was built and nowhere else. + @Test("A finished container opens after being moved") + func opensAfterMoving() throws { + let url = temporaryURL() + let moved = temporaryURL() + defer { + try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) + try? FileManager.default.removeItem(at: moved.deletingLastPathComponent()) + } + + let (writer, _) = try DictionaryWriter.open(at: url, source: source()) + try writer.append( + [DictionaryWriter.SourcedEntry(entry: entry("爱"), sourceFile: 0, sourceOffset: 1)], + checkpoint: DictionaryWriter.Checkpoint(byteOffset: 1, cardsRead: 1), + ) + try writer.finish(metadata: metadata(entryCount: 1)) + + try FileManager.default.createDirectory( + at: moved.deletingLastPathComponent(), + withIntermediateDirectories: true, + ) + try FileManager.default.copyItem(at: url, to: moved) + #expect(try DictionaryContainer(contentsOf: moved).entries(for: "爱").count == 1) + } + // MARK: - Sense payloads /// A container records which codec wrote it, so a new codec does not diff --git a/Tests/HanReaderPersistenceTests/MigrationTests.swift b/Tests/HanReaderPersistenceTests/MigrationTests.swift index a2cb09a..1cc681f 100644 --- a/Tests/HanReaderPersistenceTests/MigrationTests.swift +++ b/Tests/HanReaderPersistenceTests/MigrationTests.swift @@ -32,6 +32,8 @@ struct MigrationTests { #expect(schema == [ "appState(key,value)", "audioTrack(textId,relativePath,duration,importedAt)", + "dictionaryPreference(slug,providesGlosses,providesReadings," + + "providesLexicon,sortOrder)", "readingPosition(textId,blockIndex,tokenIndex,characterOffset,audioTime,updatedAt)", "revealedWord(textId,word,revealedAt)", "text(id,title,content,contentHash,characterCount,preview," @@ -47,7 +49,7 @@ struct MigrationTests { try migrator.hasCompletedMigrations(db) } #expect(completed) - #expect(migrator.migrations == ["v1_initial"]) + #expect(migrator.migrations == ["v1_initial", "v2_dictionary_preferences"]) } /// Running the migrator again must be a no-op. Every launch does this. @@ -62,7 +64,7 @@ struct MigrationTests { WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE 'grdb_%' """) } - #expect(tables == 5) + #expect(tables == 6) } /// The case an empty-database test cannot reach: a migration that works