From 3ae3034e53f527f35ed632696f5b57e47663b6f5 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Sun, 4 Oct 2026 21:49:17 +0300 Subject: [PATCH 1/2] feat: add the DSL markup lexer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layer three of the BKRS importer: turning a line of card body into text and markup. Pure, in the core, and validated against the real 350 MB set. Escapes are handled before tags, and that ordering is the whole design. DSL writes a literal bracket as `\[`, and the set contains 55,414 of them — a dictionary of Chinese grammar is full of bracketed glosses. The predecessor's `stripDSL` recognised tags first, so every `[` began a tag and everything to the next `]` was deleted: `\[обладать\]` lost the word inside it. Every "unknown tag" found while investigating that prototype was an escape artefact, and handling the backslash first makes the whole category vanish. Nothing is ever deleted. A span the lexer cannot identify is emitted as literal text *and* reported, because a dictionary that silently drops what it does not understand is worse than one that shows a stray bracket — the reader cannot tell "no entry here" from "the importer ate it". Running it over all three files found a bug worth the exercise. A **zero-width non-joiner binds to the character before it**, so `]` followed by U+200C is a single Swift `Character` that does not compare equal to `"]"`. A character-based scanner walks past the closing bracket and swallows the rest of the line into one unknown tag. Five tags in 13,739,066 tokens are written that way — a rate no hand-written fixture would ever reach. The lexer now scans unicode scalars, which is right anyway: markup is made of scalars, only text is made of characters. It is the same trap as CRLF being one grapheme cluster, which is what stops `split(separator: "\n")` splitting a CRLF file. Validated on the full set: 3,434,224 cards, 13,739,066 tokens, **zero** unknown tags and zero unterminated brackets, in 27 seconds. Two numbers from the plan are corrected by that scan. Escaped brackets are 55,414, not ~175k — 175,480 is the `[*]` tag count, which the plan appears to have conflated. And 62 `[*]` spans contain no `[ex]`, not 6; that path still must not crash, which is the card builder's problem. No BKRS data is committed. The plan called for a 200-card slice of the real dictionary as a fixture, which CONTRIBUTING forbids outright, so the fixtures are hand-written to cover every tag and the validation above runs against a local copy instead. The hand-written cases cannot prove the parser handles the real file; the full-set run is what proves that, and it is reproducible by anyone who has the dictionary. Co-Authored-By: Claude Opus 5 --- Sources/HanReaderCore/Format/DSLLexer.swift | 307 +++++++++++++++++++ Tests/HanReaderCoreTests/DSLLexerTests.swift | 251 +++++++++++++++ 2 files changed, 558 insertions(+) create mode 100644 Sources/HanReaderCore/Format/DSLLexer.swift create mode 100644 Tests/HanReaderCoreTests/DSLLexerTests.swift diff --git a/Sources/HanReaderCore/Format/DSLLexer.swift b/Sources/HanReaderCore/Format/DSLLexer.swift new file mode 100644 index 0000000..fcc3dc8 --- /dev/null +++ b/Sources/HanReaderCore/Format/DSLLexer.swift @@ -0,0 +1,307 @@ +// HanReader — MIT licensed. See LICENSE. + +import Foundation + +/// What a DSL markup tag means. +/// +/// The complete set, not a guess at one. A full scan of the 3,434,224-card +/// BKRS set finds exactly these and nothing else once escapes are handled +/// first — so an unknown tag is a real anomaly worth a diagnostic rather +/// than the routine occurrence a looser parser would treat it as. +/// +/// Counts from that scan, which are the reason each case exists: +/// +/// | tag | occurrences | +/// |---|---| +/// | `[m1]`…`[m4]` | 3,690,060 opens | +/// | `[i]` | 631,732 | +/// | `[p]` | 395,878 | +/// | `[ref]` | 192,496 | +/// | `[*]` | 175,480 | +/// | `[ex]` | 175,356 | +/// | `[c]` | 123,968 | +/// | `[b]` | 51,342 | +public enum DSLTagKind: String, Hashable, Sendable, CaseIterable { + /// `[mN]` — a sense at nesting level N. Closed by a bare `[/m]`. + case sense + /// `[b]` — bold, used for the Roman numeral heading a division. + case bold + /// `[i]` — italic. 631,732 of them, nearly all parenthetical + /// grammatical notes, which read far better italicised than flattened. + case italic + /// `[p]` — a part-of-speech or register label: гл., сущ., бот., уст. + case grammar + /// `[c]` — colour, with an optional argument such as `brown`. + case colour + /// `[ex]` — a usage example. + case example + /// `[*]` — a block that usually wraps an example. + case exampleBlock + /// `[ref]` — a cross-reference to another headword. + case reference + + /// The spelling that opens this tag, ignoring `[mN]`'s level. + var openingName: String { + switch self { + case .sense: "m" + case .bold: "b" + case .italic: "i" + case .grammar: "p" + case .colour: "c" + case .example: "ex" + case .exampleBlock: "*" + case .reference: "ref" + } + } +} + +/// One opening tag, with whatever it carried. +public struct DSLTag: Hashable, Sendable { + public let kind: DSLTagKind + /// The N in `[mN]`. Nil for every other tag. + /// + /// Levels observed: 1 (3,563,809), 2 (87,076), 3 (38,831), 4 (344). Most + /// cards are flat — a single `[m1]` — so nesting depth alone is not a + /// reliable signal of how deep a sense is. + public let level: Int? + /// `brown` in `[c brown]`. + public let argument: String? + + public init(kind: DSLTagKind, level: Int? = nil, argument: String? = nil) { + self.kind = kind + self.level = level + self.argument = argument + } +} + +/// One piece of a lexed DSL line. +public enum DSLToken: Hashable, Sendable { + case text(String) + case open(DSLTag) + /// `[/m]` carries no level — it closes whichever sense is open — so a + /// close is identified by kind alone. + case close(DSLTagKind) +} + +/// Something the lexer could not make sense of, reported rather than hidden. +public struct DSLDiagnostic: Hashable, Sendable { + public enum Kind: Hashable, Sendable { + /// A bracketed span whose name is not in `DSLTagKind`. + case unknownTag(String) + /// A `[` with no matching `]` before the end of the line. + case unterminatedTag + } + + public let kind: Kind + /// The text as it appeared, which is also what was emitted. Nothing is + /// dropped on the way to a diagnostic. + public let text: String + + public init(kind: Kind, text: String) { + self.kind = kind + self.text = text + } +} + +/// Splits a line of DSL body text into text and markup. +/// +/// ### Escapes are handled before tags, and that ordering is the whole thing +/// +/// DSL escapes a literal bracket as `\[` or `\]`, and the BKRS set contains +/// 55,414 of them — a dictionary of Chinese grammar is full of bracketed +/// glosses. A parser that recognises tags first and unescapes afterwards +/// sees `\[обладать\]` as an unknown tag and throws the word away. The +/// predecessor prototype's `stripDSL` did exactly that: every `[` began a +/// tag and everything to the next `]` was deleted, so the escaped brackets +/// and their contents vanished from the definition. +/// +/// Every "unknown tag" found while investigating that prototype turned out +/// to be an escape artefact. Handling the backslash first makes the whole +/// category disappear: across all three files, after unescaping, there are +/// **zero** tags outside the set above. +/// +/// ### Nothing is ever deleted +/// +/// A span the lexer cannot identify is emitted as literal text *and* +/// reported. A dictionary that silently drops what it does not understand +/// is worse than one that shows a stray bracket, because the reader cannot +/// tell the difference between "the dictionary says nothing here" and "the +/// importer ate it". +public enum DSLLexer { + /// Lexes one line of card body. + /// + /// Scans **unicode scalars**, not characters. A `Character` in Swift is + /// an extended grapheme cluster, and a zero-width non-joiner binds to + /// the character before it — so `]` followed by U+200C is a single + /// `Character` that does not compare equal to `"]"`. Five tags in the + /// BKRS set are written that way, and a character-based scanner walks + /// straight past their closing bracket, swallowing the rest of the line + /// into one unknown tag. + /// + /// This is the same trap as CRLF being one grapheme cluster, which is + /// what makes `split(separator: "\n")` fail to split a CRLF file at + /// all. Markup is made of scalars; only text is made of characters. + public static func tokens(in line: String) + -> (tokens: [DSLToken], diagnostics: [DSLDiagnostic]) + { + var scan = Scan(line) + scan.run() + return (scan.tokens, scan.diagnostics) + } + + /// One pass over a line, as a value so each step can be its own method. + private struct Scan { + let scalars: String.UnicodeScalarView + var index: String.UnicodeScalarView.Index + var tokens: [DSLToken] = [] + var diagnostics: [DSLDiagnostic] = [] + var pending = String.UnicodeScalarView() + + init(_ line: String) { + scalars = line.unicodeScalars + index = scalars.startIndex + pending.reserveCapacity(scalars.count) + } + + mutating func run() { + while index < scalars.endIndex { + switch scalars[index] { + case "\\": takeEscape() + case "[": if takeTag() { + return + } + default: + pending.append(scalars[index]) + index = scalars.index(after: index) + } + } + flushText() + } + + mutating func flushText() { + guard !pending.isEmpty else { return } + tokens.append(.text(String(pending))) + pending.removeAll(keepingCapacity: true) + } + + /// A backslash takes the next scalar literally, whatever it is. A + /// trailing backslash is just a backslash. + mutating func takeEscape() { + let next = scalars.index(after: index) + guard next < scalars.endIndex else { + pending.append(scalars[index]) + index = next + return + } + pending.append(scalars[next]) + index = scalars.index(after: next) + } + + /// Handles a `[`. Returns true when the line is finished. + mutating func takeTag() -> Bool { + guard let end = unescapedClosingBracket(in: scalars, after: index) else { + let rest = String(scalars[index...]) + diagnostics.append(DSLDiagnostic(kind: .unterminatedTag, text: rest)) + pending.append(contentsOf: scalars[index...]) + flushText() + return true + } + + let span = index ... end + let body = String(scalars[scalars.index(after: index) ..< end]) + index = scalars.index(after: end) + + switch parse(body) { + case let .open(tag): + flushText() + tokens.append(.open(tag)) + case let .close(kind): + flushText() + tokens.append(.close(kind)) + case .unknown: + diagnostics.append( + DSLDiagnostic(kind: .unknownTag(body), text: String(scalars[span])), + ) + pending.append(contentsOf: scalars[span]) + } + return false + } + } + + // MARK: - Tag parsing + + private enum Parsed { + case open(DSLTag) + case close(DSLTagKind) + case unknown + } + + /// Interprets the inside of a `[...]`. + private static func parse(_ body: String) -> Parsed { + guard !body.isEmpty else { return .unknown } + + if body.hasPrefix("/") { + let name = String(body.dropFirst()) + guard let kind = kind(forName: name) else { return .unknown } + return .close(kind) + } + + // `[c brown]` — the name is everything up to the first space, and + // the rest is the argument. Only `[c]` carries one in this data, but + // splitting generally costs nothing and avoids a special case. + let parts = body.split(separator: " ", maxSplits: 1, omittingEmptySubsequences: false) + let name = String(parts[0]) + let argument = parts.count > 1 ? String(parts[1]) : nil + + if let level = senseLevel(forName: name) { + return .open(DSLTag(kind: .sense, level: level, argument: argument)) + } + guard let kind = kind(forName: name), kind != .sense else { return .unknown } + return .open(DSLTag(kind: kind, level: nil, argument: argument)) + } + + /// `m3` → 3. Nil for anything that is not `m` followed by one digit. + private static func senseLevel(forName name: String) -> Int? { + guard name.count == 2, name.hasPrefix("m"), + let digit = name.last, let level = digit.wholeNumberValue, + level > 0 + else { return nil } + return level + } + + private static func kind(forName name: String) -> DSLTagKind? { + // A bare `m` is only ever a close: `[/m]` ends whichever sense is + // open, which is why a close carries no level. + if name == "m" { + return .sense + } + if senseLevel(forName: name) != nil { + return .sense + } + return DSLTagKind.allCases.first { $0.openingName == name } + } + + /// The next `]` that is not itself escaped. + private static func unescapedClosingBracket( + in scalars: String.UnicodeScalarView, + after start: String.UnicodeScalarView.Index, + ) + -> String.UnicodeScalarView.Index? + { + var index = scalars.index(after: start) + while index < scalars.endIndex { + let scalar = scalars[index] + if scalar == "\\" { + // Skip the escaped scalar, whatever it is. + index = scalars.index(index, offsetBy: 2, limitedBy: scalars.endIndex) + ?? scalars.endIndex + continue + } + if scalar == "]" { + return index + } + index = scalars.index(after: index) + } + return nil + } +} diff --git a/Tests/HanReaderCoreTests/DSLLexerTests.swift b/Tests/HanReaderCoreTests/DSLLexerTests.swift new file mode 100644 index 0000000..73b8f53 --- /dev/null +++ b/Tests/HanReaderCoreTests/DSLLexerTests.swift @@ -0,0 +1,251 @@ +// HanReader — MIT licensed. See LICENSE. + +import Testing +@testable import HanReaderCore + +/// Shorthands so a test reads as the markup it is about. +private func lex(_ line: String) -> [DSLToken] { + DSLLexer.tokens(in: line).tokens +} + +private func diagnostics(_ line: String) -> [DSLDiagnostic] { + DSLLexer.tokens(in: line).diagnostics +} + +/// Everything the tokens contain, concatenated. The lexer must never lose a +/// character: whatever it cannot identify comes back as text. +private func visibleText(_ line: String) -> String { + lex(line).reduce(into: "") { out, token in + if case let .text(text) = token { + out += text + } + } +} + +@Suite("DSL lexer") +struct DSLLexerTests { + // MARK: - Escapes come first + + /// The rule the whole lexer is arranged around, and the one the + /// predecessor's `stripDSL` got wrong: it treated every `[` as a tag and + /// deleted to the next `]`, so an escaped bracket took the word inside + /// it with it. There are 55,414 escaped brackets in the BKRS set. + @Test("An escaped bracket is literal text, not a tag") + func escapedBracketsSurvive() { + #expect(visibleText(#"\[обладать\]"#) == "[обладать]") + #expect(lex(#"\[обладать\]"#) == [.text("[обладать]")]) + #expect(diagnostics(#"\[обладать\]"#).isEmpty) + } + + @Test("An escaped bracket beside a real tag confuses neither") + func escapesAndTagsTogether() { + let tokens = lex(#"[i]\[книжн.\][/i]"#) + #expect(tokens == [ + .open(DSLTag(kind: .italic)), + .text("[книжн.]"), + .close(.italic), + ]) + } + + @Test("A closing bracket inside a tag does not end it early") + func escapedBracketInsideTag() { + // The `]` is escaped, so the tag runs on to the real one. + let (tokens, found) = DSLLexer.tokens(in: #"[c br\]own]x"#) + #expect(found.isEmpty, "the span should parse, not be reported") + #expect(tokens.first == .open(DSLTag(kind: .colour, argument: #"br\]own"#))) + } + + @Test("A backslash escapes any character, not only brackets") + func backslashEscapesAnything() { + #expect(visibleText(#"a\\b"#) == #"a\b"#) + #expect(visibleText(#"\x"#) == "x") + } + + @Test("A trailing backslash is just a backslash") + func trailingBackslash() { + #expect(visibleText(#"abc\"#) == #"abc\"#) + } + + // MARK: - Tags + + @Test("A sense tag carries its level, and its close does not") + func senseLevels() { + #expect(lex("[m1]") == [.open(DSLTag(kind: .sense, level: 1))]) + #expect(lex("[m4]") == [.open(DSLTag(kind: .sense, level: 4))]) + // `[/m]` ends whichever sense is open, so there is no level to carry. + #expect(lex("[/m]") == [.close(.sense)]) + } + + /// The prototype's central bug: a card's whole sense run arrives on one + /// physical line, so a parser that treats a line as a sense produces one + /// blob per entry. Two spans on one line must come out as two. + @Test("Two sense spans on one line are two spans") + func twoSensesOneLine() { + let tokens = lex("[m1]один[/m][m2]два[/m]") + #expect(tokens == [ + .open(DSLTag(kind: .sense, level: 1)), + .text("один"), + .close(.sense), + .open(DSLTag(kind: .sense, level: 2)), + .text("два"), + .close(.sense), + ]) + } + + @Test("Every tag in the data is recognised", arguments: [ + ("[b]", DSLTagKind.bold), ("[i]", .italic), ("[p]", .grammar), + ("[c]", .colour), ("[ex]", .example), ("[*]", .exampleBlock), + ("[ref]", .reference), + ]) + func knownTags(markup: String, kind: DSLTagKind) { + #expect(lex(markup) == [.open(DSLTag(kind: kind))]) + #expect(lex("[/\(markup.dropFirst().dropLast())]") == [.close(kind)]) + } + + @Test("A colour argument is kept", arguments: ["brown", "red", "violet", "green", "crimson"]) + func colourArgument(colour: String) { + #expect(lex("[c \(colour)]") == [.open(DSLTag(kind: .colour, argument: colour))]) + } + + @Test("A tag with no argument has none") + func noArgument() { + #expect(lex("[c]") == [.open(DSLTag(kind: .colour, argument: nil))]) + } + + // MARK: - Nothing is lost + + /// A dictionary that silently drops what it does not understand is worse + /// than one that shows a stray bracket: the reader cannot tell "no entry + /// here" from "the importer ate it". + @Test("An unknown tag is kept as text and reported") + func unknownTagIsKept() { + let line = "пример [zzz] текст" + #expect(visibleText(line) == line) + #expect(diagnostics(line) == [ + DSLDiagnostic(kind: .unknownTag("zzz"), text: "[zzz]"), + ]) + } + + @Test("An unterminated bracket is kept as text and reported") + func unterminatedTag() { + let line = "текст [m1 без конца" + #expect(visibleText(line) == line) + #expect(diagnostics(line).map(\.kind) == [.unterminatedTag]) + } + + @Test("An empty bracket pair is unknown, not a tag") + func emptyBrackets() { + #expect(visibleText("[]") == "[]") + #expect(diagnostics("[]").count == 1) + } + + /// `[m]` with no digit only ever appears as a close. Across all three + /// files every one of the 3,690,058 bare `m` tokens is a `[/m]`, and + /// the 3,690,060 opens all carry a level — so an opening bare `[m]` is + /// an anomaly, and the lexer reports it instead of inventing a level + /// for it. + @Test("A bare opening sense tag is reported, not given a level") + func bareSenseOpen() { + #expect(visibleText("[m]") == "[m]") + #expect(diagnostics("[m]").map(\.kind) == [.unknownTag("m")]) + } + + @Test("A sense level of zero is not a sense") + func zeroLevel() { + #expect(visibleText("[m0]") == "[m0]") + #expect(diagnostics("[m0]").count == 1) + } + + // MARK: - Scalars, not characters + + /// The bug this found in the real data, and the reason the lexer scans + /// unicode scalars rather than characters. + /// + /// A zero-width non-joiner binds to the character before it, so `]` + /// followed by U+200C is a *single* Swift `Character` that does not + /// compare equal to `"]"`. A character-based scanner walks past the + /// closing bracket and swallows the rest of the line into one unknown + /// tag. Five tags in the BKRS set are written this way — five in + /// 13,739,066 tokens, which is exactly the frequency that never shows + /// up in a hand-written fixture. + /// + /// Same trap as CRLF being one grapheme cluster, which is why + /// `split(separator: "\n")` does not split a CRLF file. + @Test("A zero-width joiner after a bracket does not hide it") + func zeroWidthAfterBracket() { + let line = "[ref]\u{200C}磁性碰锁[/ref]" + #expect(lex(line) == [ + .open(DSLTag(kind: .reference)), + .text("\u{200C}磁性碰锁"), + .close(.reference), + ]) + #expect(diagnostics(line).isEmpty) + } + + @Test("A zero-width joiner is content, and is kept") + func zeroWidthIsContent() { + #expect(visibleText("[m1]\u{200C}\u{200C}текст[/m]") == "\u{200C}\u{200C}текст") + } + + /// Proof the distinction is real rather than theoretical: these two + /// strings differ by one scalar and a `Character` scan cannot tell + /// their brackets apart. + @Test("A bracket and a bracket-plus-joiner are one Character but two scalars") + func graphemeClusterTrap() { + let joined = "]\u{200C}" + #expect(joined.count == 1, "one grapheme cluster") + #expect(joined.unicodeScalars.count == 2, "two scalars") + #expect(Array(joined)[0] != "]", "which is why a Character scan misses it") + } + + // MARK: - Ordinary text + + @Test("Text with no markup is one token") + func plainText() { + #expect(lex("любить, быть влюблённым") == [.text("любить, быть влюблённым")]) + } + + @Test("An empty line produces nothing") + func emptyLine() { + #expect(lex("").isEmpty) + #expect(diagnostics("").isEmpty) + } + + @Test("Chinese, Russian and punctuation all pass through") + func mixedScripts() { + let line = "爱儿童 любить ребёнка (детей)" + #expect(visibleText(line) == line) + } + + /// The property that matters across the whole file: for any input, the + /// text the lexer emits plus the markup it recognised accounts for every + /// character it was given. + @Test("Nothing is dropped, whatever the input", arguments: [ + "[m1][b]I[/b] [p]гл.[/p][/m]", + #"[m2]1) любить ([i]кого-л.[/i])[/m]"#, + #"[m3][*][ex]爱儿童 любить ребёнка[/ex][/*][/m]"#, + #"\[方\] диалектное"#, + "[c brown]красный[/c]", + "[zzz]неизвестно[/zzz]", + "обычный текст", + "", + ]) + func nothingIsDropped(line: String) { + let (tokens, _) = DSLLexer.tokens(in: line) + let recovered = tokens.reduce(into: "") { out, token in + switch token { + case let .text(text): out += text + case let .open(tag): + let level = tag.level.map(String.init) ?? "" + let argument = tag.argument.map { " \($0)" } ?? "" + out += "[\(tag.kind.openingName)\(level)\(argument)]" + case let .close(kind): out += "[/\(kind.openingName)]" + } + } + // Escapes are resolved, so compare against the unescaped original. + let unescaped = line + .replacingOccurrences(of: #"\["#, with: "[") + .replacingOccurrences(of: #"\]"#, with: "]") + #expect(recovered == unescaped) + } +} From c235abb73922556fba143ba4a272f3b3d75e3b42 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:00:18 +0300 Subject: [PATCH 2/2] fix: reject markup the format does not produce MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ways the lexer accepted things DSL never writes, each turning an unparsed fragment into valid-looking markup — which is the worst outcome available, because the result reads correctly and has silently dropped whatever it could not understand. `[/m1]` closed a sense. A sense ends with a bare `[/m]`; a levelled close is not one, and accepting it admitted markup this format does not produce while hiding it from the diagnostics. `[m٥]` opened a sense at level five. `Character.wholeNumberValue` answers for Arabic-Indic digits and Roman numerals too, so the level test now requires an ASCII digit. The accepted range is the format's 1–9 rather than the 1–4 BKRS happens to use: the lexer implements DSL, and which levels a dictionary uses is the card builder's business. `[b extra]` became bold with an argument. Colour is the only tag that carries one, so an argument anywhere else is now reported. All three are reported and kept as text, the same as any unknown tag. Re-running the full set confirms none of these forms occur in it: 3,434,224 cards, 13,739,066 tokens, still zero diagnostics. The stricter rules cost nothing against real data and close three holes against data that is not this one. Co-Authored-By: Claude Opus 5 --- Sources/HanReaderCore/Format/DSLLexer.swift | 68 ++++++++++++++------ Tests/HanReaderCoreTests/DSLLexerTests.swift | 43 +++++++++++++ 2 files changed, 92 insertions(+), 19 deletions(-) diff --git a/Sources/HanReaderCore/Format/DSLLexer.swift b/Sources/HanReaderCore/Format/DSLLexer.swift index fcc3dc8..d67fee0 100644 --- a/Sources/HanReaderCore/Format/DSLLexer.swift +++ b/Sources/HanReaderCore/Format/DSLLexer.swift @@ -237,50 +237,80 @@ public enum DSLLexer { } /// Interprets the inside of a `[...]`. + /// + /// Strict on purpose, in all three directions a looser reading would be + /// tempting. A sense closes with a bare `[/m]` and nothing else; a level + /// is an ASCII digit and nothing else; and only `[c]` carries an + /// argument. Anything outside that is not quietly reinterpreted — it is + /// kept as text and reported, which is the same rule the lexer applies + /// to a tag it has never heard of. + /// + /// The alternative is guessing, and a guess here is invisible: `[b bold]` + /// read as bold-with-an-argument produces markup that looks right and + /// has silently dropped whatever `bold` was meant to say. private static func parse(_ body: String) -> Parsed { guard !body.isEmpty else { return .unknown } if body.hasPrefix("/") { - let name = String(body.dropFirst()) - guard let kind = kind(forName: name) else { return .unknown } + // `[/m1]` is not a close. A sense ends with `[/m]`, and treating + // a levelled close as valid would accept markup this format does + // not produce while hiding it from the diagnostics. + guard let kind = closingKind(forName: String(body.dropFirst())) else { + return .unknown + } return .close(kind) } - // `[c brown]` — the name is everything up to the first space, and - // the rest is the argument. Only `[c]` carries one in this data, but - // splitting generally costs nothing and avoids a special case. + // `[c brown]` — the name runs to the first space, the rest is the + // argument. let parts = body.split(separator: " ", maxSplits: 1, omittingEmptySubsequences: false) let name = String(parts[0]) let argument = parts.count > 1 ? String(parts[1]) : nil if let level = senseLevel(forName: name) { - return .open(DSLTag(kind: .sense, level: level, argument: argument)) + // `[m1 extra]` means nothing in this format. + guard argument == nil else { return .unknown } + return .open(DSLTag(kind: .sense, level: level, argument: nil)) } - guard let kind = kind(forName: name), kind != .sense else { return .unknown } + + guard let kind = openingKind(forName: name) else { return .unknown } + // Colour is the only tag that takes an argument. Accepting one + // elsewhere would turn an unparsed fragment into valid-looking + // markup. + guard argument == nil || kind == .colour else { return .unknown } return .open(DSLTag(kind: kind, level: nil, argument: argument)) } - /// `m3` → 3. Nil for anything that is not `m` followed by one digit. + /// `m3` → 3. + /// + /// ASCII digits only. `Character.wholeNumberValue` also answers for + /// `٥` and `Ⅻ`, so a bare `wholeNumberValue` check turns `[m٥]` into a + /// sense at level five — markup invented from a character this format + /// never uses. + /// + /// The range is the format's 1–9, not the 1–4 that happen to occur in + /// BKRS. The lexer implements DSL; what a particular dictionary uses is + /// the card builder's business. private static func senseLevel(forName name: String) -> Int? { guard name.count == 2, name.hasPrefix("m"), - let digit = name.last, let level = digit.wholeNumberValue, - level > 0 + let digit = name.last, digit.isASCII, digit.isNumber, + let level = digit.wholeNumberValue, level > 0 else { return nil } return level } - private static func kind(forName name: String) -> DSLTagKind? { - // A bare `m` is only ever a close: `[/m]` ends whichever sense is - // open, which is why a close carries no level. - if name == "m" { - return .sense - } - if senseLevel(forName: name) != nil { - return .sense - } + /// The kind a name opens, or nil if it opens nothing. + private static func openingKind(forName name: String) -> DSLTagKind? { + // A bare `m` only ever closes; there is no level to give it. + guard name != "m" else { return nil } return DSLTagKind.allCases.first { $0.openingName == name } } + /// The kind a name closes, or nil if it closes nothing. + private static func closingKind(forName name: String) -> DSLTagKind? { + DSLTagKind.allCases.first { $0.openingName == name } + } + /// The next `]` that is not itself escaped. private static func unescapedClosingBracket( in scalars: String.UnicodeScalarView, diff --git a/Tests/HanReaderCoreTests/DSLLexerTests.swift b/Tests/HanReaderCoreTests/DSLLexerTests.swift index 73b8f53..2e09cf0 100644 --- a/Tests/HanReaderCoreTests/DSLLexerTests.swift +++ b/Tests/HanReaderCoreTests/DSLLexerTests.swift @@ -112,6 +112,49 @@ struct DSLLexerTests { #expect(lex("[c]") == [.open(DSLTag(kind: .colour, argument: nil))]) } + // MARK: - Strictness + + /// A sense ends with a bare `[/m]`. Accepting `[/m1]` would admit markup + /// this format does not produce while hiding it from the diagnostics. + @Test("A levelled close is not a close") + func levelledCloseIsRejected() { + #expect(visibleText("[/m1]текст") == "[/m1]текст") + #expect(diagnostics("[/m1]").map(\.kind) == [.unknownTag("/m1")]) + } + + /// `Character.wholeNumberValue` answers for `٥` and `Ⅻ` too, so a bare + /// check turns `[m٥]` into a sense at level five — markup invented from + /// a character the format never uses. + @Test("A sense level must be an ASCII digit", arguments: ["[m٥]", "[mⅫ]", "[m一]"]) + func nonAsciiLevelsRejected(markup: String) { + #expect(visibleText(markup) == markup) + #expect(diagnostics(markup).count == 1) + } + + /// The format's range, not the 1–4 that BKRS happens to use. The lexer + /// implements DSL; which levels a dictionary uses is the builder's + /// business. + @Test("Levels 1 to 9 open a sense", arguments: Array(1 ... 9)) + func asciiLevelsAccepted(level: Int) { + #expect(lex("[m\(level)]") == [.open(DSLTag(kind: .sense, level: level))]) + } + + /// Colour is the only tag that takes an argument. Accepting one + /// elsewhere turns an unparsed fragment into valid-looking markup and + /// silently drops whatever it said. + @Test("An argument on a tag that takes none is reported", arguments: [ + "[b extra]", "[i extra]", "[ex extra]", "[m1 extra]", "[ref extra]", + ]) + func unexpectedArgumentRejected(markup: String) { + #expect(visibleText(markup) == markup) + #expect(diagnostics(markup).count == 1) + } + + @Test("Colour still takes its argument") + func colourKeepsArgument() { + #expect(lex("[c brown]") == [.open(DSLTag(kind: .colour, argument: "brown"))]) + } + // MARK: - Nothing is lost /// A dictionary that silently drops what it does not understand is worse