diff --git a/Sources/_StringProcessing/ByteCodeGen+DSLList.swift b/Sources/_StringProcessing/ByteCodeGen+DSLList.swift index f11f1f9a7..9178a2e5b 100644 --- a/Sources/_StringProcessing/ByteCodeGen+DSLList.swift +++ b/Sources/_StringProcessing/ByteCodeGen+DSLList.swift @@ -694,7 +694,10 @@ fileprivate extension Compiler.ByteCodeGen { builder.buildQuantify(bitset: bitset, kind, minTrips, maxExtraTrips, isScalarSemantics: isScalarSemantics) } else { // Uncased character OR case-sensitive matching; match as a single scalar ascii value character - guard let val = c._singleScalarAsciiValue else { + guard let val = c._singleScalarAsciiValue, + !options.usesCanonicalEquivalence + || !val._hasNonASCIICanonicalEquivalent + else { return false } builder.buildQuantify(asciiChar: val, kind, minTrips, maxExtraTrips, isScalarSemantics: isScalarSemantics) diff --git a/Sources/_StringProcessing/ByteCodeGen.swift b/Sources/_StringProcessing/ByteCodeGen.swift index 53650abf9..1ac6e26a7 100644 --- a/Sources/_StringProcessing/ByteCodeGen.swift +++ b/Sources/_StringProcessing/ByteCodeGen.swift @@ -97,31 +97,69 @@ extension Compiler.ByteCodeGen { } mutating func emitQuotedLiteral(_ s: String) { - // ASCII is normalization-invariant, so is the safe subset for - // us to optimize + guard !s.isEmpty else { return } + + // Matching bytes directly is only safe when matching case sensitively + // and one of the following is true: + // - we're using unicodeScalar semantics, or + // - the literal is ASCII with no non-ASCII scalar equivalents if optimizationsEnabled, - !options.usesCanonicalEquivalence || s.utf8.allSatisfy(\._isASCII), - !s.isEmpty + !options.isCaseInsensitive, + s._canMatchUTF8Directly( + usesCanonicalEquivalence: options.usesCanonicalEquivalence) { + let boundaryCheck = options.semanticLevel == .graphemeCluster + builder.buildMatchUTF8(Array(s.utf8), boundaryCheck: boundaryCheck) + return + } - // TODO: Make an optimizations configuration struct, where - // we can enable/disable specific optimizations and change - // thresholds + // Special case: the literal is ASCII, but contains at least one character + // that a non-ASCII scalar is equivalent to. Here we can split around the + // special characters: for example, `"...TIKEUT..."` is emitted as: + // + // - `matchUTF8("...TI")` + // - `matchScalarEither("K", "\u{212A}")` + // - `matchUTF8("EUT...")` + if optimizationsEnabled, + !options.isCaseInsensitive, + s.utf8.allSatisfy(\._isASCII) + { + assert(options.usesCanonicalEquivalence) + assert(s.utf8.contains(where: \._hasNonASCIICanonicalEquivalent)) let longThreshold = 5 - - // Longer content will be matched against UTF-8 in contiguous - // memory - // - // TODO: case-insensitive variant (just add/subtract from - // ASCII value) - if s.utf8.count >= longThreshold, !options.isCaseInsensitive { - let boundaryCheck = options.semanticLevel == .graphemeCluster - builder.buildMatchUTF8(Array(s.utf8), boundaryCheck: boundaryCheck) - return + + var run: [UInt8] = [] + func flushRun() { + if run.count >= longThreshold { + builder.buildMatchUTF8(run, boundaryCheck: true) + } else { + // Only the last byte needs a boundary check, since the run is all ASCII. + for byte in run.dropLast() { + builder.buildMatchScalar( + Unicode.Scalar(byte), boundaryCheck: false) + } + if let byte = run.last { + builder.buildMatchScalar( + Unicode.Scalar(byte), boundaryCheck: true) + } + } + run = [] + } + + for byte in s.utf8 { + if byte._hasNonASCIICanonicalEquivalent { + flushRun() + emitCharacter(Character(Unicode.Scalar(byte))) + } else { + run.append(byte) + } } + flushRun() + return } - guard options.semanticLevel == .graphemeCluster else { + // Naive path for unicodeScalar semantics + guard options.usesCanonicalEquivalence else { for char in s { for scalar in char.unicodeScalars { emitMatchScalar(scalar) @@ -131,15 +169,21 @@ extension Compiler.ByteCodeGen { } // Fast path for eliding boundary checks for an all ascii quoted literal - if optimizationsEnabled && s.allSatisfy(\.isASCII) && !s.isEmpty { - let lastIdx = s.unicodeScalars.indices.last! - for idx in s.unicodeScalars.indices { - let boundaryCheck = idx == lastIdx - let scalar = s.unicodeScalars[idx] + if optimizationsEnabled, + s._isDirectlyComparableASCII + { + for scalar in s.unicodeScalars.dropLast() { + if options.isCaseInsensitive && scalar.properties.isCased { + builder.buildMatchScalarCaseInsensitive(scalar, boundaryCheck: false) + } else { + builder.buildMatchScalar(scalar, boundaryCheck: false) + } + } + if let scalar = s.unicodeScalars.last { if options.isCaseInsensitive && scalar.properties.isCased { - builder.buildMatchScalarCaseInsensitive(scalar, boundaryCheck: boundaryCheck) + builder.buildMatchScalarCaseInsensitive(scalar, boundaryCheck: true) } else { - builder.buildMatchScalar(scalar, boundaryCheck: boundaryCheck) + builder.buildMatchScalar(scalar, boundaryCheck: true) } } return @@ -198,7 +242,12 @@ extension Compiler.ByteCodeGen { mutating func emitMatchScalar(_ s: UnicodeScalar) { assert(options.semanticLevel == .unicodeScalar) if options.isCaseInsensitive && s.properties.isCased { - builder.buildMatchScalarCaseInsensitive(s, boundaryCheck: false) + if s.isASCII { + let otherValue = s.value ^ 32 + builder.buildMatchScalarEither(s, orScalar: UnicodeScalar(otherValue)!, boundaryCheck: false) + } else { + builder.buildMatchScalarCaseInsensitive(s, boundaryCheck: false) + } } else { builder.buildMatchScalar(s, boundaryCheck: false) } @@ -227,14 +276,28 @@ extension Compiler.ByteCodeGen { return } - if optimizationsEnabled && c.isASCII { + if optimizationsEnabled, c.isASCII, + !c._containsASCIIWithNonASCIIEquivalent { let lastIdx = c.unicodeScalars.indices.last! for idx in c.unicodeScalars.indices { builder.buildMatchScalar(c.unicodeScalars[idx], boundaryCheck: idx == lastIdx) } return } - + + // An ASCII character with a non-ASCII scalar canonically equivalent to it + // can still be matched as a scalar, by accepting either one, rather than + // falling back to materializing and comparing `Character`s. + if optimizationsEnabled, + let ascii = c._singleScalarAsciiValue, + let equivalent = ascii._nonASCIICanonicalEquivalent { + builder.buildMatchScalarEither( + Unicode.Scalar(ascii), + orScalar: equivalent, + boundaryCheck: true) + return + } + builder.buildMatch(c, isCaseInsensitive: false) } diff --git a/Sources/_StringProcessing/Engine/InstPayload.swift b/Sources/_StringProcessing/Engine/InstPayload.swift index 550fa4915..f90637707 100644 --- a/Sources/_StringProcessing/Engine/InstPayload.swift +++ b/Sources/_StringProcessing/Engine/InstPayload.swift @@ -168,6 +168,31 @@ extension Instruction.Payload { return (scalar, caseInsensitive: caseInsensitive, boundaryCheck: boundaryCheck) } + /// Two scalars plus a boundary-check flag. + /// + /// The first scalar occupies the low 21 bits, as in `scalarPayload`; the + /// second goes in bits 32-52, which that layout leaves free. The + /// boundary-check flag stays at bit 54 so it lines up with `scalarPayload`. + init( + scalar: Unicode.Scalar, + orScalar other: Unicode.Scalar, + boundaryCheck: Bool + ) { + let raw = UInt64(scalar.value) + | (UInt64(other.value) << 32) + | (boundaryCheck ? 1 << 54 : 0) + self.init(raw) + } + var scalarEitherPayload: ( + Unicode.Scalar, Unicode.Scalar, boundaryCheck: Bool + ) { + let boundaryCheck = (self.rawValue >> 54) & 1 == 1 + let first = Unicode.Scalar(_value: UInt32(self.rawValue & 0x1F_FFFF)) + let second = Unicode.Scalar( + _value: UInt32((self.rawValue >> 32) & 0x1F_FFFF)) + return (first, second, boundaryCheck: boundaryCheck) + } + init(utf8: UTF8Register, boundaryCheck: Bool) { self.init(boundaryCheck ? 1 : 0, utf8) } diff --git a/Sources/_StringProcessing/Engine/Instruction.swift b/Sources/_StringProcessing/Engine/Instruction.swift index 80bfd9b05..62c7d4a57 100644 --- a/Sources/_StringProcessing/Engine/Instruction.swift +++ b/Sources/_StringProcessing/Engine/Instruction.swift @@ -112,6 +112,13 @@ extension Instruction { /// Operands: Scalar value to match against and booleans case matchScalar + /// Match against one of two scalars, and possibly perform a boundary check. + /// + /// matchScalarEither(_: Unicode.Scalar, _: Unicode.Scalar, boundaryCheck: Bool) + /// + /// Operands: the two scalar values to match against and a boolean + case matchScalarEither + /// Match directly (binary semantics) against a series of UTF-8 bytes /// /// NOTE: Compiler should ensure to only emit this instruction when normalization diff --git a/Sources/_StringProcessing/Engine/MEBuilder.swift b/Sources/_StringProcessing/Engine/MEBuilder.swift index 6e34b9b63..d457612f8 100644 --- a/Sources/_StringProcessing/Engine/MEBuilder.swift +++ b/Sources/_StringProcessing/Engine/MEBuilder.swift @@ -207,6 +207,17 @@ extension MEProgram.Builder { instructions.append(.init(.matchScalar, .init(scalar: s, caseInsensitive: false, boundaryCheck: boundaryCheck))) } + /// Match `s`, or the non-ASCII scalar canonically equivalent to it. + mutating func buildMatchScalarEither( + _ s: Unicode.Scalar, + orScalar other: Unicode.Scalar, + boundaryCheck: Bool + ) { + instructions.append(.init( + .matchScalarEither, + .init(scalar: s, orScalar: other, boundaryCheck: boundaryCheck))) + } + mutating func buildMatchScalarCaseInsensitive(_ s: Unicode.Scalar, boundaryCheck: Bool) { instructions.append(.init(.matchScalar, .init(scalar: s, caseInsensitive: true, boundaryCheck: boundaryCheck))) } diff --git a/Sources/_StringProcessing/Engine/Processor.swift b/Sources/_StringProcessing/Engine/Processor.swift index cdd2bd3e5..29f26398e 100644 --- a/Sources/_StringProcessing/Engine/Processor.swift +++ b/Sources/_StringProcessing/Engine/Processor.swift @@ -294,6 +294,24 @@ extension Processor { return true } + mutating func matchScalarEither( + _ first: Unicode.Scalar, + _ second: Unicode.Scalar, + boundaryCheck: Bool + ) -> Bool { + guard let next = input.matchScalarEither( + first, second, + at: currentPosition, + limitedBy: end, + boundaryCheck: boundaryCheck + ) else { + signalFailure() + return false + } + currentPosition = next + return true + } + mutating func matchScalar( _ s: Unicode.Scalar, boundaryCheck: Bool, @@ -587,6 +605,14 @@ extension Processor { controller.step() } + case .matchScalarEither: + let (first, second, boundaryCheck) = payload.scalarEitherPayload + if matchScalarEither( + first, second, boundaryCheck: boundaryCheck + ) { + controller.step() + } + case .matchUTF8: let (utf8Reg, boundaryCheck) = payload.matchUTF8Payload let utf8Content = registers[utf8Reg] @@ -775,6 +801,28 @@ extension String { return cur } + /// Matches either `first` or `second` as a single scalar. + func matchScalarEither( + _ first: Unicode.Scalar, + _ second: Unicode.Scalar, + at pos: Index, + limitedBy end: String.Index, + boundaryCheck: Bool + ) -> Index? { + guard pos < end else { return nil } + let curScalar = unicodeScalars[pos] + guard curScalar == first || curScalar == second else { return nil } + + let idx = unicodeScalars.index(after: pos) + assert(idx <= end, "Input is a substring with a sub-scalar endIndex.") + + if boundaryCheck && !isOnGraphemeClusterBoundary(idx) { + return nil + } + + return idx + } + func matchScalar( _ scalar: Unicode.Scalar, at pos: Index, diff --git a/Sources/_StringProcessing/Unicode/ASCII.swift b/Sources/_StringProcessing/Unicode/ASCII.swift index 53dfe652d..0e02e8f36 100644 --- a/Sources/_StringProcessing/Unicode/ASCII.swift +++ b/Sources/_StringProcessing/Unicode/ASCII.swift @@ -80,6 +80,95 @@ extension UInt8 { } } +extension UInt8 { + /// Whether this ASCII scalar is canonically equivalent to a non-ASCII + /// scalar. + /// + /// These three non-ASCII characters compare as canonically equivalent + /// to their respective ASCII counterparts: + /// + /// U+037E GREEK QUESTION MARK == ";" + /// U+212A KELVIN SIGN == "K" + /// U+1FEF GREEK VARIA == "`" + var _hasNonASCIICanonicalEquivalent: Bool { + assert(_isASCII) + return _nonASCIICanonicalEquivalent != nil + } + + /// The non-ASCII scalar canonically equivalent to this ASCII byte, if there + /// is one. + var _nonASCIICanonicalEquivalent: Unicode.Scalar? { + switch self { + case UInt8(ascii: ";"): return "\u{037E}" // GREEK QUESTION MARK + case UInt8(ascii: "K"): return "\u{212A}" // KELVIN SIGN + case UInt8(ascii: "`"): return "\u{1FEF}" // GREEK VARIA + default: return nil + } + } +} + +extension Character { + /// Whether any of this character's scalars is an ASCII character that a + /// non-ASCII scalar is canonically equivalent to. + var _containsASCIIWithNonASCIIEquivalent: Bool { + unicodeScalars.contains { + $0.isASCII && UInt8($0.value)._hasNonASCIICanonicalEquivalent + } + } +} + +extension String { + /// Whether this literal can be matched by comparing UTF-8 bytes directly. + func _canMatchUTF8Directly(usesCanonicalEquivalence: Bool) -> Bool { + !usesCanonicalEquivalence || _isDirectlyComparableASCII + } + + /// Whether every character is ASCII with no non-ASCII canonical + /// equivalent, so each can be matched by comparing scalars directly, + /// even under canonical equivalence. + var _isDirectlyComparableASCII: Bool { + switch _isKnownASCII { + case (true, _): + return !utf8.contains(where: \._hasNonASCIICanonicalEquivalent) + case (false, true): + return false + case (false, false): + return allSatisfy(\.isASCII) + && !utf8.contains(where: \._hasNonASCIICanonicalEquivalent) + } + } + + var _isKnownASCII: (isASCII: Bool, isDefinitive: Bool) { +#if _pointerBitWidth(_64) + precondition(MemoryLayout.size == 16, + "unexpected String layout for this platform/ABI") + return withUnsafeBytes(of: self) { raw in + let word0 = raw.load(fromByteOffset: 0, as: UInt64.self) // _countAndFlagsBits + let word1 = raw.load(fromByteOffset: 8, as: UInt64.self) // discriminated object bits + let isSmall = (word1 & 0x2000_0000_0000_0000) != 0 + if isSmall { + return ((word1 & 0x4000_0000_0000_0000) != 0, true) // smallIsASCII bit + } else { + return ((word0 & 0x8000_0000_0000_0000) != 0, false) // CountAndFlags.isASCIIMask + } + } +#elseif _pointerBitWidth(_32) + precondition(MemoryLayout.size == 12, + "unexpected 32-bit String layout") + let discriminator = raw.load(fromByteOffset: 9, as: UInt8.self) + let isSmall = (discriminator & 0x20) != 0 + if isSmall { + return ((discriminator & 0x40) != 0, true) + } else { + let flags = raw.load(fromByteOffset: 10, as: UInt16.self) + return ((flags & 0x8000) != 0, false) + } +#else + #error("Unsupported platform") +#endif + } +} + extension String { /// TODO: better to take isScalarSemantics parameter, we can return more results /// and we can give the right `next` index, not requiring the caller to re-adjust it @@ -168,6 +257,5 @@ extension String { return (next, asciiValue._asciiIsWord) } } - } diff --git a/Sources/_StringProcessing/Unicode/NFC.swift b/Sources/_StringProcessing/Unicode/NFC.swift index 28ba15351..921b34274 100644 --- a/Sources/_StringProcessing/Unicode/NFC.swift +++ b/Sources/_StringProcessing/Unicode/NFC.swift @@ -15,7 +15,10 @@ internal import _RegexParser import Swift extension UInt8 { - /// Whether this is the starting byte of a sub-300 (i.e. pre-combining scalar) scalars + /// Whether this is the starting byte of a sub-0x300 (i.e. pre-combining scalar) scalars. + /// + /// Note that all UTF-8 continuation bytes return `true` for this value as well. Only + /// the starting bytes of scalars that are 0x300 and higher return `false`. var _isSub300StartingByte: Bool { self < 0xCC } } diff --git a/Tests/RegexTests/CanonicalEquivalenceTests.swift b/Tests/RegexTests/CanonicalEquivalenceTests.swift new file mode 100644 index 000000000..e5f6631ca --- /dev/null +++ b/Tests/RegexTests/CanonicalEquivalenceTests.swift @@ -0,0 +1,122 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2026 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +import Testing +@testable @_spi(RegexBenchmark) @_spi(RegexBuilder) import _StringProcessing +@testable import _RegexParser + +@Suite struct CanonicalEquivalenceTests { + /// The three ASCII characters that are canonically equivalent to a + /// non-ASCII character. + static let asciiEquivalentScalars: + [(ascii: Unicode.Scalar, nonASCII: Unicode.Scalar)] = [ + (";", "\u{037E}"), // GREEK QUESTION MARK + ("K", "\u{212A}"), // KELVIN SIGN + ("`", "\u{1FEF}"), // GREEK VARIA + ] + + @available(SwiftStdlib 5.7, *) + @Test func matchInGraphemeSemanticsOnly() throws { + for (ascii, nonASCII) in Self.asciiEquivalentScalars { + #expect("\(ascii)" == "\(nonASCII)") + let subject = "xx\(nonASCII)yy" + + let patterns = [ + String(ascii), + "\(ascii)+", + "xx\(ascii)yy", + "\(ascii)y", + "abcdef\(ascii)ghijkl", + ] + for pattern in patterns { + let input = pattern.hasPrefix("abcdef") + ? "abcdef\(nonASCII)ghijkl" + : subject + let grapheme = try Regex(pattern).matchingSemantics(.graphemeCluster) + #expect( + try grapheme.firstMatch(in: input) != nil, + """ + /\(pattern)/ should match \ + U+\(String(nonASCII.value, radix: 16, uppercase: true)) \ + under grapheme semantics + """) + + let scalarMode = try Regex(pattern).matchingSemantics(.unicodeScalar) + #expect( + try scalarMode.firstMatch(in: input) == nil, + """ + /\(pattern)/ should not match \ + U+\(String(nonASCII.value, radix: 16, uppercase: true)) \ + under scalar semantics + """) + } + } + } + + @available(SwiftStdlib 5.7, *) + @Test func caseInsensitiveLiteralsMatchEquivalents() throws { + for (ascii, nonASCII) in Self.asciiEquivalentScalars { + for pattern in ["(?i)\(ascii)", "(?i)a\(ascii)b", "(?i)\\Qa\(ascii)b\\E"] { + let subject = pattern.contains("a\(ascii)b") + ? "a\(nonASCII)b" + : String(nonASCII) + + let grapheme = try Regex(pattern).matchingSemantics(.graphemeCluster) + #expect( + try grapheme.firstMatch(in: subject) != nil, + """ + /\(pattern)/ should match \ + U+\(String(nonASCII.value, radix: 16, uppercase: true)) \ + under grapheme semantics + """) + } + } + } + + @available(SwiftStdlib 5.7, *) + @Test func combiningMarkPreventsMatch() throws { + for (ascii, nonASCII) in Self.asciiEquivalentScalars { + for spelling in [ascii, nonASCII] { + let cluster = "\(spelling)\u{0301}" + #expect( + try Regex(String(ascii)).wholeMatch(in: cluster) == nil, + "/\(ascii)/ should not whole-match \(cluster.unicodeScalars.count) scalars") + #expect( + try Regex(String(ascii)).firstMatch(in: "x\(cluster)y") == nil, + "/\(ascii)/ should not match inside a combining sequence") + #expect( + try Regex("\(ascii)+").firstMatch(in: "x\(cluster)y") == nil) + } + } + } + + @available(SwiftStdlib 5.7, *) + @Test func optimizedAndUnoptimizedAgree() throws { + for (ascii, scalar) in Self.asciiEquivalentScalars { + let subject = "xx\(scalar)yy" + for pattern in [String(ascii), "\(ascii)+", "xx\(ascii)yy"] { + for level in [RegexSemanticLevel.graphemeCluster, .unicodeScalar] { + var regex = try Regex(pattern).matchingSemantics(level) + let optimized = try regex.firstMatch(in: subject) + precondition(regex._forceAction(.addOptions(.disableOptimizations))) + let unoptimized = try regex.firstMatch(in: subject) + #expect( + optimized.map { subject[$0.range] } + == unoptimized.map { subject[$0.range] }, + """ + /\(pattern)/ diverges at \(level) for \ + U+\(String(scalar.value, radix: 16, uppercase: true)) + """) + } + } + } + } +} diff --git a/Tests/RegexTests/CompileTests.swift b/Tests/RegexTests/CompileTests.swift index 6cc872597..6ab038414 100644 --- a/Tests/RegexTests/CompileTests.swift +++ b/Tests/RegexTests/CompileTests.swift @@ -37,6 +37,8 @@ enum DecodedInstr { case matchScalarCaseInsensitiveUnchecked case matchScalarCaseInsensitive case matchScalarUnchecked + case matchScalarEither + case matchScalarEitherUnchecked case matchBitsetScalar case matchAnyNonNewline case matchBitset @@ -98,6 +100,9 @@ extension DecodedInstr { } else { return .match } + case .matchScalarEither: + let (_, _, boundaryCheck) = payload.scalarEitherPayload + return boundaryCheck ? .matchScalarEither : .matchScalarEitherUnchecked case .matchScalar: let (_, caseInsensitive, boundaryCheck) = payload.scalarPayload if caseInsensitive { @@ -422,7 +427,7 @@ extension RegexTests { // all ascii quoted literal -> elide boundary checks expectProgram( for: "abcd", - contains: [.matchScalar, .matchScalarUnchecked], + contains: [.matchUTF8], doesNotContain: [.match, .consumeBy]) // ascii character -> matchScalar with boundary check expectProgram( @@ -438,7 +443,7 @@ extension RegexTests { expectProgram( for: "abcd", semanticLevel: .unicodeScalar, - contains: [.matchScalarUnchecked], + contains: [.matchUTF8], doesNotContain: [.match, .consumeBy, .matchScalar]) expectProgram( for: "a", @@ -472,12 +477,41 @@ extension RegexTests { doesNotContain: [.matchUTF8, .consumeBy]) } + func testCanonicalEquivalentASCIICompilation() { + for literal in [";", "K", "`"] { + expectProgram( + for: literal, + semanticLevel: .graphemeCluster, + contains: [.matchScalarEither], + doesNotContain: [ + .match, .matchUTF8, .matchScalar, .matchScalarUnchecked, + ]) + expectProgram( + for: literal, + semanticLevel: .unicodeScalar, + contains: [.matchScalarUnchecked], + doesNotContain: [.match, .matchScalarEither, .matchScalarEitherUnchecked]) + // Long literals are split around the K`; characters under grapheme semantics + expectProgram( + for: "abcdef\(literal)ghijkl", + semanticLevel: .graphemeCluster, + contains: [.matchUTF8, .matchScalarEither], + doesNotContain: [.match]) + // Under scalar semantics the whole literal compares bytewise + expectProgram( + for: "abcdef\(literal)ghijkl", + semanticLevel: .unicodeScalar, + contains: [.matchUTF8], + doesNotContain: [.match, .matchScalarEither]) + } + } + func testCaseInsensitivityCompilation() { // quoted literal is all ascii -> match scalar case insensitive and skip // boundary checks expectProgram( for: "(?i)abcd", - contains: [.matchScalarCaseInsensitiveUnchecked, .matchScalarCaseInsensitive], + contains: [.matchScalarCaseInsensitive, .matchScalarCaseInsensitiveUnchecked], doesNotContain: [.match, .matchCaseInsensitive, .matchScalar, .matchScalarUnchecked]) // quoted literal is all non-cased ascii -> emit match scalar instructions expectProgram( @@ -493,9 +527,16 @@ extension RegexTests { contains: [.matchCaseInsensitive, .matchScalarCaseInsensitive], doesNotContain: [.matchScalarCaseInsensitiveUnchecked, .match, .matchScalar]) // same as before but contains ascii non cased characters -> emit matchScalar for them + // ";" is special: U+037E GREEK QUESTION MARK is canonically equivalent to + // it, so under grapheme semantics it compiles to `matchScalarEither`, which + // accepts either scalar, rather than a plain `matchScalar`. See + // UInt8._nonASCIICanonicalEquivalent. expectProgram( for: "(?i)abcd\u{301};.'!", - contains: [.matchCaseInsensitive, .matchScalarCaseInsensitive, .matchScalar], + contains: [ + .matchCaseInsensitive, .matchScalarCaseInsensitive, .matchScalar, + .matchScalarEither, + ], doesNotContain: [.matchScalarCaseInsensitiveUnchecked, .match]) // contains non-ascii non-cased characters -> emit match expectProgram(