diff --git a/spec/rewrite.spec.js b/spec/rewrite.spec.js index 0df37b4..f647b43 100644 --- a/spec/rewrite.spec.js +++ b/spec/rewrite.spec.js @@ -1,3 +1,6 @@ +import {atomic, possessive} from '../src/atomic.js'; +import {subroutines} from '../src/subroutines.js'; + describe('rewrite', () => { function toRegExp(expression, options) { const result = rewrite(expression, options); @@ -69,6 +72,33 @@ describe('rewrite', () => { it('should process subroutine definition groups', () => { expect(toRegExp('\\g(?(DEFINE)(?.))').exec('a').groups).toBeUndefined(); }); + + it('should process extended syntax around literal [ with Unicode sets disabled', () => { + const options = {disable: {v: true}, unicodeSetsPlugin: null}; + + expect('[').toMatch(toRegExp('(?>[[])', options)); + expect('[').toMatch(toRegExp('[[]++', options)); + expect('[[').toMatch(toRegExp('\\g(?(DEFINE)(?[[]))', options)); + expect('[').toMatch(toRegExp('# ignored [\n(?>[[])', options)); + expect('[').toMatch(regex(options)`(?>[${pattern`[`}])`); + expect('[').toMatch(regex(options)`(?>[${''}[])`); + }); + + it('should preserve extended syntax after a comment across interpolation', () => { + const options = {disable: {v: true}, unicodeSetsPlugin: null}; + expect('[').toMatch(regex(options)`# ignored [${pattern`(?>[[])`}`); + }); + + it('should support flag u in standalone internal plugins', () => { + const data = {flags: 'u'}; + expect('[').toMatch(new RegExp(atomic('(?>[[])', data).pattern, 'u')); + expect('[[').toMatch(new RegExp( + atomic(possessive('^[[]++$', data).pattern, data).pattern, 'u' + )); + expect('[[').toMatch(new RegExp( + subroutines('^(?[[])\\g$', data).pattern, 'u' + )); + }); }); describe('options', () => { diff --git a/src/atomic.js b/src/atomic.js index cf75bb2..e6aee39 100644 --- a/src/atomic.js +++ b/src/atomic.js @@ -1,4 +1,4 @@ -import {incrementIfAtLeast, noncapturingDelim, spliceStr} from './utils-internals.js'; +import {incrementIfAtLeast, noncapturingDelim, normalizeCharClassOpeners, spliceStr} from './utils-internals.js'; import {Context, replaceUnescaped} from 'regex-utilities'; /** @import {PluginData, PluginResult} from './regex.js'; @@ -13,6 +13,10 @@ Apply transformations for atomic groups: `(?>…)`. @returns {Required} */ function atomic(expression, data) { + expression = normalizeCharClassOpeners( + expression, + data?.unicodeSets ?? data?.flags?.includes('v') ?? true + ); const hiddenCaptures = data?.hiddenCaptures ?? []; // Capture transfer is used by let captureTransfers = data?.captureTransfers ?? new Map(); @@ -145,9 +149,14 @@ Transform posessive quantifiers into atomic groups. The posessessive quantifiers This follows Java, PCRE, Perl, and Python. Possessive quantifiers in Oniguruma and Onigmo are only: `?+`, `*+`, `++`. @param {string} expression +@param {PluginData} [data] @returns {PluginResult} */ -function possessive(expression) { +function possessive(expression, data) { + expression = normalizeCharClassOpeners( + expression, + data?.unicodeSets ?? data?.flags?.includes('v') ?? true + ); if (!(new RegExp(`${baseQuantifier}\\+`).test(expression))) { return { pattern: expression, diff --git a/src/regex.js b/src/regex.js index 41e19ec..17a8d6c 100644 --- a/src/regex.js +++ b/src/regex.js @@ -5,6 +5,7 @@ import {clean, flagXPreprocessor} from './flag-x.js'; import {Pattern, pattern} from './pattern.js'; import {RegExpSubclass} from './subclass.js'; import {subroutines} from './subroutines.js'; +import {normalizeCharClassOpeners} from './utils-internals.js'; import {adjustNumberedBackrefs, CharClassContext, containsCharClassUnion, countCaptures, enclosedTokenCharClassContexts, enclosedTokenRegexContexts, envSupportsFlagGroups, envSupportsFlagV, escapeV, getBreakoutChar, getEndContextForIncompleteExpression, preprocess, RegexContext, sandboxLoneCharClassCaret, sandboxLoneDoublePunctuatorChar, sandboxUnsafeNulls} from './utils.js'; import {Context, hasUnescaped, replaceUnescaped} from 'regex-utilities'; @@ -14,6 +15,7 @@ import {Context, hasUnescaped, replaceUnescaped} from 'regex-utilities'; flags?: string; captureTransfers?: Map>; hiddenCaptures?: Array; + unicodeSets?: boolean; }} PluginData @typedef {{ pattern: string; @@ -195,6 +197,20 @@ function getOptions(options) { }} */ function runPreprocessors(template, substitutions, options) { + if (!options.unicodeSetsPlugin && options.flags.includes('u')) { + const context = {inCharClass: false}; + ({template, substitutions} = preprocess( + template, + substitutions, + (value, runningContext) => ({ + transformed: normalizeCharClassOpeners( + String(value), false, !options.disable.x, context + ), + runningContext, + }), + options + )); + } const preprocessors = []; // Implicit flag x is handled first because otherwise some regex syntax (if unescaped) within // comments could cause problems when parsing @@ -222,6 +238,7 @@ function runPreprocessors(template, substitutions, options) { */ function runPlugins(expression, {flags, plugins, unicodeSetsPlugin, disable}) { let hiddenCaptures = []; + const unicodeSets = flags.includes('v') || !!unicodeSetsPlugin; [ ...plugins, // Run first, so provided plugins can output extended syntax ...(disable.subroutines ? [] : [subroutines]), ...(disable.atomic ? [] : [possessive, atomic]), @@ -229,7 +246,7 @@ function runPlugins(expression, {flags, plugins, unicodeSetsPlugin, disable}) { // Run last, so it doesn't have to worry about parsing extended syntax ...(!unicodeSetsPlugin ? [] : [unicodeSetsPlugin]), ].forEach(plugin => { - const result = plugin(expression, {flags, hiddenCaptures}); + const result = plugin(expression, {flags, hiddenCaptures, unicodeSets}); if (typeof result?.pattern !== 'string') { throw new Error('Plugin must return an object with a string property "pattern"'); } diff --git a/src/subroutines.js b/src/subroutines.js index 6ad569a..0c1a10a 100644 --- a/src/subroutines.js +++ b/src/subroutines.js @@ -1,5 +1,5 @@ import {capturingDelim, countCaptures, namedCapturingDelim} from './utils.js'; -import {incrementIfAtLeast, spliceStr} from './utils-internals.js'; +import {incrementIfAtLeast, normalizeCharClassOpeners, spliceStr} from './utils-internals.js'; import {Context, execUnescaped, forEachUnescaped, getGroupContents, hasUnescaped, replaceUnescaped} from 'regex-utilities'; /** @import {PluginData, PluginResult} from './regex.js'; @@ -11,6 +11,10 @@ import {Context, execUnescaped, forEachUnescaped, getGroupContents, hasUnescaped @returns {PluginResult} */ function subroutines(expression, data) { + expression = normalizeCharClassOpeners( + expression, + data?.unicodeSets ?? data?.flags?.includes('v') ?? true + ); // NOTE: subroutines and definition groups fully support numbered backreferences and unnamed // captures (from interpolated regexes or from turning implicit flag n off), and all of the // complex forward and backward backreference adjustments that can result diff --git a/src/utils-internals.js b/src/utils-internals.js index 5fe3581..ea8ed48 100644 --- a/src/utils-internals.js +++ b/src/utils-internals.js @@ -2,6 +2,47 @@ const noncapturingDelim = String.raw`\(\?(?:[:=!>A-Za-z\-]|<[=!]|\(DEFINE\))`; +/** +Escape `[` when it is a literal character within a flag-u character class. +Existing parsers can then continue to treat every unescaped `[` as the start +of a class, as required by flag v. Line comments are skipped when flag x is +enabled; comments in separate template fragments are intentionally separate. +@param {string} expression +@param {boolean} useFlagV +@param {boolean} [useFlagX] +@param {{inCharClass: boolean}} [context] +@returns {string} +*/ +function normalizeCharClassOpeners(expression, useFlagV, useFlagX = false, context = {inCharClass: false}) { + if (useFlagV) { + return expression; + } + let {inCharClass} = context; + let inComment = false; + let result = ''; + for (const [m] of expression.matchAll(/\\?./gsu)) { + if (inComment) { + result += m; + if (m === '\n') { + inComment = false; + } + } else if (useFlagX && !inCharClass && m === '#') { + inComment = true; + result += m; + } else if (m === '[') { + result += inCharClass ? '\\[' : '['; + inCharClass = true; + } else { + result += m; + if (m === ']' && inCharClass) { + inCharClass = false; + } + } + } + context.inCharClass = inCharClass; + return result; +} + /** Updates the array in place by incrementing each value greater than or equal to the threshold. @param {Array} arr @@ -29,5 +70,6 @@ function spliceStr(str, pos, oldValue, newValue) { export { incrementIfAtLeast, noncapturingDelim, + normalizeCharClassOpeners, spliceStr, };