diff --git a/spec/rewrite.spec.js b/spec/rewrite.spec.js
index 0df37b4..f647b43 100644
--- a/spec/rewrite.spec.js
+++ b/spec/rewrite.spec.js
@@ -1,3 +1,6 @@
+import {atomic, possessive} from '../src/atomic.js';
+import {subroutines} from '../src/subroutines.js';
+
describe('rewrite', () => {
function toRegExp(expression, options) {
const result = rewrite(expression, options);
@@ -69,6 +72,33 @@ describe('rewrite', () => {
it('should process subroutine definition groups', () => {
expect(toRegExp('\\g(?(DEFINE)(?.))').exec('a').groups).toBeUndefined();
});
+
+ it('should process extended syntax around literal [ with Unicode sets disabled', () => {
+ const options = {disable: {v: true}, unicodeSetsPlugin: null};
+
+ expect('[').toMatch(toRegExp('(?>[[])', options));
+ expect('[').toMatch(toRegExp('[[]++', options));
+ expect('[[').toMatch(toRegExp('\\g(?(DEFINE)(?[[]))', options));
+ expect('[').toMatch(toRegExp('# ignored [\n(?>[[])', options));
+ expect('[').toMatch(regex(options)`(?>[${pattern`[`}])`);
+ expect('[').toMatch(regex(options)`(?>[${''}[])`);
+ });
+
+ it('should preserve extended syntax after a comment across interpolation', () => {
+ const options = {disable: {v: true}, unicodeSetsPlugin: null};
+ expect('[').toMatch(regex(options)`# ignored [${pattern`(?>[[])`}`);
+ });
+
+ it('should support flag u in standalone internal plugins', () => {
+ const data = {flags: 'u'};
+ expect('[').toMatch(new RegExp(atomic('(?>[[])', data).pattern, 'u'));
+ expect('[[').toMatch(new RegExp(
+ atomic(possessive('^[[]++$', data).pattern, data).pattern, 'u'
+ ));
+ expect('[[').toMatch(new RegExp(
+ subroutines('^(?[[])\\g$', data).pattern, 'u'
+ ));
+ });
});
describe('options', () => {
diff --git a/src/atomic.js b/src/atomic.js
index cf75bb2..e6aee39 100644
--- a/src/atomic.js
+++ b/src/atomic.js
@@ -1,4 +1,4 @@
-import {incrementIfAtLeast, noncapturingDelim, spliceStr} from './utils-internals.js';
+import {incrementIfAtLeast, noncapturingDelim, normalizeCharClassOpeners, spliceStr} from './utils-internals.js';
import {Context, replaceUnescaped} from 'regex-utilities';
/**
@import {PluginData, PluginResult} from './regex.js';
@@ -13,6 +13,10 @@ Apply transformations for atomic groups: `(?>…)`.
@returns {Required}
*/
function atomic(expression, data) {
+ expression = normalizeCharClassOpeners(
+ expression,
+ data?.unicodeSets ?? data?.flags?.includes('v') ?? true
+ );
const hiddenCaptures = data?.hiddenCaptures ?? [];
// Capture transfer is used by
let captureTransfers = data?.captureTransfers ?? new Map();
@@ -145,9 +149,14 @@ Transform posessive quantifiers into atomic groups. The posessessive quantifiers
This follows Java, PCRE, Perl, and Python.
Possessive quantifiers in Oniguruma and Onigmo are only: `?+`, `*+`, `++`.
@param {string} expression
+@param {PluginData} [data]
@returns {PluginResult}
*/
-function possessive(expression) {
+function possessive(expression, data) {
+ expression = normalizeCharClassOpeners(
+ expression,
+ data?.unicodeSets ?? data?.flags?.includes('v') ?? true
+ );
if (!(new RegExp(`${baseQuantifier}\\+`).test(expression))) {
return {
pattern: expression,
diff --git a/src/regex.js b/src/regex.js
index 41e19ec..17a8d6c 100644
--- a/src/regex.js
+++ b/src/regex.js
@@ -5,6 +5,7 @@ import {clean, flagXPreprocessor} from './flag-x.js';
import {Pattern, pattern} from './pattern.js';
import {RegExpSubclass} from './subclass.js';
import {subroutines} from './subroutines.js';
+import {normalizeCharClassOpeners} from './utils-internals.js';
import {adjustNumberedBackrefs, CharClassContext, containsCharClassUnion, countCaptures, enclosedTokenCharClassContexts, enclosedTokenRegexContexts, envSupportsFlagGroups, envSupportsFlagV, escapeV, getBreakoutChar, getEndContextForIncompleteExpression, preprocess, RegexContext, sandboxLoneCharClassCaret, sandboxLoneDoublePunctuatorChar, sandboxUnsafeNulls} from './utils.js';
import {Context, hasUnescaped, replaceUnescaped} from 'regex-utilities';
@@ -14,6 +15,7 @@ import {Context, hasUnescaped, replaceUnescaped} from 'regex-utilities';
flags?: string;
captureTransfers?: Map>;
hiddenCaptures?: Array;
+ unicodeSets?: boolean;
}} PluginData
@typedef {{
pattern: string;
@@ -195,6 +197,20 @@ function getOptions(options) {
}}
*/
function runPreprocessors(template, substitutions, options) {
+ if (!options.unicodeSetsPlugin && options.flags.includes('u')) {
+ const context = {inCharClass: false};
+ ({template, substitutions} = preprocess(
+ template,
+ substitutions,
+ (value, runningContext) => ({
+ transformed: normalizeCharClassOpeners(
+ String(value), false, !options.disable.x, context
+ ),
+ runningContext,
+ }),
+ options
+ ));
+ }
const preprocessors = [];
// Implicit flag x is handled first because otherwise some regex syntax (if unescaped) within
// comments could cause problems when parsing
@@ -222,6 +238,7 @@ function runPreprocessors(template, substitutions, options) {
*/
function runPlugins(expression, {flags, plugins, unicodeSetsPlugin, disable}) {
let hiddenCaptures = [];
+ const unicodeSets = flags.includes('v') || !!unicodeSetsPlugin;
[ ...plugins, // Run first, so provided plugins can output extended syntax
...(disable.subroutines ? [] : [subroutines]),
...(disable.atomic ? [] : [possessive, atomic]),
@@ -229,7 +246,7 @@ function runPlugins(expression, {flags, plugins, unicodeSetsPlugin, disable}) {
// Run last, so it doesn't have to worry about parsing extended syntax
...(!unicodeSetsPlugin ? [] : [unicodeSetsPlugin]),
].forEach(plugin => {
- const result = plugin(expression, {flags, hiddenCaptures});
+ const result = plugin(expression, {flags, hiddenCaptures, unicodeSets});
if (typeof result?.pattern !== 'string') {
throw new Error('Plugin must return an object with a string property "pattern"');
}
diff --git a/src/subroutines.js b/src/subroutines.js
index 6ad569a..0c1a10a 100644
--- a/src/subroutines.js
+++ b/src/subroutines.js
@@ -1,5 +1,5 @@
import {capturingDelim, countCaptures, namedCapturingDelim} from './utils.js';
-import {incrementIfAtLeast, spliceStr} from './utils-internals.js';
+import {incrementIfAtLeast, normalizeCharClassOpeners, spliceStr} from './utils-internals.js';
import {Context, execUnescaped, forEachUnescaped, getGroupContents, hasUnescaped, replaceUnescaped} from 'regex-utilities';
/**
@import {PluginData, PluginResult} from './regex.js';
@@ -11,6 +11,10 @@ import {Context, execUnescaped, forEachUnescaped, getGroupContents, hasUnescaped
@returns {PluginResult}
*/
function subroutines(expression, data) {
+ expression = normalizeCharClassOpeners(
+ expression,
+ data?.unicodeSets ?? data?.flags?.includes('v') ?? true
+ );
// NOTE: subroutines and definition groups fully support numbered backreferences and unnamed
// captures (from interpolated regexes or from turning implicit flag n off), and all of the
// complex forward and backward backreference adjustments that can result
diff --git a/src/utils-internals.js b/src/utils-internals.js
index 5fe3581..ea8ed48 100644
--- a/src/utils-internals.js
+++ b/src/utils-internals.js
@@ -2,6 +2,47 @@
const noncapturingDelim = String.raw`\(\?(?:[:=!>A-Za-z\-]|<[=!]|\(DEFINE\))`;
+/**
+Escape `[` when it is a literal character within a flag-u character class.
+Existing parsers can then continue to treat every unescaped `[` as the start
+of a class, as required by flag v. Line comments are skipped when flag x is
+enabled; comments in separate template fragments are intentionally separate.
+@param {string} expression
+@param {boolean} useFlagV
+@param {boolean} [useFlagX]
+@param {{inCharClass: boolean}} [context]
+@returns {string}
+*/
+function normalizeCharClassOpeners(expression, useFlagV, useFlagX = false, context = {inCharClass: false}) {
+ if (useFlagV) {
+ return expression;
+ }
+ let {inCharClass} = context;
+ let inComment = false;
+ let result = '';
+ for (const [m] of expression.matchAll(/\\?./gsu)) {
+ if (inComment) {
+ result += m;
+ if (m === '\n') {
+ inComment = false;
+ }
+ } else if (useFlagX && !inCharClass && m === '#') {
+ inComment = true;
+ result += m;
+ } else if (m === '[') {
+ result += inCharClass ? '\\[' : '[';
+ inCharClass = true;
+ } else {
+ result += m;
+ if (m === ']' && inCharClass) {
+ inCharClass = false;
+ }
+ }
+ }
+ context.inCharClass = inCharClass;
+ return result;
+}
+
/**
Updates the array in place by incrementing each value greater than or equal to the threshold.
@param {Array} arr
@@ -29,5 +70,6 @@ function spliceStr(str, pos, oldValue, newValue) {
export {
incrementIfAtLeast,
noncapturingDelim,
+ normalizeCharClassOpeners,
spliceStr,
};