Handle flag-u character classes when Unicode sets are disabled

spec/rewrite.spec.js

@@ -1,3 +1,6 @@
+import {atomic, possessive} from '../src/atomic.js';
+import {subroutines} from '../src/subroutines.js';
+
 describe('rewrite', () => {
   function toRegExp(expression, options) {
     const result = rewrite(expression, options);
@@ -69,6 +72,33 @@ describe('rewrite', () => {
     it('should process subroutine definition groups', () => {
       expect(toRegExp('\\g<a>(?(DEFINE)(?<a>.))').exec('a').groups).toBeUndefined();
     });
+
+    it('should process extended syntax around literal [ with Unicode sets disabled', () => {
+      const options = {disable: {v: true}, unicodeSetsPlugin: null};
+
+      expect('[').toMatch(toRegExp('(?>[[])', options));
+      expect('[').toMatch(toRegExp('[[]++', options));
+      expect('[[').toMatch(toRegExp('\\g<a>(?(DEFINE)(?<a>[[]))', options));
+      expect('[').toMatch(toRegExp('# ignored [\n(?>[[])', options));
+      expect('[').toMatch(regex(options)`(?>[${pattern`[`}])`);
+      expect('[').toMatch(regex(options)`(?>[${''}[])`);
+    });
+
+    it('should preserve extended syntax after a comment across interpolation', () => {
+      const options = {disable: {v: true}, unicodeSetsPlugin: null};
+      expect('[').toMatch(regex(options)`# ignored [${pattern`(?>[[])`}`);
+    });
+
+    it('should support flag u in standalone internal plugins', () => {
+      const data = {flags: 'u'};
+      expect('[').toMatch(new RegExp(atomic('(?>[[])', data).pattern, 'u'));
+      expect('[[').toMatch(new RegExp(
+        atomic(possessive('^[[]++$', data).pattern, data).pattern, 'u'
+      ));
+      expect('[[').toMatch(new RegExp(
+        subroutines('^(?<a>[[])\\g<a>$', data).pattern, 'u'
+      ));
+    });
   });
 
   describe('options', () => {

src/atomic.js

@@ -1,4 +1,4 @@
-import {incrementIfAtLeast, noncapturingDelim, spliceStr} from './utils-internals.js';
+import {incrementIfAtLeast, noncapturingDelim, normalizeCharClassOpeners, spliceStr} from './utils-internals.js';
 import {Context, replaceUnescaped} from 'regex-utilities';
 /**
 @import {PluginData, PluginResult} from './regex.js';
@@ -13,6 +13,10 @@ Apply transformations for atomic groups: `(?>…)`.
 @returns {Required<PluginResult>}
 */
 function atomic(expression, data) {
+  expression = normalizeCharClassOpeners(
+    expression,
+    data?.unicodeSets ?? data?.flags?.includes('v') ?? true
+  );
   const hiddenCaptures = data?.hiddenCaptures ?? [];
   // Capture transfer is used by <github.com/slevithan/oniguruma-to-es>
   let captureTransfers = data?.captureTransfers ?? new Map();
@@ -145,9 +149,14 @@ Transform posessive quantifiers into atomic groups. The posessessive quantifiers
 This follows Java, PCRE, Perl, and Python.
 Possessive quantifiers in Oniguruma and Onigmo are only: `?+`, `*+`, `++`.
 @param {string} expression
+@param {PluginData} [data]
 @returns {PluginResult}
 */
-function possessive(expression) {
+function possessive(expression, data) {
+  expression = normalizeCharClassOpeners(
+    expression,
+    data?.unicodeSets ?? data?.flags?.includes('v') ?? true
+  );
   if (!(new RegExp(`${baseQuantifier}\\+`).test(expression))) {
     return {
       pattern: expression,

src/regex.js

@@ -5,6 +5,7 @@ import {clean, flagXPreprocessor} from './flag-x.js';
 import {Pattern, pattern} from './pattern.js';
 import {RegExpSubclass} from './subclass.js';
 import {subroutines} from './subroutines.js';
+import {normalizeCharClassOpeners} from './utils-internals.js';
 import {adjustNumberedBackrefs, CharClassContext, containsCharClassUnion, countCaptures, enclosedTokenCharClassContexts, enclosedTokenRegexContexts, envSupportsFlagGroups, envSupportsFlagV, escapeV, getBreakoutChar, getEndContextForIncompleteExpression, preprocess, RegexContext, sandboxLoneCharClassCaret, sandboxLoneDoublePunctuatorChar, sandboxUnsafeNulls} from './utils.js';
 import {Context, hasUnescaped, replaceUnescaped} from 'regex-utilities';
 
@@ -14,6 +15,7 @@ import {Context, hasUnescaped, replaceUnescaped} from 'regex-utilities';
   flags?: string;
   captureTransfers?: Map<number, Array<number>>;
   hiddenCaptures?: Array<number>;
+  unicodeSets?: boolean;
 }} PluginData
 @typedef {{
   pattern: string;
@@ -195,6 +197,20 @@ function getOptions(options) {
 }}
 */
 function runPreprocessors(template, substitutions, options) {
+  if (!options.unicodeSetsPlugin && options.flags.includes('u')) {
+    const context = {inCharClass: false};
+    ({template, substitutions} = preprocess(
+      template,
+      substitutions,
+      (value, runningContext) => ({
+        transformed: normalizeCharClassOpeners(
+          String(value), false, !options.disable.x, context
+        ),
+        runningContext,
+      }),
+      options
+    ));
+  }
   const preprocessors = [];
   // Implicit flag x is handled first because otherwise some regex syntax (if unescaped) within
   // comments could cause problems when parsing
@@ -222,14 +238,15 @@ function runPreprocessors(template, substitutions, options) {
 */
 function runPlugins(expression, {flags, plugins, unicodeSetsPlugin, disable}) {
   let hiddenCaptures = [];
+  const unicodeSets = flags.includes('v') || !!unicodeSetsPlugin;
   [ ...plugins, // Run first, so provided plugins can output extended syntax
     ...(disable.subroutines ? [] : [subroutines]),
     ...(disable.atomic      ? [] : [possessive, atomic]),
     ...(disable.x           ? [] : [clean]),
     // Run last, so it doesn't have to worry about parsing extended syntax
     ...(!unicodeSetsPlugin  ? [] : [unicodeSetsPlugin]),
   ].forEach(plugin => {
-    const result = plugin(expression, {flags, hiddenCaptures});
+    const result = plugin(expression, {flags, hiddenCaptures, unicodeSets});
     if (typeof result?.pattern !== 'string') {
       throw new Error('Plugin must return an object with a string property "pattern"');
     }

src/subroutines.js

@@ -1,5 +1,5 @@
 import {capturingDelim, countCaptures, namedCapturingDelim} from './utils.js';
-import {incrementIfAtLeast, spliceStr} from './utils-internals.js';
+import {incrementIfAtLeast, normalizeCharClassOpeners, spliceStr} from './utils-internals.js';
 import {Context, execUnescaped, forEachUnescaped, getGroupContents, hasUnescaped, replaceUnescaped} from 'regex-utilities';
 /**
 @import {PluginData, PluginResult} from './regex.js';
@@ -11,6 +11,10 @@ import {Context, execUnescaped, forEachUnescaped, getGroupContents, hasUnescaped
 @returns {PluginResult}
 */
 function subroutines(expression, data) {
+  expression = normalizeCharClassOpeners(
+    expression,
+    data?.unicodeSets ?? data?.flags?.includes('v') ?? true
+  );
   // NOTE: subroutines and definition groups fully support numbered backreferences and unnamed
   // captures (from interpolated regexes or from turning implicit flag n off), and all of the
   // complex forward and backward backreference adjustments that can result

src/utils-internals.js

@@ -2,6 +2,47 @@
 
 const noncapturingDelim = String.raw`\(\?(?:[:=!>A-Za-z\-]|<[=!]|\(DEFINE\))`;
 
+/**
+Escape `[` when it is a literal character within a flag-u character class.
+Existing parsers can then continue to treat every unescaped `[` as the start
+of a class, as required by flag v. Line comments are skipped when flag x is
+enabled; comments in separate template fragments are intentionally separate.
+@param {string} expression
+@param {boolean} useFlagV
+@param {boolean} [useFlagX]
+@param {{inCharClass: boolean}} [context]
+@returns {string}
+*/
+function normalizeCharClassOpeners(expression, useFlagV, useFlagX = false, context = {inCharClass: false}) {
+  if (useFlagV) {
+    return expression;
+  }
+  let {inCharClass} = context;
+  let inComment = false;
+  let result = '';
+  for (const [m] of expression.matchAll(/\\?./gsu)) {
+    if (inComment) {
+      result += m;
+      if (m === '\n') {
+        inComment = false;
+      }
+    } else if (useFlagX && !inCharClass && m === '#') {
+      inComment = true;
+      result += m;
+    } else if (m === '[') {
+      result += inCharClass ? '\\[' : '[';
+      inCharClass = true;
+    } else {
+      result += m;
+      if (m === ']' && inCharClass) {
+        inCharClass = false;
+      }
+    }
+  }
+  context.inCharClass = inCharClass;
+  return result;
+}
+
 /**
 Updates the array in place by incrementing each value greater than or equal to the threshold.
 @param {Array<number>} arr
@@ -29,5 +70,6 @@ function spliceStr(str, pos, oldValue, newValue) {
 export {
   incrementIfAtLeast,
   noncapturingDelim,
+  normalizeCharClassOpeners,
   spliceStr,
 };