DuckDB: accept digit separators in numeric literals (#2797)

* DuckDB: accept digit separators in numeric literals

`1_000_000`, `92_000.0` and `1e1_0` are numbers in DuckDB and
PostgreSQL 16, but MySQL reads `2024_01` as an identifier. The new
`Feature.allowDigitSeparators`, enabled by the PostgreSQL dialect,
drops the separators from the token image, so every consumer of number
tokens keeps working. Without it, lexing is unchanged.

* Keep the javadoc of `truncateIdentifierToken` above it

Author: guizmaii

src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java

@@ -39,7 +39,8 @@ public enum Dialect {
                         Feature.allowDoubleQuotedStrings), SQLSERVER(AdjacentStringLiterals.OFF,
                                 Feature.allowSquareBracketQuotation), POSTGRESQL(
                                         AdjacentStringLiterals.NEWLINE,
-                                        Feature.allowDollarQuotedStringTags), H2, EXASOL, BIGQUERY(
+                                        Feature.allowDollarQuotedStringTags,
+                                        Feature.allowDigitSeparators), H2, EXASOL, BIGQUERY(
                                                 AdjacentStringLiterals.WHITESPACE,
                                                 Feature.allowDoubleQuotedStrings,
                                                 Feature.allowHashLineComments,
@@ -194,6 +195,14 @@ public P withHashLineComments(boolean allowHashLineComments) {
         return withFeature(Feature.allowHashLineComments, allowHashLineComments);
     }
 
+    public P withDigitSeparators() {
+        return withFeature(Feature.allowDigitSeparators, true);
+    }
+
+    public P withDigitSeparators(boolean allowDigitSeparators) {
+        return withFeature(Feature.allowDigitSeparators, allowDigitSeparators);
+    }
+
     public P withUnparenthesizedSubSelects() {
         return withFeature(Feature.allowUnparenthesizedSubSelects, true);
     }

src/main/java/net/sf/jsqlparser/parser/feature/Feature.java

@@ -889,6 +889,12 @@ public enum Feature {
      */
     allowHashLineComments(false),
 
+    /**
+     * allows digit separators in numeric literals, e.g. {@code 1_000_000} (DuckDB, PostgreSQL 16);
+     * disabled by default, where MySQL reads {@code 2024_01} as an identifier
+     */
+    allowDigitSeparators(false),
+
     /**
      * allows sub selects without parentheses, e.g. `select * from dual where 1 = select 1`
      */

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -2467,6 +2467,38 @@ TOKEN_MGR_DECLS : {
         token.endColumn = input_stream.getEndColumn();
     }
 
+    /**
+     * DuckDB's and PostgreSQL's digit separators, as in {@code 1_000_000}.  Under
+     * Feature.allowDigitSeparators they are dropped from the image.  Otherwise the token is
+     * lexed as without separators: the longer of the number before the first {@code _} and the
+     * identifier starting there, so that MySQL's {@code 2024_01} stays an identifier.
+     */
+    private void lexDigitSeparators(Token token) {
+        int separator = token.image.indexOf('_');
+        if (separator < 0) {
+            return;
+        }
+        if (input_stream.featureConfiguration != null
+                && Boolean.TRUE.equals(configuration.getValue(Feature.allowDigitSeparators))) {
+            token.image = token.image.replace("_", "");
+            return;
+        }
+        int identifierLength = 0;
+        while (identifierLength < token.image.length()
+                && (Character.isLetterOrDigit(token.image.charAt(identifierLength))
+                        || token.image.charAt(identifierLength) == '_')) {
+            identifierLength++;
+        }
+        if (identifierLength > separator) {
+            truncateIdentifierToken(token, identifierLength);
+            token.kind = S_IDENTIFIER;
+        } else {
+            truncateIdentifierToken(token, separator);
+            token.kind = token.image.indexOf('.') >= 0 || token.image.indexOf('e') >= 0
+                    || token.image.indexOf('E') >= 0 ? S_DOUBLE : S_LONG;
+        }
+    }
+
     /** Splits a longest-match identifier without losing token positions. */
     private void truncateIdentifierToken(Token token, int length) {
         input_stream.backup(token.image.length() - length);
@@ -3163,8 +3195,8 @@ TOKEN : /* Numeric Constants */
             <S_LONG> "." (["e","E"] (["+", "-"])? <S_LONG>)?
             |
             <S_LONG> ["e","E"] (["+", "-"])? <S_LONG>
-            )>
-  |     < S_LONG: ( <DIGIT> )+ >
+            )> { lexDigitSeparators(matchedToken); }
+  |     < S_LONG: ( <DIGIT> )+ ( "_" ( <DIGIT> )+ )* > { lexDigitSeparators(matchedToken); }
   |     < #DIGIT: ["0" - "9"] >
   |     < S_HEX: ("X" <QUOTED_HEX> ( (" ")* <QUOTED_HEX> )* | "0x" ( <HEX_DIGIT> )+ ) >
   |     < #QUOTED_HEX: "'" ( <HEX_VALUE> )* "'" >

src/test/java/net/sf/jsqlparser/expression/DigitSeparatorsTest.java

@@ -0,0 +1,64 @@
+/*-
+ * #%L
+ * JSQLParser library
+ * %%
+ * Copyright (C) 2004 - 2026 JSQLParser
+ * %%
+ * Dual licensed under GNU LGPL 2.1 or Apache License 2.0
+ * #L%
+ */
+package net.sf.jsqlparser.expression;
+
+import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.statement.select.PlainSelect;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+class DigitSeparatorsTest {
+
+    private static Expression firstItem(String sql, boolean digitSeparators)
+            throws JSQLParserException {
+        PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse(sql,
+                parser -> parser.withDigitSeparators(digitSeparators));
+        return select.getSelectItem(0).getExpression();
+    }
+
+    @ParameterizedTest
+    @CsvSource({"1_000, 1000", "1_000_000, 1000000", "92_000.0, 92000.0", "1_000.5_5, 1000.55",
+            ".5_5, .55", "1e1_0, 1e10", "1_0e10, 10e10", "1.5e+1_0, 1.5e+10"})
+    void testSeparatorsAreDropped(String literal, String expected) throws JSQLParserException {
+        assertEquals(expected, firstItem("SELECT " + literal, true).toString());
+    }
+
+    @Test
+    void testSeparatedNumbersAreNumbers() throws JSQLParserException {
+        assertEquals(1000L, ((LongValue) firstItem("SELECT 1_000", true)).getValue());
+        assertEquals(92000.0, ((DoubleValue) firstItem("SELECT 92_000.0", true)).getValue());
+        assertEquals("SELECT * FROM t LIMIT 1000", CCJSqlParserUtil
+                .parse("SELECT * FROM t LIMIT 1_000", parser -> parser.withDigitSeparators())
+                .toString());
+    }
+
+    @Test
+    void testPostgreSqlAllowsSeparators() throws JSQLParserException {
+        PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse("SELECT 1_000",
+                parser -> parser.withDialect(Dialect.POSTGRESQL));
+        assertInstanceOf(LongValue.class, select.getSelectItem(0).getExpression());
+    }
+
+    // MySQL identifiers may start with digits
+    @Test
+    void testWithoutSeparatorsDigitLedNamesStayIdentifiers() throws JSQLParserException {
+        assertInstanceOf(Column.class, firstItem("SELECT 1_000", false));
+        assertInstanceOf(Column.class, firstItem("SELECT 2024_01.c FROM 2024_01", false));
+        assertEquals("SELECT .5 _5", CCJSqlParserUtil.parse("SELECT .5_5").toString());
+        assertEquals("SELECT 1.5e+1 _0", CCJSqlParserUtil.parse("SELECT 1.5e+1_0").toString());
+    }
+}