address review: handle more cases (exponent, underscore)

Author: skirpichev

Doc/faq/programming.rst

@@ -854,7 +854,7 @@ a :exc:`SyntaxError` because the period is seen as a decimal point::
    >>> 1.__class__
      File "<stdin>", line 1
      1.__class__
-      ^
+       ^
    SyntaxError: invalid decimal literal
 
 The solution is to separate the literal from the period

Lib/test/test_exceptions.py

@@ -299,7 +299,7 @@ def testSyntaxErrorOffset(self):
         check('(0x+1)', 1, 4)
         check('x = 0xI', 1, 7)
         check('0010 + 2', 1, 1)
-        check('x = 32e-+4', 1, 8)
+        check('x = 32e-+4', 1, 9)
         check('x = 0o9', 1, 7)
         check('\u03b1 = 0xI', 1, 7)
         check(b'\xce\xb1 = 0xI', 1, 7)
@@ -322,15 +322,15 @@ def baz():
         check("""f'''
             {
             (123_a)
-            }'''""", 3, 17)
+            }'''""", 3, 18)
         check("""f'''
             {
             f\"\"\"
             {
             (123_a)
             }
             \"\"\"
-            }'''""", 5, 17)
+            }'''""", 5, 18)
         check('''f"""
 
 

Lib/test/test_grammar.py

@@ -193,6 +193,9 @@ def check(test, error=False):
                ("1jz", 3, "imaginary"),
                ("0xI", 3, "hexadecimal"),
                ("0bz", 3, "binary"),
+               ("100_a", 5, " decimal"),
+               ("1.__class__", 3, " decimal"),
+               ("1e+-3", 4, " decimal"),
                # SyntaxWarning's currently:
                ("1or 0", 2, " decimal"),
                ("0o1or 0", 4, "octal"),

Parser/lexer/number.c

@@ -47,11 +47,14 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
      * other keyword or identifier.
      */
     int r = 0;
+    int in_exp = 0; /* do we handle exponent now? */
+
     if (c == 'a') {
         r = lookahead(tok, "nd");
     }
     else if (c == 'e') {
         r = lookahead(tok, "lse");
+        in_exp = 1;
     }
     else if (c == 'f') {
         r = lookahead(tok, "or");
@@ -69,6 +72,9 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
     else if (c == 'n') {
         r = lookahead(tok, "ot");
     }
+    if (in_exp) {
+        c = tok_nextc(tok);
+    }
     if (r) {
         if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
                 "invalid %s literal", kind))
@@ -81,6 +87,9 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
         _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
         return 0;
     }
+    if (in_exp) {
+        tok_backup(tok, c);
+    }
     return 1;
 }
 
@@ -98,7 +107,6 @@ tok_decimal_tail(struct tok_state *tok)
         }
         c = tok_nextc(tok);
         if (!Py_ISDIGIT(c)) {
-            tok_backup(tok, c);
             _PyTokenizer_syntaxerror(tok, "invalid decimal literal");
             return 0;
         }
@@ -269,7 +277,6 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
                 if (c == '+' || c == '-') {
                     c = tok_nextc(tok);
                     if (!Py_ISDIGIT(c)) {
-                        tok_backup(tok, c);
                         return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
                     }
                 } else if (!Py_ISDIGIT(c)) {