diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 4e1157e6ebe7296..1adc4714e8efc88 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -854,7 +854,7 @@ a :exc:`SyntaxError` because the period is seen as a decimal point:: >>> 1.__class__ File "", line 1 1.__class__ - ^ + ^ SyntaxError: invalid decimal literal The solution is to separate the literal from the period diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 0c02b38dd3c0a89..feaea32b85675f9 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -296,14 +296,14 @@ def testSyntaxErrorOffset(self): check('try:\n pass\nexcept*:\n pass\nexcept* ValueError:\n pass', 3, 8) # Errors thrown by the tokenizer - check('(0x+1)', 1, 3) - check('x = 0xI', 1, 6) + check('(0x+1)', 1, 4) + check('x = 0xI', 1, 7) check('0010 + 2', 1, 1) - check('x = 32e-+4', 1, 8) + check('x = 32e-+4', 1, 9) check('x = 0o9', 1, 7) - check('\u03b1 = 0xI', 1, 6) - check(b'\xce\xb1 = 0xI', 1, 6) - check(b'# -*- coding: iso8859-7 -*-\n\xe1 = 0xI', 2, 6, + check('\u03b1 = 0xI', 1, 7) + check(b'\xce\xb1 = 0xI', 1, 7) + check(b'# -*- coding: iso8859-7 -*-\n\xe1 = 0xI', 2, 7, encoding='iso8859-7') check(b"""if 1: def foo(): @@ -322,7 +322,7 @@ def baz(): check("""f''' { (123_a) - }'''""", 3, 17) + }'''""", 3, 18) check("""f''' { f\"\"\" @@ -330,7 +330,7 @@ def baz(): (123_a) } \"\"\" - }'''""", 5, 17) + }'''""", 5, 18) check('''f""" diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index ebcd98a0a37776d..98c49fb4b433827 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -21,6 +21,7 @@ import_helper, skip_emscripten_stack_overflow, skip_wasi_stack_overflow, + subTests, ) from test.support.numbers import ( VALID_UNDERSCORE_LITERALS, @@ -180,6 +181,35 @@ def check(test, error=False): check("[0x1for x in ()]") check("[0xfor x in ()]") + @subTests('source,offset,msg', + [("0xfg", 4, "hexadecimal"), + ("0x9g", 4, "hexadecimal"), + ("0b1z", 4, "binary"), + ("0o7q", 4, "octal"), + ("9spam", 2, " decimal"), + ("0xfspam", 4, "hexadecimal"), + ("1.0x", 4, " decimal"), + ("1e3w", 4, " decimal"), + ("1jz", 3, "imaginary"), + ("0xI", 3, "hexadecimal"), + ("0bz", 3, "binary"), + ("100_a", 5, " decimal"), + ("1.__class__", 3, " decimal"), + ("1e+-3", 4, " decimal"), + # SyntaxWarning's currently: + ("1or 0", 2, " decimal"), + ("0o1or 0", 4, "octal"), + ]) + def test_end_of_numerical_literals_offset(self, source, offset, msg): + # gh-149277: verify the error caret points at the first invalid + # character, not the last valid digit. + with warnings.catch_warnings(): + warnings.simplefilter("error", SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + compile(source, "", "eval") + self.assertEqual(cm.exception.offset, offset) + self.assertIn(msg, cm.exception.msg) + def test_string_literals(self): x = ''; y = ""; self.assertTrue(len(x) == 0 and x == y) x = '\''; y = "'"; self.assertTrue(len(x) == 1 and x == y and ord(x) == 39) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 948b341a5dd72ec..5bfb54228d0af1d 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2455,7 +2455,7 @@ def test_extra_tokens_relaxes_lexer_errors(self): cases = [ ( "2sin(x)", - ("invalid decimal literal", (1, 1)), + ("invalid decimal literal", (1, 2)), [ (token.NUMBER, "2", (1, 0), (1, 1)), (token.NAME, "sin", (1, 1), (1, 4)), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-11-08-16-20.gh-issue-149277.dYRtaF.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-11-08-16-20.gh-issue-149277.dYRtaF.rst new file mode 100644 index 000000000000000..c8f4bc682f526f4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-11-08-16-20.gh-issue-149277.dYRtaF.rst @@ -0,0 +1,2 @@ +Fix the :exc:`SyntaxError` caret position for invalid numeric literals to +point at the first invalid character. diff --git a/Parser/lexer/number.c b/Parser/lexer/number.c index 8bca8cbb9adfe5e..c32c8af33371e38 100644 --- a/Parser/lexer/number.c +++ b/Parser/lexer/number.c @@ -47,11 +47,14 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) { * other keyword or identifier. */ int r = 0; + int in_exp = 0; /* do we handle exponent now? */ + if (c == 'a') { r = lookahead(tok, "nd"); } else if (c == 'e') { r = lookahead(tok, "lse"); + in_exp = 1; } else if (c == 'f') { r = lookahead(tok, "or"); @@ -69,21 +72,24 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) { else if (c == 'n') { r = lookahead(tok, "ot"); } + if (in_exp) { + c = tok_nextc(tok); + } if (r) { - tok_backup(tok, c); if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning, "invalid %s literal", kind)) { return 0; } - tok_nextc(tok); } else /* In future releases, only error will remain. */ if (c < 128 && is_potential_identifier_char(c)) { - tok_backup(tok, c); _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind); return 0; } + if (in_exp) { + tok_backup(tok, c); + } return 1; } @@ -101,7 +107,6 @@ tok_decimal_tail(struct tok_state *tok) } c = tok_nextc(tok); if (!Py_ISDIGIT(c)) { - tok_backup(tok, c); _PyTokenizer_syntaxerror(tok, "invalid decimal literal"); return 0; } @@ -130,7 +135,6 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c, c = tok_nextc(tok); } if (!Py_ISXDIGIT(c)) { - tok_backup(tok, c); return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal")); } do { @@ -154,7 +158,6 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c, "invalid digit '%c' in octal literal", c)); } else { - tok_backup(tok, c); return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal")); } } @@ -182,7 +185,6 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c, return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c)); } else { - tok_backup(tok, c); return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal")); } } @@ -275,7 +277,6 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c, if (c == '+' || c == '-') { c = tok_nextc(tok); if (!Py_ISDIGIT(c)) { - tok_backup(tok, c); return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal")); } } else if (!Py_ISDIGIT(c)) {