From 7fb8fb6d46b761fad7219f756bba083975d7dcfb Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 11 Aug 2026 08:28:41 +0300 Subject: [PATCH 1/5] gh-149277: Fix error position for invalid numeric literals --- Lib/test/test_grammar.py | 22 +++++++++++++++++++ Lib/test/test_tokenize.py | 2 +- ...-08-11-08-16-20.gh-issue-149277.dYRtaF.rst | 2 ++ Parser/lexer/number.c | 4 ---- 4 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-11-08-16-20.gh-issue-149277.dYRtaF.rst diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index ebcd98a0a37776d..1861249f1409fb3 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -21,6 +21,7 @@ import_helper, skip_emscripten_stack_overflow, skip_wasi_stack_overflow, + subTests, ) from test.support.numbers import ( VALID_UNDERSCORE_LITERALS, @@ -180,6 +181,27 @@ def check(test, error=False): check("[0x1for x in ()]") check("[0xfor x in ()]") + @subTests('source,offset,msg', + [("0xfg", 4, "hexadecimal"), + ("0x9g", 4, "hexadecimal"), + ("0b1z", 4, "binary"), + ("0o7q", 4, "octal"), + ("9spam", 2, " decimal"), + ("0xfspam", 4, "hexadecimal"), + ("1.0x", 4, " decimal"), + ("1e3w", 4, " decimal"), + ("1jz", 3, "imaginary"), + ("0xI", 3, "hexadecimal"), + ("0bz", 3, "binary"), + ]) + def test_end_of_numerical_literals_offset(self, source, offset, msg): + # gh-149277: verify the error caret points at the first invalid + # character, not the last valid digit. + with self.assertRaises(SyntaxError) as cm: + compile(source, "", "eval") + self.assertEqual(cm.exception.offset, offset) + self.assertIn(msg, cm.exception.msg) + def test_string_literals(self): x = ''; y = ""; self.assertTrue(len(x) == 0 and x == y) x = '\''; y = "'"; self.assertTrue(len(x) == 1 and x == y and ord(x) == 39) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index e2db09d61f409bd..39e70bd82c4c556 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2277,7 +2277,7 @@ def test_extra_tokens_relaxes_lexer_errors(self): cases = [ ( "2sin(x)", - ("invalid decimal literal", (1, 1)), + ("invalid decimal literal", (1, 2)), [ (token.NUMBER, "2", (1, 0), (1, 1)), (token.NAME, "sin", (1, 1), (1, 4)), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-11-08-16-20.gh-issue-149277.dYRtaF.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-11-08-16-20.gh-issue-149277.dYRtaF.rst new file mode 100644 index 000000000000000..c8f4bc682f526f4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-11-08-16-20.gh-issue-149277.dYRtaF.rst @@ -0,0 +1,2 @@ +Fix the :exc:`SyntaxError` caret position for invalid numeric literals to +point at the first invalid character. diff --git a/Parser/lexer/number.c b/Parser/lexer/number.c index 8bca8cbb9adfe5e..02e11d12616b04a 100644 --- a/Parser/lexer/number.c +++ b/Parser/lexer/number.c @@ -80,7 +80,6 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) { } else /* In future releases, only error will remain. */ if (c < 128 && is_potential_identifier_char(c)) { - tok_backup(tok, c); _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind); return 0; } @@ -130,7 +129,6 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c, c = tok_nextc(tok); } if (!Py_ISXDIGIT(c)) { - tok_backup(tok, c); return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal")); } do { @@ -154,7 +152,6 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c, "invalid digit '%c' in octal literal", c)); } else { - tok_backup(tok, c); return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal")); } } @@ -182,7 +179,6 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c, return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c)); } else { - tok_backup(tok, c); return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal")); } } From 105d36ca47883a0db7de7fa22b6cfa1049a94422 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 11 Aug 2026 10:42:55 +0300 Subject: [PATCH 2/5] +1 --- Lib/test/test_exceptions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 0cee756958f3ad1..c53bb35c4987769 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -279,14 +279,14 @@ def testSyntaxErrorOffset(self): check('try:\n pass\nexcept*:\n pass\nexcept* ValueError:\n pass', 3, 8) # Errors thrown by the tokenizer - check('(0x+1)', 1, 3) - check('x = 0xI', 1, 6) + check('(0x+1)', 1, 4) + check('x = 0xI', 1, 7) check('0010 + 2', 1, 1) check('x = 32e-+4', 1, 8) check('x = 0o9', 1, 7) - check('\u03b1 = 0xI', 1, 6) - check(b'\xce\xb1 = 0xI', 1, 6) - check(b'# -*- coding: iso8859-7 -*-\n\xe1 = 0xI', 2, 6, + check('\u03b1 = 0xI', 1, 7) + check(b'\xce\xb1 = 0xI', 1, 7) + check(b'# -*- coding: iso8859-7 -*-\n\xe1 = 0xI', 2, 7, encoding='iso8859-7') check(b"""if 1: def foo(): From 80b51343423ef54d83459aa17c4edec5f2b974d1 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Sat, 5 Sep 2026 06:46:55 +0300 Subject: [PATCH 3/5] Handle more cases (SyntaxWarning's currently) --- Lib/test/test_grammar.py | 9 +++++++-- Parser/lexer/number.c | 2 -- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 1861249f1409fb3..2f6c8129843f608 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -193,12 +193,17 @@ def check(test, error=False): ("1jz", 3, "imaginary"), ("0xI", 3, "hexadecimal"), ("0bz", 3, "binary"), + # SyntaxWarning's currently: + ("1or 0", 2, " decimal"), + ("0or 0", 3, "octal"), ]) def test_end_of_numerical_literals_offset(self, source, offset, msg): # gh-149277: verify the error caret points at the first invalid # character, not the last valid digit. - with self.assertRaises(SyntaxError) as cm: - compile(source, "", "eval") + with warnings.catch_warnings(): + warnings.simplefilter("error", SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + compile(source, "", "eval") self.assertEqual(cm.exception.offset, offset) self.assertIn(msg, cm.exception.msg) diff --git a/Parser/lexer/number.c b/Parser/lexer/number.c index 02e11d12616b04a..c38840fd4327431 100644 --- a/Parser/lexer/number.c +++ b/Parser/lexer/number.c @@ -70,13 +70,11 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) { r = lookahead(tok, "ot"); } if (r) { - tok_backup(tok, c); if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning, "invalid %s literal", kind)) { return 0; } - tok_nextc(tok); } else /* In future releases, only error will remain. */ if (c < 128 && is_potential_identifier_char(c)) { From afe200a75a5bf36b9d75929ab92bf82ea6298309 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 8 Sep 2026 06:00:24 +0300 Subject: [PATCH 4/5] address review: add SyntaxWarning's test for octal --- Lib/test/test_grammar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 2f6c8129843f608..83e945e3b873582 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -195,7 +195,7 @@ def check(test, error=False): ("0bz", 3, "binary"), # SyntaxWarning's currently: ("1or 0", 2, " decimal"), - ("0or 0", 3, "octal"), + ("0o1or 0", 4, "octal"), ]) def test_end_of_numerical_literals_offset(self, source, offset, msg): # gh-149277: verify the error caret points at the first invalid From 74e0063252d37daf6f0bcd4f8aa4c428d28a5a66 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 8 Sep 2026 07:41:58 +0300 Subject: [PATCH 5/5] address review: handle more cases (exponent, underscore) --- Doc/faq/programming.rst | 2 +- Lib/test/test_exceptions.py | 6 +++--- Lib/test/test_grammar.py | 3 +++ Parser/lexer/number.c | 11 +++++++++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 4e1157e6ebe7296..1adc4714e8efc88 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -854,7 +854,7 @@ a :exc:`SyntaxError` because the period is seen as a decimal point:: >>> 1.__class__ File "", line 1 1.__class__ - ^ + ^ SyntaxError: invalid decimal literal The solution is to separate the literal from the period diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index cd2550545d7c303..feaea32b85675f9 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -299,7 +299,7 @@ def testSyntaxErrorOffset(self): check('(0x+1)', 1, 4) check('x = 0xI', 1, 7) check('0010 + 2', 1, 1) - check('x = 32e-+4', 1, 8) + check('x = 32e-+4', 1, 9) check('x = 0o9', 1, 7) check('\u03b1 = 0xI', 1, 7) check(b'\xce\xb1 = 0xI', 1, 7) @@ -322,7 +322,7 @@ def baz(): check("""f''' { (123_a) - }'''""", 3, 17) + }'''""", 3, 18) check("""f''' { f\"\"\" @@ -330,7 +330,7 @@ def baz(): (123_a) } \"\"\" - }'''""", 5, 17) + }'''""", 5, 18) check('''f""" diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 83e945e3b873582..98c49fb4b433827 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -193,6 +193,9 @@ def check(test, error=False): ("1jz", 3, "imaginary"), ("0xI", 3, "hexadecimal"), ("0bz", 3, "binary"), + ("100_a", 5, " decimal"), + ("1.__class__", 3, " decimal"), + ("1e+-3", 4, " decimal"), # SyntaxWarning's currently: ("1or 0", 2, " decimal"), ("0o1or 0", 4, "octal"), diff --git a/Parser/lexer/number.c b/Parser/lexer/number.c index c38840fd4327431..c32c8af33371e38 100644 --- a/Parser/lexer/number.c +++ b/Parser/lexer/number.c @@ -47,11 +47,14 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) { * other keyword or identifier. */ int r = 0; + int in_exp = 0; /* do we handle exponent now? */ + if (c == 'a') { r = lookahead(tok, "nd"); } else if (c == 'e') { r = lookahead(tok, "lse"); + in_exp = 1; } else if (c == 'f') { r = lookahead(tok, "or"); @@ -69,6 +72,9 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) { else if (c == 'n') { r = lookahead(tok, "ot"); } + if (in_exp) { + c = tok_nextc(tok); + } if (r) { if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning, "invalid %s literal", kind)) @@ -81,6 +87,9 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) { _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind); return 0; } + if (in_exp) { + tok_backup(tok, c); + } return 1; } @@ -98,7 +107,6 @@ tok_decimal_tail(struct tok_state *tok) } c = tok_nextc(tok); if (!Py_ISDIGIT(c)) { - tok_backup(tok, c); _PyTokenizer_syntaxerror(tok, "invalid decimal literal"); return 0; } @@ -269,7 +277,6 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c, if (c == '+' || c == '-') { c = tok_nextc(tok); if (!Py_ISDIGIT(c)) { - tok_backup(tok, c); return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal")); } } else if (!Py_ISDIGIT(c)) {