gh-153569: report tokenizer diagnostics without rewinding the scanner

Author: pablogsal

Lib/test/test_codeop.py

@@ -113,6 +113,17 @@ def test_valid(self, compiler):
         av("def f():\n pass\n#foo\n")
         av("@a.b.c\ndef f():\n pass\n")
 
+    @subTests('symbol', ('single', 'exec'))
+    @subTests('prefix', ('', 'f', 't'))
+    def test_incomplete_string_diagnostics(self, symbol, prefix):
+        opening = f'    á = {prefix}"""first\n'
+        source = 'if True:\n' + opening + 'second'
+        with self.assertRaises(_IncompleteInputError) as cm:
+            Compile()(source, '<input>', symbol)
+        text = opening + 'second' + ('\n' if symbol == 'exec' else '')
+        self.assertEqual(cm.exception.args, (
+            'incomplete input', ('<input>', 2, 9, text, 2, -1)))
+
     @subTests('compiler', COMPILERS)
     def test_incomplete(self, compiler):
         ai = functools.partial(self.assertIncomplete, compiler=compiler)

Lib/test/test_source_encoding.py

@@ -3,8 +3,8 @@
 import unittest
 from test import support
 from test.support import script_helper
-from test.support.os_helper import TESTFN, unlink, rmtree
-from test.support.import_helper import unload
+from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree
+from test.support.import_helper import import_module, unload
 import importlib
 import os
 import sys
@@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self):
                 self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
 
     def test_invalid_utf8_offset_after_non_ascii(self):
+        for name in ('é', 'éé', '𝒜'):
+            with self.subTest(name=name):
+                source = ('x = ' + name).encode() + b'\xff\n'
+                with self.assertRaises(SyntaxError) as caught:
+                    compile(source, '<test>', 'exec')
+                error = caught.exception
+                self.assertEqual(
+                    (error.lineno, error.offset, error.end_lineno, error.end_offset),
+                    (1, 5 + len(name), 1, 5 + len(name)),
+                )
+
+    @support.cpython_only
+    def test_invalid_utf8_file_offset_after_non_ascii(self):
+        _testcapi = import_module('_testcapi')
+        self.addCleanup(unlink, TESTFN_ASCII)
+        with open(TESTFN_ASCII, 'wb') as f:
+            f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n')
         with self.assertRaises(SyntaxError) as caught:
-            compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
+            _testcapi.run_file(
+                os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {})
         error = caught.exception
         self.assertEqual(
             (error.lineno, error.offset, error.end_lineno, error.end_offset),
-            (1, 6, 1, 6),
+            (2, 7, 2, 7),
         )
 
     def test_long_bom_conflict_message_is_not_truncated(self):

Lib/test/test_tstring.py

@@ -215,6 +215,8 @@ def test_nested_templates(self):
 
     def test_syntax_errors(self):
         for case, err in (
+            ('t"""{(\n1\n)}\ntail', "unterminated triple-quoted t-string literal"),
+            ('f"""{(\n1\n)}\ntail', "unterminated triple-quoted f-string literal"),
             ("t'", "unterminated t-string literal"),
             ("t'''", "unterminated triple-quoted t-string literal"),
             ("t''''", "unterminated triple-quoted t-string literal"),

Parser/lexer/lexer.c

@@ -89,6 +89,7 @@ verify_identifier(struct tok_state *tok)
     assert(PyUnicode_GET_LENGTH(s) > 0);
     if (invalid < PyUnicode_GET_LENGTH(s)) {
         Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
+        const char *error_cursor = tok->cur;
         if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
             /* Determine the offset in UTF-8 encoded input */
             Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
@@ -99,14 +100,20 @@ verify_identifier(struct tok_state *tok)
                 tok->done = E_ERROR;
                 return 0;
             }
-            tok->cur = tok->start + PyBytes_GET_SIZE(s);
+            error_cursor = tok->start + PyBytes_GET_SIZE(s);
         }
         Py_DECREF(s);
         if (Py_UNICODE_ISPRINTABLE(ch)) {
-            _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch);
+            _PyTokenizer_syntaxerror_at(
+                tok, tok->line_start,
+                error_cursor - tok->line_start, tok->lineno, -1, -1,
+                "invalid character '%c' (U+%04X)", ch, ch);
         }
         else {
-            _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch);
+            _PyTokenizer_syntaxerror_at(
+                tok, tok->line_start,
+                error_cursor - tok->line_start, tok->lineno, -1, -1,
+                "invalid non-printable character U+%04X", ch);
         }
         return 0;
     }

Parser/lexer/state.h

@@ -72,6 +72,14 @@ typedef struct {
     indentation_level stack[MAXINDENT];
 } lexer_layout_state;
 
+/* Supplemental source context for a terminal error. location is the reporting
+   cursor, independent of the scanner cursor; lineno == 0 means absent.
+   The text span may cover multiple physical lines. */
+typedef struct {
+    _PyTok_Loc location;
+    _PyTok_Span text_span;
+} _PyTokenizer_Diagnostic;
+
 /* Tokenizer state */
 struct tok_state {
     _PyTok_Off buf_offset;
@@ -86,6 +94,7 @@ struct tok_state {
     lexer_layout_state layout;
     int lineno;         /* Current line number */
     _PyTok_Loc start_loc;
+    _PyTokenizer_Diagnostic diagnostic;
     int level;          /* () [] {} Parentheses nesting level */
             /* Used to allow free continuations inside them */
     char parenstack[MAXLEVEL];

Parser/lexer/string.c

@@ -7,13 +7,18 @@
 
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
 
-static void
-rewind_to_string_start(struct tok_state *tok, _PyTok_Off start,
-                       _PyTok_Loc location)
+static int
+string_error_token(struct tok_state *tok, struct token *token,
+                   _PyTok_Off start, _PyTok_Loc location)
 {
-    tok->cur = start + 1;
-    tok->line_start = start - location.byte_col;
-    tok->lineno = location.lineno;
+    tok->diagnostic = (_PyTokenizer_Diagnostic){
+        .location = {location.lineno, location.byte_col + 1},
+        .text_span = _PyTok_SpanFromBounds(start - location.byte_col, tok->inp),
+    };
+    int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1);
+    token->start_loc = location;
+    token->end_loc = (_PyTok_Loc){location.lineno, -1};
+    return type;
 }
 
 int
@@ -351,44 +356,51 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
         }
         if (c == EOF || (quote_size == 1 && c == '\n')) {
             int end_lineno = tok->lineno;
-            rewind_to_string_start(tok, tok->start, tok->start_loc);
+            _PyTok_Loc location = tok->start_loc;
+            const char *line = tok->start - location.byte_col;
+            Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
 
             const ftstring_state *state = _PyLexer_CurrentFTString(tok);
             if (state != NULL) {
                 /* A matching quote belongs to the surrounding formatted
                  * string, so the expression is missing its closing brace. */
                 if (state->quote == quote && state->quote_size == quote_size) {
-                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+                    _PyTokenizer_syntaxerror_at(
+                        tok, line, cursor_offset, location.lineno, -1, -1,
                         "%c-string: expecting '}'",
-                        _PyLexer_StringPrefix(state->kind)));
+                        _PyLexer_StringPrefix(state->kind));
+                    return string_error_token(tok, token, tok->start, location);
                 }
             }
 
             if (quote_size == 3) {
-                _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
-                                 " (detected at line %d)", end_lineno);
+                _PyTokenizer_syntaxerror_at(
+                    tok, line, cursor_offset, location.lineno, -1, -1,
+                    "unterminated triple-quoted string literal"
+                    " (detected at line %d)", end_lineno);
                 if (c != '\n') {
                     tok->done = E_EOFS;
                 }
-                return MAKE_TOKEN(ERRORTOKEN);
+                return string_error_token(tok, token, tok->start, location);
             }
             else {
                 if (has_escaped_quote) {
-                    _PyTokenizer_syntaxerror(
-                        tok,
+                    _PyTokenizer_syntaxerror_at(
+                        tok, line, cursor_offset, location.lineno, -1, -1,
                         "unterminated string literal (detected at line %d); "
                         "perhaps you escaped the end quote?",
                         end_lineno
                     );
                 } else {
-                    _PyTokenizer_syntaxerror(
-                        tok, "unterminated string literal (detected at line %d)", end_lineno
+                    _PyTokenizer_syntaxerror_at(
+                        tok, line, cursor_offset, location.lineno, -1, -1,
+                        "unterminated string literal (detected at line %d)", end_lineno
                     );
                 }
                 if (c != '\n') {
                     tok->done = E_EOLS;
                 }
-                return MAKE_TOKEN(ERRORTOKEN);
+                return string_error_token(tok, token, tok->start, location);
             }
         }
         if (c == quote) {
@@ -452,25 +464,29 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
             }
 
             int end_lineno = tok->lineno;
-            rewind_to_string_start(tok,
-                current->start,
-                current->start_loc);
+            _PyTok_Loc location = current->start_loc;
+            const char *line = _PyLexer_BufferPointer(tok, current->start) - location.byte_col;
+            Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
 
             if (quote_size == 3) {
-                _PyTokenizer_syntaxerror(tok,
-                                    "unterminated triple-quoted %c-string literal"
-                                    " (detected at line %d)",
-                                    _PyLexer_StringPrefix(current->kind), end_lineno);
+                _PyTokenizer_syntaxerror_at(
+                    tok, line, cursor_offset, location.lineno, -1, -1,
+                    "unterminated triple-quoted %c-string literal"
+                    " (detected at line %d)",
+                    _PyLexer_StringPrefix(current->kind), end_lineno);
                 if (c != '\n') {
                     tok->done = E_EOFS;
                 }
-                return MAKE_TOKEN(ERRORTOKEN);
+                return string_error_token(tok, token,
+                    _PyLexer_BufferPointer(tok, current->start), location);
             }
             else {
-                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                                    "unterminated %c-string literal (detected at"
-                                    " line %d)",
-                                    _PyLexer_StringPrefix(current->kind), end_lineno));
+                _PyTokenizer_syntaxerror_at(
+                    tok, line, cursor_offset, location.lineno, -1, -1,
+                    "unterminated %c-string literal (detected at line %d)",
+                    _PyLexer_StringPrefix(current->kind), end_lineno);
+                return string_error_token(tok, token,
+                    _PyLexer_BufferPointer(tok, current->start), location);
             }
         }
 

Parser/tokenizer/decoder.c

@@ -246,7 +246,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
             end_col--;
         }
         _PyTokenizer_syntaxerror_at(
-            tok, line_data, 0, cookie_line, 0, end_col, "encoding problem: %s with BOM", cookie);
+            tok, line_data, 0, cookie_line, 0, end_col,
+            "encoding problem: %s with BOM", cookie);
         PyMem_Free(cookie);
         return _PYTOK_ENCODING_ERROR;
     }

Parser/tokenizer/helpers.c

@@ -321,26 +321,21 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
     const char *badchar = NULL;
     const char *c;
     int length;
-    int col_offset = 0;
     const char *line_start = line;
     for (c = line; *c; c += length) {
         if (!(length = valid_utf8((const unsigned char *)c))) {
             badchar = c;
             break;
         }
-        col_offset++;
         if (*c == '\n') {
             lineno++;
-            col_offset = 0;
             line_start = c + 1;
         }
     }
     if (badchar) {
-        tok->lineno = lineno;
-        tok->line_start = _PyLexer_BufferOffset(tok, line_start);
-        tok->cur = _PyLexer_BufferOffset(tok, badchar);
-        _PyTokenizer_syntaxerror_known_range(tok,
-                col_offset + 1, col_offset + 1,
+        _PyTokenizer_syntaxerror_at(
+                tok, line_start, badchar - line_start + 1, lineno,
+                -1, -1,
                 "Non-UTF-8 code starting with '\\x%.2x'"
                 "%s%V on line %i, "
                 "but no encoding declared; "

Parser/tokenizer/helpers.h

@@ -8,10 +8,17 @@
 int _PyTokenizer_syntaxerror_at(struct tok_state *, const char *,
     Py_ssize_t, int, int, int, const char *, ...);
 int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...);
+/* Positive range columns are 1-based byte columns. A start column of -1
+   derives the character column from the reporting cursor; an end column of
+   -1 uses the start column. */
 int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...);
+int _PyTokenizer_syntaxerror_at(
+    struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset,
+    int lineno, int col_offset, int end_col_offset, const char *format, ...);
 int _PyTokenizer_indenterror(struct tok_state *tok);
 int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char);
 int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...);
+
 void _PyTokenizer_raise_init_error(PyObject *filename);
 
 int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno);