Derive encoding error columns from the reporting cursor

Author: pablogsal

Parser/tokenizer/helpers.c

@@ -334,10 +334,9 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
         }
     }
     if (badchar) {
-        int col_offset = (int)(badchar - line_start) + 1;
         _PyTokenizer_syntaxerror_at(
-                tok, line_start, badchar - line_start, lineno,
-                col_offset, col_offset,
+                tok, line_start, badchar - line_start + 1, lineno,
+                -1, -1,
                 "Non-UTF-8 code starting with '\\x%.2x'"
                 "%s%V on line %i, "
                 "but no encoding declared; "

Parser/tokenizer/helpers.h

@@ -6,6 +6,9 @@
 #include "../lexer/state.h"
 
 int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...);
+/* Positive range columns are 1-based byte columns. A start column of -1
+   derives the character column from the reporting cursor; an end column of
+   -1 uses the start column. */
 int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...);
 int _PyTokenizer_syntaxerror_at(
     struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset,