gh-153569: move tokenizer input state and relocation into the reader

Author: pablogsal

Lib/test/test_repl.py

@@ -184,9 +184,8 @@ def read_until(marker, start=0):
 
     @cpython_only
     def test_lexer_buffer_realloc_with_null_start(self):
-        # gh-144759: NULL pointer arithmetic in the lexer when start and
-        # multi_line_start are NULL (uninitialized in tok_mode_stack[0])
-        # and the lexer buffer is reallocated while parsing long input.
+        # gh-144759: NULL pointer arithmetic when the lexer buffer grows
+        # while parsing long input.
         long_value = "a" * 2000
         user_input = dedent(f"""\
         x = f'{{{long_value!r}}}'

Parser/lexer/lexer.c

@@ -197,15 +197,16 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
         }
         tok_backup(tok, c);
         if (c == '#' || c == '\n' || c == '\r') {
+            int interactive = _PyTok_ReaderIsInteractive(tok);
             /* Lines with only whitespace and/or comments
                shouldn't affect the indentation and are
                not passed to the parser as NEWLINE tokens,
                except *totally* empty lines in interactive
                mode, which signal the end of a command group. */
-            if (col == 0 && c == '\n' && tok->prompt != NULL) {
+            if (col == 0 && c == '\n' && interactive) {
                 blankline = 0; /* Let it through */
             }
-            else if (tok->prompt != NULL && tok->lineno == 1) {
+            else if (interactive && tok->lineno == 1) {
                 /* In interactive mode, if the first line contains
                    only spaces and/or a comment, let it through. */
                 blankline = 0;

Parser/lexer/state.c

@@ -1,61 +1,11 @@
 #include "Python.h"
-#include "pycore_pystate.h"
 #include "pycore_token.h"
 #include "errcode.h"
 
 #include "state.h"
 #include "../tokenizer/helpers.h"
 #include "../tokenizer/reader.h"
 
-/* Create and initialize a new tok_state structure */
-struct tok_state *
-_PyTokenizer_tok_new(void)
-{
-    struct tok_state *tok = (struct tok_state *)PyMem_Calloc(
-                                            1,
-                                            sizeof(struct tok_state));
-    if (tok == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-
-    tok->cur = tok->inp = 0;
-    tok->line_start = -1;
-    tok->fp_interactive = 0;
-    tok->interactive_src_start = NULL;
-    tok->interactive_src_end = NULL;
-    tok->start = -1;
-    tok->done = E_OK;
-    tok->fp = NULL;
-    tok->indent = 0;
-    tok->indstack[0] = 0;
-    tok->atbol = 1;
-    tok->pendin = 0;
-    tok->prompt = NULL;
-    tok->lineno = 0;
-    tok->start_loc = (_PyTok_Loc){-1, -1};
-    tok->level = 0;
-    tok->altindstack[0] = 0;
-    tok->encoding = NULL;
-    tok->filename = NULL;
-    tok->module = NULL;
-    tok->type_comments = 0;
-    tok->interactive_underflow = IUNDERFLOW_NORMAL;
-    tok->str = NULL;
-    tok->report_warnings = 1;
-    tok->tok_extra_tokens = 0;
-    tok->comment_newline = 0;
-    tok->implicit_newline = 0;
-    _PyTok_SourceInit(&tok->source);
-    tok->reader = NULL;
-    tok->ftstring_stack = tok->ftstring_stack_inline;
-    tok->ftstring_capacity = FTSTRING_STACK_INLINE_CAPACITY;
-#ifdef Py_DEBUG
-    tok->debug = _Py_GetConfig()->parser_debug;
-#endif
-    return tok;
-}
-
 ftstring_state *
 _PyLexer_PushFTString(struct tok_state *tok)
 {

Parser/lexer/state.h

@@ -10,13 +10,6 @@
 #define MAXFTSTRINGLEVEL 150
 #define FTSTRING_STACK_INLINE_CAPACITY 1
 
-enum interactive_underflow_t {
-    /* Normal mode of operation: return a new token when asked in interactive mode */
-    IUNDERFLOW_NORMAL,
-    /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This
-     * can be used to prevent the tokenizer to prompt the user for new tokens */
-    IUNDERFLOW_STOP,
-};
 
 
 typedef enum {
@@ -74,17 +67,13 @@ struct tok_state {
     _PyTok_Off start;
     _PyTok_Off line_start;
     _PyTok_SourceText source;
-    int fp_interactive; /* If the file descriptor is interactive */
-    char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
-    char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
     int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
     /* NB If done != E_OK, cur must be == inp!!! */
     FILE *fp;           /* Rest of input; NULL if tokenizing a string */
     int indent;         /* Current indentation index */
     int indstack[MAXINDENT];            /* Stack of indents */
     int atbol;          /* Nonzero if at begin of new line */
     int pendin;         /* Pending indents (if > 0) or dedents (if < 0) */
-    const char *prompt;          /* For interactive prompting */
     int lineno;         /* Current line number */
     _PyTok_Loc start_loc;
     int level;          /* () [] {} Parentheses nesting level */
@@ -98,15 +87,11 @@ struct tok_state {
     int altindstack[MAXINDENT];         /* Stack of alternate indents */
     /* Stuff for PEP 0263 */
     char *encoding;         /* Source encoding. */
-    char* str;          /* Source string being tokenized (if tokenizing from a string)*/
 
     struct _PyTok_Reader *reader;
 
     int type_comments;      /* Whether to look for type comments */
 
-    /* How to proceed when asked for a new token in interactive mode */
-    enum interactive_underflow_t interactive_underflow;
-    int report_warnings;
     ftstring_state *ftstring_stack;
     ftstring_state ftstring_stack_inline[FTSTRING_STACK_INLINE_CAPACITY];
     int ftstring_depth;
@@ -182,7 +167,6 @@ _PyLexer_ByteColumn(const struct tok_state *tok)
 
 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, _PyTok_Off start, _PyTok_Off end);
 
-struct tok_state *_PyTokenizer_tok_new(void);
 void _PyTokenizer_Free(struct tok_state *);
 ftstring_state *_PyLexer_PushFTString(struct tok_state *);
 void _PyLexer_PopFTString(struct tok_state *);

Parser/tokenizer/api.c

@@ -160,11 +160,11 @@ _PyTokenizer_HasTrailingStatement(const struct tok_state *tok)
 int
 _PyTokenizer_IsInteractive(const struct tok_state *tok)
 {
-    return tok->prompt != NULL;
+    return _PyTok_ReaderIsInteractive(tok);
 }
 
 void
 _PyTokenizer_StopInteractive(struct tok_state *tok)
 {
-    tok->interactive_underflow = IUNDERFLOW_STOP;
+    _PyTok_ReaderStopInteractive(tok);
 }

Parser/tokenizer/decoder.c

@@ -103,7 +103,9 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf,
     }
     result[write] = '\0';
     *out_len = write;
-    *implicit_newline = implicit;
+    if (implicit_newline != NULL) {
+        *implicit_newline = implicit;
+    }
     return result;
 }
 
@@ -390,10 +392,9 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only,
     if (stored < 0) {
         return -1;
     }
-    tok->str = tok->source.bytes != NULL ? tok->source.bytes : (char *)"";
     if (!utf8_only &&
             (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
-            !_PyTokenizer_ensure_utf8(tok->str, tok, 1)) {
+            !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) {
         return -1;
     }
     return 0;

Parser/tokenizer/helpers.c

@@ -137,10 +137,6 @@ _PyTokenizer_indenterror(struct tok_state *tok)
 int
 _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char)
 {
-    if (!tok->report_warnings) {
-        return 0;
-    }
-
     PyObject *msg = PyUnicode_FromFormat(
         "\"\\%c\" is an invalid escape sequence. "
         "Such sequences will not work in the future. "
@@ -226,10 +222,6 @@ _PyTokenizer_raise_init_error(PyObject *filename)
 int
 _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...)
 {
-    if (!tok->report_warnings) {
-        return 0;
-    }
-
     PyObject *errmsg;
     va_list vargs;
     va_start(vargs, format);