gh-153569: simplify tokenizer input state

Author: pablogsal

Modules/_testinternalcapi/tokenizer.c

@@ -31,7 +31,11 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
     }
 
     _PyTok_SourceClear(&source);
-    if (_PyTok_SourceAppend(&source, "tail", 4) < 0) {
+    if (check(_PyTok_SourceAppend(&source, "tail", 4) == 0,
+              "cannot reuse source") < 0 ||
+            check(source.len == 4 &&
+                      memcmp(source.bytes, "tail\0", 5) == 0,
+                  "wrong reused source contents") < 0) {
         goto error;
     }
 

Parser/lexer/state.h

@@ -74,7 +74,6 @@ struct tok_state {
     _PyTok_Off buf_offset; /* Logical offset of buf[0]. */
     const char *start;  /* Start of current token if not NULL */
     int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
-    /* NB If done != E_OK, cur must be == inp!!! */
     FILE *fp;           /* Rest of input; NULL if tokenizing a string */
     int indent;         /* Current indentation index */
     int indstack[MAXINDENT];            /* Stack of indents */
@@ -146,7 +145,7 @@ _PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
     return tok->buf_offset + offset;
 }
 
-static inline char *
+static inline const char *
 _PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
 {
     assert(tok->buf != NULL);

Parser/tokenizer/decoder.c

@@ -103,7 +103,9 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf,
     }
     result[write] = '\0';
     *out_len = write;
-    *implicit_newline = implicit;
+    if (implicit_newline != NULL) {
+        *implicit_newline = implicit;
+    }
     return result;
 }
 

Parser/tokenizer/reader.c

@@ -191,7 +191,11 @@ chunk_is_line(const _PyTok_Chunk *chunk)
 static _PyTok_ReadResult
 next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
 {
-    const char *source_end = _PyTok_SourceData(&tok->source) + tok->source.len;
+    if (tok->source.len == 0) {
+        return _PYTOK_READ_EOF;
+    }
+    assert(tok->source.bytes != NULL);
+    const char *source_end = tok->source.bytes + tok->source.len;
     if (tok->inp == source_end) {
         return _PYTOK_READ_EOF;
     }
@@ -553,18 +557,15 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
         _PyTok_ChunkClear(&decoded);
         return _PYTOK_READ_ERROR;
     }
-    int implicit_newline;
     chunk->data = _PyTok_NormalizeNewlines(
         decoded.data, decoded.len, 0, 0,
-        &chunk->len, &implicit_newline);
+        &chunk->len, NULL);
     _PyTok_ChunkClear(&decoded);
     if (chunk->data == NULL) {
-        PyErr_NoMemory();
         tok->done = E_NOMEM;
         return _PYTOK_READ_ERROR;
     }
     chunk->ownership = _PYTOK_CHUNK_PYMEM;
-    chunk->implicit_newline = implicit_newline;
     return _PYTOK_READ_LINE;
 }
 
@@ -656,10 +657,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
     }
 
     Py_ssize_t scan_len = chunk.len;
-    if (kind == _PYTOK_READER_INTERACTIVE &&
-            chunk.implicit_newline) {
-        scan_len--;
-    }
     if (streaming) {
         if (reset_buffer) {
             reset_streaming_buffer(tok);