gh-153569: simplify tokenizer source storage

Author: pablogsal

Modules/_testinternalcapi/tokenizer.c

@@ -12,64 +12,26 @@ check(int condition, const char *message)
     return -1;
 }
 
-static int
-check_system_error(int failed, const char *message)
-{
-    if (!failed || !PyErr_ExceptionMatches(PyExc_SystemError)) {
-        PyErr_SetString(PyExc_AssertionError, message);
-        return -1;
-    }
-    PyErr_Clear();
-    return 0;
-}
-
 static PyObject *
 test_tokenizer_source(PyObject *Py_UNUSED(module),
                       PyObject *Py_UNUSED(args))
 {
-    _PyTok_SourceText source;
-    _PyTok_SourceInit(&source);
+    _PyTok_SourceText source = {0};
 
-    if (check_system_error(
-            _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
-            "accepted empty source line") < 0 ||
-            check_system_error(
-                _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
-                "accepted multiple source lines") < 0 ||
-            check_system_error(
-                _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
-                "accepted missing implicit newline") < 0 ||
-            check(_PyTok_SourceAppendLine(
-                      &source, "alpha\n", 6, 0) == 0,
+    if (check(_PyTok_SourceAppend(
+                      &source, "alpha\n", 6) == 0,
                   "wrong first source offset") < 0 ||
-            check(_PyTok_SourceAppendLine(
-                      &source, "\xce\xb2\n", 3, 1) == 6,
+            check(_PyTok_SourceAppend(
+                      &source, "\xce\xb2\n", 3) == 6,
                   "wrong second source offset") < 0 ||
-            check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
-                      _PyTok_SourceLineIsImplicit(&source, 2),
-                  "wrong implicit newline flags") < 0) {
-        goto error;
-    }
-
-    Py_ssize_t view_len;
-    const char *view = _PyTok_SourceSpanView(
-        &source, _PyTok_SpanFromBounds(6, 8), &view_len);
-    if (check(view != NULL && view_len == 2 &&
-                  memcmp(view, "\xce\xb2", 2) == 0,
-              "wrong source span view") < 0 ||
-            check_system_error(
-                _PyTok_SourceSpanView(
-                    &source, _PyTok_SpanFromBounds(0, source.len + 1),
-                    &view_len) == NULL,
-                "accepted invalid source span") < 0) {
+            check(source.len == 9 &&
+                      memcmp(source.bytes, "alpha\n\xce\xb2\n", 9) == 0,
+                  "wrong source contents") < 0) {
         goto error;
     }
 
     _PyTok_SourceClear(&source);
-    if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
-            check_system_error(
-                _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
-                "appended after unterminated source line") < 0) {
+    if (_PyTok_SourceAppend(&source, "tail", 4) < 0) {
         goto error;
     }
 

Parser/tokenizer/decoder.c

@@ -297,51 +297,34 @@ static int
 store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len,
                       int preserve_crlf, int add_final_newline)
 {
-    Py_ssize_t pos = 0;
-    while (pos < len) {
-        Py_ssize_t raw_line_len;
-        if (preserve_crlf) {
-            const char *newline = memchr(data + pos, '\n', len - pos);
-            raw_line_len = newline == NULL
-                ? len - pos : newline - data - pos + 1;
-        }
-        else {
-            raw_line_len = raw_line_length(data + pos, len - pos);
-        }
-        int terminated = preserve_crlf
-            ? data[pos + raw_line_len - 1] == '\n'
-            : data[pos + raw_line_len - 1] == '\n' ||
-              data[pos + raw_line_len - 1] == '\r';
-        int add_newline = add_final_newline &&
-            pos + raw_line_len == len && !terminated;
-        int normalize = add_newline ||
-            (!preserve_crlf &&
-             memchr(data + pos, '\r', raw_line_len) != NULL);
-
-        const char *line = data + pos;
-        Py_ssize_t line_len = raw_line_len;
-        char *normalized = NULL;
-        int implicit = 0;
-        if (normalize) {
-            normalized = _PyTok_NormalizeNewlines(
-                line, line_len, preserve_crlf, add_newline,
-                &line_len, &implicit);
-            if (normalized == NULL) {
-                tok->done = E_NOMEM;
-                return -1;
-            }
-            line = normalized;
-        }
-        _PyTok_Off appended = _PyTok_SourceAppendLine(
-            &tok->source, line, line_len, implicit);
-        PyMem_Free(normalized);
-        if (appended < 0) {
-            tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
-                ? E_NOMEM : E_ERROR;
+    int normalize = (!preserve_crlf && memchr(data, '\r', len) != NULL) ||
+        (add_final_newline && len > 0 && data[len - 1] != '\n');
+    char *normalized = NULL;
+    int implicit = 0;
+    if (normalize) {
+        normalized = _PyTok_NormalizeNewlines(
+            data, len, preserve_crlf, add_final_newline,
+            &len, &implicit);
+        if (normalized == NULL) {
+            tok->done = E_NOMEM;
             return -1;
         }
-        pos += raw_line_len;
+        data = normalized;
+    }
+    if (normalized != NULL) {
+        assert(tok->source.bytes == NULL);
+        tok->source = (_PyTok_SourceText){
+            .bytes = normalized,
+            .len = len,
+            .cap = len + 1,
+        };
+    }
+    else if (_PyTok_SourceAppend(&tok->source, data, len) < 0) {
+        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
+            ? E_NOMEM : E_ERROR;
+        return -1;
     }
+    tok->reader->prepared_implicit_newline = implicit;
     return 0;
 }
 

Parser/tokenizer/reader.c

@@ -191,20 +191,18 @@ chunk_is_line(const _PyTok_Chunk *chunk)
 static _PyTok_ReadResult
 next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
 {
-    int lineno = tok->lineno + 1;
-    if (lineno > tok->source.nlines) {
+    const char *source_end = _PyTok_SourceData(&tok->source) + tok->source.len;
+    if (tok->inp == source_end) {
         return _PYTOK_READ_EOF;
     }
     const char *start = tok->inp;
-    const char *newline = memchr(
-        start, '\n', tok->source.bytes + tok->source.len - start);
-    _PyTok_Off end = newline != NULL
-        ? newline - tok->source.bytes + 1 : tok->source.len;
+    const char *newline = memchr(start, '\n', source_end - start);
+    const char *end = newline != NULL ? newline + 1 : source_end;
     chunk->data = (char *)start;
-    chunk->len = tok->source.bytes + end - start;
+    chunk->len = end - start;
     chunk->ownership = _PYTOK_CHUNK_BORROWED;
-    chunk->implicit_newline = _PyTok_SourceLineIsImplicit(
-        &tok->source, lineno);
+    chunk->implicit_newline = end == source_end &&
+        tok->reader->prepared_implicit_newline;
     return _PYTOK_READ_LINE;
 }
 
@@ -260,7 +258,6 @@ initialize_file(struct tok_state *tok)
     if (result != _PYTOK_READ_LINE) {
         return -1;
     }
-    reader->prefetched_count = 1;
     Py_ssize_t bom_len;
     _PyTok_EncodingResult detection = _PyTok_DetectEncoding(
         tok, &reader->prefetched_lines[0], NULL, 0, &bom_len);
@@ -278,16 +275,13 @@ initialize_file(struct tok_state *tok)
         reader->prefetched_lines[0].data = first;
         reader->prefetched_lines[0].ownership = _PYTOK_CHUNK_PYMEM;
         result = read_file_line(tok, &reader->prefetched_lines[1]);
-        if (result == _PYTOK_READ_LINE) {
-            reader->prefetched_count = 2;
-        }
-        else if (result == _PYTOK_READ_EOF) {
+        if (result == _PYTOK_READ_EOF) {
             reader->file_eof = 1;
         }
-        else {
+        else if (result != _PYTOK_READ_LINE) {
             return -1;
         }
-        _PyTok_Chunk *second = reader->prefetched_count == 2
+        _PyTok_Chunk *second = reader->prefetched_lines[1].data != NULL
             ? &reader->prefetched_lines[1] : NULL;
         detection = _PyTok_DetectEncoding(
             tok, &reader->prefetched_lines[0], second, 1, &bom_len);
@@ -357,10 +351,13 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk)
             return _PYTOK_READ_LINE;
         }
         _PyTok_Chunk input = {0};
-        if (reader->prefetched_index < reader->prefetched_count) {
-            input = reader->prefetched_lines[reader->prefetched_index];
-            reader->prefetched_lines[reader->prefetched_index++] =
-                (_PyTok_Chunk){0};
+        if (reader->prefetched_lines[0].data != NULL) {
+            input = reader->prefetched_lines[0];
+            reader->prefetched_lines[0] = (_PyTok_Chunk){0};
+        }
+        else if (reader->prefetched_lines[1].data != NULL) {
+            input = reader->prefetched_lines[1];
+            reader->prefetched_lines[1] = (_PyTok_Chunk){0};
         }
         else if (!reader->file_eof) {
             _PyTok_ReadResult result = read_file_line(tok, &input);
@@ -556,16 +553,18 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
         _PyTok_ChunkClear(&decoded);
         return _PYTOK_READ_ERROR;
     }
+    int implicit_newline;
     chunk->data = _PyTok_NormalizeNewlines(
         decoded.data, decoded.len, 0, 0,
-        &chunk->len, &chunk->implicit_newline);
+        &chunk->len, &implicit_newline);
     _PyTok_ChunkClear(&decoded);
     if (chunk->data == NULL) {
         PyErr_NoMemory();
         tok->done = E_NOMEM;
         return _PYTOK_READ_ERROR;
     }
     chunk->ownership = _PYTOK_CHUNK_PYMEM;
+    chunk->implicit_newline = implicit_newline;
     return _PYTOK_READ_LINE;
 }
 
@@ -648,6 +647,13 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         }
         return 0;
     }
+    if (tok->lineno == INT_MAX) {
+        _PyTok_ChunkClear(&chunk);
+        PyErr_SetString(PyExc_OverflowError,
+                        "too many tokenizer source lines");
+        tok->done = E_ERROR;
+        return 0;
+    }
 
     Py_ssize_t scan_len = chunk.len;
     if (kind == _PYTOK_READER_INTERACTIVE &&
@@ -678,9 +684,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         if (!reset_buffer) {
             offsets = save_buffer_offsets(tok, tok->source.bytes);
         }
-        _PyTok_Off source_start = _PyTok_SourceAppendLine(
-            &tok->source, chunk.data, chunk.len,
-            chunk.implicit_newline);
+        _PyTok_Off source_start = _PyTok_SourceAppend(
+            &tok->source, chunk.data, chunk.len);
         if (source_start < 0) {
             _PyTok_ChunkClear(&chunk);
             tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)

Parser/tokenizer/reader_internal.h

@@ -32,14 +32,13 @@ typedef enum {
 
 typedef struct {
     char *data;
-    Py_ssize_t len;
-    int implicit_newline;
     PyObject *owner;
+    Py_ssize_t len;
     _PyTok_ChunkOwnership ownership;
+    unsigned char implicit_newline;
 } _PyTok_Chunk;
 
 typedef struct _PyTok_Reader {
-    _PyTok_ReaderKind kind;
     PyObject *readline;
     PyObject *decoder;
     const char *prompt;
@@ -50,19 +49,18 @@ typedef struct _PyTok_Reader {
     char *file_buffer;
     Py_ssize_t file_buffer_cap;
     _PyTok_Chunk prefetched_lines[2];
-    int prefetched_index;
-    int prefetched_count;
 
     char *decoded;
     Py_ssize_t decoded_pos;
     Py_ssize_t decoded_len;
     Py_ssize_t decoded_cap;
-    int decoded_tail_is_implicit;
-
-    int file_initialized;
-    int file_eof;
-    int decoder_finalized;
-    int stop_interactive;
+    _PyTok_ReaderKind kind;
+    unsigned char decoded_tail_is_implicit;
+    unsigned char file_initialized;
+    unsigned char file_eof;
+    unsigned char decoder_finalized;
+    unsigned char stop_interactive;
+    unsigned char prepared_implicit_newline;
 } _PyTok_Reader;
 
 struct tok_state;