From 09117bc3173b6854f3d614fb0efe79bca4b4cc63 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 6 Sep 2026 21:15:44 +0100 Subject: [PATCH] gh-153569: return token spans and unify decoded source storage (#156482) Represent token boundaries as logical source offsets with explicit start and end locations, and adapt pegen and _tokenize to consume them. Use SourceText to own decoded input across all reader modes. Streaming readers discard consumed windows while preserving logical offsets and reusing the allocation. Prepared and interactive input retain their source. Force source relocation on growth in debug builds and add coverage for formatted-string growth, interactive input, storage reuse, and offset limits. Scanner positions remain pointers, rebased when storage moves. --- Lib/test/test_capi/test_tokenizer.py | 3 + Lib/test/test_repl.py | 16 ++++ Lib/test/test_tokenize.py | 24 ++++++ Modules/_testinternalcapi/tokenizer.c | 105 ++++++++++++++++++++++++ Parser/lexer/buffer.c | 82 ++++++++----------- Parser/lexer/buffer.h | 18 ++++- Parser/lexer/lexer.c | 14 ++-- Parser/lexer/lexer.h | 21 +++++ Parser/lexer/state.c | 52 ++++++------ Parser/lexer/state.h | 13 ++- Parser/lexer/string.c | 8 +- Parser/pegen.c | 41 +++++----- Parser/tokenizer/cursor.c | 9 ++- Parser/tokenizer/cursor.h | 16 +++- Parser/tokenizer/reader.c | 112 ++++++++++++++++---------- Parser/tokenizer/source.c | 48 ++++++++--- Parser/tokenizer/source.h | 25 ++++-- Python/Python-tokenize.c | 45 ++++++----- 18 files changed, 453 insertions(+), 199 deletions(-) diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py index 2fe1fef241e90ae..eb04f6c0136022d 100644 --- a/Lib/test/test_capi/test_tokenizer.py +++ b/Lib/test/test_capi/test_tokenizer.py @@ -9,6 +9,9 @@ class TokenizerTests(unittest.TestCase): def test_source(self): _testinternalcapi.test_tokenizer_source() + def test_source_discard(self): + _testinternalcapi.test_tokenizer_source_discard() + def test_cursor(self): _testinternalcapi.test_tokenizer_cursor() diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index ea0f72e0e8d4568..372c110783bce7a 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -198,6 +198,22 @@ def test_lexer_buffer_realloc_with_null_start(self): self.assertEqual(p.returncode, 0) self.assertIn(long_value, output) + @cpython_only + def test_multiline_fstring_source_reallocation(self): + long_line = " " * 9000 + "+ 2" + user_input = ( + 'value = f"""{(\n' + '1\n' + f'{long_line}\n' + ')}"""\n' + 'print(value)\n' + ) + p = spawn_repl() + p.stdin.write(user_input) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(">>> 3\n>>> ", output) + def test_close_stdin(self): user_input = dedent(''' import os diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index c471f857660ec90..948b341a5dd72ec 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2427,6 +2427,30 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self): (token.ENDMARKER, "", (1, 0), (1, 0), ""), ) + def test_fstring_offsets_survive_buffer_reallocation(self): + for prefix in ("f", "t"): + for extra_tokens in (False, True): + with self.subTest(prefix=prefix, extra_tokens=extra_tokens): + physical_lines = [ + prefix + '"""\n', + "{(\n", + " " * 9000 + "1\n", + ")=:>{2}}\n", + '"""\n', + ] + source = "".join(physical_lines) + chunks = iter([ + "".join(physical_lines[:2]), + "".join(physical_lines[2:4]), + physical_lines[4], + "", + ]) + expected = self._get_tokens( + source, extra_tokens=extra_tokens) + tokens = list(tokenize._generate_tokens_from_c_tokenizer( + chunks.__next__, extra_tokens=extra_tokens)) + self.assertEqual(tokens, expected) + def test_extra_tokens_relaxes_lexer_errors(self): cases = [ ( diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 0b292410d3eb4ef..df481cb832a4363 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -195,6 +195,28 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } + _PyTok_SourceDiscard(&source); + if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4, + "wrong retained source offset") < 0 || + _PyTok_SourceLine(&source, 1, &line) < 0 || + check(line.start == 4 && line.end == 6, + "wrong retained source line") < 0 || + _PyTok_SourceLocation( + &source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 || + check(loc.lineno == 1 && loc.byte_col == 0, + "wrong retained source location") < 0) { + goto error; + } + view = _PyTok_SourceSpanView( + &source, _PyTok_SpanFromBounds(4, 5), &view_len); + if (check(view != NULL && view_len == 1 && view[0] == 'a', + "wrong retained source span") < 0 || + check_system_error(_PyTok_SourceSpanView( + &source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL, + "accepted discarded source span") < 0) { + goto error; + } + _PyTok_SourceClear(&source); Py_RETURN_NONE; @@ -296,6 +318,88 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module), } #endif + _PyTok_Off base = source.len; + _PyTok_SourceDiscard(&source); + if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 || + _PyTok_SourceAppendLine(&source, "cd", 2, 0) < 0) { + goto error; + } + _PyTok_CursorInit(&cursor, &source); + if (_PyTok_CursorSetLine(&cursor, 1) < 0 || + check(cursor.pos == base && _PyTok_CursorPeek(&cursor, 1) == 'b', + "wrong retained cursor line") < 0 || + _PyTok_CursorSetLine(&cursor, 2) < 0 || + check(_PyTok_CursorAdvance(&cursor) == 'c', + "wrong retained cursor byte") < 0 || + _PyTok_CursorSetOffset(&cursor, base + 5) < 0 || + check(cursor.lineno == 2 && _PyTok_CursorAdvance(&cursor) == EOF, + "wrong retained cursor EOF") < 0) { + goto error; + } + + _PyTok_SourceClear(&source); + Py_RETURN_NONE; + +error: + _PyTok_SourceClear(&source); + return NULL; +} + +static PyObject * +test_tokenizer_source_discard(PyObject *Py_UNUSED(module), + PyObject *Py_UNUSED(args)) +{ + _PyTok_SourceText source; + _PyTok_SourceInit(&source); + for (int i = 0; i < 260; i++) { + if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) { + goto error; + } + } + char *bytes = source.bytes; + _PyTok_Off capacity = source.cap; + _PyTok_SourceDiscard(&source); + if (check(source.base_offset == 520 && source.len == 0 && + source.nlines == 0 && source.bytes == bytes && + source.cap == capacity && source.bytes[0] == '\0', + "discard did not preserve source allocation") < 0) { + goto error; + } + for (int i = 0; i < 260; i++) { + if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i, + "wrong source offset after discard") < 0 || + check(!_PyTok_SourceLineIsImplicit(&source, i + 1), + "discard preserved implicit newline flag") < 0) { + goto error; + } + } + if (check(source.bytes == bytes && source.cap == capacity, + "discarded allocation was not reused") < 0) { + goto error; + } + _PyTok_SourceDiscard(&source); + if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040, + "wrong source offset after repeated discard") < 0) { + goto error; + } + _PyTok_SourceDiscard(&source); + if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044, + "cannot append after discarding unterminated line") < 0) { + goto error; + } + _PyTok_SourceDiscard(&source); + source.base_offset = PY_SSIZE_T_MAX - 1; + if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 && + PyErr_ExceptionMatches(PyExc_MemoryError), + "accepted overflowing logical source offset") < 0) { + goto error; + } + PyErr_Clear(); + if (check(source.len == 0 && source.nlines == 0 && + source.base_offset == PY_SSIZE_T_MAX - 1, + "overflow changed retained source") < 0) { + goto error; + } _PyTok_SourceClear(&source); Py_RETURN_NONE; @@ -307,6 +411,7 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module), static PyMethodDef test_methods[] = { {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS}, {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS}, + {"test_tokenizer_source_discard", test_tokenizer_source_discard, METH_NOARGS}, {NULL}, }; diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c index cd6885a7d01040a..9c39544ca7c4790 100644 --- a/Parser/lexer/buffer.c +++ b/Parser/lexer/buffer.c @@ -1,62 +1,46 @@ #include "Python.h" -#include "errcode.h" - +#include "buffer.h" #include "state.h" -/* Traverse and remember all f-string buffers, in order to be able to restore - them after reallocating tok->buf */ void -_PyLexer_remember_fstring_buffers(struct tok_state *tok) +_PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base, + _PyLexer_BufferPointers *pointers) { - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); + pointers->buf_from_base = tok->buf - base; + pointers->cur_from_buf = tok->cur - tok->buf; + pointers->inp_from_buf = tok->inp - tok->buf; + pointers->start_from_buf = tok->start == NULL + ? -1 : tok->start - tok->buf; + pointers->line_start_from_buf = tok->line_start == NULL + ? -1 : tok->line_start - tok->buf; + pointers->multi_line_start_from_buf = tok->multi_line_start == NULL + ? -1 : tok->multi_line_start - tok->buf; + for (int index = tok->tok_mode_stack_index; index > 0; --index) { + tokenizer_mode *mode = &tok->tok_mode_stack[index]; mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; - mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf; + mode->multi_line_start_offset = mode->multi_line_start == NULL + ? -1 : mode->multi_line_start - tok->buf; } } -/* Traverse and restore all f-string buffers after reallocating tok->buf */ void -_PyLexer_restore_fstring_buffers(struct tok_state *tok) -{ - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); - mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset; - mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset; - } -} - -int -_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) +_PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base, + const _PyLexer_BufferPointers *pointers) { - Py_ssize_t cur = tok->cur - tok->buf; - Py_ssize_t oldsize = tok->inp - tok->buf; - Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1); - if (newsize > tok->end - tok->buf) { - char *newbuf = tok->buf; - Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf; - Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf; - Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf; - _PyLexer_remember_fstring_buffers(tok); - newbuf = (char *)PyMem_Realloc(newbuf, newsize); - if (newbuf == NULL) { - tok->done = E_NOMEM; - return 0; - } - tok->buf = newbuf; - tok->cur = tok->buf + cur; - tok->inp = tok->buf + oldsize; - tok->end = tok->buf + newsize; - tok->start = start < 0 ? NULL : tok->buf + start; - tok->line_start = line_start < 0 ? NULL : tok->buf + line_start; - tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start; - _PyLexer_restore_fstring_buffers(tok); + tok->buf = base + pointers->buf_from_base; + tok->cur = tok->buf + pointers->cur_from_buf; + tok->inp = tok->buf + pointers->inp_from_buf; + tok->start = pointers->start_from_buf < 0 + ? NULL : tok->buf + pointers->start_from_buf; + tok->line_start = pointers->line_start_from_buf < 0 + ? NULL : tok->buf + pointers->line_start_from_buf; + tok->multi_line_start = pointers->multi_line_start_from_buf < 0 + ? NULL : tok->buf + pointers->multi_line_start_from_buf; + for (int index = tok->tok_mode_stack_index; index > 0; --index) { + tokenizer_mode *mode = &tok->tok_mode_stack[index]; + mode->start = mode->start_offset < 0 + ? NULL : tok->buf + mode->start_offset; + mode->multi_line_start = mode->multi_line_start_offset < 0 + ? NULL : tok->buf + mode->multi_line_start_offset; } - return 1; } diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h index bb218162ff48453..285da124226d50e 100644 --- a/Parser/lexer/buffer.h +++ b/Parser/lexer/buffer.h @@ -3,8 +3,20 @@ #include "pyport.h" -void _PyLexer_remember_fstring_buffers(struct tok_state *tok); -void _PyLexer_restore_fstring_buffers(struct tok_state *tok); -int _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size); +struct tok_state; + +typedef struct { + Py_ssize_t buf_from_base; + Py_ssize_t cur_from_buf; + Py_ssize_t inp_from_buf; + Py_ssize_t start_from_buf; + Py_ssize_t line_start_from_buf; + Py_ssize_t multi_line_start_from_buf; +} _PyLexer_BufferPointers; + +void _PyLexer_SaveBufferPointers( + struct tok_state *, const char *, _PyLexer_BufferPointers *); +void _PyLexer_RestoreBufferPointers( + struct tok_state *, char *, const _PyLexer_BufferPointers *); #endif diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index a96362c8961023a..f96b31b9d2f38a1 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -12,8 +12,6 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) -#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\ - _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end)) /* Spaces in this constant are treated as "zero or more spaces or tabs" when tokenizing. */ @@ -360,21 +358,25 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str && !(tok->cur > ignore_end && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0])))); + int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT; + int start_col_offset = is_type_ignore + ? ignore_end_col_offset : current_starting_col_offset; + p_end = tok->cur; if (is_type_ignore) { p_start = ignore_end; - p_end = tok->cur; /* If this type ignore is the only thing on the line, consume the newline also. */ if (blankline) { tok_nextc(tok); tok->atbol = 1; } - return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset); } else { p_start = type_start; - p_end = tok->cur; - return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset); } + _PyLexer_token_setup(tok, token, type, p_start, p_end); + token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset}; + token->end_loc = (_PyTok_Loc){tok->lineno, tok->col_offset}; + return type; } } if (tok->tok_extra_tokens) { diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index 1d97ac57b745b09..040935a7e689138 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -7,4 +7,25 @@ int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur); int _PyTokenizer_Get(struct tok_state *, struct token *); +/* The view points into the current input window. The next + _PyTokenizer_Get() call may discard it. */ +static inline const char * +_PyToken_TextView(const struct tok_state *tok, const struct token *token, + Py_ssize_t *length) +{ + assert(length != NULL); + if (token->span.start < 0) { + assert(token->span.start == -1 && token->span.end == -1); + *length = 0; + return ""; + } + assert(_PyTok_SpanIsValid(token->span)); + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + assert(token->span.start >= tok->buf_offset); + assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf); + *length = token->span.end - token->span.start; + return tok->buf + (token->span.start - tok->buf_offset); +} + #endif diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 2a6408bef927a36..d82a7d0f296bac0 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -26,7 +26,6 @@ _PyTokenizer_tok_new(void) tok->interactive_src_start = NULL; tok->interactive_src_end = NULL; tok->start = NULL; - tok->end = NULL; tok->done = E_OK; tok->fp = NULL; tok->tabsize = TABSIZE; @@ -101,41 +100,46 @@ _PyToken_Free(struct token *token) { void _PyToken_Init(struct token *token) { +#ifdef Py_DEBUG + token->span = (_PyTok_Span){-1, -1}; + token->start_loc = (_PyTok_Loc){-1, -1}; + token->end_loc = (_PyTok_Loc){-1, -1}; +#endif token->metadata = NULL; } -int -_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, - int end_col_offset, const char *start, const char *end) +static inline _PyTok_Span +buffer_span(const struct tok_state *tok, const char *start, const char *end) { - token->level = tok->level; - token->lineno = token->end_lineno = tok->lineno; - token->col_offset = col_offset; - token->end_col_offset = end_col_offset; - token->start = start; - token->end = end; - return type; + if (start == NULL) { + assert(end == NULL); + return (_PyTok_Span){-1, -1}; + } + assert(end != NULL); + const char *base = tok->buf; + assert(base != NULL); + assert(tok->inp >= base); + Py_ssize_t start_offset = start - base; + Py_ssize_t end_offset = end - base; + assert(start_offset >= 0 && start_offset <= end_offset); + assert(end_offset <= tok->inp - base); + assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset); + return _PyTok_SpanFromBounds( + tok->buf_offset + start_offset, tok->buf_offset + end_offset); } int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end) { - assert((start == NULL && end == NULL) || (start != NULL && end != NULL)); token->level = tok->level; - if (ISSTRINGLIT(type)) { - token->lineno = tok->first_lineno; - } - else { - token->lineno = tok->lineno; - } - token->end_lineno = tok->lineno; - token->col_offset = token->end_col_offset = -1; - token->start = start; - token->end = end; + token->span = buffer_span(tok, start, end); + int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno; + token->start_loc = (_PyTok_Loc){lineno, -1}; + token->end_loc = (_PyTok_Loc){tok->lineno, -1}; if (start != NULL && end != NULL) { - token->col_offset = tok->starting_col_offset; - token->end_col_offset = tok->col_offset; + token->start_loc.byte_col = tok->starting_col_offset; + token->end_loc.byte_col = tok->col_offset; } return type; } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 0824785195491ee..6d19e685bd7f81b 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -23,8 +23,9 @@ enum interactive_underflow_t { struct token { int level; - int lineno, col_offset, end_lineno, end_col_offset; - const char *start, *end; + _PyTok_Span span; + _PyTok_Loc start_loc; + _PyTok_Loc end_loc; PyObject *metadata; }; @@ -67,15 +68,15 @@ typedef struct _tokenizer_mode { /* Tokenizer state */ struct tok_state { - /* Input state; buf <= cur <= inp <= end */ + /* Input state; buf <= cur <= inp */ /* NB an entire line is held in the buffer */ - char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */ + char *buf; char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ + _PyTok_Off buf_offset; /* Logical offset of buf[0]. */ int fp_interactive; /* If the file descriptor is interactive */ char *interactive_src_start; /* The start of the source parsed so far in interactive mode */ char *interactive_src_end; /* The end of the source parsed so far in interactive mode */ - const char *end; /* End of input buffer if buf != NULL */ const char *start; /* Start of current token if not NULL */ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ @@ -128,8 +129,6 @@ struct tok_state { #endif }; -int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, - int end_col_offset, const char *start, const char *end); int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); struct tok_state *_PyTokenizer_tok_new(void); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index d67c48f7f678eda..fc0299c5c7c592f 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -125,7 +125,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) { assert(tok->cur != NULL); - Py_ssize_t size = strlen(tok->cur); + Py_ssize_t size = cur == 0 + ? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur); tokenizer_mode *tok_mode = TOK_GET_MODE(tok); switch (cur) { @@ -142,7 +143,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) goto error; } tok_mode->last_expr_buffer = new_buffer; - strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size); + memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, + tok->cur, size); tok_mode->last_expr_size += size; break; case '{': @@ -155,7 +157,7 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) } tok_mode->last_expr_size = size; tok_mode->last_expr_end = -1; - strncpy(tok_mode->last_expr_buffer, tok->cur, size); + memcpy(tok_mode->last_expr_buffer, tok->cur, size); break; case '}': case '!': diff --git a/Parser/pegen.c b/Parser/pegen.c index fcec810037e98d4..d86dd22444e6a7b 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -171,18 +171,17 @@ growable_comment_array_deallocate(growable_comment_array *arr) { } static int -_get_keyword_or_name_type(Parser *p, struct token *new_token) +_get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length) { - Py_ssize_t name_len = new_token->end_col_offset - new_token->col_offset; - assert(name_len > 0); + assert(length > 0); - if (name_len >= p->n_keyword_lists || - p->keywords[name_len] == NULL || - p->keywords[name_len]->type == -1) { + if (length >= p->n_keyword_lists || + p->keywords[length] == NULL || + p->keywords[length]->type == -1) { return NAME; } - for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) { - if (strncmp(k->str, new_token->start, (size_t)name_len) == 0) { + for (KeywordToken *k = p->keywords[length]; k != NULL && k->type != -1; k++) { + if (memcmp(k->str, text, (size_t)length) == 0) { return k->type; } } @@ -193,8 +192,11 @@ static int initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) { assert(parser_token != NULL); - parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type; - parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start); + Py_ssize_t length; + const char *text = _PyToken_TextView(p->tok, new_token, &length); + parser_token->type = token_type == NAME + ? _get_keyword_or_name_type(p, text, length) : token_type; + parser_token->bytes = PyBytes_FromStringAndSize(text, length); if (parser_token->bytes == NULL) { return -1; } @@ -214,12 +216,14 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to } parser_token->level = new_token->level; - parser_token->lineno = new_token->lineno; - parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset - : new_token->col_offset; - parser_token->end_lineno = new_token->end_lineno; - parser_token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_col_offset - : new_token->end_col_offset; + parser_token->lineno = new_token->start_loc.lineno; + parser_token->col_offset = p->tok->lineno == p->starting_lineno + ? p->starting_col_offset + new_token->start_loc.byte_col + : new_token->start_loc.byte_col; + parser_token->end_lineno = new_token->end_loc.lineno; + parser_token->end_col_offset = p->tok->lineno == p->starting_lineno + ? p->starting_col_offset + new_token->end_loc.byte_col + : new_token->end_loc.byte_col; p->fill += 1; @@ -261,13 +265,14 @@ _PyPegen_fill_token(Parser *p) // Record and skip '# type: ignore' comments while (type == TYPE_IGNORE) { - Py_ssize_t len = new_token.end_col_offset - new_token.col_offset; + Py_ssize_t len; + const char *text = _PyToken_TextView(p->tok, &new_token, &len); char *tag = PyMem_Malloc((size_t)len + 1); if (tag == NULL) { PyErr_NoMemory(); goto error; } - strncpy(tag, new_token.start, (size_t)len); + memcpy(tag, text, (size_t)len); tag[len] = '\0'; // Ownership of tag passes to the growable array if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) { diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c index 698a26a740fd249..523b99dedc6160a 100644 --- a/Parser/tokenizer/cursor.c +++ b/Parser/tokenizer/cursor.c @@ -23,7 +23,7 @@ _PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno) if (lineno > 0 && cursor->lineno == lineno - 1 && lineno <= source->nlines) { _PyTok_Off start = cursor->line_end; - _PyTok_Off end = source->len; + _PyTok_Off end = source->base_offset + source->len; if (lineno < source->nlines) { end = _PyTok_SourceFindLineEnd(source, start); if (end < 0) { @@ -53,8 +53,9 @@ _PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset) int stays_on_line = cursor->lineno > 0 && offset >= cursor->line_start && offset < cursor->line_end; if (!stays_on_line && cursor->lineno > 0 && - offset == cursor->line_end && offset == source->len && - (offset == 0 || source->bytes[offset - 1] != '\n')) { + offset == cursor->line_end && + offset - source->base_offset == source->len && + (source->len == 0 || source->bytes[source->len - 1] != '\n')) { stays_on_line = 1; } if (stays_on_line) { @@ -68,7 +69,7 @@ _PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset) return -1; } _PyTok_Off start = offset - loc.byte_col; - _PyTok_Off end = source->len; + _PyTok_Off end = source->base_offset + source->len; if (loc.lineno < source->nlines) { end = _PyTok_SourceFindLineEnd(source, start); if (end < 0) { diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h index d0fd9cf80b77b71..18e404251316f0c 100644 --- a/Parser/tokenizer/cursor.h +++ b/Parser/tokenizer/cursor.h @@ -21,8 +21,12 @@ PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off); static inline void _PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source) { + _PyTok_Off base = source != NULL ? source->base_offset : 0; *cursor = (_PyTok_Cursor){ .source = source, + .pos = base, + .line_start = base, + .line_end = base, }; } @@ -35,14 +39,16 @@ _PyTok_CursorAdvance(_PyTok_Cursor *cursor) assert(cursor->source != NULL); assert(cursor->pos >= cursor->line_start); assert(cursor->pos <= cursor->line_end); - assert(cursor->line_end <= cursor->source->len); + assert(cursor->line_start >= cursor->source->base_offset); + assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len); if (cursor->pos >= cursor->line_end) { return EOF; } if (cursor->pos - cursor->line_start >= INT_MAX) { return EOF; } - return Py_CHARMASK(cursor->source->bytes[cursor->pos++]); + return Py_CHARMASK(cursor->source->bytes[ + cursor->pos++ - cursor->source->base_offset]); } /* Return the byte at a nonnegative distance within the current line, or EOF @@ -53,13 +59,15 @@ _PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance) assert(cursor->source != NULL); assert(cursor->pos >= cursor->line_start); assert(cursor->pos <= cursor->line_end); - assert(cursor->line_end <= cursor->source->len); + assert(cursor->line_start >= cursor->source->base_offset); + assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len); assert(distance >= 0); if (distance < 0 || distance >= cursor->line_end - cursor->pos) { return EOF; } - return Py_CHARMASK(cursor->source->bytes[cursor->pos + distance]); + return Py_CHARMASK(cursor->source->bytes[ + cursor->pos - cursor->source->base_offset + distance]); } #endif diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 82b824f56374fcf..b9b4a4610874419 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -13,6 +13,12 @@ # include #endif +static inline int +reader_is_streaming(_PyTok_ReaderKind kind) +{ + return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE; +} + void _PyTok_ReaderFree(struct tok_state *tok) { @@ -28,10 +34,7 @@ _PyTok_ReaderFree(struct tok_state *tok) } PyMem_Free(reader->file_buffer); PyMem_Free(reader->decoded); - if (reader->kind != _PYTOK_READER_PREPARED) { - PyMem_Free(tok->buf); - tok->buf = NULL; - } + tok->buf = NULL; PyMem_Free(reader); tok->reader = NULL; } @@ -529,19 +532,31 @@ reader_next(struct tok_state *tok, _PyTok_Chunk *chunk) Py_UNREACHABLE(); } +static void +reset_streaming_buffer(struct tok_state *tok) +{ + assert(tok->buf != NULL); + assert(tok->cur >= tok->buf && tok->cur <= tok->inp); + _PyTok_SourceDiscard(&tok->source); + tok->buf_offset = tok->source.base_offset; + tok->buf = tok->cur = tok->inp = (char *)_PyTok_SourceData(&tok->source); + tok->line_start = tok->buf; +} + int _PyTok_ReaderUnderflow(struct tok_state *tok) { - int prepared = tok->reader->kind == _PYTOK_READER_PREPARED; + _PyTok_ReaderKind kind = tok->reader->kind; + int prepared = kind == _PYTOK_READER_PREPARED; + int streaming = reader_is_streaming(kind); int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok); - if (reset_buffer && tok->reader->kind != _PYTOK_READER_INTERACTIVE) { - tok->cur = tok->inp = tok->buf; - } - _PyTok_Chunk chunk; _PyTok_ReadResult result = reader_next(tok, &chunk); if (result != _PYTOK_READ_LINE) { + if (reset_buffer && streaming) { + reset_streaming_buffer(tok); + } if (result == _PYTOK_READ_EOF) { tok->done = E_EOF; } @@ -558,34 +573,51 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) ? E_NOMEM : E_ERROR; } } - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && + if (kind == _PYTOK_READER_INTERACTIVE && result != _PYTOK_READ_STOPPED) { PySys_WriteStderr("\n"); } return 0; } - Py_ssize_t copy_len = chunk.len; - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && + Py_ssize_t scan_len = chunk.len; + if (kind == _PYTOK_READER_INTERACTIVE && chunk.implicit_newline) { - copy_len--; - } - if (reset_buffer && tok->reader->kind == _PYTOK_READER_INTERACTIVE) { - tok->cur = tok->inp = tok->buf; - } - if (!prepared && !_PyLexer_tok_reserve_buf(tok, copy_len + 1)) { - _PyTok_ChunkClear(&chunk); - tok->input_error = 1; - return 0; - } - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && - _PyTok_SourceAppendLine(&tok->source, chunk.data, chunk.len, - chunk.implicit_newline) < 0) { - _PyTok_ChunkClear(&chunk); - tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) - ? E_NOMEM : E_ERROR; - tok->input_error = 1; - return 0; + scan_len--; + } + if (!prepared) { + if (streaming && reset_buffer) { + reset_streaming_buffer(tok); + } + _PyLexer_BufferPointers pointers; + if (!reset_buffer) { + _PyLexer_SaveBufferPointers( + tok, tok->source.bytes, &pointers); + } + _PyTok_Off source_start = _PyTok_SourceAppendLine( + &tok->source, chunk.data, chunk.len, + chunk.implicit_newline); + if (source_start < 0) { + _PyTok_ChunkClear(&chunk); + tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) + ? E_NOMEM : E_ERROR; + tok->input_error = 1; + return 0; + } + if (reset_buffer) { + tok->buf = tok->cur = + tok->source.bytes + (source_start - tok->source.base_offset); + tok->buf_offset = source_start; + tok->line_start = tok->buf; + tok->start = NULL; + tok->multi_line_start = NULL; + } + else { + _PyLexer_RestoreBufferPointers( + tok, tok->source.bytes, &pointers); + } + tok->inp = tok->source.bytes + + (source_start - tok->source.base_offset) + scan_len; } if (tok->fp_interactive) { tok->interactive_src_start = tok->source.bytes; @@ -594,14 +626,11 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) if (prepared) { if (tok->start == NULL) { tok->buf = tok->cur; + tok->buf_offset = tok->source.base_offset + + (chunk.data - tok->source.bytes); } tok->inp = chunk.data + chunk.len; } - else { - memcpy(tok->inp, chunk.data, (size_t)copy_len); - tok->inp += copy_len; - *tok->inp = '\0'; - } tok->implicit_newline = chunk.implicit_newline; if (!prepared && tok->tok_mode_stack_index && @@ -611,7 +640,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) return 0; } ADVANCE_LINENO(); - if (tok->reader->kind == _PYTOK_READER_FILE && + if (kind == _PYTOK_READER_FILE && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) { _PyTok_ChunkClear(&chunk); @@ -639,14 +668,10 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind) if (kind == _PYTOK_READER_PREPARED) { return tok; } - tok->buf = PyMem_Malloc(BUFSIZ); - if (tok->buf == NULL) { - PyErr_NoMemory(); - _PyTokenizer_Free(tok); - return NULL; + if (reader_is_streaming(kind)) { + tok->buf = tok->cur = tok->inp = + (char *)_PyTok_SourceData(&tok->source); } - tok->cur = tok->inp = tok->buf; - tok->end = tok->buf + BUFSIZ; return tok; } @@ -664,7 +689,6 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input, return NULL; } tok->buf = tok->cur = tok->inp = tok->str; - tok->end = tok->buf; return tok; } diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index c0f7925e33f8b97..2f2aaf2589246d9 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -19,6 +19,22 @@ _PyTok_SourceClear(_PyTok_SourceText *source) _PyTok_SourceInit(source); } +void +_PyTok_SourceDiscard(_PyTok_SourceText *source) +{ + assert(source->base_offset <= PY_SSIZE_T_MAX - source->len); + source->base_offset += source->len; + source->len = 0; + if (source->bytes != NULL) { + source->bytes[0] = '\0'; + } + if (source->implicit_lines != NULL) { + Py_ssize_t used = source->nlines / 8 + (source->nlines % 8 != 0); + memset(source->implicit_lines, 0, Py_MIN(used, source->implicit_cap)); + } + source->nlines = 0; +} + static int reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) { @@ -89,7 +105,7 @@ reserve_checkpoints(_PyTok_SourceText *source, int needed) static int reserve_implicit_lines(_PyTok_SourceText *source, int nlines) { - Py_ssize_t needed = ((Py_ssize_t)nlines + 7) / 8; + Py_ssize_t needed = nlines / 8 + (nlines % 8 != 0); if (needed <= source->implicit_cap) { return 0; } @@ -143,7 +159,8 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, if (validate_line(source, bytes, len, implicit_newline) < 0) { return -1; } - if (source->len > PY_SSIZE_T_MAX - len - 1) { + if (source->len > PY_SSIZE_T_MAX - len - 1 || + source->base_offset > PY_SSIZE_T_MAX - source->len - len) { PyErr_NoMemory(); return -1; } @@ -162,26 +179,28 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, source->len += len; source->bytes[source->len] = '\0'; if (checkpoint) { - source->line_checkpoints[checkpoint_count - 1] = start; + source->line_checkpoints[checkpoint_count - 1] = + source->base_offset + start; } if (implicit_newline) { source->implicit_lines[(nlines - 1) / 8] |= (unsigned char)(1U << ((nlines - 1) & 7)); } source->nlines = nlines; - return start; + return source->base_offset + start; } const char * _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span, Py_ssize_t *len) { - if (!_PyTok_SpanIsValid(span) || span.end > source->len || len == NULL) { + if (!_PyTok_SpanIsValid(span) || span.start < source->base_offset || + span.end - source->base_offset > source->len || len == NULL) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source span"); return NULL; } *len = span.end - span.start; - return source->bytes == NULL ? "" : source->bytes + span.start; + return _PyTok_SourceData(source) + (span.start - source->base_offset); } int @@ -220,8 +239,8 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, } if (lineno > source->nlines) { *line = (_PyTok_Line){ - .start = source->len, - .end = source->len, + .start = source->base_offset + source->len, + .end = source->base_offset + source->len, }; return 0; } @@ -236,7 +255,7 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, } current++; } - _PyTok_Off end = source->len; + _PyTok_Off end = source->base_offset + source->len; if (lineno < source->nlines) { end = _PyTok_SourceFindLineEnd(source, start); if (end < 0) { @@ -248,7 +267,8 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, .end = end, .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno), .contains_nul = memchr( - source->bytes + start, 0, end - start) != NULL, + source->bytes + (start - source->base_offset), + 0, end - start) != NULL, }; return 0; } @@ -257,21 +277,23 @@ int _PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset, _PyTok_Affinity affinity, _PyTok_Loc *loc) { - if (offset < 0 || offset > source->len || loc == NULL || + if (offset < source->base_offset || + offset - source->base_offset > source->len || loc == NULL || (affinity != _PYTOK_AFFINITY_LEFT && affinity != _PYTOK_AFFINITY_RIGHT)) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset"); return -1; } if (source->nlines == 0 || - (offset == source->len && source_ends_in_newline(source) && + (offset - source->base_offset == source->len && + source_ends_in_newline(source) && affinity == _PYTOK_AFFINITY_RIGHT)) { *loc = (_PyTok_Loc){eof_lineno(source), 0}; return 0; } _PyTok_Off key = offset; - if (affinity == _PYTOK_AFFINITY_LEFT && key > 0) { + if (affinity == _PYTOK_AFFINITY_LEFT && key > source->base_offset) { key--; } int low = 0; diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index b42ecda1b31aa50..7a2f46f73aff470 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -5,7 +5,8 @@ typedef Py_ssize_t _PyTok_Off; -/* Half-open byte offsets into a _PyTok_SourceText. */ +/* Spans use half-open logical byte offsets into decoded input. Their backing + storage may retain only the current input window. */ typedef struct { _PyTok_Off start; _PyTok_Off end; @@ -32,6 +33,7 @@ typedef struct { typedef struct { char *bytes; + _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; _PyTok_Off *line_checkpoints; @@ -41,9 +43,18 @@ typedef struct { Py_ssize_t implicit_cap; } _PyTok_SourceText; +static inline const char * +_PyTok_SourceData(const _PyTok_SourceText *source) +{ + return source->bytes != NULL ? source->bytes : ""; +} + PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *); /* Clear invalidates all cursors, spans, and views for the source. */ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); +/* Discard the retained window and invalidate its cursors, spans, and views. + Keep its allocation and advance the logical base to the end of the window. */ +PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may contain one newline, as its final byte. An unterminated line must be the final line. implicit_newline means that the final newline was synthesized. @@ -54,8 +65,8 @@ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( /* The returned view is invalidated by SourceAppendLine and SourceClear. */ PyAPI_FUNC(const char *) _PyTok_SourceSpanView( const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *); -/* Look up a 1-based line. Empty and newline-terminated sources have an empty - virtual line at EOF. */ +/* Look up a 1-based line in the retained window. Empty and newline-terminated + sources have an empty virtual line at EOF. */ PyAPI_FUNC(int) _PyTok_SourceLine( const _PyTok_SourceText *, int, _PyTok_Line *); /* Return false for invalid line numbers and the virtual EOF line. */ @@ -81,19 +92,21 @@ _PyTok_SpanIsValid(_PyTok_Span span) static inline _PyTok_Off _PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start) { - if (source->bytes == NULL || start < 0 || start >= source->len) { + if (source->bytes == NULL || start < source->base_offset || + start - source->base_offset >= source->len) { PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index"); return -1; } + _PyTok_Off relative_start = start - source->base_offset; const char *newline = memchr( - source->bytes + start, '\n', source->len - start); + source->bytes + relative_start, '\n', source->len - relative_start); if (newline == NULL) { PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index"); return -1; } - return newline - source->bytes + 1; + return source->base_offset + (newline - source->bytes) + 1; } #endif diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 762b7b3e4c8d71d..71f236b08d93c8f 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -203,14 +203,19 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si } static void -_get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_start, - PyObject *line, int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno, +_get_col_offsets(tokenizeriterobject *it, const struct token *token, + const char *token_start, const char *line_start, + PyObject *line, int line_changed, Py_ssize_t *col_offset, Py_ssize_t *end_col_offset) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); + const char *token_end = token_start == NULL + ? NULL : token_start + token->span.end - token->span.start; + Py_ssize_t lineno = token->start_loc.lineno; + Py_ssize_t end_lineno = token->end_loc.lineno; Py_ssize_t byte_offset = -1; - if (token.start != NULL && token.start >= line_start) { - byte_offset = token.start - line_start; + if (token_start != NULL && token_start >= line_start) { + byte_offset = token_start - line_start; if (line_changed) { *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); it->byte_col_offset_diff = byte_offset - *col_offset; @@ -220,15 +225,13 @@ _get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_s } } - if (token.end != NULL && token.end >= it->tok->line_start) { - Py_ssize_t end_byte_offset = token.end - it->tok->line_start; + if (token_end != NULL && token_end >= it->tok->line_start) { + Py_ssize_t end_byte_offset = token_end - it->tok->line_start; if (lineno == end_lineno) { - // If the whole token is at the same line, we can just use the token.start - // buffer for figuring out the new column offset, since using line is not - // performant for very long lines. + // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); *end_col_offset = *col_offset + token_col_offset; - it->byte_col_offset_diff += token.end - token.start - token_col_offset; + it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset); @@ -263,12 +266,17 @@ tokenizeriter_next(PyObject *op) it->done = 1; goto exit; } - PyObject *str = NULL; - if (token.start == NULL || token.end == NULL) { + const char *token_start = NULL; + PyObject *str; + if (token.span.start < 0) { + assert(token.span.start == -1 && token.span.end == -1); str = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } else { - str = PyUnicode_FromStringAndSize(token.start, token.end - token.start); + Py_ssize_t token_length; + token_start = _PyToken_TextView( + it->tok, &token, &token_length); + str = PyUnicode_FromStringAndSize(token_start, token_length); } if (str == NULL) { goto exit; @@ -297,12 +305,12 @@ tokenizeriter_next(PyObject *op) goto exit; } - Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno; - Py_ssize_t end_lineno = it->tok->lineno; + Py_ssize_t lineno = token.start_loc.lineno; + Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - _get_col_offsets(it, token, line_start, line, line_changed, - lineno, end_lineno, &col_offset, &end_col_offset); + _get_col_offsets(it, &token, token_start, line_start, line, line_changed, + &col_offset, &end_col_offset); if (it->tok->tok_extra_tokens) { if (is_trailing_token) { @@ -317,7 +325,8 @@ tokenizeriter_next(PyObject *op) else if (type == NEWLINE) { Py_DECREF(str); if (!it->tok->implicit_newline) { - if (it->tok->start[0] == '\r') { + assert(token_start != NULL); + if (token_start[0] == '\r') { str = PyUnicode_FromString("\r\n"); } else { str = PyUnicode_FromString("\n");