From e757c06a61ac6d73d9d0c3c08955d86f176573a1 Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Tue, 8 Sep 2026 21:28:40 +0300 Subject: [PATCH 1/2] gh-157137: Mark the PEP 820 soft deprecations as 3.15, not `next` (#157138) --- Doc/c-api/module.rst | 6 +++--- Doc/c-api/type.rst | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 31df4451f15e66..e4664ef815f567 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -853,7 +853,7 @@ struct: .. versionadded:: 3.5 - .. soft-deprecated:: next + .. soft-deprecated:: 3.15 Prefer :c:func:`PyModule_FromSlotsAndSpec` in new code. @@ -877,7 +877,7 @@ struct: .. versionadded:: 3.5 - .. soft-deprecated:: next + .. soft-deprecated:: 3.15 Prefer :c:func:`PyModule_FromSlotsAndSpec` in new code. @@ -887,7 +887,7 @@ struct: .. versionadded:: 3.5 - .. soft-deprecated:: next + .. soft-deprecated:: 3.15 To run a module's own execution slots, prefer :c:func:`PyModule_Exec`, which works on modules that were not created from a diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 2e48c56bd2c5a5..5216e09dd11bee 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -828,7 +828,7 @@ They will continue to work, but new features will be added as slots for .. versionadded:: 3.12 - .. soft-deprecated:: next + .. soft-deprecated:: 3.15 Prefer :c:func:`PyType_FromSlots` in new code. @@ -859,7 +859,7 @@ They will continue to work, but new features will be added as slots for Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. - .. soft-deprecated:: next + .. soft-deprecated:: 3.15 Prefer :c:func:`PyType_FromSlots` in new code. @@ -885,7 +885,7 @@ They will continue to work, but new features will be added as slots for Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. - .. soft-deprecated:: next + .. soft-deprecated:: 3.15 Prefer :c:func:`PyType_FromSlots` in new code. @@ -910,7 +910,7 @@ They will continue to work, but new features will be added as slots for Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. - .. soft-deprecated:: next + .. soft-deprecated:: 3.15 Prefer :c:func:`PyType_FromSlots` in new code. From 2ddc218b20625ef4a6293aee3696e605c6bac5b9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 9 Sep 2026 00:15:01 +0100 Subject: [PATCH 2/2] gh-153569: centralize formatted-string state and source spans (#156484) Formatted-string expressions and comments can outlive the buffer window where scanning began. Pointer boundaries require buffer relocation to repair tokenizer state. Record logical source ranges instead. Retain the active input window while a formatted string is open, give each mode ownership of its comment spans, and materialize text through shared span views. --- Lib/test/test_fstring.py | 15 + Lib/test/test_syntax.py | 4 + Lib/test/test_tokenize.py | 34 ++ Lib/test/test_tstring.py | 26 ++ Parser/action_helpers.c | 34 +- Parser/lexer/buffer.c | 17 - Parser/lexer/buffer.h | 1 - Parser/lexer/lexer.c | 151 ++++----- Parser/lexer/lexer.h | 10 +- Parser/lexer/lexer_internal.h | 47 ++- Parser/lexer/state.c | 109 +++--- Parser/lexer/state.h | 174 +++++++--- Parser/lexer/string.c | 612 +++++++++++++++------------------- Parser/pegen.c | 1 - Parser/pegen.h | 3 - Parser/pegen_errors.c | 8 +- Parser/tokenizer/helpers.h | 4 - Parser/tokenizer/reader.c | 22 +- Python/Python-tokenize.c | 3 +- 19 files changed, 668 insertions(+), 607 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index c1ef1a73f05c20..460adde4956a9f 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1657,6 +1657,7 @@ def __repr__(self): self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x') self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********') self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20') + self.assertEqual(f"{C():{C():{4=}}}", 'FORMAT-FORMAT-4=4') self.assertRaises(SyntaxError, eval, "f'{C=]'") @@ -1679,6 +1680,20 @@ def __repr__(self): self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'') self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'') + self.assertEqual(f'{"""a" # inside"""=}', + '"""a" # inside"""=\'a" # inside\'') + self.assertEqual(f"{'''a' # inside'''=}", + "'''a' # inside'''=\"a' # inside\"") + self.assertEqual(f'{"""a""""#" # outside +=}', '"""a""""#" \n=\'a#\'') + + x, y = 1, 2 + self.assertEqual(f'{x != y # outside +=}', 'x != y \n=True') + + d = {'a#b': 42} + self.assertEqual(f'''{f"{d["a#b"]}"=}''', + 'f"{d["a#b"]}"=\'42\'') self.assertEqual(f'{ # some comment goes here """hello"""=}', ' \n """hello"""=\'hello\'') diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index dbd518707c1826..f667707f2bb08b 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -3393,6 +3393,10 @@ def test_invalid_line_continuation_error_position(self): self._check_error('\nfgdfgf\n1,\\#\n2\n', "unexpected character after line continuation character", lineno=3, offset=4) + for prefix in ("f", "t"): + self._check_error(f'{prefix}"""{{\n\\ x}}"""', + "unexpected character after line continuation character", + lineno=2, offset=2) def test_invalid_line_continuation_left_recursive(self): # Check bpo-42218: SyntaxErrors following left-recursive rules diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 948b341a5dd72e..53215eceeb8aed 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2573,6 +2573,40 @@ def test_degraded_fstring_format_spec(self): ("f-string: single '}' is not allowed", (1, 11)), ) + def test_carriage_return_after_debug_comment(self): + for prefix in ("f", "t"): + with self.subTest(prefix=prefix): + tokens = self._get_tokens(f"{prefix}'''{{x=# comment\r}}'''") + self.assertEqual(tokens[4].string, "# comment\r}") + + def test_incomplete_formatted_string_comment_after_carriage_return(self): + for prefix in ("f", "t"): + with self.subTest(prefix=prefix): + for extra_tokens in (False, True): + with self.assertRaises(tokenize.TokenError) as caught: + self._get_tokens( + f"{prefix}'{{#\r!", extra_tokens=extra_tokens + ) + self.assertEqual( + caught.exception.args, + ("unexpected EOF in multi-line statement", (1, 7)), + ) + + def test_formatted_string_nesting_limit(self): + def nested_string(depth, prefix): + source = "'x'" + for _ in range(depth): + source = f'{prefix}"{{{source}}}"' + return source + + for prefix in ("f", "t"): + with self.subTest(prefix=prefix): + self._get_tokens(nested_string(149, prefix)) + with self.assertRaisesRegex( + tokenize.TokenError, + "too many nested f-strings or t-strings"): + self._get_tokens(nested_string(150, prefix)) + def test_escaped_fstring_brace_has_a_position_gap(self): tokens = self._get_tokens('f"a{{"', extra_tokens=True) self.assertEqual( diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index 74653c77c55de1..75d7085c6731d9 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -140,6 +140,15 @@ def test_debug_specifier(self): ) self.assertEqual(fstring(t), "Value: value = 42") + class C: + def __format__(self, spec): + return f"FORMAT-{spec}" + + x = y = C() + t = t"{x:{y:{value=}}}" + self.assertEqual(t.interpolations[0].format_spec, + "FORMAT-value=42") + def test_raw_tstrings(self): path = r"C:\Users" t = rt"{path}\Documents" @@ -287,5 +296,22 @@ def test_triple_quoted(self): ) self.assertEqual(fstring(t), "\n Hello,\n Python\n ") + t = t'{"""a" # inside"""}' + self.assertEqual(t.interpolations[0].expression, + '"""a" # inside"""') + + t = t'{"""a""""#" # outside +}' + self.assertEqual(t.interpolations[0].expression, '"""a""""#"') + + x, y = 1, 2 + t = t'{x != y # outside +}' + self.assertEqual(t.interpolations[0].expression, 'x != y') + + d = {'a#b': 42} + t = t'''{f"{d["a#b"]}"}''' + self.assertEqual(t.interpolations[0].expression, 'f"{d["a#b"]}"') + if __name__ == '__main__': unittest.main() diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 8690dca8331b5c..8ca6898711f72b 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1001,6 +1001,13 @@ result_token_with_metadata(Parser *p, void *result, PyObject *metadata) return res; } +static char +formatted_string_prefix(const Parser *p) +{ + const ftstring_state *state = _PyLexer_CurrentFTString(p->tok); + return state == NULL ? 'f' : _PyLexer_StringPrefix(state->kind); +} + ResultTokenWithMetadata * _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv) { @@ -1008,7 +1015,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv) return RAISE_SYNTAX_ERROR_KNOWN_RANGE( conv_token, conv, "%c-string: conversion type must come right after the exclamation mark", - TOK_GET_STRING_PREFIX(p->tok) + formatted_string_prefix(p) ); } @@ -1017,7 +1024,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv) !(first == 's' || first == 'r' || first == 'a')) { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(conv, "%c-string: invalid conversion character %R: expected 's', 'r', or 'a'", - TOK_GET_STRING_PREFIX(p->tok), + formatted_string_prefix(p), conv->v.Name.id); return NULL; } @@ -1344,7 +1351,8 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* tok } static asdl_expr_seq * -_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b, enum string_kind_t string_kind) +_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, + Token *b, ftstring_kind string_kind) { Py_ssize_t n_items = asdl_seq_LEN(raw_expressions); Py_ssize_t total_items = n_items; @@ -1370,15 +1378,13 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b for (Py_ssize_t i = 0; i < n_items; i++) { expr_ty item = asdl_seq_GET(raw_expressions, i); - // This should correspond to a JoinedStr node of two elements - // created _PyPegen_formatted_value. This situation can only be the result of - // a (f|t)-string debug expression where the first element is a constant with the text and the second - // a formatted value with the expression. + /* Debug expressions arrive as JoinedStr(text, value); flatten them + into the surrounding string. */ if (item->kind == JoinedStr_kind) { asdl_expr_seq *values = item->v.JoinedStr.values; if (asdl_seq_LEN(values) != 2) { PyErr_Format(PyExc_SystemError, - string_kind == TSTRING + _PyLexer_IsTString(string_kind) ? "unexpected TemplateStr node without debug data in t-string at line %d" : "unexpected JoinedStr node without debug data in f-string at line %d", item->lineno); @@ -1390,7 +1396,9 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b asdl_seq_SET(seq, index++, first); expr_ty second = asdl_seq_GET(values, 1); - assert((string_kind == TSTRING && second->kind == Interpolation_kind) || second->kind == FormattedValue_kind); + assert((_PyLexer_IsTString(string_kind) && + second->kind == Interpolation_kind) || + second->kind == FormattedValue_kind); asdl_seq_SET(seq, index++, second); continue; @@ -1460,12 +1468,8 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) { return NULL; } - // Check if we're inside a raw f-string for format spec decoding - int is_raw = 0; - if (INSIDE_FSTRING(p->tok)) { - tokenizer_mode *mode = TOK_GET_MODE(p->tok); - is_raw = mode->raw; - } + const ftstring_state *state = _PyLexer_CurrentFTString(p->tok); + int is_raw = state != NULL && _PyLexer_IsRawString(state->kind); PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok); if (str == NULL) { diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c index 9c39544ca7c479..ead973b0e85234 100644 --- a/Parser/lexer/buffer.c +++ b/Parser/lexer/buffer.c @@ -13,14 +13,6 @@ _PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base, ? -1 : tok->start - tok->buf; pointers->line_start_from_buf = tok->line_start == NULL ? -1 : tok->line_start - tok->buf; - pointers->multi_line_start_from_buf = tok->multi_line_start == NULL - ? -1 : tok->multi_line_start - tok->buf; - for (int index = tok->tok_mode_stack_index; index > 0; --index) { - tokenizer_mode *mode = &tok->tok_mode_stack[index]; - mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; - mode->multi_line_start_offset = mode->multi_line_start == NULL - ? -1 : mode->multi_line_start - tok->buf; - } } void @@ -34,13 +26,4 @@ _PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base, ? NULL : tok->buf + pointers->start_from_buf; tok->line_start = pointers->line_start_from_buf < 0 ? NULL : tok->buf + pointers->line_start_from_buf; - tok->multi_line_start = pointers->multi_line_start_from_buf < 0 - ? NULL : tok->buf + pointers->multi_line_start_from_buf; - for (int index = tok->tok_mode_stack_index; index > 0; --index) { - tokenizer_mode *mode = &tok->tok_mode_stack[index]; - mode->start = mode->start_offset < 0 - ? NULL : tok->buf + mode->start_offset; - mode->multi_line_start = mode->multi_line_start_offset < 0 - ? NULL : tok->buf + mode->multi_line_start_offset; - } } diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h index 285da124226d50..af57732a69363f 100644 --- a/Parser/lexer/buffer.h +++ b/Parser/lexer/buffer.h @@ -11,7 +11,6 @@ typedef struct { Py_ssize_t inp_from_buf; Py_ssize_t start_from_buf; Py_ssize_t line_start_from_buf; - Py_ssize_t multi_line_start_from_buf; } _PyLexer_BufferPointers; void _PyLexer_SaveBufferPointers( diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index f96b31b9d2f38a..c5157c1c021e01 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -7,7 +7,7 @@ #include "../tokenizer/helpers.h" #include "../tokenizer/reader.h" -/* Alternate tab spacing */ +#define TABSIZE 8 #define ALTTABSIZE 1 @@ -30,11 +30,10 @@ _PyLexer_nextc(struct tok_state *tok) int rc; for (;;) { if (tok->cur != tok->inp) { - if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) { + if (tok->cur - tok->line_start >= INT_MAX) { tok->done = E_COLUMNOVERFLOW; return EOF; } - tok->col_offset++; return Py_CHARMASK(*tok->cur++); /* Fast path */ } if (tok->done != E_OK) { @@ -74,7 +73,6 @@ _PyLexer_backup(struct tok_state *tok, int c) if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) { Py_FatalError("tok_backup: wrong character"); } - tok->col_offset--; } } @@ -88,7 +86,7 @@ verify_identifier(struct tok_state *tok) return 1; } PyObject *s; - if (tok->input_error) + if (tok_failed(tok)) return 0; s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL); if (s == NULL) { @@ -156,8 +154,11 @@ tok_continuation_line(struct tok_state *tok) { int -_PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token) +_PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token *token) { + assert(current == NULL || + (current->mode == FTSTRING_MODE_EXPRESSION && + current->replacement_depth > 0)); int c; int blankline, nonascii; @@ -165,7 +166,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str const char *p_end = NULL; nextline: tok->start = NULL; - tok->starting_col_offset = -1; + tok->start_loc = (_PyTok_Loc){tok->lineno, -1}; blankline = 0; @@ -181,7 +182,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str col++, altcol++; } else if (c == '\t') { - col = (col / tok->tabsize + 1) * tok->tabsize; + col = (col / TABSIZE + 1) * TABSIZE; altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; } else if (c == '\014') {/* Control-L (formfeed) */ @@ -269,7 +270,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str } tok->start = tok->cur; - tok->starting_col_offset = tok->col_offset; + tok->start_loc = (_PyTok_Loc){ + tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) : -1}; /* Return pending indents/dedents */ if (tok->pendin != 0) { @@ -304,7 +306,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str /* Set start of current token */ tok->start = tok->cur == NULL ? NULL : tok->cur - 1; - tok->starting_col_offset = tok->col_offset - 1; + tok->start_loc = (_PyTok_Loc){ + tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) - 1 : -1}; /* Skip comment, unless it's a type comment */ if (c == '#') { @@ -317,13 +320,25 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str c = tok_nextc(tok); } + if (current != NULL) { + const char *comment_end = tok->cur; + if (c == '\n' || c == '\r') { + comment_end--; + } + if (_PyLexer_record_ftstring_comment( + tok, current, tok->start, comment_end) < 0) { + tok->done = E_NOMEM; + return MAKE_TOKEN(ERRORTOKEN); + } + } + if (tok->tok_extra_tokens) { p = tok->start; } if (tok->type_comments) { p = tok->start; - current_starting_col_offset = tok->starting_col_offset; + current_starting_col_offset = tok->start_loc.byte_col; prefix = type_comment_prefix; while (*prefix && p < tok->cur) { if (*prefix == ' ') { @@ -375,7 +390,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str } _PyLexer_token_setup(tok, token, type, p_start, p_end); token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset}; - token->end_loc = (_PyTok_Loc){tok->lineno, tok->col_offset}; + token->end_loc = (_PyTok_Loc){tok->lineno, + _PyLexer_ByteColumn(tok)}; return type; } } @@ -534,29 +550,16 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str } /* Punctuation character */ - int is_punctuation = (c == ':' || c == '}' || c == '!' || c == '{'); - if (is_punctuation && INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) { - /* This code block gets executed before the curly_bracket_depth is incremented - * by the `{` case, so for ensuring that we are on the 0th level, we need - * to adjust it manually */ - int cursor = current_tok->curly_bracket_depth - (c != '{'); - int in_format_spec = current_tok->in_format_spec; - int cursor_in_format_with_debug = - cursor == 1 && (current_tok->in_debug || in_format_spec); - int cursor_valid = cursor == 0 || cursor_in_format_with_debug; - if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) { - return MAKE_TOKEN(ENDMARKER); - } - if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) { + int is_punctuation = (c == ':' || c == '}' || c == '!'); + if (is_punctuation && current != NULL) { + int type = _PyLexer_ftstring_punctuation(tok, current, token, c); + if (type < 0) { return MAKE_TOKEN(ERRORTOKEN); } - - if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) { - current_tok->kind = TOK_FSTRING_MODE; - current_tok->in_format_spec = 1; + if (type != 0) { p_start = tok->start; p_end = tok->cur; - return MAKE_TOKEN(_PyToken_OneChar(c)); + return MAKE_TOKEN(type); } } @@ -592,16 +595,20 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str tok->parenlinenostack[tok->level] = tok->lineno; tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start); tok->level++; - if (INSIDE_FSTRING(tok)) { - current_tok->curly_bracket_depth++; - } break; case ')': case ']': case '}': - if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "%c-string: single '}' is not allowed", TOK_GET_STRING_PREFIX(tok))); + if (current != NULL && + _PyLexer_FTStringBracketDepth(tok, current) == 0) { + if (c == '}') { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + "%c-string: single '}' is not allowed", + _PyLexer_StringPrefix(current->kind))); + } + return MAKE_TOKEN(_PyTokenizer_syntaxerror( + tok, "%c-string: unmatched '%c'", + _PyLexer_StringPrefix(current->kind), c)); } if (!tok->tok_extra_tokens && !tok->level) { return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c)); @@ -612,17 +619,15 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') || (opening == '[' && c == ']') || (opening == '{' && c == '}'))) { - /* If the opening bracket belongs to an f-string's expression - part (e.g. f"{)}") and the closing bracket is an arbitrary - nested expression, then instead of matching a different - syntactical construct with it; we'll throw an unmatched - parentheses error. */ - if (INSIDE_FSTRING(tok) && opening == '{') { - assert(current_tok->curly_bracket_depth >= 0); - int previous_bracket = current_tok->curly_bracket_depth - 1; - if (previous_bracket == current_tok->curly_bracket_expr_start_depth) { + /* Do not match a closer against the brace that opened the + * current replacement field. */ + if (current != NULL && opening == '{') { + int bracket_depth = + _PyLexer_FTStringBracketDepth(tok, current); + if (bracket_depth == current->replacement_depth - 1) { return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "%c-string: unmatched '%c'", TOK_GET_STRING_PREFIX(tok), c)); + "%c-string: unmatched '%c'", + _PyLexer_StringPrefix(current->kind), c)); } } if (tok->parenlinenostack[tok->level] != tok->lineno) { @@ -639,19 +644,9 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str } } } - - if (INSIDE_FSTRING(tok)) { - current_tok->curly_bracket_depth--; - if (current_tok->curly_bracket_depth < 0) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'", - TOK_GET_STRING_PREFIX(tok), c)); - } - if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) { - current_tok->curly_bracket_expr_start_depth--; - current_tok->kind = TOK_FSTRING_MODE; - current_tok->in_format_spec = 0; - current_tok->in_debug = 0; - } + if (current != NULL && + _PyLexer_close_ftstring_expr(tok, current, c) < 0) { + return MAKE_TOKEN(ERRORTOKEN); } break; default: @@ -662,8 +657,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c)); } - if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) { - current_tok->in_debug = 1; + if (c == '=' && current != NULL) { + _PyLexer_mark_ftstring_debug(tok, current); } /* Punctuation character */ @@ -673,22 +668,28 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str } -static int -tok_get(struct tok_state *tok, struct token *token) -{ - tokenizer_mode *current_tok = TOK_GET_MODE(tok); - if (current_tok->kind == TOK_REGULAR_MODE) { - return _PyLexer_get_normal_mode(tok, current_tok, token); - } else { - return _PyLexer_get_fstring_mode(tok, current_tok, token); - } -} - int _PyTokenizer_Get(struct tok_state *tok, struct token *token) { - int result = tok_get(tok, token); - if (tok->input_error) { + ftstring_state *current = _PyLexer_CurrentFTString(tok); + int result; + if (current == NULL) { + result = _PyLexer_get_normal(tok, NULL, token); + } + else { + switch (current->mode) { + case FTSTRING_MODE_EXPRESSION: + result = _PyLexer_get_normal(tok, current, token); + break; + case FTSTRING_MODE_MIDDLE: + case FTSTRING_MODE_FORMAT_SPEC: + result = _PyLexer_get_ftstring(tok, current, token); + break; + default: + Py_UNREACHABLE(); + } + } + if (tok_failed(tok)) { result = ERRORTOKEN; } return result; diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index 040935a7e68913..63f6e628640107 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -3,8 +3,6 @@ #include "state.h" -int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur); - int _PyTokenizer_Get(struct tok_state *, struct token *); /* The view points into the current input window. The next @@ -19,13 +17,7 @@ _PyToken_TextView(const struct tok_state *tok, const struct token *token, *length = 0; return ""; } - assert(_PyTok_SpanIsValid(token->span)); - assert(tok->buf != NULL); - assert(tok->inp >= tok->buf); - assert(token->span.start >= tok->buf_offset); - assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf); - *length = token->span.end - token->span.start; - return tok->buf + (token->span.start - tok->buf_offset); + return _PyLexer_BufferSpanView(tok, token->span, length); } #endif diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index c6d3b9045c7292..95d9754080bedb 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -1,6 +1,7 @@ #ifndef _PY_LEXER_INTERNAL_H_ #define _PY_LEXER_INTERNAL_H_ +#include "errcode.h" #include "lexer.h" #define is_potential_identifier_start(c) (\ @@ -16,42 +17,34 @@ || c == '_'\ || (c >= 128)) -#ifdef Py_DEBUG -static inline tokenizer_mode * -TOK_GET_MODE(struct tok_state *tok) -{ - assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL); - return &tok->tok_mode_stack[tok->tok_mode_stack_index]; -} +#define FTSTRING_MIDDLE(state) \ + (_PyLexer_IsTString((state)->kind) ? TSTRING_MIDDLE : FSTRING_MIDDLE) +#define FTSTRING_END(state) \ + (_PyLexer_IsTString((state)->kind) ? TSTRING_END : FSTRING_END) +#define tok_nextc _PyLexer_nextc +#define tok_backup _PyLexer_backup -static inline tokenizer_mode * -TOK_NEXT_MODE(struct tok_state *tok) +static inline int +tok_failed(const struct tok_state *tok) { - assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); - return &tok->tok_mode_stack[++tok->tok_mode_stack_index]; + return tok->done != E_OK && tok->done != E_EOF && + tok->done != E_INTERACT_STOP; } -#else -#define TOK_GET_MODE(tok) (&(tok)->tok_mode_stack[(tok)->tok_mode_stack_index]) -#define TOK_NEXT_MODE(tok) (&(tok)->tok_mode_stack[++(tok)->tok_mode_stack_index]) -#endif - -#define FTSTRING_MIDDLE(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_MIDDLE : FSTRING_MIDDLE) -#define FTSTRING_END(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_END : FSTRING_END) -#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f') - -#define tok_nextc _PyLexer_nextc -#define tok_backup _PyLexer_backup int _PyLexer_nextc(struct tok_state *); void _PyLexer_backup(struct tok_state *, int); -int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char); +int _PyLexer_record_ftstring_comment( + struct tok_state *, ftstring_state *, const char *, const char *); +int _PyLexer_ftstring_punctuation( + struct tok_state *, ftstring_state *, struct token *, int); +int _PyLexer_close_ftstring_expr( + struct tok_state *, ftstring_state *, int); +void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *); int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int); int _PyLexer_scan_string(struct tok_state *, struct token *, int); -int _PyLexer_get_normal_mode(struct tok_state *, tokenizer_mode *, struct token *); -int _PyLexer_get_fstring_mode(struct tok_state *, tokenizer_mode *, struct token *); +int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *); +int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *); #endif diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index d82a7d0f296bac..cdf997363b0003 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -4,11 +4,9 @@ #include "errcode.h" #include "state.h" +#include "../tokenizer/helpers.h" #include "../tokenizer/reader.h" -/* Never change this */ -#define TABSIZE 8 - /* Create and initialize a new tok_state structure */ struct tok_state * _PyTokenizer_tok_new(void) @@ -28,18 +26,15 @@ _PyTokenizer_tok_new(void) tok->start = NULL; tok->done = E_OK; tok->fp = NULL; - tok->tabsize = TABSIZE; tok->indent = 0; tok->indstack[0] = 0; tok->atbol = 1; tok->pendin = 0; tok->prompt = NULL; tok->lineno = 0; - tok->starting_col_offset = -1; - tok->col_offset = -1; + tok->start_loc = (_PyTok_Loc){-1, -1}; tok->level = 0; tok->altindstack[0] = 0; - tok->input_error = 0; tok->encoding = NULL; tok->filename = NULL; tok->module = NULL; @@ -52,30 +47,60 @@ _PyTokenizer_tok_new(void) tok->implicit_newline = 0; _PyTok_SourceInit(&tok->source); tok->reader = NULL; - tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .quote='\0', .quote_size = 0, .in_debug=0}; - tok->tok_mode_stack_index = 0; + tok->ftstring_stack = tok->ftstring_stack_inline; + tok->ftstring_capacity = FTSTRING_STACK_INLINE_CAPACITY; #ifdef Py_DEBUG tok->debug = _Py_GetConfig()->parser_debug; #endif return tok; } -static void -free_fstring_expressions(struct tok_state *tok) +ftstring_state * +_PyLexer_PushFTString(struct tok_state *tok) { - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); - if (mode->last_expr_buffer != NULL) { - PyMem_Free(mode->last_expr_buffer); - mode->last_expr_buffer = NULL; - mode->last_expr_size = 0; - mode->last_expr_end = -1; - mode->in_format_spec = 0; + assert(tok->ftstring_depth >= 0 && tok->ftstring_depth <= tok->ftstring_capacity); + int next_depth = tok->ftstring_depth + 1; + if (next_depth >= MAXFTSTRINGLEVEL) { + _PyTokenizer_syntaxerror( + tok, "too many nested f-strings or t-strings"); + return NULL; + } + if (tok->ftstring_depth == tok->ftstring_capacity) { + int capacity = Py_MIN(Py_MAX(tok->ftstring_capacity * 2, 4), + MAXFTSTRINGLEVEL); + size_t size = (size_t)capacity * sizeof(*tok->ftstring_stack); + ftstring_state *stack; + if (tok->ftstring_stack == tok->ftstring_stack_inline) { + stack = PyMem_Malloc(size); + if (stack != NULL) { + memcpy(stack, tok->ftstring_stack, + (size_t)tok->ftstring_depth * sizeof(*stack)); + } + } + else { + stack = PyMem_Realloc(tok->ftstring_stack, size); } + if (stack == NULL) { + PyErr_NoMemory(); + tok->done = E_NOMEM; + return NULL; + } + tok->ftstring_stack = stack; + tok->ftstring_capacity = capacity; } + ftstring_state *state = &tok->ftstring_stack[tok->ftstring_depth]; + tok->ftstring_depth = next_depth; + *state = (ftstring_state){0}; + return state; +} + +void +_PyLexer_PopFTString(struct tok_state *tok) +{ + ftstring_state *state = _PyLexer_CurrentFTString(tok); + assert(state != NULL); + PyMem_Free(state->comments); + tok->ftstring_depth--; } /* Free a tok_state structure */ @@ -89,7 +114,12 @@ _PyTokenizer_Free(struct tok_state *tok) Py_XDECREF(tok->module); _PyTok_ReaderFree(tok); _PyTok_SourceClear(&tok->source); - free_fstring_expressions(tok); + for (int i = 0; i < tok->ftstring_depth; i++) { + PyMem_Free(tok->ftstring_stack[i].comments); + } + if (tok->ftstring_stack != tok->ftstring_stack_inline) { + PyMem_Free(tok->ftstring_stack); + } PyMem_Free(tok); } @@ -108,38 +138,17 @@ _PyToken_Init(struct token *token) { token->metadata = NULL; } -static inline _PyTok_Span -buffer_span(const struct tok_state *tok, const char *start, const char *end) -{ - if (start == NULL) { - assert(end == NULL); - return (_PyTok_Span){-1, -1}; - } - assert(end != NULL); - const char *base = tok->buf; - assert(base != NULL); - assert(tok->inp >= base); - Py_ssize_t start_offset = start - base; - Py_ssize_t end_offset = end - base; - assert(start_offset >= 0 && start_offset <= end_offset); - assert(end_offset <= tok->inp - base); - assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset); - return _PyTok_SpanFromBounds( - tok->buf_offset + start_offset, tok->buf_offset + end_offset); -} - int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end) { token->level = tok->level; - token->span = buffer_span(tok, start, end); - int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno; - token->start_loc = (_PyTok_Loc){lineno, -1}; - token->end_loc = (_PyTok_Loc){tok->lineno, -1}; - + token->span = _PyLexer_BufferSpan(tok, start, end); if (start != NULL && end != NULL) { - token->start_loc.byte_col = tok->starting_col_offset; - token->end_loc.byte_col = tok->col_offset; + token->start_loc = tok->start_loc; + token->end_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)}; + } + else { + token->start_loc = token->end_loc = (_PyTok_Loc){tok->lineno, -1}; } return type; } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 6d19e685bd7f81..6203084f19b54f 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -6,12 +6,8 @@ #define MAXINDENT 100 /* Max indentation level */ #define MAXLEVEL 200 /* Max parentheses level */ -#define MAXFSTRINGLEVEL 150 /* Max f-string nesting level */ - -#define INSIDE_FSTRING(tok) (tok->tok_mode_stack_index > 0) -#define INSIDE_FSTRING_EXPR(tok) (tok->curly_bracket_expr_start_depth >= 0) -#define INSIDE_FSTRING_EXPR_AT_TOP(tok) \ - (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1) +#define MAXFTSTRINGLEVEL 150 +#define FTSTRING_STACK_INLINE_CAPACITY 1 enum interactive_underflow_t { /* Normal mode of operation: return a new token when asked in interactive mode */ @@ -29,42 +25,52 @@ struct token { PyObject *metadata; }; -enum tokenizer_mode_kind_t { - TOK_REGULAR_MODE, - TOK_FSTRING_MODE, -}; +typedef enum { + FTSTRING_MODE_MIDDLE, + FTSTRING_MODE_EXPRESSION, + FTSTRING_MODE_FORMAT_SPEC, +} ftstring_mode; -enum string_kind_t { +typedef enum { FSTRING, + RAW_FSTRING, TSTRING, -}; + RAW_TSTRING, +} ftstring_kind; #define MAX_EXPR_NESTING 3 -typedef struct _tokenizer_mode { - enum tokenizer_mode_kind_t kind; - - int curly_bracket_depth; - int curly_bracket_expr_start_depth; +typedef struct _tokenizer_comments { + Py_ssize_t count; + Py_ssize_t capacity; + _PyTok_Span spans[]; +} tokenizer_comments; +typedef struct _ftstring_state { + ftstring_mode mode; + ftstring_kind kind; char quote; - int quote_size; - int raw; - const char* start; - const char* multi_line_start; - int first_line; - - Py_ssize_t start_offset; - Py_ssize_t multi_line_start_offset; - - Py_ssize_t last_expr_size; - Py_ssize_t last_expr_end; - char* last_expr_buffer; - int in_debug; - int in_format_spec; - - enum string_kind_t string_kind; -} tokenizer_mode; + unsigned char quote_size; + unsigned char debug_expr; + unsigned char replacement_depth; + int paren_level; + _PyTok_Off start; + _PyTok_Loc start_loc; + _PyTok_Span expr_span; + tokenizer_comments *comments; +} ftstring_state; + +static inline int +_PyLexer_IsTString(ftstring_kind kind) +{ + return kind == TSTRING || kind == RAW_TSTRING; +} + +static inline int +_PyLexer_IsRawString(ftstring_kind kind) +{ + return kind == RAW_FSTRING || kind == RAW_TSTRING; +} /* Tokenizer state */ struct tok_state { @@ -81,17 +87,13 @@ struct tok_state { int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ FILE *fp; /* Rest of input; NULL if tokenizing a string */ - int tabsize; /* Tab spacing */ int indent; /* Current indentation index */ int indstack[MAXINDENT]; /* Stack of indents */ int atbol; /* Nonzero if at begin of new line */ int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ const char *prompt; /* For interactive prompting */ int lineno; /* Current line number */ - int first_lineno; /* First line of a single line or multi line string - expression (cf. issue 16806) */ - int starting_col_offset; /* The column offset at the beginning of a token */ - int col_offset; /* Current col offset */ + _PyTok_Loc start_loc; int level; /* () [] {} Parentheses nesting level */ /* Used to allow free continuations inside them */ char parenstack[MAXLEVEL]; @@ -102,12 +104,8 @@ struct tok_state { /* Stuff for checking on different tab sizes */ int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ - int input_error; char *encoding; /* Source encoding. */ const char* line_start; /* pointer to start of current line */ - const char* multi_line_start; /* pointer to start of first line of - a single line or multi line string - expression (cf. issue 16806) */ char* str; /* Source string being tokenized (if tokenizing from a string)*/ _PyTok_SourceText source; @@ -118,9 +116,10 @@ struct tok_state { /* How to proceed when asked for a new token in interactive mode */ enum interactive_underflow_t interactive_underflow; int report_warnings; - // TODO: Factor this into its own thing - tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL]; - int tok_mode_stack_index; + ftstring_state *ftstring_stack; + ftstring_state ftstring_stack_inline[FTSTRING_STACK_INLINE_CAPACITY]; + int ftstring_depth; + int ftstring_capacity; int tok_extra_tokens; int comment_newline; int implicit_newline; @@ -129,10 +128,93 @@ struct tok_state { #endif }; +static inline ftstring_state * +_PyLexer_CurrentFTString(struct tok_state *tok) +{ + assert(tok->ftstring_stack != NULL); + assert(tok->ftstring_depth >= 0 && tok->ftstring_depth <= tok->ftstring_capacity); + if (tok->ftstring_depth == 0) { + return NULL; + } + return &tok->ftstring_stack[tok->ftstring_depth - 1]; +} + +static inline char +_PyLexer_StringPrefix(ftstring_kind kind) +{ + return _PyLexer_IsTString(kind) ? 't' : 'f'; +} + +static inline int +_PyLexer_FTStringBracketDepth(const struct tok_state *tok, + const ftstring_state *state) +{ + return tok->level - state->paren_level; +} + +static inline _PyTok_Off +_PyLexer_BufferOffset(const struct tok_state *tok, const char *position) +{ + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + assert(position >= tok->buf && position <= tok->inp); + Py_ssize_t offset = position - tok->buf; + assert(tok->buf_offset <= PY_SSIZE_T_MAX - offset); + return tok->buf_offset + offset; +} + +static inline char * +_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset) +{ + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + assert(offset >= tok->buf_offset); + assert(offset - tok->buf_offset <= tok->inp - tok->buf); + return tok->buf + (offset - tok->buf_offset); +} + +static inline const char * +_PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span, + Py_ssize_t *length) +{ + assert(length != NULL); + assert(_PyTok_SpanIsValid(span)); + *length = span.end - span.start; + (void)_PyLexer_BufferPointer(tok, span.end); + return _PyLexer_BufferPointer(tok, span.start); +} + +static inline int +_PyLexer_ByteColumn(const struct tok_state *tok) +{ + assert(tok->line_start != NULL); + assert(tok->cur >= tok->line_start); + Py_ssize_t column = tok->cur - tok->line_start; + assert(column <= INT_MAX); + return (int)column; +} + +static inline _PyTok_Span +_PyLexer_BufferSpan(const struct tok_state *tok, const char *start, + const char *end) +{ + if (start == NULL) { + assert(end == NULL); + return (_PyTok_Span){-1, -1}; + } + assert(end != NULL); + assert(start <= end); + return _PyTok_SpanFromBounds( + _PyLexer_BufferOffset(tok, start), + _PyLexer_BufferOffset(tok, end)); +} + int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); struct tok_state *_PyTokenizer_tok_new(void); void _PyTokenizer_Free(struct tok_state *); +ftstring_state *_PyLexer_PushFTString(struct tok_state *); +void _PyLexer_PopFTString(struct tok_state *); void _PyToken_Free(struct token *); void _PyToken_Init(struct token *); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index fc0299c5c7c592..ac775e9d61ba09 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -7,110 +7,117 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) -int -_PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { - assert(token != NULL); - assert(c == '}' || c == ':' || c == '!'); - tokenizer_mode *tok_mode = TOK_GET_MODE(tok); +static void +rewind_to_string_start(struct tok_state *tok, const char *start, + _PyTok_Loc location) +{ + tok->cur = (char *)start + 1; + tok->line_start = start - location.byte_col; + tok->lineno = location.lineno; +} - if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) { +int +_PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state, + const char *start, const char *end) +{ + assert(state == _PyLexer_CurrentFTString(tok) && state->mode == FTSTRING_MODE_EXPRESSION); + if (state->expr_span.end >= 0) { return 0; } - PyObject *res = NULL; - - // Look for a # character outside of string literals - int hash_detected = 0; - int in_string = 0; - char quote_char = 0; - - for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { - char ch = tok_mode->last_expr_buffer[i]; - - // Skip escaped characters - if (ch == '\\') { - i++; - continue; + assert(state->expr_span.start >= 0); + tokenizer_comments *comments = state->comments; + if (comments == NULL || comments->count == comments->capacity) { + Py_ssize_t max_capacity = (PY_SSIZE_T_MAX - + (Py_ssize_t)sizeof(*comments)) / + (Py_ssize_t)sizeof(*comments->spans); + if (comments != NULL && comments->capacity > max_capacity / 2) { + PyErr_NoMemory(); + return -1; } - - // Handle quotes - if (ch == '"' || ch == '\'') { - // The following if/else block works becase there is an off number - // of quotes in STRING tokens and the lexer only ever reaches this - // function with valid STRING tokens. - // For example: """hello""" - // First quote: in_string = 1 - // Second quote: in_string = 0 - // Third quote: in_string = 1 - if (!in_string) { - in_string = 1; - quote_char = ch; - } - else if (ch == quote_char) { - in_string = 0; - } - continue; + Py_ssize_t capacity = comments == NULL ? 4 : comments->capacity * 2; + size_t size = sizeof(*comments) + + (size_t)capacity * sizeof(*comments->spans); + tokenizer_comments *resized = PyMem_Realloc(comments, size); + if (resized == NULL) { + PyErr_NoMemory(); + return -1; } - - // Check for # outside strings - if (ch == '#' && !in_string) { - hash_detected = 1; - break; + comments = resized; + if (state->comments == NULL) { + comments->count = 0; } + comments->capacity = capacity; + state->comments = comments; } - // If we found a # character in the expression, we need to handle comments - if (hash_detected) { - // Allocate buffer for processed result - char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char)); - if (!result) { - return -1; - } + comments->spans[comments->count++] = + _PyLexer_BufferSpan(tok, start, end); + return 0; +} - Py_ssize_t i = 0; // Input position - Py_ssize_t j = 0; // Output position - in_string = 0; // Whether we're in a string - quote_char = 0; // Current string quote char - - // Process each character - while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { - char ch = tok_mode->last_expr_buffer[i]; - - // Handle string quotes - if (ch == '"' || ch == '\'') { - // See comment above to understand this part - if (!in_string) { - in_string = 1; - quote_char = ch; - } else if (ch == quote_char) { - in_string = 0; - } - result[j++] = ch; - } - // Skip comments - else if (ch == '#' && !in_string) { - while (i < tok_mode->last_expr_size - tok_mode->last_expr_end && - tok_mode->last_expr_buffer[i] != '\n') { - i++; - } - if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { - result[j++] = '\n'; - } - } - // Copy other chars - else { - result[j++] = ch; +static int +finish_ftstring_expr(struct tok_state *tok, ftstring_state *state, + struct token *token) +{ + assert(token != NULL && state == _PyLexer_CurrentFTString(tok)); + assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start != NULL); + + if (state->expr_span.end >= 0) { + return 0; + } + assert(state->expr_span.start >= 0); + state->expr_span.end = _PyLexer_BufferOffset(tok, tok->start); + int tstring_interpolation = _PyLexer_IsTString(state->kind) && + state->replacement_depth == 1; + if (!(state->debug_expr || tstring_interpolation) || token->metadata) { + return 0; + } + Py_ssize_t expr_len; + const char *expr = _PyLexer_BufferSpanView( + tok, state->expr_span, &expr_len); + tokenizer_comments *comments = state->comments; + PyObject *res; + if (comments != NULL && comments->count > 0) { + Py_ssize_t stripped_size = expr_len; + _PyTok_Off previous_end = state->expr_span.start; + Py_ssize_t comment_count = 0; + for (Py_ssize_t i = 0; i < comments->count; i++) { + _PyTok_Span comment = comments->spans[i]; + assert(_PyTok_SpanIsValid(comment)); + assert(comment.start >= previous_end); + if (comment.start >= state->expr_span.end) { + break; } - i++; + assert(comment.end <= state->expr_span.end); + stripped_size -= comment.end - comment.start; + previous_end = comment.end; + comment_count++; } - - result[j] = '\0'; // Null-terminate the result string - res = PyUnicode_DecodeUTF8(result, j, NULL); - PyMem_Free(result); - } else { - res = PyUnicode_DecodeUTF8( - tok_mode->last_expr_buffer, - tok_mode->last_expr_size - tok_mode->last_expr_end, - NULL - ); + char *stripped = PyMem_Malloc((size_t)stripped_size); + if (stripped == NULL) { + PyErr_NoMemory(); + return -1; + } + _PyTok_Off copied_to = state->expr_span.start; + Py_ssize_t stripped_len = 0; + for (Py_ssize_t i = 0; i < comment_count; i++) { + _PyTok_Span comment = comments->spans[i]; + Py_ssize_t length = comment.start - copied_to; + memcpy(stripped + stripped_len, + expr + copied_to - state->expr_span.start, + (size_t)length); + stripped_len += length; + copied_to = comment.end; + } + Py_ssize_t length = state->expr_span.end - copied_to; + memcpy(stripped + stripped_len, + expr + copied_to - state->expr_span.start, + (size_t)length); + stripped_len += length; + res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL); + PyMem_Free(stripped); + } + else { + res = PyUnicode_DecodeUTF8(expr, expr_len, NULL); } if (!res) { @@ -120,63 +127,82 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { return 0; } -int -_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) +static int +begin_ftstring_expr(struct tok_state *tok, ftstring_state *state, + _PyTok_Off expr_start) { - assert(tok->cur != NULL); + assert(state->mode != FTSTRING_MODE_EXPRESSION); + state->expr_span = (_PyTok_Span){expr_start, -1}; + if (state->comments != NULL) { + state->comments->count = 0; + } + if (state->replacement_depth >= MAX_EXPR_NESTING) { + _PyTokenizer_syntaxerror( + tok, "%c-string: expressions nested too deeply", + _PyLexer_StringPrefix(state->kind)); + return -1; + } + state->replacement_depth++; + state->mode = FTSTRING_MODE_EXPRESSION; + state->debug_expr = 0; + return 0; +} - Py_ssize_t size = cur == 0 - ? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur); - tokenizer_mode *tok_mode = TOK_GET_MODE(tok); +int +_PyLexer_ftstring_punctuation(struct tok_state *tok, ftstring_state *state, + struct token *token, int c) +{ + assert(state->mode == FTSTRING_MODE_EXPRESSION); + assert(c == ':' || c == '}' || c == '!'); + if (_PyLexer_FTStringBracketDepth(tok, state) != state->replacement_depth) { + return 0; + } + if (c == '!') { + int next = tok_nextc(tok); + tok_backup(tok, next); + if (next == '=') { + return 0; + } + } + if (finish_ftstring_expr(tok, state, token) < 0) { + return -1; + } + if (c == ':') { + state->mode = FTSTRING_MODE_FORMAT_SPEC; + return COLON; + } + return 0; +} - switch (cur) { - case 0: - if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) { - return 1; - } - char *new_buffer = PyMem_Realloc( - tok_mode->last_expr_buffer, - tok_mode->last_expr_size + size - ); - if (new_buffer == NULL) { - PyMem_Free(tok_mode->last_expr_buffer); - goto error; - } - tok_mode->last_expr_buffer = new_buffer; - memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, - tok->cur, size); - tok_mode->last_expr_size += size; - break; - case '{': - if (tok_mode->last_expr_buffer != NULL) { - PyMem_Free(tok_mode->last_expr_buffer); - } - tok_mode->last_expr_buffer = PyMem_Malloc(size); - if (tok_mode->last_expr_buffer == NULL) { - goto error; - } - tok_mode->last_expr_size = size; - tok_mode->last_expr_end = -1; - memcpy(tok_mode->last_expr_buffer, tok->cur, size); - break; - case '}': - case '!': - tok_mode->last_expr_end = strlen(tok->start); - break; - case ':': - if (tok_mode->last_expr_end == -1) { - tok_mode->last_expr_end = strlen(tok->start); - } - break; - default: - Py_UNREACHABLE(); +int +_PyLexer_close_ftstring_expr(struct tok_state *tok, ftstring_state *state, + int c) +{ + assert(state->mode == FTSTRING_MODE_EXPRESSION); + assert(c == ')' || c == ']' || c == '}'); + int depth = _PyLexer_FTStringBracketDepth(tok, state); + if (depth < 0) { + _PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'", + _PyLexer_StringPrefix(state->kind), c); + return -1; + } + if (c == '}' && depth == state->replacement_depth - 1) { + state->replacement_depth--; + state->mode = FTSTRING_MODE_MIDDLE; + state->debug_expr = 0; } - return 1; -error: - tok->done = E_NOMEM; return 0; } +void +_PyLexer_mark_ftstring_debug(struct tok_state *tok, ftstring_state *state) +{ + assert(state->mode == FTSTRING_MODE_EXPRESSION); + if (_PyLexer_FTStringBracketDepth(tok, state) == state->replacement_depth) { + state->debug_expr = 1; + } +} + int _PyLexer_check_string_prefixes(struct tok_state *tok, int saw_b, int saw_r, int saw_u, @@ -231,13 +257,6 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) int quote = c; int quote_size = 1; /* 1 or 3 */ - /* Nodes of type STRING, especially multi line strings - must be handled differently in order to get both - the starting line number and the column offset right. - (cf. issue 16806) */ - tok->first_lineno = tok->lineno; - tok->multi_line_start = tok->line_start; - /* Find the quote size and start of string */ int after_quote = tok_nextc(tok); if (after_quote == quote) { @@ -255,53 +274,44 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) tok_backup(tok, after_quote); } - p_start = tok->start; p_end = tok->cur; - if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested f-strings or t-strings")); - } - tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok); - the_current_tok->kind = TOK_FSTRING_MODE; - the_current_tok->quote = quote; - the_current_tok->quote_size = quote_size; - the_current_tok->start = tok->start; - the_current_tok->multi_line_start = tok->line_start; - the_current_tok->first_line = tok->lineno; - the_current_tok->start_offset = -1; - the_current_tok->multi_line_start_offset = -1; - the_current_tok->last_expr_buffer = NULL; - the_current_tok->last_expr_size = 0; - the_current_tok->last_expr_end = -1; - the_current_tok->in_format_spec = 0; - the_current_tok->in_debug = 0; - - enum string_kind_t string_kind = FSTRING; + ftstring_state *state = _PyLexer_PushFTString(tok); + if (state == NULL) { + return MAKE_TOKEN(ERRORTOKEN); + } + state->mode = FTSTRING_MODE_MIDDLE; + state->quote = quote; + state->quote_size = quote_size; + state->paren_level = tok->level; + state->start = _PyLexer_BufferOffset(tok, tok->start); + state->start_loc = tok->start_loc; + state->expr_span = (_PyTok_Span){-1, -1}; + + int raw = 0; + int tstring = 0; switch (*tok->start) { case 'T': case 't': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - string_kind = TSTRING; + raw = Py_TOLOWER(tok->start[1]) == 'r'; + tstring = 1; break; case 'F': case 'f': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; + raw = Py_TOLOWER(tok->start[1]) == 'r'; break; case 'R': case 'r': - the_current_tok->raw = 1; - if (Py_TOLOWER(*(tok->start + 1)) == 't') { - string_kind = TSTRING; - } + raw = 1; + tstring = Py_TOLOWER(tok->start[1]) == 't'; break; default: Py_UNREACHABLE(); } - - the_current_tok->string_kind = string_kind; - the_current_tok->curly_bracket_depth = 0; - the_current_tok->curly_bracket_expr_start_depth = -1; - return string_kind == TSTRING ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); + state->kind = tstring + ? (raw ? RAW_TSTRING : TSTRING) + : (raw ? RAW_FSTRING : FSTRING); + return tstring ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); } int @@ -315,13 +325,6 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) int end_quote_size = 0; int has_escaped_quote = 0; - /* Nodes of type STRING, especially multi line strings - must be handled differently in order to get both - the starting line number and the column offset right. - (cf. issue 16806) */ - tok->first_lineno = tok->lineno; - tok->multi_line_start = tok->line_start; - /* Find the quote size and start of string */ c = tok_nextc(tok); if (c == quote) { @@ -347,33 +350,23 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) break; } if (c == EOF || (quote_size == 1 && c == '\n')) { - assert(tok->multi_line_start != NULL); - // shift the tok_state's location into - // the start of string, and report the error - // from the initial quote character - tok->cur = (char *)tok->start; - tok->cur++; - tok->line_start = tok->multi_line_start; - int start = tok->lineno; - tok->lineno = tok->first_lineno; - - if (INSIDE_FSTRING(tok)) { - /* When we are in an f-string, before raising the - * unterminated string literal error, check whether - * does the initial quote matches with f-strings quotes - * and if it is, then this must be a missing '}' token - * so raise the proper error */ - tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); - if (the_current_tok->quote == quote && - the_current_tok->quote_size == quote_size) { + int end_lineno = tok->lineno; + rewind_to_string_start(tok, tok->start, tok->start_loc); + + const ftstring_state *state = _PyLexer_CurrentFTString(tok); + if (state != NULL) { + /* A matching quote belongs to the surrounding formatted + * string, so the expression is missing its closing brace. */ + if (state->quote == quote && state->quote_size == quote_size) { return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok))); + "%c-string: expecting '}'", + _PyLexer_StringPrefix(state->kind))); } } if (quote_size == 3) { _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal" - " (detected at line %d)", start); + " (detected at line %d)", end_lineno); if (c != '\n') { tok->done = E_EOFS; } @@ -385,11 +378,11 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) tok, "unterminated string literal (detected at line %d); " "perhaps you escaped the end quote?", - start + end_lineno ); } else { _PyTokenizer_syntaxerror( - tok, "unterminated string literal (detected at line %d)", start + tok, "unterminated string literal (detected at line %d)", end_lineno ); } if (c != '\n') { @@ -421,118 +414,52 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) } int -_PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token) +_PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct token *token) { + assert(current == _PyLexer_CurrentFTString(tok) && current->mode != FTSTRING_MODE_EXPRESSION); + assert((current->quote_size == 1 || current->quote_size == 3) && + current->replacement_depth <= MAX_EXPR_NESTING); const char *p_start = NULL; const char *p_end = NULL; int end_quote_size = 0; int unicode_escape = 0; + int quote = current->quote; + int quote_size = current->quote_size; + int in_format_spec = current->mode == FTSTRING_MODE_FORMAT_SPEC; + int raw = _PyLexer_IsRawString(current->kind); tok->start = tok->cur; - tok->first_lineno = tok->lineno; - tok->starting_col_offset = tok->col_offset; - - // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize - // before it. - int start_char = tok_nextc(tok); - if (start_char == '{') { - int peek1 = tok_nextc(tok); - tok_backup(tok, peek1); - tok_backup(tok, start_char); - if (peek1 != '{') { - current_tok->curly_bracket_expr_start_depth++; - if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok))); - } - TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; - return _PyLexer_get_normal_mode(tok, current_tok, token); - } - } - else { - tok_backup(tok, start_char); - } - - // Check if we are at the end of the string - for (int i = 0; i < current_tok->quote_size; i++) { - int quote = tok_nextc(tok); - if (quote != current_tok->quote) { - tok_backup(tok, quote); - goto f_string_middle; - } - } + tok->start_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)}; - if (current_tok->last_expr_buffer != NULL) { - PyMem_Free(current_tok->last_expr_buffer); - current_tok->last_expr_buffer = NULL; - current_tok->last_expr_size = 0; - current_tok->last_expr_end = -1; - } - - p_start = tok->start; - p_end = tok->cur; - tok->tok_mode_stack_index--; - return MAKE_TOKEN(FTSTRING_END(current_tok)); - -f_string_middle: - - // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle - // this. - tok->multi_line_start = tok->line_start; - while (end_quote_size != current_tok->quote_size) { + while (end_quote_size != quote_size) { int c = tok_nextc(tok); if (tok->done == E_ERROR || tok->done == E_DECODE) { return MAKE_TOKEN(ERRORTOKEN); } - int in_format_spec = ( - current_tok->in_format_spec - && - INSIDE_FSTRING_EXPR(current_tok) - ); - - if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) { - if (tok->input_error) { + + if (c == EOF || (quote_size == 1 && c == '\n')) { + if (tok_failed(tok)) { return MAKE_TOKEN(ERRORTOKEN); } - // If we are in a format spec and we found a newline, - // it means that the format spec ends here and we should - // return to the regular mode. if (in_format_spec && c == '\n') { - if (current_tok->quote_size == 1) { - return MAKE_TOKEN( - _PyTokenizer_syntaxerror( - tok, - "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings", - TOK_GET_STRING_PREFIX(tok), TOK_GET_STRING_PREFIX(tok) - ) - ); - } - tok_backup(tok, c); - TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; - current_tok->in_format_spec = 0; - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); + return MAKE_TOKEN(_PyTokenizer_syntaxerror( + tok, + "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings", + _PyLexer_StringPrefix(current->kind), + _PyLexer_StringPrefix(current->kind))); } - assert(tok->multi_line_start != NULL); - // shift the tok_state's location into - // the start of string, and report the error - // from the initial quote character - tok->cur = (char *)current_tok->start; - tok->cur++; - tok->line_start = current_tok->multi_line_start; - int start = tok->lineno; - - tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); - tok->lineno = the_current_tok->first_line; + int end_lineno = tok->lineno; + rewind_to_string_start(tok, + _PyLexer_BufferPointer(tok, current->start), + current->start_loc); - if (current_tok->quote_size == 3) { + if (quote_size == 3) { _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted %c-string literal" " (detected at line %d)", - TOK_GET_STRING_PREFIX(tok), start); + _PyLexer_StringPrefix(current->kind), end_lineno); if (c != '\n') { tok->done = E_EOFS; } @@ -541,11 +468,12 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st else { return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unterminated %c-string literal (detected at" - " line %d)", TOK_GET_STRING_PREFIX(tok), start)); + " line %d)", + _PyLexer_StringPrefix(current->kind), end_lineno)); } } - if (c == current_tok->quote) { + if (c == quote) { end_quote_size += 1; continue; } else { @@ -553,62 +481,63 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st } if (c == '{') { - if (!_PyLexer_update_ftstring_expr(tok, c)) { - return MAKE_TOKEN(ENDMARKER); - } int peek = tok_nextc(tok); if (peek != '{' || in_format_spec) { tok_backup(tok, peek); + _PyTok_Off expr_start = _PyLexer_BufferOffset(tok, tok->cur); tok_backup(tok, c); - current_tok->curly_bracket_expr_start_depth++; - if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok))); + if (begin_ftstring_expr(tok, current, expr_start) < 0) { + return MAKE_TOKEN(ERRORTOKEN); } - TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; - current_tok->in_format_spec = 0; p_start = tok->start; p_end = tok->cur; + if (p_start == p_end) { + return _PyLexer_get_normal(tok, current, token); + } } else { p_start = tok->start; p_end = tok->cur - 1; } - return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); + goto emit_middle; } else if (c == '}') { if (unicode_escape) { p_start = tok->start; p_end = tok->cur; - return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); + goto emit_middle; } int peek = tok_nextc(tok); - // The tokenizer can only be in the format spec if we have already completed the expression - // scanning (indicated by the end of the expression being set) and we are not at the top level - // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double - // brackets, we can bypass it here. - int cursor = current_tok->curly_bracket_depth; - if (peek == '}' && !in_format_spec && cursor == 0) { + int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current); + if (peek == '}' && !in_format_spec && bracket_depth == 0) { p_start = tok->start; p_end = tok->cur - 1; - } else { + } + else { tok_backup(tok, peek); + if (!in_format_spec && bracket_depth == 0) { + if (tok->start == tok->cur - 1) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror( + tok, "%c-string: single '}' is not allowed", + _PyLexer_StringPrefix(current->kind))); + } + tok_backup(tok, c); + p_start = tok->start; + p_end = tok->cur; + goto emit_middle; + } tok_backup(tok, c); - TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; - current_tok->in_format_spec = 0; + current->mode = FTSTRING_MODE_EXPRESSION; p_start = tok->start; p_end = tok->cur; } - return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); + goto emit_middle; } else if (c == '\\') { int peek = tok_nextc(tok); if (peek == '\r') { peek = tok_nextc(tok); } - // Special case when the backslash is right before a curly - // brace. We have to restore and return the control back - // to the loop for the next iteration. if (peek == '{' || peek == '}') { - if (!current_tok->raw) { + if (!raw) { if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) { return MAKE_TOKEN(ERRORTOKEN); } @@ -617,7 +546,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st continue; } - if (!current_tok->raw) { + if (!raw) { if (peek == 'N') { /* Handle named unicode escapes (\N{BULLET}) */ peek = tok_nextc(tok); @@ -627,18 +556,21 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st tok_backup(tok, peek); } } - } /* else { - skip the escaped character - }*/ + } } } - // Backup the f-string quotes to emit a final FSTRING_MIDDLE and - // add the quotes to the FSTRING_END in the next tokenizer iteration. - for (int i = 0; i < current_tok->quote_size; i++) { - tok_backup(tok, current_tok->quote); - } p_start = tok->start; p_end = tok->cur; - return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); + if (p_end - quote_size == p_start) { + int end_token = FTSTRING_END(current); + _PyLexer_PopFTString(tok); + return MAKE_TOKEN(end_token); + } + for (int i = 0; i < quote_size; i++) { + tok_backup(tok, quote); + } + p_end = tok->cur; +emit_middle: + return MAKE_TOKEN(FTSTRING_MIDDLE(current)); } diff --git a/Parser/pegen.c b/Parser/pegen.c index d86dd22444e6a7..20dbbcabec379e 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1035,7 +1035,6 @@ _PyPegen_run_parser(Parser *p) } if (p->start_rule == Py_single_input && bad_single_statement(p)) { - p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement"); } diff --git a/Parser/pegen.h b/Parser/pegen.h index 3cca698692bf6b..81137782621daf 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -26,9 +26,6 @@ #define CURRENT_POS (-5) -#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index])) -#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f') - typedef struct _memo { int type; void *node; diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index b13e1c079220a9..f4c3f988706259 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -62,7 +62,7 @@ _Pypegen_tokenizer_error(Parser *p) msg = "too many levels of indentation"; break; case E_LINECONT: { - col_offset = p->tok->cur - p->tok->buf - 1; + col_offset = p->tok->cur - p->tok->line_start - 1; msg = "unexpected character after line continuation character"; break; } @@ -163,10 +163,8 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { exit: _PyToken_Free(&new_token); - // If we're in an f-string, we want the syntax error in the expression part - // to propagate, so that tokenizer errors (like expecting '}') that happen afterwards - // do not swallow it. - if (PyErr_Occurred() && p->tok->tok_mode_stack_index <= 0) { + // Preserve expression errors over later formatted-string errors. + if (PyErr_Occurred() && _PyLexer_CurrentFTString(p->tok) == NULL) { Py_XDECREF(value); Py_XDECREF(type); Py_XDECREF(traceback); diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 5edf5a3dfd2e0d..51de0cbb156f83 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -5,10 +5,6 @@ #include "../lexer/state.h" -#define ADVANCE_LINENO() \ - tok->lineno++; \ - tok->col_offset = 0; - int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...); int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); int _PyTokenizer_indenterror(struct tok_state *tok); diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index b9b4a461087441..6eb528ca985ed6 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -102,6 +102,7 @@ append_implicit_newline(_PyTok_Reader *reader) static int pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) { + assert(reader->decoded_pos >= 0 && reader->decoded_pos <= reader->decoded_len); if (reader->decoded_pos == reader->decoded_len) { return 0; } @@ -546,10 +547,13 @@ reset_streaming_buffer(struct tok_state *tok) int _PyTok_ReaderUnderflow(struct tok_state *tok) { + assert(tok->cur == tok->inp || (tok->buf != NULL && + tok->cur >= tok->buf && tok->cur < tok->inp)); _PyTok_ReaderKind kind = tok->reader->kind; int prepared = kind == _PYTOK_READER_PREPARED; int streaming = reader_is_streaming(kind); - int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok); + int reset_buffer = !prepared && tok->start == NULL && + _PyLexer_CurrentFTString(tok) == NULL; _PyTok_Chunk chunk; _PyTok_ReadResult result = reader_next(tok, &chunk); @@ -567,7 +571,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) tok->done = E_INTR; } else { - tok->input_error = 1; if (tok->done == E_OK) { tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) ? E_NOMEM : E_ERROR; @@ -601,7 +604,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) _PyTok_ChunkClear(&chunk); tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) ? E_NOMEM : E_ERROR; - tok->input_error = 1; return 0; } if (reset_buffer) { @@ -610,7 +612,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) tok->buf_offset = source_start; tok->line_start = tok->buf; tok->start = NULL; - tok->multi_line_start = NULL; } else { _PyLexer_RestoreBufferPointers( @@ -624,7 +625,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) tok->interactive_src_end = tok->source.bytes + tok->source.len; } if (prepared) { - if (tok->start == NULL) { + if (tok->start == NULL && _PyLexer_CurrentFTString(tok) == NULL) { tok->buf = tok->cur; tok->buf_offset = tok->source.base_offset + (chunk.data - tok->source.bytes); @@ -633,18 +634,11 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) } tok->implicit_newline = chunk.implicit_newline; - if (!prepared && tok->tok_mode_stack_index && - !_PyLexer_update_ftstring_expr(tok, 0)) { - _PyTok_ChunkClear(&chunk); - tok->input_error = 1; - return 0; - } - ADVANCE_LINENO(); + tok->lineno++; if (kind == _PYTOK_READER_FILE && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) { _PyTok_ChunkClear(&chunk); - tok->input_error = 1; return 0; } _PyTok_ChunkClear(&chunk); @@ -671,6 +665,7 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind) if (reader_is_streaming(kind)) { tok->buf = tok->cur = tok->inp = (char *)_PyTok_SourceData(&tok->source); + tok->line_start = tok->buf; } return tok; } @@ -689,6 +684,7 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input, return NULL; } tok->buf = tok->cur = tok->inp = tok->str; + tok->line_start = tok->str; return tok; } diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 71f236b08d93c8..eb5c0b86a8fbe4 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -287,7 +287,8 @@ tokenizeriter_next(PyObject *op) is_trailing_token = 1; } - const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start; + const char *line_start = ISSTRINGLIT(type) + ? token_start - token.start_loc.byte_col : it->tok->line_start; PyObject* line = NULL; int line_changed = 1; if (it->tok->tok_extra_tokens && is_trailing_token) {