gh-153569: simplify formatted-string state

Author: pablogsal

Lib/test/test_fstring.py

@@ -663,7 +663,10 @@ def create_nested_fstring(n):
             prev = create_nested_fstring(n-1)
             return f'f"{{{prev}}}"'
 
-        raises_syntax_or_memory_error(create_nested_fstring(160))
+        compile(create_nested_fstring(149), "<string>", "eval")
+        with self.assertRaisesRegex(
+                SyntaxError, "too many nested f-strings or t-strings"):
+            compile(create_nested_fstring(150), "<string>", "eval")
         raises_syntax_or_memory_error("f'{" + "("*100 + "}'")
         raises_syntax_or_memory_error("f'{" + "("*1000 + "}'")
         raises_syntax_or_memory_error("f'{" + "("*10_000 + "}'")

Lib/test/test_repl.py

@@ -184,9 +184,8 @@ def read_until(marker, start=0):
 
     @cpython_only
     def test_lexer_buffer_realloc_with_null_start(self):
-        # gh-144759: NULL pointer arithmetic in the lexer when start and
-        # multi_line_start are NULL (uninitialized in tok_mode_stack[0])
-        # and the lexer buffer is reallocated while parsing long input.
+        # gh-144759: NULL pointer arithmetic when the lexer buffer grows
+        # while parsing long input.
         long_value = "a" * 2000
         user_input = dedent(f"""\
         x = f'{{{long_value!r}}}'

Lib/test/test_tokenize.py

@@ -2574,6 +2574,12 @@ def test_degraded_fstring_format_spec(self):
             ("f-string: single '}' is not allowed", (1, 11)),
         )
 
+    def test_carriage_return_after_debug_comment(self):
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                tokens = self._get_tokens(f"{prefix}'''{{x=# comment\r}}'''")
+                self.assertEqual(tokens[4].string, "# comment\r}")
+
     def test_escaped_fstring_brace_has_a_position_gap(self):
         tokens = self._get_tokens('f"a{{"', extra_tokens=True)
         self.assertEqual(

Parser/action_helpers.c

@@ -1001,14 +1001,33 @@ result_token_with_metadata(Parser *p, void *result, PyObject *metadata)
     return res;
 }
 
+static char
+formatted_string_prefix(const Parser *p)
+{
+    int nested = 0;
+    for (int i = p->mark - 1; i >= 0; i--) {
+        int type = p->tokens[i]->type;
+        if (type == FSTRING_END || type == TSTRING_END) {
+            nested++;
+        }
+        else if (type == FSTRING_START || type == TSTRING_START) {
+            if (nested == 0) {
+                return type == TSTRING_START ? 't' : 'f';
+            }
+            nested--;
+        }
+    }
+    Py_UNREACHABLE();
+}
+
 ResultTokenWithMetadata *
 _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
 {
     if (conv_token->lineno != conv->lineno || conv_token->end_col_offset != conv->col_offset) {
         return RAISE_SYNTAX_ERROR_KNOWN_RANGE(
             conv_token, conv,
             "%c-string: conversion type must come right after the exclamation mark",
-            _PyLexer_CurrentStringPrefix(p->tok)
+            formatted_string_prefix(p)
         );
     }
 
@@ -1017,7 +1036,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
             !(first == 's' || first == 'r' || first == 'a')) {
         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(conv,
                                             "%c-string: invalid conversion character %R: expected 's', 'r', or 'a'",
-                                            _PyLexer_CurrentStringPrefix(p->tok),
+                                            formatted_string_prefix(p),
                                             conv->v.Name.id);
         return NULL;
     }
@@ -1379,7 +1398,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
             asdl_expr_seq *values = item->v.JoinedStr.values;
             if (asdl_seq_LEN(values) != 2) {
                 PyErr_Format(PyExc_SystemError,
-                             _PyLexer_IsTStringKind(string_kind)
+                             _PyLexer_IsTString(string_kind)
                              ? "unexpected TemplateStr node without debug data in t-string at line %d"
                              : "unexpected JoinedStr node without debug data in f-string at line %d",
                              item->lineno);
@@ -1391,7 +1410,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
             asdl_seq_SET(seq, index++, first);
 
             expr_ty second = asdl_seq_GET(values, 1);
-            assert((_PyLexer_IsTStringKind(string_kind) &&
+            assert((_PyLexer_IsTString(string_kind) &&
                     second->kind == Interpolation_kind) ||
                    second->kind == FormattedValue_kind);
             asdl_seq_SET(seq, index++, second);
@@ -1463,12 +1482,8 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
         return NULL;
     }
 
-    // Check if we're inside a raw f-string for format spec decoding
-    int is_raw = 0;
-    if (_PyLexer_InsideFString(p->tok)) {
-        tokenizer_mode *mode = _PyLexer_CurrentMode(p->tok);
-        is_raw = _PyLexer_IsRawString(mode);
-    }
+    const ftstring_state *state = _PyLexer_CurrentFTString(p->tok);
+    int is_raw = state != NULL && _PyLexer_IsRawString(state->kind);
 
     PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);
     if (str == NULL) {

Parser/lexer/lexer.c

@@ -169,8 +169,11 @@ tok_continuation_line(struct tok_state *tok) {
 
 
 int
-_PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
+_PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token *token)
 {
+    assert(current == NULL ||
+           (current->mode == FTSTRING_MODE_EXPRESSION &&
+            current->replacement_depth > 0));
     int c;
     int blankline, nonascii;
 
@@ -333,13 +336,13 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             c = tok_nextc(tok);
         }
 
-        if (_PyLexer_InsideFString(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
+        if (current != NULL) {
             const char *comment_end = tok->cur;
             if (c == '\n') {
                 comment_end--;
             }
             if (_PyLexer_record_ftstring_comment(
-                    tok, tok->start, comment_end) < 0) {
+                    tok, current, tok->start, comment_end) < 0) {
                 tok->done = E_NOMEM;
                 return MAKE_TOKEN(ERRORTOKEN);
             }
@@ -545,12 +548,11 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
 
     /* Punctuation character */
     int is_punctuation = (c == ':' || c == '}' || c == '!');
-    if (is_punctuation && _PyLexer_InsideFString(tok) &&
-            INSIDE_FSTRING_EXPR(current_tok)) {
-        int cursor = current_tok->curly_bracket_depth - 1;
-        int in_format_spec = current_tok->in_format_spec;
+    if (is_punctuation && current != NULL) {
+        int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
+        int cursor = bracket_depth - 1;
         int cursor_in_format_with_debug =
-            cursor == 1 && (current_tok->in_debug || in_format_spec);
+            cursor == 1 && current->debug_expr;
         int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
         if (cursor_valid && c == '!') {
             int c2 = tok_nextc(tok);
@@ -560,17 +562,15 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             tok_backup(tok, c2);
         }
         if (cursor_valid) {
-            _PyLexer_finish_ftstring_expr(tok);
+            _PyLexer_finish_ftstring_expr(tok, current);
         }
         if (cursor_valid &&
-                _PyLexer_set_ftstring_expr_metadata(tok, token)) {
+                _PyLexer_set_ftstring_expr_metadata(tok, current, token)) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
 
-        if (c == ':' &&
-                cursor == current_tok->curly_bracket_expr_start_depth) {
-            current_tok->kind = TOK_FSTRING_MODE;
-            current_tok->in_format_spec = 1;
+        if (c == ':' && bracket_depth == current->replacement_depth) {
+            current->mode = FTSTRING_MODE_FORMAT_SPEC;
             p_start = tok->start;
             p_end = tok->cur;
             return MAKE_TOKEN(_PyToken_OneChar(c));
@@ -609,18 +609,20 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         tok->parenlinenostack[tok->level] = tok->lineno;
         tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
         tok->level++;
-        if (_PyLexer_InsideFString(tok)) {
-            current_tok->curly_bracket_depth++;
-        }
         break;
     case ')':
     case ']':
     case '}':
-        if (_PyLexer_InsideFString(tok) &&
-                !current_tok->curly_bracket_depth && c == '}') {
-            return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                "%c-string: single '}' is not allowed",
-                _PyLexer_CurrentStringPrefix(tok)));
+        if (current != NULL &&
+                _PyLexer_FTStringBracketDepth(tok, current) == 0) {
+            if (c == '}') {
+                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+                    "%c-string: single '}' is not allowed",
+                    _PyLexer_StringPrefix(current->kind)));
+            }
+            return MAKE_TOKEN(_PyTokenizer_syntaxerror(
+                tok, "%c-string: unmatched '%c'",
+                _PyLexer_StringPrefix(current->kind), c));
         }
         if (!tok->tok_extra_tokens && !tok->level) {
             return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
@@ -631,18 +633,15 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
                                             (opening == '[' && c == ']') ||
                                             (opening == '{' && c == '}'))) {
-                /* If the opening bracket belongs to an f-string's expression
-                part (e.g. f"{)}") and the closing bracket is an arbitrary
-                nested expression, then instead of matching a different
-                syntactical construct with it; we'll throw an unmatched
-                parentheses error. */
-                if (_PyLexer_InsideFString(tok) && opening == '{') {
-                    assert(current_tok->curly_bracket_depth >= 0);
-                    int previous_bracket = current_tok->curly_bracket_depth - 1;
-                    if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
+                /* Do not match a closer against the brace that opened the
+                 * current replacement field. */
+                if (current != NULL && opening == '{') {
+                    int bracket_depth =
+                        _PyLexer_FTStringBracketDepth(tok, current);
+                    if (bracket_depth == current->replacement_depth - 1) {
                         return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                             "%c-string: unmatched '%c'",
-                            _PyLexer_CurrentStringPrefix(tok), c));
+                            _PyLexer_StringPrefix(current->kind), c));
                     }
                 }
                 if (tok->parenlinenostack[tok->level] != tok->lineno) {
@@ -660,18 +659,16 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             }
         }
 
-        if (_PyLexer_InsideFString(tok)) {
-            current_tok->curly_bracket_depth--;
-            if (current_tok->curly_bracket_depth < 0) {
+        if (current != NULL) {
+            int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
+            if (bracket_depth < 0) {
                 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
-                    _PyLexer_CurrentStringPrefix(tok), c));
+                    _PyLexer_StringPrefix(current->kind), c));
             }
-            if (c == '}' && current_tok->curly_bracket_depth ==
-                    current_tok->curly_bracket_expr_start_depth) {
-                current_tok->curly_bracket_expr_start_depth--;
-                current_tok->kind = TOK_FSTRING_MODE;
-                current_tok->in_format_spec = 0;
-                current_tok->in_debug = 0;
+            if (c == '}' && bracket_depth == current->replacement_depth - 1) {
+                current->replacement_depth--;
+                current->mode = FTSTRING_MODE_MIDDLE;
+                current->debug_expr = 0;
             }
         }
         break;
@@ -683,8 +680,9 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
     }
 
-    if (c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) {
-        current_tok->in_debug = 1;
+    if (c == '=' && current != NULL &&
+            _PyLexer_FTStringBracketDepth(tok, current) == current->replacement_depth) {
+        current->debug_expr = 1;
     }
 
     /* Punctuation character */
@@ -697,12 +695,11 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
 static int
 tok_get(struct tok_state *tok, struct token *token)
 {
-    tokenizer_mode *current_tok = _PyLexer_CurrentMode(tok);
-    if (current_tok->kind == TOK_REGULAR_MODE) {
-        return _PyLexer_get_normal_mode(tok, current_tok, token);
-    } else {
-        return _PyLexer_get_fstring_mode(tok, current_tok, token);
+    ftstring_state *current = _PyLexer_CurrentFTString(tok);
+    if (current == NULL || current->mode == FTSTRING_MODE_EXPRESSION) {
+        return _PyLexer_get_normal(tok, current, token);
     }
+    return _PyLexer_get_ftstring(tok, current, token);
 }
 
 int

Parser/lexer/lexer_internal.h

@@ -25,10 +25,10 @@ enum string_prefix_flags {
     STRING_PREFIX_T = 1 << 4,
 };
 
-#define FTSTRING_MIDDLE(mode) \
-    (_PyLexer_IsTString(mode) ? TSTRING_MIDDLE : FSTRING_MIDDLE)
-#define FTSTRING_END(mode) \
-    (_PyLexer_IsTString(mode) ? TSTRING_END : FSTRING_END)
+#define FTSTRING_MIDDLE(state) \
+    (_PyLexer_IsTString((state)->kind) ? TSTRING_MIDDLE : FSTRING_MIDDLE)
+#define FTSTRING_END(state) \
+    (_PyLexer_IsTString((state)->kind) ? TSTRING_END : FSTRING_END)
 #define tok_nextc _PyLexer_nextc
 #define tok_backup _PyLexer_backup
 
@@ -41,16 +41,16 @@ tok_failed(const struct tok_state *tok)
 
 int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
-void _PyLexer_begin_ftstring_expr(struct tok_state *);
-void _PyLexer_finish_ftstring_expr(struct tok_state *);
+void _PyLexer_finish_ftstring_expr(struct tok_state *, ftstring_state *);
 int _PyLexer_record_ftstring_comment(
-    struct tok_state *, const char *, const char *);
-int _PyLexer_set_ftstring_expr_metadata(struct tok_state *, struct token *);
+    struct tok_state *, ftstring_state *, const char *, const char *);
+int _PyLexer_set_ftstring_expr_metadata(
+    struct tok_state *, const ftstring_state *, struct token *);
 int _PyLexer_scan_prefixed_string(
     struct tok_state *, struct token *, int, unsigned int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
 int _PyLexer_scan_string(struct tok_state *, struct token *, int);
-int _PyLexer_get_normal_mode(struct tok_state *, tokenizer_mode *, struct token *);
-int _PyLexer_get_fstring_mode(struct tok_state *, tokenizer_mode *, struct token *);
+int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *);
+int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *);
 
 #endif