gh-153569: preserve debug text in format specs

Author: pablogsal

Parser/lexer/lexer.c

@@ -544,10 +544,10 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
     }
 
     /* Punctuation character */
-    int is_punctuation = (c == ':' || c == '}' || c == '!' || c == '{');
+    int is_punctuation = (c == ':' || c == '}' || c == '!');
     if (is_punctuation && _PyLexer_InsideFString(tok) &&
             INSIDE_FSTRING_EXPR(current_tok)) {
-        int cursor = current_tok->curly_bracket_depth - (c != '{');
+        int cursor = current_tok->curly_bracket_depth - 1;
         int in_format_spec = current_tok->in_format_spec;
         int cursor_in_format_with_debug =
             cursor == 1 && (current_tok->in_debug || in_format_spec);
@@ -560,14 +560,9 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             tok_backup(tok, c2);
         }
         if (cursor_valid) {
-            if (c == '{') {
-                _PyLexer_begin_ftstring_expr(tok);
-            }
-            else {
-                _PyLexer_finish_ftstring_expr(tok);
-            }
+            _PyLexer_finish_ftstring_expr(tok);
         }
-        if (cursor_valid && c != '{' &&
+        if (cursor_valid &&
                 _PyLexer_set_ftstring_expr_metadata(tok, token)) {
             return MAKE_TOKEN(ERRORTOKEN);
         }

Parser/lexer/string.c

@@ -386,13 +386,15 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
     if (start_char == '{') {
         int peek1 = tok_nextc(tok);
         tok_backup(tok, peek1);
-        tok_backup(tok, start_char);
         if (peek1 != '{') {
+            _PyLexer_begin_ftstring_expr(tok);
+            tok_backup(tok, start_char);
             if (enter_ftstring_expr(tok, current_tok) < 0) {
                 return MAKE_TOKEN(ERRORTOKEN);
             }
             return _PyLexer_get_normal_mode(tok, current_tok, token);
         }
+        tok_backup(tok, start_char);
     }
     else {
         tok_backup(tok, start_char);
@@ -490,6 +492,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
             int peek = tok_nextc(tok);
             if (peek != '{' || in_format_spec) {
                 tok_backup(tok, peek);
+                _PyLexer_begin_ftstring_expr(tok);
                 tok_backup(tok, c);
                 if (enter_ftstring_expr(tok, current_tok) < 0) {
                     return MAKE_TOKEN(ERRORTOKEN);