gh-153569: finish nested formatted-string expressions

Author: pablogsal

Lib/test/test_fstring.py

@@ -1657,6 +1657,7 @@ def __repr__(self):
         self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x')
         self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********')
         self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20')
+        self.assertEqual(f"{C():{C():{4=}}}", 'FORMAT-FORMAT-4=4')
 
         self.assertRaises(SyntaxError, eval, "f'{C=]'")
 

Lib/test/test_tstring.py

@@ -140,6 +140,15 @@ def test_debug_specifier(self):
         )
         self.assertEqual(fstring(t), "Value: value = 42")
 
+        class C:
+            def __format__(self, spec):
+                return f"FORMAT-{spec}"
+
+        x = y = C()
+        t = t"{x:{y:{value=}}}"
+        self.assertEqual(t.interpolations[0].format_spec,
+                         "FORMAT-value=42")
+
     def test_raw_tstrings(self):
         path = r"C:\Users"
         t = rt"{path}\Documents"

Parser/action_helpers.c

@@ -1390,10 +1390,8 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
     for (Py_ssize_t i = 0; i < n_items; i++) {
         expr_ty item = asdl_seq_GET(raw_expressions, i);
 
-        // This should correspond to a JoinedStr node of two elements
-        // created _PyPegen_formatted_value. This situation can only be the result of
-        // a (f|t)-string debug expression where the first element is a constant with the text and the second
-        // a formatted value with the expression.
+        /* Debug expressions arrive as JoinedStr(text, value); flatten them
+           into the surrounding string. */
         if (item->kind == JoinedStr_kind) {
             asdl_expr_seq *values = item->v.JoinedStr.values;
             if (asdl_seq_LEN(values) != 2) {

Parser/lexer/lexer.c

@@ -550,26 +550,21 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
     int is_punctuation = (c == ':' || c == '}' || c == '!');
     if (is_punctuation && current != NULL) {
         int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
-        int cursor = bracket_depth - 1;
-        int cursor_in_format_with_debug =
-            cursor == 1 && current->debug_expr;
-        int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
-        if (cursor_valid && c == '!') {
+        int at_expression_boundary =
+            bracket_depth == current->replacement_depth;
+        if (at_expression_boundary && c == '!') {
             int c2 = tok_nextc(tok);
             if (c2 == '=') {
-                cursor_valid = 0;
+                at_expression_boundary = 0;
             }
             tok_backup(tok, c2);
         }
-        if (cursor_valid) {
-            _PyLexer_finish_ftstring_expr(tok, current);
-        }
-        if (cursor_valid &&
-                _PyLexer_set_ftstring_expr_metadata(tok, current, token)) {
+        if (at_expression_boundary &&
+                _PyLexer_finish_ftstring_expr(tok, current, token)) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
 
-        if (c == ':' && bracket_depth == current->replacement_depth) {
+        if (c == ':' && at_expression_boundary) {
             current->mode = FTSTRING_MODE_FORMAT_SPEC;
             p_start = tok->start;
             p_end = tok->cur;

Parser/lexer/lexer_internal.h

@@ -41,11 +41,10 @@ tok_failed(const struct tok_state *tok)
 
 int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
-void _PyLexer_finish_ftstring_expr(struct tok_state *, ftstring_state *);
 int _PyLexer_record_ftstring_comment(
     struct tok_state *, ftstring_state *, const char *, const char *);
-int _PyLexer_set_ftstring_expr_metadata(
-    struct tok_state *, const ftstring_state *, struct token *);
+int _PyLexer_finish_ftstring_expr(
+    struct tok_state *, ftstring_state *, struct token *);
 int _PyLexer_scan_prefixed_string(
     struct tok_state *, struct token *, int, unsigned int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);

Parser/lexer/string.c

@@ -54,14 +54,19 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
 }
 
 int
-_PyLexer_set_ftstring_expr_metadata(struct tok_state *tok,
-                                    const ftstring_state *state,
-                                    struct token *token)
+_PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
+                              struct token *token)
 {
     assert(token != NULL);
 
-    if (!(state->debug_expr || _PyLexer_IsTString(state->kind)) ||
-            token->metadata) {
+    if (state->expr_span.end >= 0) {
+        return 0;
+    }
+    assert(state->expr_span.start >= 0);
+    state->expr_span.end = _PyLexer_BufferOffset(tok, tok->start);
+    int tstring_interpolation = _PyLexer_IsTString(state->kind) &&
+        state->replacement_depth == 1;
+    if (!(state->debug_expr || tstring_interpolation) || token->metadata) {
         return 0;
     }
     Py_ssize_t expr_len;
@@ -120,14 +125,6 @@ _PyLexer_set_ftstring_expr_metadata(struct tok_state *tok,
     return 0;
 }
 
-void
-_PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state)
-{
-    if (state->expr_span.end < 0) {
-        state->expr_span.end = _PyLexer_BufferOffset(tok, tok->start);
-    }
-}
-
 static int
 check_string_prefixes(struct tok_state *tok, unsigned int prefixes)
 {