gh-153569: hide tokenizer state behind token and source operations

Author: pablogsal

Makefile.pre.in

@@ -401,6 +401,7 @@ TOKENIZER_OBJS=		\
 		Parser/lexer/string.o \
 		Parser/tokenizer/cursor.o \
 		Parser/tokenizer/decoder.o \
+		Parser/tokenizer/api.o \
 		Parser/tokenizer/reader.o \
 		Parser/tokenizer/source.o \
 		Parser/tokenizer/helpers.o
@@ -419,6 +420,7 @@ TOKENIZER_HEADERS= \
 		Parser/tokenizer/reader.h \
 		Parser/tokenizer/reader_internal.h \
 		Parser/tokenizer/source.h \
+		Parser/tokenizer/types.h \
 		Parser/tokenizer/tokenizer.h \
 		Parser/tokenizer/helpers.h
 

PCbuild/_freeze_module.vcxproj

@@ -187,6 +187,7 @@
     <ClCompile Include="..\Parser\lexer\number.c" />
     <ClCompile Include="..\Parser\lexer\string.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />
+    <ClCompile Include="..\Parser\tokenizer\api.c" />
     <ClCompile Include="..\Parser\tokenizer\reader.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\helpers.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -478,6 +478,9 @@
     <ClCompile Include="..\Parser\tokenizer\decoder.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Parser\tokenizer\api.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\reader.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -427,6 +427,7 @@
     <ClInclude Include="..\Parser\tokenizer\cursor.h" />
     <ClInclude Include="..\Parser\tokenizer\reader.h" />
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
+    <ClInclude Include="..\Parser\tokenizer\types.h" />
     <ClInclude Include="..\Parser\tokenizer\source.h" />
     <ClInclude Include="..\Parser\tokenizer\helpers.h" />
     <ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
@@ -597,6 +598,7 @@
     <ClCompile Include="..\Parser\tokenizer\cursor.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />
+    <ClCompile Include="..\Parser\tokenizer\api.c" />
     <ClCompile Include="..\Parser\tokenizer\reader.c" />
     <ClCompile Include="..\Parser\tokenizer\helpers.c" />
     <ClCompile Include="..\Parser\token.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -342,6 +342,9 @@
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h">
       <Filter>Parser</Filter>
     </ClInclude>
+    <ClInclude Include="..\Parser\tokenizer\types.h">
+      <Filter>Parser</Filter>
+    </ClInclude>
     <ClInclude Include="..\Parser\tokenizer\source.h">
       <Filter>Parser</Filter>
     </ClInclude>
@@ -1373,6 +1376,9 @@
     <ClCompile Include="..\Parser\tokenizer\decoder.c">
       <Filter>Parser</Filter>
     </ClCompile>
+    <ClCompile Include="..\Parser\tokenizer\api.c">
+      <Filter>Parser</Filter>
+    </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\reader.c">
       <Filter>Parser</Filter>
     </ClCompile>

Parser/action_helpers.c

@@ -1001,14 +1001,33 @@ result_token_with_metadata(Parser *p, void *result, PyObject *metadata)
     return res;
 }
 
+static char
+formatted_string_prefix(const Parser *p)
+{
+    int nested = 0;
+    for (int i = p->mark - 1; i >= 0; i--) {
+        int type = p->tokens[i]->type;
+        if (type == FSTRING_END || type == TSTRING_END) {
+            nested++;
+        }
+        else if (type == FSTRING_START || type == TSTRING_START) {
+            if (nested == 0) {
+                return type == TSTRING_START ? 't' : 'f';
+            }
+            nested--;
+        }
+    }
+    Py_UNREACHABLE();
+}
+
 ResultTokenWithMetadata *
 _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
 {
     if (conv_token->lineno != conv->lineno || conv_token->end_col_offset != conv->col_offset) {
         return RAISE_SYNTAX_ERROR_KNOWN_RANGE(
             conv_token, conv,
             "%c-string: conversion type must come right after the exclamation mark",
-            TOK_GET_STRING_PREFIX(p->tok)
+            formatted_string_prefix(p)
         );
     }
 
@@ -1017,7 +1036,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
             !(first == 's' || first == 'r' || first == 'a')) {
         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(conv,
                                             "%c-string: invalid conversion character %R: expected 's', 'r', or 'a'",
-                                            TOK_GET_STRING_PREFIX(p->tok),
+                                            formatted_string_prefix(p),
                                             conv->v.Name.id);
         return NULL;
     }
@@ -1344,7 +1363,8 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* tok
 }
 
 static asdl_expr_seq *
-_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b, enum string_kind_t string_kind)
+_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
+                   Token *b, int is_tstring)
 {
     Py_ssize_t n_items = asdl_seq_LEN(raw_expressions);
     Py_ssize_t total_items = n_items;
@@ -1370,15 +1390,13 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
     for (Py_ssize_t i = 0; i < n_items; i++) {
         expr_ty item = asdl_seq_GET(raw_expressions, i);
 
-        // This should correspond to a JoinedStr node of two elements
-        // created _PyPegen_formatted_value. This situation can only be the result of
-        // a (f|t)-string debug expression where the first element is a constant with the text and the second
-        // a formatted value with the expression.
+        /* Debug expressions arrive as JoinedStr(text, value); flatten them
+           into the surrounding string. */
         if (item->kind == JoinedStr_kind) {
             asdl_expr_seq *values = item->v.JoinedStr.values;
             if (asdl_seq_LEN(values) != 2) {
                 PyErr_Format(PyExc_SystemError,
-                             string_kind == TSTRING
+                             is_tstring
                              ? "unexpected TemplateStr node without debug data in t-string at line %d"
                              : "unexpected JoinedStr node without debug data in f-string at line %d",
                              item->lineno);
@@ -1390,7 +1408,9 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
             asdl_seq_SET(seq, index++, first);
 
             expr_ty second = asdl_seq_GET(values, 1);
-            assert((string_kind == TSTRING && second->kind == Interpolation_kind) || second->kind == FormattedValue_kind);
+            assert((is_tstring &&
+                    second->kind == Interpolation_kind) ||
+                   second->kind == FormattedValue_kind);
             asdl_seq_SET(seq, index++, second);
 
             continue;
@@ -1432,7 +1452,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
 expr_ty
 _PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b) {
 
-    asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, TSTRING);
+    asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, 1);
     if (resized_exprs == NULL) {
         return NULL;
     }
@@ -1444,7 +1464,7 @@ _PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token
 expr_ty
 _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) {
 
-    asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, FSTRING);
+    asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, 0);
     if (resized_exprs == NULL) {
         return NULL;
     }
@@ -1460,12 +1480,7 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
         return NULL;
     }
 
-    // Check if we're inside a raw f-string for format spec decoding
-    int is_raw = 0;
-    if (INSIDE_FSTRING(p->tok)) {
-        tokenizer_mode *mode = TOK_GET_MODE(p->tok);
-        is_raw = mode->raw;
-    }
+    int is_raw = tok->is_raw;
 
     PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);
     if (str == NULL) {
@@ -2047,13 +2062,14 @@ _warn_relative_import_of_lazy(Parser *p, asdl_seq *dots, expr_ty module)
         return -1;
     }
 
+    _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
     int res = _PyErr_EmitSyntaxWarning(msg,
-                                       p->tok->filename,
+                                       info.filename,
                                        module->lineno,
                                        module->col_offset + 1,
                                        module->end_lineno,
                                        module->end_col_offset + 1,
-                                       p->tok->module);
+                                       info.module);
     Py_DECREF(msg);
     return res;
 }

Parser/lexer/lexer.h

@@ -5,27 +5,4 @@
 
 int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
 
-int _PyTokenizer_Get(struct tok_state *, struct token *);
-
-/* The view points into the current input window. The next
-   _PyTokenizer_Get() call may discard it. */
-static inline const char *
-_PyToken_TextView(const struct tok_state *tok, const struct token *token,
-                  Py_ssize_t *length)
-{
-    assert(length != NULL);
-    if (token->span.start < 0) {
-        assert(token->span.start == -1 && token->span.end == -1);
-        *length = 0;
-        return "";
-    }
-    assert(_PyTok_SpanIsValid(token->span));
-    assert(tok->buf != NULL);
-    assert(tok->inp >= tok->buf);
-    assert(token->span.start >= tok->buf_offset);
-    assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf);
-    *length = token->span.end - token->span.start;
-    return tok->buf + (token->span.start - tok->buf_offset);
-}
-
 #endif

Parser/lexer/state.c

@@ -132,6 +132,8 @@ int
 _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
 {
     token->level = tok->level;
+    token->is_raw = ISSTRINGLIT(type)
+        && tok->tok_mode_stack[tok->tok_mode_stack_index].raw;
     token->span = buffer_span(tok, start, end);
     int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
     token->start_loc = (_PyTok_Loc){lineno, -1};

Parser/lexer/state.h

@@ -3,6 +3,7 @@
 
 #include "object.h"
 #include "../tokenizer/source.h"
+#include "../tokenizer/tokenizer.h"
 
 #define MAXINDENT 100       /* Max indentation level */
 #define MAXLEVEL 200        /* Max parentheses level */
@@ -21,13 +22,6 @@ enum interactive_underflow_t {
     IUNDERFLOW_STOP,
 };
 
-struct token {
-    int level;
-    _PyTok_Span span;
-    _PyTok_Loc start_loc;
-    _PyTok_Loc end_loc;
-    PyObject *metadata;
-};
 
 enum tokenizer_mode_kind_t {
     TOK_REGULAR_MODE,
@@ -129,12 +123,37 @@ struct tok_state {
 #endif
 };
 
+static inline _PyTok_Off
+_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
+{
+    const char *base = _PyTok_SourceData(&tok->source);
+    assert(position >= base && position <= base + tok->source.len);
+    return tok->source.base_offset + (position - base);
+}
+
+static inline const char *
+_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
+{
+    assert(offset >= tok->source.base_offset);
+    assert(offset - tok->source.base_offset <= tok->source.len);
+    return _PyTok_SourceData(&tok->source) + (offset - tok->source.base_offset);
+}
+
+static inline const char *
+_PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span,
+                        Py_ssize_t *length)
+{
+    assert(length != NULL);
+    assert(_PyTok_SpanIsValid(span));
+    *length = span.end - span.start;
+    (void)_PyLexer_BufferPointer(tok, span.end);
+    return _PyLexer_BufferPointer(tok, span.start);
+}
+
 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
 
 struct tok_state *_PyTokenizer_tok_new(void);
 void _PyTokenizer_Free(struct tok_state *);
-void _PyToken_Free(struct token *);
-void _PyToken_Init(struct token *);
 
 
 #endif