gh-153569: finish tokenizer offsets, diagnostics, and opaque API

Author: pablogsal

Lib/test/test_codeop.py

@@ -113,6 +113,17 @@ def test_valid(self, compiler):
         av("def f():\n pass\n#foo\n")
         av("@a.b.c\ndef f():\n pass\n")
 
+    @subTests('symbol', ('single', 'exec'))
+    @subTests('prefix', ('', 'f', 't'))
+    def test_incomplete_string_diagnostics(self, symbol, prefix):
+        opening = f'    á = {prefix}"""first\n'
+        source = 'if True:\n' + opening + 'second'
+        with self.assertRaises(_IncompleteInputError) as cm:
+            Compile()(source, '<input>', symbol)
+        text = opening + 'second' + ('\n' if symbol == 'exec' else '')
+        self.assertEqual(cm.exception.args, (
+            'incomplete input', ('<input>', 2, 9, text, 2, -1)))
+
     @subTests('compiler', COMPILERS)
     def test_incomplete(self, compiler):
         ai = functools.partial(self.assertIncomplete, compiler=compiler)

Lib/test/test_fstring.py

@@ -1657,6 +1657,7 @@ def __repr__(self):
         self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x')
         self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********')
         self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20')
+        self.assertEqual(f"{C():{C():{4=}}}", 'FORMAT-FORMAT-4=4')
 
         self.assertRaises(SyntaxError, eval, "f'{C=]'")
 
@@ -1679,6 +1680,20 @@ def __repr__(self):
 
         self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
         self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
+        self.assertEqual(f'{"""a" # inside"""=}',
+                         '"""a" # inside"""=\'a" # inside\'')
+        self.assertEqual(f"{'''a' # inside'''=}",
+                         "'''a' # inside'''=\"a' # inside\"")
+        self.assertEqual(f'{"""a""""#" # outside
+=}', '"""a""""#" \n=\'a#\'')
+
+        x, y = 1, 2
+        self.assertEqual(f'{x != y # outside
+=}', 'x != y \n=True')
+
+        d = {'a#b': 42}
+        self.assertEqual(f'''{f"{d["a#b"]}"=}''',
+                         'f"{d["a#b"]}"=\'42\'')
 
         self.assertEqual(f'{ # some comment goes here
   """hello"""=}',  ' \n  """hello"""=\'hello\'')
@@ -1876,6 +1891,9 @@ def __format__(self, format):
         self.assertEqual(f"{UnchangedFormat():{r'\xFF'}}", '\\xFF')
         self.assertEqual(rf"{UnchangedFormat():{r'\xFF'}}", '\\xFF')
 
+        self.assertEqual(rf"{UnchangedFormat():{f'\xFF'}}\n", 'ÿ\\n')
+        self.assertEqual(f"{UnchangedFormat():{rf'\xFF'}}\n", '\\xFF\n')
+
         # Test continuation character in format specs
         self.assertEqual(f"""{UnchangedFormat():{'a'\
                         'b'}}""", 'ab')

Lib/test/test_repl.py

@@ -184,9 +184,8 @@ def read_until(marker, start=0):
 
     @cpython_only
     def test_lexer_buffer_realloc_with_null_start(self):
-        # gh-144759: NULL pointer arithmetic in the lexer when start and
-        # multi_line_start are NULL (uninitialized in tok_mode_stack[0])
-        # and the lexer buffer is reallocated while parsing long input.
+        # gh-144759: NULL pointer arithmetic when the lexer buffer grows
+        # while parsing long input.
         long_value = "a" * 2000
         user_input = dedent(f"""\
         x = f'{{{long_value!r}}}'

Lib/test/test_syntax.py

@@ -3393,6 +3393,10 @@ def test_invalid_line_continuation_error_position(self):
         self._check_error('\nfgdfgf\n1,\\#\n2\n',
                           "unexpected character after line continuation character",
                           lineno=3, offset=4)
+        for prefix in ("f", "t"):
+            self._check_error(f'{prefix}"""{{\n\\ x}}"""',
+                              "unexpected character after line continuation character",
+                              lineno=2, offset=2)
 
     def test_invalid_line_continuation_left_recursive(self):
         # Check bpo-42218: SyntaxErrors following left-recursive rules

Lib/test/test_tokenize.py

@@ -2573,6 +2573,40 @@ def test_degraded_fstring_format_spec(self):
             ("f-string: single '}' is not allowed", (1, 11)),
         )
 
+    def test_carriage_return_after_debug_comment(self):
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                tokens = self._get_tokens(f"{prefix}'''{{x=# comment\r}}'''")
+                self.assertEqual(tokens[4].string, "# comment\r}")
+
+    def test_incomplete_formatted_string_comment_after_carriage_return(self):
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                for extra_tokens in (False, True):
+                    with self.assertRaises(tokenize.TokenError) as caught:
+                        self._get_tokens(
+                            f"{prefix}'{{#\r!", extra_tokens=extra_tokens
+                        )
+                    self.assertEqual(
+                        caught.exception.args,
+                        ("unexpected EOF in multi-line statement", (1, 7)),
+                    )
+
+    def test_formatted_string_nesting_limit(self):
+        def nested_string(depth, prefix):
+            source = "'x'"
+            for _ in range(depth):
+                source = f'{prefix}"{{{source}}}"'
+            return source
+
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                self._get_tokens(nested_string(149, prefix))
+                with self.assertRaisesRegex(
+                        tokenize.TokenError,
+                        "too many nested f-strings or t-strings"):
+                    self._get_tokens(nested_string(150, prefix))
+
     def test_escaped_fstring_brace_has_a_position_gap(self):
         tokens = self._get_tokens('f"a{{"', extra_tokens=True)
         self.assertEqual(

Lib/test/test_tstring.py

@@ -140,6 +140,15 @@ def test_debug_specifier(self):
         )
         self.assertEqual(fstring(t), "Value: value = 42")
 
+        class C:
+            def __format__(self, spec):
+                return f"FORMAT-{spec}"
+
+        x = y = C()
+        t = t"{x:{y:{value=}}}"
+        self.assertEqual(t.interpolations[0].format_spec,
+                         "FORMAT-value=42")
+
     def test_raw_tstrings(self):
         path = r"C:\Users"
         t = rt"{path}\Documents"
@@ -150,6 +159,14 @@ def test_raw_tstrings(self):
         t = tr"{path}\Documents"
         self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])
 
+        value = 42
+        t = rt"{value:{f'\xFF'}}\n"
+        self.assertTStringEqual(
+            t, ("", "\\n"), [(value, "value", None, 'ÿ')])
+        t = t"{value:{rf'\xFF'}}\n"
+        self.assertTStringEqual(
+            t, ("", "\n"), [(value, "value", None, '\\xFF')])
+
     def test_template_concatenation(self):
         # Test template + template
         t1 = t"Hello, "
@@ -217,6 +234,10 @@ def test_syntax_errors(self):
             ("t'{x=!}'", "t-string: missing conversion character"),
             ("t'{x!z}'", "t-string: invalid conversion character 'z': "
                          "expected 's', 'r', or 'a'"),
+            ("f\"{t'{x!z}'}\"", "t-string: invalid conversion character 'z': "
+                                "expected 's', 'r', or 'a'"),
+            ("t'{f\"{x!z}\"}'", "f-string: invalid conversion character 'z': "
+                                "expected 's', 'r', or 'a'"),
             ("t'{lambda:1}'", "t-string: lambda expressions are not allowed "
                               "without parentheses"),
             ("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"),
@@ -287,5 +308,22 @@ def test_triple_quoted(self):
         )
         self.assertEqual(fstring(t), "\n        Hello,\n        Python\n        ")
 
+        t = t'{"""a" # inside"""}'
+        self.assertEqual(t.interpolations[0].expression,
+                         '"""a" # inside"""')
+
+        t = t'{"""a""""#" # outside
+}'
+        self.assertEqual(t.interpolations[0].expression, '"""a""""#"')
+
+        x, y = 1, 2
+        t = t'{x != y # outside
+}'
+        self.assertEqual(t.interpolations[0].expression, 'x != y')
+
+        d = {'a#b': 42}
+        t = t'''{f"{d["a#b"]}"}'''
+        self.assertEqual(t.interpolations[0].expression, 'f"{d["a#b"]}"')
+
 if __name__ == '__main__':
     unittest.main()

Makefile.pre.in

@@ -394,11 +394,11 @@ PEGEN_OBJS=		\
 		Parser/peg_api.o
 
 TOKENIZER_OBJS=		\
-		Parser/lexer/buffer.o \
 		Parser/lexer/lexer.o \
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
 		Parser/lexer/string.o \
+		Parser/tokenizer/api.o \
 		Parser/tokenizer/decoder.o \
 		Parser/tokenizer/reader.o \
 		Parser/tokenizer/source.o \
@@ -410,10 +410,9 @@ PEGEN_HEADERS= \
 		$(srcdir)/Parser/string_parser.h
 
 TOKENIZER_HEADERS= \
-		Parser/lexer/buffer.h \
-		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \
+		Parser/tokenizer/types.h \
 		Parser/tokenizer/reader.h \
 		Parser/tokenizer/reader_internal.h \
 		Parser/tokenizer/source.h \
@@ -3460,7 +3459,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
 MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
 MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
 MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
 MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
 MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
 

Modules/_testinternalcapi/tokenizer.c

@@ -23,13 +23,28 @@ check_system_error(int failed, const char *message)
     return 0;
 }
 
+static int
+check_line_view(const _PyTok_SourceText *source, Py_ssize_t lineno,
+                const char *expected)
+{
+    Py_ssize_t len;
+    const char *line = _PyTok_SourceLineView(source, lineno, &len);
+    return check(len == (Py_ssize_t)strlen(expected) &&
+                 memcmp(line, expected, len) == 0,
+                 "wrong source line view");
+}
+
 static PyObject *
 test_tokenizer_source(PyObject *Py_UNUSED(module),
                       PyObject *Py_UNUSED(args))
 {
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
 
+    if (check_line_view(&source, 1, "") < 0) {
+        goto error;
+    }
+
     if (check_system_error(
             _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
             "accepted empty source line") < 0 ||
@@ -51,6 +66,14 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
         goto error;
     }
 
+    if (check_line_view(&source, PY_SSIZE_T_MIN, "alpha") < 0 ||
+            check_line_view(&source, 1, "alpha") < 0 ||
+            check_line_view(&source, 2, "\xce\xb2") < 0 ||
+            check_line_view(&source, 3, "") < 0 ||
+            check_line_view(&source, PY_SSIZE_T_MAX, "") < 0) {
+        goto error;
+    }
+
     if (check(source.len == 9 &&
                   memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0,
               "wrong source contents") < 0) {
@@ -64,6 +87,10 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
                 "appended after unterminated source line") < 0) {
         goto error;
     }
+    if (check_line_view(&source, 1, "tail") < 0 ||
+            check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) {
+        goto error;
+    }
 
     _PyTok_SourceClear(&source);
     Py_RETURN_NONE;

PCbuild/_freeze_module.vcxproj

@@ -181,12 +181,12 @@
     <ClCompile Include="..\Parser\action_helpers.c" />
     <ClCompile Include="..\Parser\string_parser.c" />
     <ClCompile Include="..\Parser\token.c" />
-    <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\lexer\state.c" />
     <ClCompile Include="..\Parser\lexer\lexer.c" />
     <ClCompile Include="..\Parser\lexer\number.c" />
     <ClCompile Include="..\Parser\lexer\string.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />
+    <ClCompile Include="..\Parser\tokenizer\api.c" />
     <ClCompile Include="..\Parser\tokenizer\reader.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\helpers.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -469,15 +469,15 @@
     <ClCompile Include="..\Parser\lexer\string.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\lexer\buffer.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Parser\lexer\state.c">
       <Filter>Source Files</Filter>
     </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\decoder.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Parser\tokenizer\api.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\reader.c">
       <Filter>Source Files</Filter>
     </ClCompile>