diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py
index eb04f6c0136022..57d0a3c2f2e99e 100644
--- a/Lib/test/test_capi/test_tokenizer.py
+++ b/Lib/test/test_capi/test_tokenizer.py
@@ -12,9 +12,6 @@ def test_source(self):
def test_source_discard(self):
_testinternalcapi.test_tokenizer_source_discard()
- def test_cursor(self):
- _testinternalcapi.test_tokenizer_cursor()
-
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
index d57452602ce557..c75c0f8627bf08 100644
--- a/Lib/test/test_codeop.py
+++ b/Lib/test/test_codeop.py
@@ -113,6 +113,17 @@ def test_valid(self, compiler):
av("def f():\n pass\n#foo\n")
av("@a.b.c\ndef f():\n pass\n")
+ @subTests('symbol', ('single', 'exec'))
+ @subTests('prefix', ('', 'f', 't'))
+ def test_incomplete_string_diagnostics(self, symbol, prefix):
+ opening = f' á = {prefix}"""first\n'
+ source = 'if True:\n' + opening + 'second'
+ with self.assertRaises(_IncompleteInputError) as cm:
+ Compile()(source, '', symbol)
+ text = opening + 'second' + ('\n' if symbol == 'exec' else '')
+ self.assertEqual(cm.exception.args, (
+ 'incomplete input', ('', 2, 9, text, 2, -1)))
+
@subTests('compiler', COMPILERS)
def test_incomplete(self, compiler):
ai = functools.partial(self.assertIncomplete, compiler=compiler)
diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py
index 372c110783bce7..e6eb7df16d2321 100644
--- a/Lib/test/test_repl.py
+++ b/Lib/test/test_repl.py
@@ -184,9 +184,8 @@ def read_until(marker, start=0):
@cpython_only
def test_lexer_buffer_realloc_with_null_start(self):
- # gh-144759: NULL pointer arithmetic in the lexer when start and
- # multi_line_start are NULL (uninitialized in tok_mode_stack[0])
- # and the lexer buffer is reallocated while parsing long input.
+ # gh-144759: NULL pointer arithmetic when the lexer buffer grows
+ # while parsing long input.
long_value = "a" * 2000
user_input = dedent(f"""\
x = f'{{{long_value!r}}}'
diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
index 862a20a058be75..ec98e609c4e98f 100644
--- a/Lib/test/test_source_encoding.py
+++ b/Lib/test/test_source_encoding.py
@@ -3,8 +3,8 @@
import unittest
from test import support
from test.support import script_helper
-from test.support.os_helper import TESTFN, unlink, rmtree
-from test.support.import_helper import unload
+from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree
+from test.support.import_helper import import_module, unload
import importlib
import os
import sys
@@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self):
self.assertRaises(SyntaxError, compile, seq, '', 'exec')
def test_invalid_utf8_offset_after_non_ascii(self):
+ for name in ('é', 'éé', '𝒜'):
+ with self.subTest(name=name):
+ source = ('x = ' + name).encode() + b'\xff\n'
+ with self.assertRaises(SyntaxError) as caught:
+ compile(source, '', 'exec')
+ error = caught.exception
+ self.assertEqual(
+ (error.lineno, error.offset, error.end_lineno, error.end_offset),
+ (1, 5 + len(name), 1, 5 + len(name)),
+ )
+
+ @support.cpython_only
+ def test_invalid_utf8_file_offset_after_non_ascii(self):
+ _testcapi = import_module('_testcapi')
+ self.addCleanup(unlink, TESTFN_ASCII)
+ with open(TESTFN_ASCII, 'wb') as f:
+ f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n')
with self.assertRaises(SyntaxError) as caught:
- compile(b"x = \xc3\xa9\xff\n", "", "exec")
+ _testcapi.run_file(
+ os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {})
error = caught.exception
self.assertEqual(
(error.lineno, error.offset, error.end_lineno, error.end_offset),
- (1, 6, 1, 6),
+ (2, 7, 2, 7),
)
def test_long_bom_conflict_message_is_not_truncated(self):
diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py
index 74653c77c55de1..c5813849cac7c8 100644
--- a/Lib/test/test_tstring.py
+++ b/Lib/test/test_tstring.py
@@ -198,6 +198,8 @@ def test_nested_templates(self):
def test_syntax_errors(self):
for case, err in (
+ ('t"""{(\n1\n)}\ntail', "unterminated triple-quoted t-string literal"),
+ ('f"""{(\n1\n)}\ntail', "unterminated triple-quoted f-string literal"),
("t'", "unterminated t-string literal"),
("t'''", "unterminated triple-quoted t-string literal"),
("t''''", "unterminated triple-quoted t-string literal"),
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 78a486623181fa..5eb8d4099f7c4b 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -394,12 +394,11 @@ PEGEN_OBJS= \
Parser/peg_api.o
TOKENIZER_OBJS= \
- Parser/lexer/buffer.o \
Parser/lexer/lexer.o \
+ Parser/lexer/layout.o \
Parser/lexer/number.o \
Parser/lexer/state.o \
Parser/lexer/string.o \
- Parser/tokenizer/cursor.o \
Parser/tokenizer/decoder.o \
Parser/tokenizer/reader.o \
Parser/tokenizer/source.o \
@@ -411,11 +410,9 @@ PEGEN_HEADERS= \
$(srcdir)/Parser/string_parser.h
TOKENIZER_HEADERS= \
- Parser/lexer/buffer.h \
Parser/lexer/lexer.h \
Parser/lexer/lexer_internal.h \
Parser/lexer/state.h \
- Parser/tokenizer/cursor.h \
Parser/tokenizer/reader.h \
Parser/tokenizer/reader_internal.h \
Parser/tokenizer/source.h \
@@ -3463,7 +3460,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c
index df481cb832a436..1f89c12f223c7d 100644
--- a/Modules/_testinternalcapi/tokenizer.c
+++ b/Modules/_testinternalcapi/tokenizer.c
@@ -1,6 +1,6 @@
#include "parts.h"
-#include "../../Parser/tokenizer/cursor.h"
+#include "../../Parser/tokenizer/source.h"
static int
check(int condition, const char *message)
@@ -24,13 +24,14 @@ check_system_error(int failed, const char *message)
}
static int
-same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right)
+check_line_view(const _PyTok_SourceText *source, Py_ssize_t lineno,
+ const char *expected)
{
- return left->source == right->source &&
- left->pos == right->pos &&
- left->line_start == right->line_start &&
- left->line_end == right->line_end &&
- left->lineno == right->lineno;
+ Py_ssize_t len;
+ const char *line = _PyTok_SourceLineView(source, lineno, &len);
+ return check(len == (Py_ssize_t)strlen(expected) &&
+ memcmp(line, expected, len) == 0,
+ "wrong source line view");
}
static PyObject *
@@ -40,300 +41,54 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
_PyTok_SourceText source;
_PyTok_SourceInit(&source);
- _PyTok_Loc loc;
- _PyTok_Line line;
- if (check(_PyTok_SourceLocation(
- &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate empty source") < 0 ||
- check(loc.lineno == 1 && loc.byte_col == 0,
- "wrong empty source location") < 0 ||
- check(_PyTok_SourceLine(&source, 1, &line) == 0,
- "cannot find empty source line") < 0 ||
- check(line.start == 0 && line.end == 0,
- "wrong empty source line") < 0 ||
- check_system_error(
- _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
- "accepted empty source line") < 0 ||
+ if (check_line_view(&source, 1, "") < 0) {
+ goto error;
+ }
+
+ if (check_system_error(
+ _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
+ "accepted empty source line") < 0 ||
check_system_error(
_PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
"accepted multiple source lines") < 0 ||
check_system_error(
_PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
- "accepted missing implicit newline") < 0) {
- goto error;
- }
-
- if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0,
- "wrong first source offset") < 0 ||
+ "accepted missing implicit newline") < 0 ||
+ check(_PyTok_SourceAppendLine(
+ &source, "alpha\n", 6, 0) == 0,
+ "wrong first source offset") < 0 ||
check(_PyTok_SourceAppendLine(
&source, "\xce\xb2\n", 3, 1) == 6,
"wrong second source offset") < 0 ||
- check(_PyTok_SourceAppendLine(
- &source, "nul\0x\n", 6, 0) == 9,
- "wrong third source offset") < 0) {
- goto error;
- }
-
- int marker_line = 257;
- int final_line = 300;
- _PyTok_Off marker_start = -1;
- for (int lineno = 4; lineno <= final_line; lineno++) {
- const char *text = lineno == marker_line ? "marker\n" : "x\n";
- Py_ssize_t len = (Py_ssize_t)strlen(text);
- _PyTok_Off start = _PyTok_SourceAppendLine(
- &source, text, len, lineno == final_line);
- if (start < 0) {
- goto error;
- }
- if (lineno == marker_line) {
- marker_start = start;
- }
- }
-
- if (check(source.nlines == final_line, "wrong source line count") < 0 ||
- check(_PyTok_SourceLine(&source, marker_line, &line) == 0,
- "cannot find late source line") < 0 ||
- check(line.start == marker_start &&
- line.end == marker_start + 7,
- "wrong late source line") < 0 ||
- check(!line.implicit_newline && !line.contains_nul,
- "wrong late source flags") < 0 ||
- check(_PyTok_SourceLine(&source, 2, &line) == 0,
- "cannot find second source line") < 0 ||
- check(line.start == 6 && line.end == 9 &&
- line.implicit_newline && !line.contains_nul,
- "wrong second source line") < 0 ||
check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
_PyTok_SourceLineIsImplicit(&source, 2),
- "wrong early implicit newline flags") < 0 ||
- check(_PyTok_SourceLine(&source, 3, &line) == 0,
- "cannot find third source line") < 0 ||
- check(line.contains_nul, "missing null byte flag") < 0 ||
- check(_PyTok_SourceLine(&source, final_line, &line) == 0,
- "cannot find final source line") < 0 ||
- check(line.implicit_newline &&
- _PyTok_SourceLineIsImplicit(&source, final_line),
- "missing late implicit newline flag") < 0) {
+ "wrong implicit newline flags") < 0) {
goto error;
}
- Py_ssize_t view_len;
- const char *view = _PyTok_SourceSpanView(
- &source, _PyTok_SpanFromBounds(6, 8), &view_len);
- if (check(view != NULL && view_len == 2 &&
- memcmp(view, "\xce\xb2", 2) == 0,
- "wrong source span view") < 0 ||
- check(_PyTok_SourceLocation(
- &source, marker_start,
- _PYTOK_AFFINITY_LEFT, &loc) == 0,
- "cannot locate left line boundary") < 0 ||
- check(loc.lineno == marker_line - 1 && loc.byte_col == 2,
- "wrong left boundary location") < 0 ||
- check(_PyTok_SourceLocation(
- &source, marker_start,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate right line boundary") < 0 ||
- check(loc.lineno == marker_line && loc.byte_col == 0,
- "wrong right boundary location") < 0 ||
- check(_PyTok_SourceLocation(
- &source, marker_start + 1,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate late source byte") < 0 ||
- check(loc.lineno == marker_line && loc.byte_col == 1,
- "wrong late source location") < 0) {
+ if (check_line_view(&source, PY_SSIZE_T_MIN, "alpha") < 0 ||
+ check_line_view(&source, 1, "alpha") < 0 ||
+ check_line_view(&source, 2, "\xce\xb2") < 0 ||
+ check_line_view(&source, 3, "") < 0 ||
+ check_line_view(&source, PY_SSIZE_T_MAX, "") < 0) {
goto error;
}
- if (check(_PyTok_SourceLocation(
- &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0,
- "cannot locate left EOF") < 0 ||
- check(loc.lineno == final_line && loc.byte_col == 2,
- "wrong left EOF location") < 0 ||
- check(_PyTok_SourceLocation(
- &source, source.len,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate right EOF") < 0 ||
- check(loc.lineno == final_line + 1 && loc.byte_col == 0,
- "wrong right EOF location") < 0 ||
- check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0,
- "cannot find virtual EOF line") < 0 ||
- check(line.start == source.len && line.end == source.len,
- "wrong virtual EOF line") < 0 ||
- check(!_PyTok_SourceLineIsImplicit(&source, 0) &&
- !_PyTok_SourceLineIsImplicit(
- &source, final_line + 1),
- "virtual or invalid line is implicit") < 0) {
- goto error;
- }
-
- view = _PyTok_SourceSpanView(
- &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len);
- if (check_system_error(view == NULL, "accepted invalid source span") < 0 ||
- check_system_error(
- _PyTok_SourceLocation(
- &source, source.len + 1,
- _PYTOK_AFFINITY_RIGHT, &loc) < 0,
- "accepted invalid source offset") < 0 ||
- check_system_error(
- _PyTok_SourceLine(&source, final_line + 2, &line) < 0,
- "accepted invalid source line") < 0) {
+ if (check(source.len == 9 &&
+ memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0,
+ "wrong source contents") < 0) {
goto error;
}
_PyTok_SourceClear(&source);
- _PyTok_SourceInit(&source);
if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
check_system_error(
_PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
- "appended after unterminated source line") < 0 ||
- check(_PyTok_SourceLocation(
- &source, source.len,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate unterminated EOF") < 0 ||
- check(loc.lineno == 1 && loc.byte_col == 4,
- "wrong unterminated EOF location") < 0) {
- goto error;
- }
-
- _PyTok_SourceDiscard(&source);
- if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4,
- "wrong retained source offset") < 0 ||
- _PyTok_SourceLine(&source, 1, &line) < 0 ||
- check(line.start == 4 && line.end == 6,
- "wrong retained source line") < 0 ||
- _PyTok_SourceLocation(
- &source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 ||
- check(loc.lineno == 1 && loc.byte_col == 0,
- "wrong retained source location") < 0) {
- goto error;
- }
- view = _PyTok_SourceSpanView(
- &source, _PyTok_SpanFromBounds(4, 5), &view_len);
- if (check(view != NULL && view_len == 1 && view[0] == 'a',
- "wrong retained source span") < 0 ||
- check_system_error(_PyTok_SourceSpanView(
- &source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL,
- "accepted discarded source span") < 0) {
- goto error;
- }
-
- _PyTok_SourceClear(&source);
- Py_RETURN_NONE;
-
-error:
- _PyTok_SourceClear(&source);
- return NULL;
-}
-
-static PyObject *
-test_tokenizer_cursor(PyObject *Py_UNUSED(module),
- PyObject *Py_UNUSED(args))
-{
- _PyTok_SourceText source;
- _PyTok_SourceInit(&source);
- if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
- _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) {
- goto error;
- }
-
- _PyTok_Cursor cursor;
- _PyTok_CursorInit(&cursor, &source);
- if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 ||
- check(cursor.lineno == 3 && cursor.pos == source.len,
- "wrong cursor at virtual EOF") < 0 ||
- _PyTok_CursorSetLine(&cursor, 1) < 0) {
- goto error;
- }
-
- char large[BUFSIZ + 1];
- memset(large, 'z', sizeof(large));
- large[sizeof(large) - 1] = '\n';
- if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) {
- goto error;
- }
-
- if (check(_PyTok_CursorPeek(&cursor, 0) == 'a',
- "wrong cursor peek after relocation") < 0 ||
- check(_PyTok_CursorPeek(&cursor, 1) == 'b',
- "wrong distant cursor peek") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == 'a',
- "wrong first cursor byte") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == 'b',
- "wrong second cursor byte") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == '\n',
- "wrong final cursor byte") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == EOF,
- "cursor advanced past line") < 0 ||
- check(_PyTok_CursorSetOffset(&cursor, 2) == 0,
- "cannot seek cursor offset") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == '\n',
- "wrong cursor byte after seek") < 0 ||
- check(_PyTok_CursorSetOffset(&cursor, 3) == 0,
- "cannot seek line boundary") < 0 ||
- check(cursor.lineno == 2 && cursor.line_start == 3 &&
- _PyTok_CursorAdvance(&cursor) == 'c',
- "wrong cursor at line boundary") < 0 ||
- check(_PyTok_CursorSetLine(&cursor, 3) == 0,
- "cannot advance cursor to final line") < 0 ||
- check(cursor.line_start == 6 &&
- _PyTok_CursorAdvance(&cursor) == 'z',
- "wrong cursor byte on final line") < 0) {
- goto error;
- }
-
- _PyTok_Cursor saved = cursor;
- if (check_system_error(
- _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0,
- "accepted invalid cursor offset") < 0 ||
- check(same_cursor(&cursor, &saved),
- "invalid offset changed cursor") < 0 ||
- check_system_error(
- _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0,
- "accepted invalid cursor line") < 0 ||
- check(same_cursor(&cursor, &saved),
- "invalid line changed cursor") < 0 ||
- check(_PyTok_CursorSetOffset(&cursor, source.len) == 0,
- "cannot set cursor to EOF") < 0 ||
- check(cursor.lineno == 4 && cursor.pos == source.len,
- "wrong cursor at EOF") < 0) {
- goto error;
- }
-
-#if SIZEOF_VOID_P > 4
- char byte = 0;
- _PyTok_SourceText huge_source = {
- .bytes = &byte,
- .len = (_PyTok_Off)INT_MAX + 1,
- };
- _PyTok_Cursor huge_cursor = {
- .source = &huge_source,
- .pos = INT_MAX,
- .line_end = (_PyTok_Off)INT_MAX + 1,
- .lineno = 1,
- };
- if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF &&
- huge_cursor.pos == INT_MAX,
- "cursor advanced past maximum column") < 0) {
- goto error;
- }
-#endif
-
- _PyTok_Off base = source.len;
- _PyTok_SourceDiscard(&source);
- if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
- _PyTok_SourceAppendLine(&source, "cd", 2, 0) < 0) {
+ "appended after unterminated source line") < 0) {
goto error;
}
- _PyTok_CursorInit(&cursor, &source);
- if (_PyTok_CursorSetLine(&cursor, 1) < 0 ||
- check(cursor.pos == base && _PyTok_CursorPeek(&cursor, 1) == 'b',
- "wrong retained cursor line") < 0 ||
- _PyTok_CursorSetLine(&cursor, 2) < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == 'c',
- "wrong retained cursor byte") < 0 ||
- _PyTok_CursorSetOffset(&cursor, base + 5) < 0 ||
- check(cursor.lineno == 2 && _PyTok_CursorAdvance(&cursor) == EOF,
- "wrong retained cursor EOF") < 0) {
+ if (check_line_view(&source, 1, "tail") < 0 ||
+ check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) {
goto error;
}
@@ -410,7 +165,6 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
static PyMethodDef test_methods[] = {
{"test_tokenizer_source", test_tokenizer_source, METH_NOARGS},
- {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS},
{"test_tokenizer_source_discard", test_tokenizer_source_discard, METH_NOARGS},
{NULL},
};
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index 469fd77cc8be9d..9374a9a6687e49 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -181,9 +181,9 @@
-
+
diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters
index 976c99b7d24bdf..2264b5312f526e 100644
--- a/PCbuild/_freeze_module.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -463,13 +463,13 @@
Source Files
-
+
Source Files
-
+
Source Files
-
+
Source Files
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index 79dfc9ccf39ec2..c90eeb95ff60da 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -423,8 +423,6 @@
-
-
@@ -591,10 +589,9 @@
+
-
-
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index 765b4d46b12dd0..4a1bd70c499ff2 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -330,12 +330,6 @@
Parser
-
- Parser
-
-
- Parser
-
Parser
@@ -1352,6 +1346,9 @@
Parser
+
+ Parser
+
Parser
@@ -1361,12 +1358,6 @@
Parser
-
- Parser
-
-
- Parser
-
Parser
diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c
deleted file mode 100644
index 9c39544ca7c479..00000000000000
--- a/Parser/lexer/buffer.c
+++ /dev/null
@@ -1,46 +0,0 @@
-#include "Python.h"
-#include "buffer.h"
-#include "state.h"
-
-void
-_PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base,
- _PyLexer_BufferPointers *pointers)
-{
- pointers->buf_from_base = tok->buf - base;
- pointers->cur_from_buf = tok->cur - tok->buf;
- pointers->inp_from_buf = tok->inp - tok->buf;
- pointers->start_from_buf = tok->start == NULL
- ? -1 : tok->start - tok->buf;
- pointers->line_start_from_buf = tok->line_start == NULL
- ? -1 : tok->line_start - tok->buf;
- pointers->multi_line_start_from_buf = tok->multi_line_start == NULL
- ? -1 : tok->multi_line_start - tok->buf;
- for (int index = tok->tok_mode_stack_index; index > 0; --index) {
- tokenizer_mode *mode = &tok->tok_mode_stack[index];
- mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
- mode->multi_line_start_offset = mode->multi_line_start == NULL
- ? -1 : mode->multi_line_start - tok->buf;
- }
-}
-
-void
-_PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base,
- const _PyLexer_BufferPointers *pointers)
-{
- tok->buf = base + pointers->buf_from_base;
- tok->cur = tok->buf + pointers->cur_from_buf;
- tok->inp = tok->buf + pointers->inp_from_buf;
- tok->start = pointers->start_from_buf < 0
- ? NULL : tok->buf + pointers->start_from_buf;
- tok->line_start = pointers->line_start_from_buf < 0
- ? NULL : tok->buf + pointers->line_start_from_buf;
- tok->multi_line_start = pointers->multi_line_start_from_buf < 0
- ? NULL : tok->buf + pointers->multi_line_start_from_buf;
- for (int index = tok->tok_mode_stack_index; index > 0; --index) {
- tokenizer_mode *mode = &tok->tok_mode_stack[index];
- mode->start = mode->start_offset < 0
- ? NULL : tok->buf + mode->start_offset;
- mode->multi_line_start = mode->multi_line_start_offset < 0
- ? NULL : tok->buf + mode->multi_line_start_offset;
- }
-}
diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h
deleted file mode 100644
index 285da124226d50..00000000000000
--- a/Parser/lexer/buffer.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef _LEXER_BUFFER_H_
-#define _LEXER_BUFFER_H_
-
-#include "pyport.h"
-
-struct tok_state;
-
-typedef struct {
- Py_ssize_t buf_from_base;
- Py_ssize_t cur_from_buf;
- Py_ssize_t inp_from_buf;
- Py_ssize_t start_from_buf;
- Py_ssize_t line_start_from_buf;
- Py_ssize_t multi_line_start_from_buf;
-} _PyLexer_BufferPointers;
-
-void _PyLexer_SaveBufferPointers(
- struct tok_state *, const char *, _PyLexer_BufferPointers *);
-void _PyLexer_RestoreBufferPointers(
- struct tok_state *, char *, const _PyLexer_BufferPointers *);
-
-#endif
diff --git a/Parser/lexer/layout.c b/Parser/lexer/layout.c
new file mode 100644
index 00000000000000..e460574b6e97c4
--- /dev/null
+++ b/Parser/lexer/layout.c
@@ -0,0 +1,195 @@
+#include "Python.h"
+#include "errcode.h"
+#include "pycore_token.h"
+
+#include "lexer_internal.h"
+#include "../tokenizer/helpers.h"
+#include "../tokenizer/reader.h"
+
+#define TABSIZE 8
+#define ALTTABSIZE 1
+
+int
+_PyLexer_ContinueLine(struct tok_state *tok)
+{
+ int c = tok_nextc(tok);
+ if (c == '\r') {
+ c = tok_nextc(tok);
+ }
+ if (c != '\n') {
+ tok->done = E_LINECONT;
+ return -1;
+ }
+ c = tok_nextc(tok);
+ if (c == EOF) {
+ tok->done = E_EOF;
+ tok->cur = tok->inp;
+ return -1;
+ } else {
+ tok_backup(tok, c);
+ }
+ return c;
+}
+
+
+static int
+update_indentation(struct tok_state *tok, int col, int altcol)
+{
+ lexer_layout_state *layout = &tok->layout;
+ if (col == layout->stack[layout->depth].column) {
+ if (altcol != layout->stack[layout->depth].alternate_column) {
+ _PyTokenizer_indenterror(tok);
+ return -1;
+ }
+ }
+ else if (col > layout->stack[layout->depth].column) {
+ if (layout->depth + 1 >= MAXINDENT) {
+ tok->done = E_TOODEEP;
+ tok->cur = tok->inp;
+ return -1;
+ }
+ if (altcol <= layout->stack[layout->depth].alternate_column) {
+ _PyTokenizer_indenterror(tok);
+ return -1;
+ }
+ layout->pending++;
+ layout->stack[++layout->depth] = (indentation_level){col, altcol};
+ }
+ else {
+ while (layout->depth > 0 &&
+ col < layout->stack[layout->depth].column) {
+ layout->pending--;
+ layout->depth--;
+ }
+ if (col != layout->stack[layout->depth].column) {
+ tok->done = E_DEDENT;
+ tok->cur = tok->inp;
+ return -1;
+ }
+ if (altcol != layout->stack[layout->depth].alternate_column) {
+ _PyTokenizer_indenterror(tok);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+int
+_PyLexer_BeginLine(struct tok_state *tok)
+{
+ assert(tok->layout.at_bol);
+ int c;
+ int blankline = 0;
+ int col = 0;
+ int altcol = 0;
+ tok->layout.at_bol = 0;
+ int cont_line_col = 0;
+ for (;;) {
+ c = tok_nextc(tok);
+ if (c == ' ') {
+ col++, altcol++;
+ }
+ else if (c == '\t') {
+ col = (col / TABSIZE + 1) * TABSIZE;
+ altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
+ }
+ else if (c == '\014') {/* Control-L (formfeed) */
+ col = altcol = 0; /* For Emacs users */
+ }
+ else if (c == '\\') {
+ // Indentation cannot be split over multiple physical lines
+ // using backslashes. This means that if we found a backslash
+ // preceded by whitespace, **the first one we find** determines
+ // the level of indentation of whatever comes next.
+ cont_line_col = cont_line_col ? cont_line_col : col;
+ if ((c = _PyLexer_ContinueLine(tok)) == -1) {
+ return -1;
+ }
+ }
+ else if (c == EOF && PyErr_Occurred()) {
+ return -1;
+ }
+ else {
+ break;
+ }
+ }
+ tok_backup(tok, c);
+ if (c == '#' || c == '\n' || c == '\r') {
+ int interactive = _PyTok_ReaderIsInteractive(tok);
+ /* Lines with only whitespace and/or comments
+ shouldn't affect the indentation and are
+ not passed to the parser as NEWLINE tokens,
+ except *totally* empty lines in interactive
+ mode, which signal the end of a command group. */
+ if (col == 0 && c == '\n' && interactive) {
+ blankline = 0; /* Let it through */
+ }
+ else if (interactive && tok->lineno == 1) {
+ /* In interactive mode, if the first line contains
+ only spaces and/or a comment, let it through. */
+ blankline = 0;
+ col = altcol = 0;
+ }
+ else {
+ blankline = 1; /* Ignore completely */
+ }
+ }
+ if (!blankline && tok->level == 0) {
+ col = cont_line_col ? cont_line_col : col;
+ altcol = cont_line_col ? cont_line_col : altcol;
+ if (update_indentation(tok, col, altcol) < 0) {
+ return -1;
+ }
+ }
+ return blankline;
+}
+
+int
+_PyLexer_IndentationToken(struct tok_state *tok, struct token *token)
+{
+ assert(tok->layout.pending != 0);
+ const char *p_start = NULL;
+ const char *p_end = NULL;
+ if (tok->layout.pending < 0) {
+ if (tok->tok_extra_tokens) {
+ p_start = tok->cur;
+ p_end = tok->cur;
+ }
+ tok->layout.pending++;
+ return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end);
+ }
+ else {
+ if (tok->tok_extra_tokens) {
+ p_start = tok->buf;
+ p_end = tok->cur;
+ }
+ tok->layout.pending--;
+ return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end);
+ }
+}
+
+int
+_PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline)
+{
+ tok->layout.at_bol = 1;
+ if (blankline || tok->level > 0) {
+ if (!tok->tok_extra_tokens) {
+ return 0;
+ }
+ }
+ else if (!tok->layout.comment_newline || !tok->tok_extra_tokens) {
+ return _PyLexer_token_setup(tok, token, NEWLINE,
+ tok->start, tok->cur - 1);
+ }
+ tok->layout.comment_newline = 0;
+ return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur);
+}
+
+void
+_PyLexer_ImplyDedents(struct tok_state *tok)
+{
+ if (tok->layout.depth != 0) {
+ tok->layout.pending = -tok->layout.depth;
+ tok->layout.depth = 0;
+ }
+}
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index f96b31b9d2f38a..59c2d896962c61 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -7,10 +7,6 @@
#include "../tokenizer/helpers.h"
#include "../tokenizer/reader.h"
-/* Alternate tab spacing */
-#define ALTTABSIZE 1
-
-
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
/* Spaces in this constant are treated as "zero or more spaces or tabs" when
@@ -105,6 +101,7 @@ verify_identifier(struct tok_state *tok)
assert(PyUnicode_GET_LENGTH(s) > 0);
if (invalid < PyUnicode_GET_LENGTH(s)) {
Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
+ const char *error_cursor = tok->cur;
if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
/* Determine the offset in UTF-8 encoded input */
Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
@@ -115,14 +112,20 @@ verify_identifier(struct tok_state *tok)
tok->done = E_ERROR;
return 0;
}
- tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
+ error_cursor = tok->start + PyBytes_GET_SIZE(s);
}
Py_DECREF(s);
if (Py_UNICODE_ISPRINTABLE(ch)) {
- _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch);
+ _PyTokenizer_syntaxerror_at(
+ tok, tok->line_start,
+ error_cursor - tok->line_start, tok->lineno, -1, -1,
+ "invalid character '%c' (U+%04X)", ch, ch);
}
else {
- _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch);
+ _PyTokenizer_syntaxerror_at(
+ tok, tok->line_start,
+ error_cursor - tok->line_start, tok->lineno, -1, -1,
+ "invalid non-printable character U+%04X", ch);
}
return 0;
}
@@ -130,31 +133,6 @@ verify_identifier(struct tok_state *tok)
return 1;
}
-
-
-static inline int
-tok_continuation_line(struct tok_state *tok) {
- int c = tok_nextc(tok);
- if (c == '\r') {
- c = tok_nextc(tok);
- }
- if (c != '\n') {
- tok->done = E_LINECONT;
- return -1;
- }
- c = tok_nextc(tok);
- if (c == EOF) {
- tok->done = E_EOF;
- tok->cur = tok->inp;
- return -1;
- } else {
- tok_backup(tok, c);
- }
- return c;
-}
-
-
-
int
_PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
{
@@ -169,126 +147,18 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
blankline = 0;
- /* Get indentation level */
- if (tok->atbol) {
- int col = 0;
- int altcol = 0;
- tok->atbol = 0;
- int cont_line_col = 0;
- for (;;) {
- c = tok_nextc(tok);
- if (c == ' ') {
- col++, altcol++;
- }
- else if (c == '\t') {
- col = (col / tok->tabsize + 1) * tok->tabsize;
- altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
- }
- else if (c == '\014') {/* Control-L (formfeed) */
- col = altcol = 0; /* For Emacs users */
- }
- else if (c == '\\') {
- // Indentation cannot be split over multiple physical lines
- // using backslashes. This means that if we found a backslash
- // preceded by whitespace, **the first one we find** determines
- // the level of indentation of whatever comes next.
- cont_line_col = cont_line_col ? cont_line_col : col;
- if ((c = tok_continuation_line(tok)) == -1) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- else if (c == EOF && PyErr_Occurred()) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- else {
- break;
- }
- }
- tok_backup(tok, c);
- if (c == '#' || c == '\n' || c == '\r') {
- /* Lines with only whitespace and/or comments
- shouldn't affect the indentation and are
- not passed to the parser as NEWLINE tokens,
- except *totally* empty lines in interactive
- mode, which signal the end of a command group. */
- if (col == 0 && c == '\n' && tok->prompt != NULL) {
- blankline = 0; /* Let it through */
- }
- else if (tok->prompt != NULL && tok->lineno == 1) {
- /* In interactive mode, if the first line contains
- only spaces and/or a comment, let it through. */
- blankline = 0;
- col = altcol = 0;
- }
- else {
- blankline = 1; /* Ignore completely */
- }
- /* We can't jump back right here since we still
- may need to skip to the end of a comment */
- }
- if (!blankline && tok->level == 0) {
- col = cont_line_col ? cont_line_col : col;
- altcol = cont_line_col ? cont_line_col : altcol;
- if (col == tok->indstack[tok->indent]) {
- /* No change */
- if (altcol != tok->altindstack[tok->indent]) {
- return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
- }
- }
- else if (col > tok->indstack[tok->indent]) {
- /* Indent -- always one */
- if (tok->indent+1 >= MAXINDENT) {
- tok->done = E_TOODEEP;
- tok->cur = tok->inp;
- return MAKE_TOKEN(ERRORTOKEN);
- }
- if (altcol <= tok->altindstack[tok->indent]) {
- return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
- }
- tok->pendin++;
- tok->indstack[++tok->indent] = col;
- tok->altindstack[tok->indent] = altcol;
- }
- else /* col < tok->indstack[tok->indent] */ {
- /* Dedent -- any number, must be consistent */
- while (tok->indent > 0 &&
- col < tok->indstack[tok->indent]) {
- tok->pendin--;
- tok->indent--;
- }
- if (col != tok->indstack[tok->indent]) {
- tok->done = E_DEDENT;
- tok->cur = tok->inp;
- return MAKE_TOKEN(ERRORTOKEN);
- }
- if (altcol != tok->altindstack[tok->indent]) {
- return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
- }
- }
+ if (tok->layout.at_bol) {
+ blankline = _PyLexer_BeginLine(tok);
+ if (blankline < 0) {
+ return MAKE_TOKEN(ERRORTOKEN);
}
}
tok->start = tok->cur;
tok->starting_col_offset = tok->col_offset;
- /* Return pending indents/dedents */
- if (tok->pendin != 0) {
- if (tok->pendin < 0) {
- if (tok->tok_extra_tokens) {
- p_start = tok->cur;
- p_end = tok->cur;
- }
- tok->pendin++;
- return MAKE_TOKEN(DEDENT);
- }
- else {
- if (tok->tok_extra_tokens) {
- p_start = tok->buf;
- p_end = tok->cur;
- }
- tok->pendin--;
- return MAKE_TOKEN(INDENT);
- }
+ if (tok->layout.pending != 0) {
+ return _PyLexer_IndentationToken(tok, token);
}
/* Peek ahead at the next character */
@@ -368,7 +238,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
/* If this type ignore is the only thing on the line, consume the newline also. */
if (blankline) {
tok_nextc(tok);
- tok->atbol = 1;
+ tok->layout.at_bol = 1;
}
} else {
p_start = type_start;
@@ -383,7 +253,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
tok_backup(tok, c); /* don't eat the newline or EOF */
p_start = p;
p_end = tok->cur;
- tok->comment_newline = blankline;
+ tok->layout.comment_newline = blankline;
return MAKE_TOKEN(COMMENT);
}
}
@@ -464,29 +334,12 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
c = tok_nextc(tok);
}
- /* Newline */
if (c == '\n') {
- tok->atbol = 1;
- if (blankline || tok->level > 0) {
- if (tok->tok_extra_tokens) {
- if (tok->comment_newline) {
- tok->comment_newline = 0;
- }
- p_start = tok->start;
- p_end = tok->cur;
- return MAKE_TOKEN(NL);
- }
+ int type = _PyLexer_Newline(tok, token, blankline);
+ if (type == 0) {
goto nextline;
}
- if (tok->comment_newline && tok->tok_extra_tokens) {
- tok->comment_newline = 0;
- p_start = tok->start;
- p_end = tok->cur;
- return MAKE_TOKEN(NL);
- }
- p_start = tok->start;
- p_end = tok->cur - 1; /* Leave '\n' out of the string */
- return MAKE_TOKEN(NEWLINE);
+ return type;
}
/* Period or number starting with period? */
@@ -527,7 +380,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
/* Line continuation */
if (c == '\\') {
- if ((c = tok_continuation_line(tok)) == -1) {
+ if ((c = _PyLexer_ContinueLine(tok)) == -1) {
return MAKE_TOKEN(ERRORTOKEN);
}
goto again; /* Read next line */
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index c6d3b9045c7292..d0e10d40e43c8b 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -44,6 +44,12 @@ TOK_NEXT_MODE(struct tok_state *tok)
#define tok_nextc _PyLexer_nextc
#define tok_backup _PyLexer_backup
+/* Return -1 on error, otherwise whether the line is blank. */
+int _PyLexer_BeginLine(struct tok_state *);
+int _PyLexer_ContinueLine(struct tok_state *);
+int _PyLexer_IndentationToken(struct tok_state *, struct token *);
+/* Return zero when the newline is suppressed, otherwise its token type. */
+int _PyLexer_Newline(struct tok_state *, struct token *, int);
int _PyLexer_nextc(struct tok_state *);
void _PyLexer_backup(struct tok_state *, int);
int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char);
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index d82a7d0f296bac..b8f308e2e81211 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -1,65 +1,10 @@
#include "Python.h"
-#include "pycore_pystate.h"
#include "pycore_token.h"
#include "errcode.h"
#include "state.h"
#include "../tokenizer/reader.h"
-/* Never change this */
-#define TABSIZE 8
-
-/* Create and initialize a new tok_state structure */
-struct tok_state *
-_PyTokenizer_tok_new(void)
-{
- struct tok_state *tok = (struct tok_state *)PyMem_Calloc(
- 1,
- sizeof(struct tok_state));
- if (tok == NULL) {
- PyErr_NoMemory();
- return NULL;
- }
-
- tok->buf = tok->cur = tok->inp = NULL;
- tok->fp_interactive = 0;
- tok->interactive_src_start = NULL;
- tok->interactive_src_end = NULL;
- tok->start = NULL;
- tok->done = E_OK;
- tok->fp = NULL;
- tok->tabsize = TABSIZE;
- tok->indent = 0;
- tok->indstack[0] = 0;
- tok->atbol = 1;
- tok->pendin = 0;
- tok->prompt = NULL;
- tok->lineno = 0;
- tok->starting_col_offset = -1;
- tok->col_offset = -1;
- tok->level = 0;
- tok->altindstack[0] = 0;
- tok->input_error = 0;
- tok->encoding = NULL;
- tok->filename = NULL;
- tok->module = NULL;
- tok->type_comments = 0;
- tok->interactive_underflow = IUNDERFLOW_NORMAL;
- tok->str = NULL;
- tok->report_warnings = 1;
- tok->tok_extra_tokens = 0;
- tok->comment_newline = 0;
- tok->implicit_newline = 0;
- _PyTok_SourceInit(&tok->source);
- tok->reader = NULL;
- tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .quote='\0', .quote_size = 0, .in_debug=0};
- tok->tok_mode_stack_index = 0;
-#ifdef Py_DEBUG
- tok->debug = _Py_GetConfig()->parser_debug;
-#endif
- return tok;
-}
-
static void
free_fstring_expressions(struct tok_state *tok)
{
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 6d19e685bd7f81..99407158cc8bf2 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -13,14 +13,6 @@
#define INSIDE_FSTRING_EXPR_AT_TOP(tok) \
(tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1)
-enum interactive_underflow_t {
- /* Normal mode of operation: return a new token when asked in interactive mode */
- IUNDERFLOW_NORMAL,
- /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This
- * can be used to prevent the tokenizer to prompt the user for new tokens */
- IUNDERFLOW_STOP,
-};
-
struct token {
int level;
_PyTok_Span span;
@@ -66,6 +58,27 @@ typedef struct _tokenizer_mode {
enum string_kind_t string_kind;
} tokenizer_mode;
+typedef struct {
+ int column;
+ int alternate_column;
+} indentation_level;
+
+typedef struct {
+ int depth;
+ int pending;
+ int at_bol;
+ int comment_newline;
+ indentation_level stack[MAXINDENT];
+} lexer_layout_state;
+
+/* Supplemental source context for a terminal error. location is the reporting
+ cursor, independent of the scanner cursor; lineno == 0 means absent.
+ The text span may cover multiple physical lines. */
+typedef struct {
+ _PyTok_Loc location;
+ _PyTok_Span text_span;
+} _PyTokenizer_Diagnostic;
+
/* Tokenizer state */
struct tok_state {
/* Input state; buf <= cur <= inp */
@@ -74,24 +87,17 @@ struct tok_state {
char *cur; /* Next character in buffer */
char *inp; /* End of data in buffer */
_PyTok_Off buf_offset; /* Logical offset of buf[0]. */
- int fp_interactive; /* If the file descriptor is interactive */
- char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
- char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
const char *start; /* Start of current token if not NULL */
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
/* NB If done != E_OK, cur must be == inp!!! */
FILE *fp; /* Rest of input; NULL if tokenizing a string */
- int tabsize; /* Tab spacing */
- int indent; /* Current indentation index */
- int indstack[MAXINDENT]; /* Stack of indents */
- int atbol; /* Nonzero if at begin of new line */
- int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
- const char *prompt; /* For interactive prompting */
+ lexer_layout_state layout;
int lineno; /* Current line number */
int first_lineno; /* First line of a single line or multi line string
expression (cf. issue 16806) */
int starting_col_offset; /* The column offset at the beginning of a token */
int col_offset; /* Current col offset */
+ _PyTokenizer_Diagnostic diagnostic;
int level; /* () [] {} Parentheses nesting level */
/* Used to allow free continuations inside them */
char parenstack[MAXLEVEL];
@@ -99,8 +105,6 @@ struct tok_state {
int parencolstack[MAXLEVEL];
PyObject *filename;
PyObject *module;
- /* Stuff for checking on different tab sizes */
- int altindstack[MAXINDENT]; /* Stack of alternate indents */
/* Stuff for PEP 0263 */
int input_error;
char *encoding; /* Source encoding. */
@@ -108,30 +112,82 @@ struct tok_state {
const char* multi_line_start; /* pointer to start of first line of
a single line or multi line string
expression (cf. issue 16806) */
- char* str; /* Source string being tokenized (if tokenizing from a string)*/
_PyTok_SourceText source;
struct _PyTok_Reader *reader;
int type_comments; /* Whether to look for type comments */
- /* How to proceed when asked for a new token in interactive mode */
- enum interactive_underflow_t interactive_underflow;
- int report_warnings;
- // TODO: Factor this into its own thing
tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL];
int tok_mode_stack_index;
int tok_extra_tokens;
- int comment_newline;
int implicit_newline;
#ifdef Py_DEBUG
int debug;
#endif
};
+static inline _PyTok_Off
+_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
+{
+ assert(tok->buf != NULL);
+ assert(tok->inp >= tok->buf);
+ assert(position >= tok->buf && position <= tok->inp);
+ Py_ssize_t offset = position - tok->buf;
+ assert(tok->buf_offset <= PY_SSIZE_T_MAX - offset);
+ return tok->buf_offset + offset;
+}
+
+static inline char *
+_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
+{
+ assert(tok->buf != NULL);
+ assert(tok->inp >= tok->buf);
+ assert(offset >= tok->buf_offset);
+ assert(offset - tok->buf_offset <= tok->inp - tok->buf);
+ return tok->buf + (offset - tok->buf_offset);
+}
+
+static inline const char *
+_PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span,
+ Py_ssize_t *length)
+{
+ assert(length != NULL);
+ assert(_PyTok_SpanIsValid(span));
+ *length = span.end - span.start;
+ (void)_PyLexer_BufferPointer(tok, span.end);
+ return _PyLexer_BufferPointer(tok, span.start);
+}
+
+static inline int
+_PyLexer_ByteColumn(const struct tok_state *tok)
+{
+ assert(tok->line_start != NULL);
+ assert(tok->cur >= tok->line_start);
+ Py_ssize_t column = tok->cur - tok->line_start;
+ assert(column <= INT_MAX);
+ return (int)column;
+}
+
+static inline _PyTok_Span
+_PyLexer_BufferSpan(const struct tok_state *tok, const char *start,
+ const char *end)
+{
+ if (start == NULL) {
+ assert(end == NULL);
+ return (_PyTok_Span){-1, -1};
+ }
+ assert(end != NULL);
+ assert(start <= end);
+ return _PyTok_SpanFromBounds(
+ _PyLexer_BufferOffset(tok, start),
+ _PyLexer_BufferOffset(tok, end));
+}
+
int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
-struct tok_state *_PyTokenizer_tok_new(void);
+void _PyLexer_ImplyDedents(struct tok_state *);
+
void _PyTokenizer_Free(struct tok_state *);
void _PyToken_Free(struct token *);
void _PyToken_Init(struct token *);
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index fc0299c5c7c592..397c90c2e5b9e3 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -7,6 +7,20 @@
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
+static int
+string_error_token(struct tok_state *tok, struct token *token,
+ const char *start, _PyTok_Loc location)
+{
+ tok->diagnostic = (_PyTokenizer_Diagnostic){
+ .location = {location.lineno, location.byte_col + 1},
+ .text_span = _PyLexer_BufferSpan(tok, start - location.byte_col, tok->inp),
+ };
+ int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, NULL, NULL);
+ token->start_loc = location;
+ token->end_loc = (_PyTok_Loc){location.lineno, -1};
+ return type;
+}
+
int
_PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
assert(token != NULL);
@@ -347,55 +361,52 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
break;
}
if (c == EOF || (quote_size == 1 && c == '\n')) {
- assert(tok->multi_line_start != NULL);
- // shift the tok_state's location into
- // the start of string, and report the error
- // from the initial quote character
- tok->cur = (char *)tok->start;
- tok->cur++;
- tok->line_start = tok->multi_line_start;
- int start = tok->lineno;
- tok->lineno = tok->first_lineno;
-
- if (INSIDE_FSTRING(tok)) {
- /* When we are in an f-string, before raising the
- * unterminated string literal error, check whether
- * does the initial quote matches with f-strings quotes
- * and if it is, then this must be a missing '}' token
- * so raise the proper error */
- tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
- if (the_current_tok->quote == quote &&
- the_current_tok->quote_size == quote_size) {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
- "%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok)));
+ int end_lineno = tok->lineno;
+ _PyTok_Loc location = {tok->first_lineno, tok->starting_col_offset};
+ const char *line = tok->start - location.byte_col;
+ Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
+
+ const tokenizer_mode *state = INSIDE_FSTRING(tok) ? TOK_GET_MODE(tok) : NULL;
+ if (state != NULL) {
+ /* A matching quote belongs to the surrounding formatted
+ * string, so the expression is missing its closing brace. */
+ if (state->quote == quote && state->quote_size == quote_size) {
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
+ "%c-string: expecting '}'",
+ TOK_GET_STRING_PREFIX(tok));
+ return string_error_token(tok, token, tok->start, location);
}
}
if (quote_size == 3) {
- _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
- " (detected at line %d)", start);
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
+ "unterminated triple-quoted string literal"
+ " (detected at line %d)", end_lineno);
if (c != '\n') {
tok->done = E_EOFS;
}
- return MAKE_TOKEN(ERRORTOKEN);
+ return string_error_token(tok, token, tok->start, location);
}
else {
if (has_escaped_quote) {
- _PyTokenizer_syntaxerror(
- tok,
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
"unterminated string literal (detected at line %d); "
"perhaps you escaped the end quote?",
- start
+ end_lineno
);
} else {
- _PyTokenizer_syntaxerror(
- tok, "unterminated string literal (detected at line %d)", start
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
+ "unterminated string literal (detected at line %d)", end_lineno
);
}
if (c != '\n') {
tok->done = E_EOLS;
}
- return MAKE_TOKEN(ERRORTOKEN);
+ return string_error_token(tok, token, tok->start, location);
}
}
if (c == quote) {
@@ -516,32 +527,31 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
}
- assert(tok->multi_line_start != NULL);
- // shift the tok_state's location into
- // the start of string, and report the error
- // from the initial quote character
- tok->cur = (char *)current_tok->start;
- tok->cur++;
- tok->line_start = current_tok->multi_line_start;
- int start = tok->lineno;
-
- tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
- tok->lineno = the_current_tok->first_line;
+ int end_lineno = tok->lineno;
+ _PyTok_Loc location = {current_tok->first_line,
+ (int)(current_tok->start - current_tok->multi_line_start)};
+ const char *line = current_tok->multi_line_start;
+ Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
if (current_tok->quote_size == 3) {
- _PyTokenizer_syntaxerror(tok,
- "unterminated triple-quoted %c-string literal"
- " (detected at line %d)",
- TOK_GET_STRING_PREFIX(tok), start);
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
+ "unterminated triple-quoted %c-string literal"
+ " (detected at line %d)",
+ TOK_GET_STRING_PREFIX(tok), end_lineno);
if (c != '\n') {
tok->done = E_EOFS;
}
- return MAKE_TOKEN(ERRORTOKEN);
+ return string_error_token(tok, token,
+ current_tok->start, location);
}
else {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
- "unterminated %c-string literal (detected at"
- " line %d)", TOK_GET_STRING_PREFIX(tok), start));
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
+ "unterminated %c-string literal (detected at line %d)",
+ TOK_GET_STRING_PREFIX(tok), end_lineno);
+ return string_error_token(tok, token,
+ current_tok->start, location);
}
}
diff --git a/Parser/pegen.c b/Parser/pegen.c
index d86dd22444e6a7..300cb32424c2d8 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -8,8 +8,9 @@
#include
#include "lexer/lexer.h"
-#include "tokenizer/tokenizer.h"
#include "tokenizer/helpers.h"
+#include "tokenizer/reader.h"
+#include "tokenizer/tokenizer.h"
#include "pegen.h"
#define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two.
@@ -217,11 +218,11 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
parser_token->level = new_token->level;
parser_token->lineno = new_token->start_loc.lineno;
- parser_token->col_offset = p->tok->lineno == p->starting_lineno
+ parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno
? p->starting_col_offset + new_token->start_loc.byte_col
: new_token->start_loc.byte_col;
parser_token->end_lineno = new_token->end_loc.lineno;
- parser_token->end_col_offset = p->tok->lineno == p->starting_lineno
+ parser_token->end_col_offset = new_token->end_loc.lineno == p->starting_lineno
? p->starting_col_offset + new_token->end_loc.byte_col
: new_token->end_loc.byte_col;
@@ -287,9 +288,8 @@ _PyPegen_fill_token(Parser *p)
type = NEWLINE; /* Add an extra newline */
p->parsing_started = 0;
- if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
- p->tok->pendin = -p->tok->indent;
- p->tok->indent = 0;
+ if (!(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
+ _PyLexer_ImplyDedents(p->tok);
}
}
else {
@@ -943,9 +943,7 @@ reset_parser_state_for_error_pass(Parser *p)
}
p->mark = 0;
p->call_invalid_rules = 1;
- // Don't try to get extra tokens in interactive mode when trying to
- // raise specialized errors in the second pass.
- p->tok->interactive_underflow = IUNDERFLOW_STOP;
+ _PyTok_ReaderStopInteractive(p->tok);
}
static inline int
@@ -961,13 +959,7 @@ _PyPegen_set_syntax_error_metadata(Parser *p) {
PyErr_SetRaisedException(exc);
return;
}
- const char *source = NULL;
- if (p->tok->str != NULL) {
- source = p->tok->str;
- }
- if (!source && p->tok->fp_interactive && p->tok->interactive_src_start) {
- source = p->tok->interactive_src_start;
- }
+ const char *source = _PyTokenizer_RetainedSource(p->tok);
PyObject* the_source = NULL;
if (source) {
if (p->tok->encoding == NULL) {
@@ -1070,10 +1062,6 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
}
return NULL;
}
- if (!tok->fp || ps1 != NULL || ps2 != NULL ||
- PyUnicode_CompareWithASCIIString(filename_ob, "") == 0) {
- tok->fp_interactive = 1;
- }
// This transfers the ownership to the tokenizer
tok->filename = Py_NewRef(filename_ob);
@@ -1095,8 +1083,9 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
result = _PyPegen_run_parser(p);
_PyPegen_Parser_Free(p);
- if (tok->fp_interactive && tok->interactive_src_start && result && interactive_src != NULL) {
- *interactive_src = PyUnicode_FromString(tok->interactive_src_start);
+ const char *source = _PyTokenizer_RetainedSource(tok);
+ if (source != NULL && result && interactive_src != NULL) {
+ *interactive_src = PyUnicode_FromString(source);
if (!*interactive_src || _PyArena_AddPyObject(arena, *interactive_src) < 0) {
Py_XDECREF(*interactive_src);
result = NULL;
diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index b13e1c079220a9..0c787cefa8d73e 100644
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -7,6 +7,8 @@
#include "lexer/state.h"
#include "lexer/lexer.h"
#include "pegen.h"
+#include "tokenizer/reader.h"
+#include "tokenizer/tokenizer.h"
// TOKENIZER ERRORS
@@ -122,7 +124,7 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
// before the one that we had for the generic error.
// We don't want to tokenize to the end for interactive input
- if (p->tok->prompt != NULL) {
+ if (_PyTok_ReaderIsInteractive(p->tok)) {
return 0;
}
@@ -202,7 +204,10 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err
Py_ssize_t col_offset;
Py_ssize_t end_col_offset = -1;
if (t->col_offset == -1) {
- if (p->tok->cur == p->tok->buf) {
+ _PyTokenizer_Diagnostic diagnostic = p->tok->diagnostic;
+ if (diagnostic.location.lineno != 0) {
+ col_offset = diagnostic.location.byte_col;
+ } else if (p->tok->cur == p->tok->buf) {
col_offset = 0;
} else {
const char* start = p->tok->buf ? p->tok->line_start : p->tok->buf;
@@ -225,46 +230,15 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err
}
static PyObject *
-get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno)
+get_error_line_from_source(Parser *p, Py_ssize_t lineno)
{
- /* If the file descriptor is interactive, the source lines of the current
- * (multi-line) statement are stored in p->tok->interactive_src_start.
- * If not, we're parsing from a string, which means that the whole source
- * is stored in p->tok->str. */
- assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp != NULL);
-
- char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str;
- if (cur_line == NULL) {
- assert(p->tok->fp_interactive);
- // We can reach this point if the tokenizer buffers for interactive source have not been
- // initialized because we failed to decode the original source with the given locale.
+ if (_PyTokenizer_RetainedSource(p->tok) == NULL) {
return Py_GetConstant(Py_CONSTANT_EMPTY_STR);
}
-
Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno;
- const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp;
-
- if (buf_end < cur_line) {
- buf_end = cur_line + strlen(cur_line);
- }
-
- for (int i = 0; i < relative_lineno - 1; i++) {
- char *new_line = strchr(cur_line, '\n');
- // The assert is here for debug builds but the conditional that
- // follows is there so in release builds we do not crash at the cost
- // to report a potentially wrong line.
- assert(new_line != NULL && new_line + 1 < buf_end);
- if (new_line == NULL || new_line + 1 > buf_end) {
- break;
- }
- cur_line = new_line + 1;
- }
-
- char *next_newline;
- if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line
- next_newline = cur_line + strlen(cur_line);
- }
- return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
+ Py_ssize_t len;
+ const char *line = _PyTok_SourceLineView(&p->tok->source, relative_lineno, &len);
+ return PyUnicode_DecodeUTF8(line, len, "replace");
}
void *
@@ -282,12 +256,19 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
PyObject *error_line = NULL;
PyObject *tmp = NULL;
p->error_indicator = 1;
+ _PyTokenizer_Diagnostic diagnostic = p->tok->diagnostic;
+ _PyTok_Loc location = diagnostic.location.lineno != 0
+ ? diagnostic.location : (_PyTok_Loc){p->tok->lineno,
+ p->tok->line_start == NULL ? -1 : _PyLexer_ByteColumn(p->tok)};
+ _PyTok_Span text_span = diagnostic.location.lineno != 0
+ ? diagnostic.text_span : _PyLexer_BufferSpan(
+ p->tok, p->tok->line_start, p->tok->inp);
if (end_lineno == CURRENT_POS) {
- end_lineno = p->tok->lineno;
+ end_lineno = location.lineno;
}
if (end_col_offset == CURRENT_POS) {
- end_col_offset = p->tok->cur - p->tok->line_start;
+ end_col_offset = location.byte_col;
}
errstr = PyUnicode_FromFormatV(errmsg, va);
@@ -295,8 +276,8 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
goto error;
}
- if (p->tok->fp_interactive && p->tok->interactive_src_start != NULL) {
- error_line = get_error_line_from_tokenizer_buffers(p, lineno);
+ if (_PyTok_ReaderIsInteractive(p->tok) && _PyTokenizer_RetainedSource(p->tok) != NULL) {
+ error_line = get_error_line_from_source(p, lineno);
}
else if (p->start_rule == Py_file_input) {
error_line = _PyErr_ProgramDecodedTextObject(p->tok->filename,
@@ -311,14 +292,17 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
we're actually parsing from a file, which has an E_EOF SyntaxError and in that case
`PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which
does not physically exist */
- assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);
-
- if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) {
- Py_ssize_t size = p->tok->inp - p->tok->line_start;
- error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace");
+ assert((p->tok->fp == NULL || p->tok->fp == stdin) || p->tok->done == E_EOF);
+
+ if (location.lineno <= lineno &&
+ p->tok->inp > p->tok->buf) {
+ Py_ssize_t size;
+ const char *line = _PyLexer_BufferSpanView(
+ p->tok, text_span, &size);
+ error_line = PyUnicode_DecodeUTF8(line, size, "replace");
}
- else if (p->tok->fp == NULL || p->tok->fp == stdin) {
- error_line = get_error_line_from_tokenizer_buffers(p, lineno);
+ else if ((p->tok->fp == NULL || p->tok->fp == stdin)) {
+ error_line = get_error_line_from_source(p, lineno);
}
else {
error_line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c
deleted file mode 100644
index 523b99dedc6160..00000000000000
--- a/Parser/tokenizer/cursor.c
+++ /dev/null
@@ -1,82 +0,0 @@
-#include "Python.h"
-
-#include "cursor.h"
-
-static void
-set_line(_PyTok_Cursor *cursor, int lineno, _PyTok_Off start,
- _PyTok_Off end)
-{
- cursor->pos = start;
- cursor->line_start = start;
- cursor->line_end = end;
- cursor->lineno = lineno;
-}
-
-int
-_PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno)
-{
- if (cursor->source == NULL) {
- PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source");
- return -1;
- }
- const _PyTok_SourceText *source = cursor->source;
- if (lineno > 0 && cursor->lineno == lineno - 1 &&
- lineno <= source->nlines) {
- _PyTok_Off start = cursor->line_end;
- _PyTok_Off end = source->base_offset + source->len;
- if (lineno < source->nlines) {
- end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- }
- set_line(cursor, lineno, start, end);
- return 0;
- }
-
- _PyTok_Line line;
- if (_PyTok_SourceLine(source, lineno, &line) < 0) {
- return -1;
- }
- set_line(cursor, lineno, line.start, line.end);
- return 0;
-}
-
-int
-_PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset)
-{
- if (cursor->source == NULL) {
- PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source");
- return -1;
- }
- const _PyTok_SourceText *source = cursor->source;
- int stays_on_line = cursor->lineno > 0 &&
- offset >= cursor->line_start && offset < cursor->line_end;
- if (!stays_on_line && cursor->lineno > 0 &&
- offset == cursor->line_end &&
- offset - source->base_offset == source->len &&
- (source->len == 0 || source->bytes[source->len - 1] != '\n')) {
- stays_on_line = 1;
- }
- if (stays_on_line) {
- cursor->pos = offset;
- return 0;
- }
-
- _PyTok_Loc loc;
- if (_PyTok_SourceLocation(
- source, offset, _PYTOK_AFFINITY_RIGHT, &loc) < 0) {
- return -1;
- }
- _PyTok_Off start = offset - loc.byte_col;
- _PyTok_Off end = source->base_offset + source->len;
- if (loc.lineno < source->nlines) {
- end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- }
- set_line(cursor, loc.lineno, start, end);
- cursor->pos = offset;
- return 0;
-}
diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h
deleted file mode 100644
index 18e404251316f0..00000000000000
--- a/Parser/tokenizer/cursor.h
+++ /dev/null
@@ -1,73 +0,0 @@
-#ifndef Py_TOKENIZER_CURSOR_H
-#define Py_TOKENIZER_CURSOR_H
-
-#include "source.h"
-
-typedef struct {
- /* The source must remain initialized at this address while in use. */
- const _PyTok_SourceText *source;
- _PyTok_Off pos;
- _PyTok_Off line_start;
- _PyTok_Off line_end;
- int lineno;
-} _PyTok_Cursor;
-
-/* Move to the start of a 1-based line. Both setters preserve the cursor on
- error. */
-PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int);
-/* Move to an offset. A line boundary selects the following line. */
-PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off);
-
-static inline void
-_PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source)
-{
- _PyTok_Off base = source != NULL ? source->base_offset : 0;
- *cursor = (_PyTok_Cursor){
- .source = source,
- .pos = base,
- .line_start = base,
- .line_end = base,
- };
-}
-
-/* Read one byte from the current line, including its terminating newline.
- EOF marks the line boundary, not necessarily the end of the source. It is
- also returned if advancing would make the byte column unrepresentable. */
-static inline int
-_PyTok_CursorAdvance(_PyTok_Cursor *cursor)
-{
- assert(cursor->source != NULL);
- assert(cursor->pos >= cursor->line_start);
- assert(cursor->pos <= cursor->line_end);
- assert(cursor->line_start >= cursor->source->base_offset);
- assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len);
- if (cursor->pos >= cursor->line_end) {
- return EOF;
- }
- if (cursor->pos - cursor->line_start >= INT_MAX) {
- return EOF;
- }
- return Py_CHARMASK(cursor->source->bytes[
- cursor->pos++ - cursor->source->base_offset]);
-}
-
-/* Return the byte at a nonnegative distance within the current line, or EOF
- if the distance reaches or crosses the line boundary. */
-static inline int
-_PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance)
-{
- assert(cursor->source != NULL);
- assert(cursor->pos >= cursor->line_start);
- assert(cursor->pos <= cursor->line_end);
- assert(cursor->line_start >= cursor->source->base_offset);
- assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len);
- assert(distance >= 0);
- if (distance < 0 ||
- distance >= cursor->line_end - cursor->pos) {
- return EOF;
- }
- return Py_CHARMASK(cursor->source->bytes[
- cursor->pos - cursor->source->base_offset + distance]);
-}
-
-#endif
diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c
index af17b8b63235f5..c36860237fce0b 100644
--- a/Parser/tokenizer/decoder.c
+++ b/Parser/tokenizer/decoder.c
@@ -103,7 +103,9 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf,
}
result[write] = '\0';
*out_len = write;
- *implicit_newline = implicit;
+ if (implicit_newline != NULL) {
+ *implicit_newline = implicit;
+ }
return result;
}
@@ -238,22 +240,14 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
const _PyTok_Chunk *line = cookie_line == 2 ? second : first;
const char *line_data = line->data + (cookie_line == 1 ? 3 : 0);
Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0);
- const char *saved_line_start = tok->line_start;
- char *saved_cur = tok->cur;
- int saved_lineno = tok->lineno;
- tok->line_start = line_data;
- tok->cur = (char *)line_data;
- tok->lineno = cookie_line;
int end_col = (int)Py_MIN(line_len, INT_MAX);
if (end_col > 0 && (line_data[end_col - 1] == '\n' ||
line_data[end_col - 1] == '\r')) {
end_col--;
}
- _PyTokenizer_syntaxerror_known_range(
- tok, 0, end_col, "encoding problem: %s with BOM", cookie);
- tok->line_start = saved_line_start;
- tok->cur = saved_cur;
- tok->lineno = saved_lineno;
+ _PyTokenizer_syntaxerror_at(
+ tok, line_data, 0, cookie_line, 0, end_col,
+ "encoding problem: %s with BOM", cookie);
PyMem_Free(cookie);
return _PYTOK_ENCODING_ERROR;
}
@@ -399,10 +393,9 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only,
if (stored < 0) {
return -1;
}
- tok->str = tok->source.bytes != NULL ? tok->source.bytes : (char *)"";
if (!utf8_only &&
(tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
- !_PyTokenizer_ensure_utf8(tok->str, tok, 1)) {
+ !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) {
return -1;
}
return 0;
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index e99be70ea5e980..66a8b160f68787 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -24,7 +24,8 @@ byte_col_to_char_col(const char *line, int byte_col)
}
static int
-_syntaxerror_range(struct tok_state *tok, const char *format,
+_syntaxerror_range(struct tok_state *tok, const char *line_start,
+ Py_ssize_t cursor_offset, int lineno, const char *format,
int col_offset, int end_col_offset,
va_list vargs)
{
@@ -40,7 +41,7 @@ _syntaxerror_range(struct tok_state *tok, const char *format,
goto error;
}
- errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start,
+ errtext = PyUnicode_DecodeUTF8(line_start, cursor_offset,
"replace");
if (!errtext) {
goto error;
@@ -50,19 +51,19 @@ _syntaxerror_range(struct tok_state *tok, const char *format,
col_offset = (int)PyUnicode_GET_LENGTH(errtext);
}
else if (col_offset > 0) {
- col_offset = byte_col_to_char_col(tok->line_start, col_offset);
+ col_offset = byte_col_to_char_col(line_start, col_offset);
}
if (end_col_offset == -1) {
end_col_offset = col_offset;
}
else if (end_col_offset > 0) {
- end_col_offset = byte_col_to_char_col(tok->line_start, end_col_offset);
+ end_col_offset = byte_col_to_char_col(line_start, end_col_offset);
}
- Py_ssize_t line_len = strcspn(tok->line_start, "\n");
- if (line_len != tok->cur - tok->line_start) {
+ Py_ssize_t line_len = strcspn(line_start, "\n");
+ if (line_len != cursor_offset) {
Py_DECREF(errtext);
- errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len,
+ errtext = PyUnicode_DecodeUTF8(line_start, line_len,
"replace");
}
if (!errtext) {
@@ -71,8 +72,8 @@ _syntaxerror_range(struct tok_state *tok, const char *format,
args = Py_BuildValue("(O(OiiNii))", errmsg,
tok->filename ? tok->filename : Py_None,
- tok->lineno, col_offset, errtext,
- tok->lineno, end_col_offset);
+ lineno, col_offset, errtext,
+ lineno, end_col_offset);
if (args) {
PyErr_SetObject(PyExc_SyntaxError, args);
Py_DECREF(args);
@@ -90,7 +91,9 @@ _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...)
// These errors are cleaned on startup. Todo: Fix it.
va_list vargs;
va_start(vargs, format);
- int ret = _syntaxerror_range(tok, format, -1, -1, vargs);
+ int ret = _syntaxerror_range(
+ tok, tok->line_start,
+ tok->cur - tok->line_start, tok->lineno, format, -1, -1, vargs);
va_end(vargs);
return ret;
}
@@ -102,7 +105,24 @@ _PyTokenizer_syntaxerror_known_range(struct tok_state *tok,
{
va_list vargs;
va_start(vargs, format);
- int ret = _syntaxerror_range(tok, format, col_offset, end_col_offset, vargs);
+ int ret = _syntaxerror_range(
+ tok, tok->line_start,
+ tok->cur - tok->line_start, tok->lineno, format,
+ col_offset, end_col_offset, vargs);
+ va_end(vargs);
+ return ret;
+}
+
+int
+_PyTokenizer_syntaxerror_at(
+ struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset,
+ int lineno, int col_offset, int end_col_offset, const char *format, ...)
+{
+ va_list vargs;
+ va_start(vargs, format);
+ int ret = _syntaxerror_range(
+ tok, line_start, cursor_offset, lineno, format,
+ col_offset, end_col_offset, vargs);
va_end(vargs);
return ret;
}
@@ -118,10 +138,6 @@ _PyTokenizer_indenterror(struct tok_state *tok)
int
_PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char)
{
- if (!tok->report_warnings) {
- return 0;
- }
-
PyObject *msg = PyUnicode_FromFormat(
"\"\\%c\" is an invalid escape sequence. "
"Such sequences will not work in the future. "
@@ -207,10 +223,6 @@ _PyTokenizer_raise_init_error(PyObject *filename)
int
_PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...)
{
- if (!tok->report_warnings) {
- return 0;
- }
-
PyObject *errmsg;
va_list vargs;
va_start(vargs, format);
@@ -310,26 +322,21 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
const char *badchar = NULL;
const char *c;
int length;
- int col_offset = 0;
const char *line_start = line;
for (c = line; *c; c += length) {
if (!(length = valid_utf8((const unsigned char *)c))) {
badchar = c;
break;
}
- col_offset++;
if (*c == '\n') {
lineno++;
- col_offset = 0;
line_start = c + 1;
}
}
if (badchar) {
- tok->lineno = lineno;
- tok->line_start = line_start;
- tok->cur = (char *)badchar;
- _PyTokenizer_syntaxerror_known_range(tok,
- col_offset + 1, col_offset + 1,
+ _PyTokenizer_syntaxerror_at(
+ tok, line_start, badchar - line_start + 1, lineno,
+ -1, -1,
"Non-UTF-8 code starting with '\\x%.2x'"
"%s%V on line %i, "
"but no encoding declared; "
diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h
index 5edf5a3dfd2e0d..71f725c0b4c792 100644
--- a/Parser/tokenizer/helpers.h
+++ b/Parser/tokenizer/helpers.h
@@ -10,10 +10,17 @@
tok->col_offset = 0;
int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...);
+/* Positive range columns are 1-based byte columns. A start column of -1
+ derives the character column from the reporting cursor; an end column of
+ -1 uses the start column. */
int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...);
+int _PyTokenizer_syntaxerror_at(
+ struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset,
+ int lineno, int col_offset, int end_col_offset, const char *format, ...);
int _PyTokenizer_indenterror(struct tok_state *tok);
int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char);
int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...);
+
void _PyTokenizer_raise_init_error(PyObject *filename);
int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno);
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index b9b4a461087441..9c1865234637f7 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -1,13 +1,13 @@
#include "Python.h"
#include "pycore_fileutils.h"
+#include "pycore_pystate.h"
#include "errcode.h"
#include "helpers.h"
#include "reader.h"
#include "reader_internal.h"
-#include "../lexer/buffer.h"
-#include "../lexer/lexer.h"
#include "../lexer/state.h"
+#include "../lexer/lexer.h"
#ifdef HAVE_UNISTD_H
# include
@@ -19,6 +19,56 @@ reader_is_streaming(_PyTok_ReaderKind kind)
return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE;
}
+typedef struct {
+ Py_ssize_t buf;
+ Py_ssize_t cur;
+ Py_ssize_t inp;
+ Py_ssize_t start;
+ Py_ssize_t line_start;
+ Py_ssize_t multi_line_start;
+} BufferOffsets;
+
+static BufferOffsets
+save_buffer_offsets(struct tok_state *tok, const char *base)
+{
+ for (int index = tok->tok_mode_stack_index; index > 0; --index) {
+ tokenizer_mode *mode = &tok->tok_mode_stack[index];
+ mode->start_offset = mode->start == NULL ? -1 : mode->start - base;
+ mode->multi_line_start_offset = mode->multi_line_start == NULL
+ ? -1 : mode->multi_line_start - base;
+ }
+ return (BufferOffsets) {
+ .buf = tok->buf - base,
+ .cur = tok->cur - base,
+ .inp = tok->inp - base,
+ .start = tok->start == NULL ? -1 : tok->start - base,
+ .line_start = tok->line_start == NULL
+ ? -1 : tok->line_start - base,
+ .multi_line_start = tok->multi_line_start == NULL
+ ? -1 : tok->multi_line_start - base,
+ };
+}
+
+static void
+restore_buffer_offsets(struct tok_state *tok, char *base,
+ const BufferOffsets *offsets)
+{
+ tok->buf = base + offsets->buf;
+ tok->cur = base + offsets->cur;
+ tok->inp = base + offsets->inp;
+ tok->start = offsets->start < 0 ? NULL : base + offsets->start;
+ tok->line_start = offsets->line_start < 0
+ ? NULL : base + offsets->line_start;
+ tok->multi_line_start = offsets->multi_line_start < 0
+ ? NULL : base + offsets->multi_line_start;
+ for (int index = tok->tok_mode_stack_index; index > 0; --index) {
+ tokenizer_mode *mode = &tok->tok_mode_stack[index];
+ mode->start = mode->start_offset < 0 ? NULL : base + mode->start_offset;
+ mode->multi_line_start = mode->multi_line_start_offset < 0
+ ? NULL : base + mode->multi_line_start_offset;
+ }
+}
+
void
_PyTok_ReaderFree(struct tok_state *tok)
{
@@ -137,10 +187,10 @@ chunk_is_line(const _PyTok_Chunk *chunk)
static _PyTok_ReadResult
next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
{
- int lineno = tok->lineno + 1;
- if (lineno > tok->source.nlines) {
+ if (tok->lineno >= tok->source.nlines) {
return _PYTOK_READ_EOF;
}
+ int lineno = tok->lineno + 1;
const char *start = tok->inp;
const char *newline = memchr(
start, '\n', tok->source.bytes + tok->source.len - start);
@@ -206,7 +256,6 @@ initialize_file(struct tok_state *tok)
if (result != _PYTOK_READ_LINE) {
return -1;
}
- reader->prefetched_count = 1;
Py_ssize_t bom_len;
_PyTok_EncodingResult detection = _PyTok_DetectEncoding(
tok, &reader->prefetched_lines[0], NULL, 0, &bom_len);
@@ -224,16 +273,13 @@ initialize_file(struct tok_state *tok)
reader->prefetched_lines[0].data = first;
reader->prefetched_lines[0].ownership = _PYTOK_CHUNK_PYMEM;
result = read_file_line(tok, &reader->prefetched_lines[1]);
- if (result == _PYTOK_READ_LINE) {
- reader->prefetched_count = 2;
- }
- else if (result == _PYTOK_READ_EOF) {
+ if (result == _PYTOK_READ_EOF) {
reader->file_eof = 1;
}
- else {
+ else if (result != _PYTOK_READ_LINE) {
return -1;
}
- _PyTok_Chunk *second = reader->prefetched_count == 2
+ _PyTok_Chunk *second = reader->prefetched_lines[1].data != NULL
? &reader->prefetched_lines[1] : NULL;
detection = _PyTok_DetectEncoding(
tok, &reader->prefetched_lines[0], second, 1, &bom_len);
@@ -303,10 +349,13 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk)
return _PYTOK_READ_LINE;
}
_PyTok_Chunk input = {0};
- if (reader->prefetched_index < reader->prefetched_count) {
- input = reader->prefetched_lines[reader->prefetched_index];
- reader->prefetched_lines[reader->prefetched_index++] =
- (_PyTok_Chunk){0};
+ if (reader->prefetched_lines[0].data != NULL) {
+ input = reader->prefetched_lines[0];
+ reader->prefetched_lines[0] = (_PyTok_Chunk){0};
+ }
+ else if (reader->prefetched_lines[1].data != NULL) {
+ input = reader->prefetched_lines[1];
+ reader->prefetched_lines[1] = (_PyTok_Chunk){0};
}
else if (!reader->file_eof) {
_PyTok_ReadResult result = read_file_line(tok, &input);
@@ -475,13 +524,13 @@ static _PyTok_ReadResult
next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
{
_PyTok_Reader *reader = tok->reader;
- if (tok->interactive_underflow == IUNDERFLOW_STOP) {
+ if (reader->stop_interactive) {
return _PYTOK_READ_STOPPED;
}
char *input = PyOS_Readline(
- tok->fp != NULL ? tok->fp : stdin, stdout, tok->prompt);
+ tok->fp != NULL ? tok->fp : stdin, stdout, reader->prompt);
if (reader->nextprompt != NULL) {
- tok->prompt = reader->nextprompt;
+ reader->prompt = reader->nextprompt;
}
if (input == NULL) {
return _PYTOK_READ_INTERRUPT;
@@ -504,7 +553,7 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
}
chunk->data = _PyTok_NormalizeNewlines(
decoded.data, decoded.len, 0, 0,
- &chunk->len, &chunk->implicit_newline);
+ &chunk->len, NULL);
_PyTok_ChunkClear(&decoded);
if (chunk->data == NULL) {
PyErr_NoMemory();
@@ -515,6 +564,32 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
return _PYTOK_READ_LINE;
}
+int
+_PyTok_ReaderIsInteractive(const struct tok_state *tok)
+{
+ return tok->reader->kind == _PYTOK_READER_INTERACTIVE;
+}
+
+const char *
+_PyTokenizer_RetainedSource(const struct tok_state *tok)
+{
+ if (tok->reader->kind == _PYTOK_READER_PREPARED) {
+ return _PyTok_SourceData(&tok->source);
+ }
+ if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
+ return tok->source.bytes;
+ }
+ return NULL;
+}
+
+void
+_PyTok_ReaderStopInteractive(struct tok_state *tok)
+{
+ if (_PyTok_ReaderIsInteractive(tok)) {
+ tok->reader->stop_interactive = 1;
+ }
+}
+
static _PyTok_ReadResult
reader_next(struct tok_state *tok, _PyTok_Chunk *chunk)
{
@@ -579,6 +654,12 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
}
return 0;
}
+ if (tok->lineno == INT_MAX) {
+ PyErr_SetString(PyExc_OverflowError, "too many tokenizer source lines");
+ tok->done = E_ERROR;
+ _PyTok_ChunkClear(&chunk);
+ return 0;
+ }
Py_ssize_t scan_len = chunk.len;
if (kind == _PYTOK_READER_INTERACTIVE &&
@@ -589,10 +670,9 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
if (streaming && reset_buffer) {
reset_streaming_buffer(tok);
}
- _PyLexer_BufferPointers pointers;
+ BufferOffsets offsets;
if (!reset_buffer) {
- _PyLexer_SaveBufferPointers(
- tok, tok->source.bytes, &pointers);
+ offsets = save_buffer_offsets(tok, tok->source.bytes);
}
_PyTok_Off source_start = _PyTok_SourceAppendLine(
&tok->source, chunk.data, chunk.len,
@@ -613,18 +693,13 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
tok->multi_line_start = NULL;
}
else {
- _PyLexer_RestoreBufferPointers(
- tok, tok->source.bytes, &pointers);
+ restore_buffer_offsets(tok, tok->source.bytes, &offsets);
}
tok->inp = tok->source.bytes +
(source_start - tok->source.base_offset) + scan_len;
}
- if (tok->fp_interactive) {
- tok->interactive_src_start = tok->source.bytes;
- tok->interactive_src_end = tok->source.bytes + tok->source.len;
- }
if (prepared) {
- if (tok->start == NULL) {
+ if (tok->start == NULL && !INSIDE_FSTRING(tok)) {
tok->buf = tok->cur;
tok->buf_offset = tok->source.base_offset +
(chunk.data - tok->source.bytes);
@@ -654,10 +729,19 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
static struct tok_state *
tokenizer_new_with_reader(_PyTok_ReaderKind kind)
{
- struct tok_state *tok = _PyTokenizer_tok_new();
+ struct tok_state *tok = PyMem_Calloc(1, sizeof(*tok));
if (tok == NULL) {
+ PyErr_NoMemory();
return NULL;
}
+ tok->done = E_OK;
+ tok->layout.at_bol = 1;
+ tok->starting_col_offset = -1;
+ tok->col_offset = -1;
+ tok->tok_mode_stack[0].kind = TOK_REGULAR_MODE;
+#ifdef Py_DEBUG
+ tok->debug = _Py_GetConfig()->parser_debug;
+#endif
tok->reader = PyMem_Calloc(1, sizeof(*tok->reader));
if (tok->reader == NULL) {
PyErr_NoMemory();
@@ -688,7 +772,9 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input,
_PyTokenizer_Free(tok);
return NULL;
}
- tok->buf = tok->cur = tok->inp = tok->str;
+ char *source = (char *)_PyTok_SourceData(&tok->source);
+ tok->buf = tok->cur = tok->inp = source;
+ tok->line_start = source;
return tok;
}
@@ -734,7 +820,7 @@ _PyTokenizer_FromFile(FILE *fp, const char *encoding,
return NULL;
}
tok->fp = fp;
- tok->prompt = ps1;
+ tok->reader->prompt = ps1;
tok->reader->nextprompt = ps2;
return tok;
}
@@ -788,13 +874,10 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
_PyTokenizer_Free(tok);
return NULL;
}
- /* Reporting a warning here could recursively ask for the encoding. */
- tok->report_warnings = 0;
- while (tok->lineno < 2 && tok->done == E_OK) {
- struct token token;
- _PyToken_Init(&token);
- _PyTokenizer_Get(tok, &token);
- _PyToken_Free(&token);
+ if (initialize_file(tok) < 0) {
+ fclose(fp);
+ _PyTokenizer_Free(tok);
+ return NULL;
}
fclose(fp);
char *encoding = tok->encoding == NULL
diff --git a/Parser/tokenizer/reader.h b/Parser/tokenizer/reader.h
index c27bc2aa3fb819..2913e52b9d563b 100644
--- a/Parser/tokenizer/reader.h
+++ b/Parser/tokenizer/reader.h
@@ -5,5 +5,7 @@ struct tok_state;
void _PyTok_ReaderFree(struct tok_state *);
int _PyTok_ReaderUnderflow(struct tok_state *);
+int _PyTok_ReaderIsInteractive(const struct tok_state *);
+void _PyTok_ReaderStopInteractive(struct tok_state *);
#endif
diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h
index 121d0f96f6698a..b4f6807c207a4a 100644
--- a/Parser/tokenizer/reader_internal.h
+++ b/Parser/tokenizer/reader_internal.h
@@ -32,33 +32,32 @@ typedef enum {
typedef struct {
char *data;
- Py_ssize_t len;
- int implicit_newline;
PyObject *owner;
+ Py_ssize_t len;
_PyTok_ChunkOwnership ownership;
+ unsigned char implicit_newline;
} _PyTok_Chunk;
typedef struct _PyTok_Reader {
- _PyTok_ReaderKind kind;
PyObject *readline;
PyObject *decoder;
+ const char *prompt;
const char *nextprompt;
char *file_buffer;
Py_ssize_t file_buffer_cap;
_PyTok_Chunk prefetched_lines[2];
- int prefetched_index;
- int prefetched_count;
char *decoded;
Py_ssize_t decoded_pos;
Py_ssize_t decoded_len;
Py_ssize_t decoded_cap;
- int decoded_tail_is_implicit;
-
- int file_initialized;
- int file_eof;
- int decoder_finalized;
+ _PyTok_ReaderKind kind;
+ unsigned char decoded_tail_is_implicit;
+ unsigned char file_initialized;
+ unsigned char file_eof;
+ unsigned char decoder_finalized;
+ unsigned char stop_interactive;
} _PyTok_Reader;
struct tok_state;
diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c
index 2f2aaf2589246d..d69eab93923e81 100644
--- a/Parser/tokenizer/source.c
+++ b/Parser/tokenizer/source.c
@@ -2,8 +2,6 @@
#include "source.h"
-#define LINE_CHECKPOINT_INTERVAL 256
-
void
_PyTok_SourceInit(_PyTok_SourceText *source)
{
@@ -14,7 +12,6 @@ void
_PyTok_SourceClear(_PyTok_SourceText *source)
{
PyMem_Free(source->bytes);
- PyMem_Free(source->line_checkpoints);
PyMem_Free(source->implicit_lines);
_PyTok_SourceInit(source);
}
@@ -74,34 +71,6 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed)
return 0;
}
-static int
-reserve_checkpoints(_PyTok_SourceText *source, int needed)
-{
- if (needed <= source->checkpoints_cap) {
- return 0;
- }
- int cap;
- if (source->checkpoints_cap == 0) {
- cap = 16;
- }
- else if (source->checkpoints_cap <= INT_MAX / 2) {
- cap = source->checkpoints_cap * 2;
- }
- else {
- PyErr_NoMemory();
- return -1;
- }
- _PyTok_Off *checkpoints = source->line_checkpoints;
- PyMem_Resize(checkpoints, _PyTok_Off, cap);
- if (checkpoints == NULL) {
- PyErr_NoMemory();
- return -1;
- }
- source->line_checkpoints = checkpoints;
- source->checkpoints_cap = cap;
- return 0;
-}
-
static int
reserve_implicit_lines(_PyTok_SourceText *source, int nlines)
{
@@ -165,11 +134,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
return -1;
}
int nlines = source->nlines + 1;
- int checkpoint = ((nlines - 1) % LINE_CHECKPOINT_INTERVAL) == 0;
- int checkpoint_count = (nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1;
- if ((checkpoint &&
- reserve_checkpoints(source, checkpoint_count) < 0) ||
- (implicit_newline && reserve_implicit_lines(source, nlines) < 0) ||
+ if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) ||
reserve_bytes(source, source->len + len + 1) < 0) {
return -1;
}
@@ -178,10 +143,6 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
memcpy(source->bytes + start, bytes, len);
source->len += len;
source->bytes[source->len] = '\0';
- if (checkpoint) {
- source->line_checkpoints[checkpoint_count - 1] =
- source->base_offset + start;
- }
if (implicit_newline) {
source->implicit_lines[(nlines - 1) / 8] |=
(unsigned char)(1U << ((nlines - 1) & 7));
@@ -191,16 +152,23 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
}
const char *
-_PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
+_PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno,
Py_ssize_t *len)
{
- if (!_PyTok_SpanIsValid(span) || span.start < source->base_offset ||
- span.end - source->base_offset > source->len || len == NULL) {
- PyErr_SetString(PyExc_SystemError, "invalid tokenizer source span");
- return NULL;
+ assert(len != NULL);
+ const char *line = _PyTok_SourceData(source);
+ const char *end = line + source->len;
+ while (lineno > 1) {
+ const char *newline = memchr(line, '\n', end - line);
+ if (newline == NULL) {
+ break;
+ }
+ line = newline + 1;
+ lineno--;
}
- *len = span.end - span.start;
- return _PyTok_SourceData(source) + (span.start - source->base_offset);
+ const char *newline = memchr(line, '\n', end - line);
+ *len = (newline != NULL ? newline : end) - line;
+ return line;
}
int
@@ -213,124 +181,3 @@ _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno)
return (source->implicit_lines[(lineno - 1) / 8] >>
((lineno - 1) & 7)) & 1;
}
-
-static int
-source_ends_in_newline(const _PyTok_SourceText *source)
-{
- return source->len > 0 && source->bytes[source->len - 1] == '\n';
-}
-
-static int
-eof_lineno(const _PyTok_SourceText *source)
-{
- if (source->nlines == 0) {
- return 1;
- }
- return source->nlines + source_ends_in_newline(source);
-}
-
-int
-_PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
- _PyTok_Line *line)
-{
- if (line == NULL || lineno < 1 || lineno > eof_lineno(source)) {
- PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line");
- return -1;
- }
- if (lineno > source->nlines) {
- *line = (_PyTok_Line){
- .start = source->base_offset + source->len,
- .end = source->base_offset + source->len,
- };
- return 0;
- }
-
- int checkpoint = (lineno - 1) / LINE_CHECKPOINT_INTERVAL;
- int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
- _PyTok_Off start = source->line_checkpoints[checkpoint];
- while (current < lineno) {
- start = _PyTok_SourceFindLineEnd(source, start);
- if (start < 0) {
- return -1;
- }
- current++;
- }
- _PyTok_Off end = source->base_offset + source->len;
- if (lineno < source->nlines) {
- end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- }
- *line = (_PyTok_Line){
- .start = start,
- .end = end,
- .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno),
- .contains_nul = memchr(
- source->bytes + (start - source->base_offset),
- 0, end - start) != NULL,
- };
- return 0;
-}
-
-int
-_PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset,
- _PyTok_Affinity affinity, _PyTok_Loc *loc)
-{
- if (offset < source->base_offset ||
- offset - source->base_offset > source->len || loc == NULL ||
- (affinity != _PYTOK_AFFINITY_LEFT &&
- affinity != _PYTOK_AFFINITY_RIGHT)) {
- PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset");
- return -1;
- }
- if (source->nlines == 0 ||
- (offset - source->base_offset == source->len &&
- source_ends_in_newline(source) &&
- affinity == _PYTOK_AFFINITY_RIGHT)) {
- *loc = (_PyTok_Loc){eof_lineno(source), 0};
- return 0;
- }
-
- _PyTok_Off key = offset;
- if (affinity == _PYTOK_AFFINITY_LEFT && key > source->base_offset) {
- key--;
- }
- int low = 0;
- int high = (source->nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1;
- while (low < high) {
- int middle = low + (high - low) / 2;
- if (source->line_checkpoints[middle] <= key) {
- low = middle + 1;
- }
- else {
- high = middle;
- }
- }
- int checkpoint = low - 1;
- if (checkpoint < 0) {
- PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index");
- return -1;
- }
- int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
- _PyTok_Off start = source->line_checkpoints[checkpoint];
- while (lineno < source->nlines) {
- _PyTok_Off end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- if (offset < end ||
- (offset == end && affinity == _PYTOK_AFFINITY_LEFT)) {
- break;
- }
- start = end;
- lineno++;
- }
- _PyTok_Off byte_col = offset - start;
- if (byte_col > INT_MAX) {
- PyErr_SetString(PyExc_OverflowError, "tokenizer column is too large");
- return -1;
- }
- *loc = (_PyTok_Loc){lineno, (int)byte_col};
- return 0;
-}
diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h
index 7a2f46f73aff47..6972caf8a36aa2 100644
--- a/Parser/tokenizer/source.h
+++ b/Parser/tokenizer/source.h
@@ -18,28 +18,13 @@ typedef struct {
int byte_col;
} _PyTok_Loc;
-typedef enum {
- _PYTOK_AFFINITY_LEFT,
- _PYTOK_AFFINITY_RIGHT,
-} _PyTok_Affinity;
-
-/* The half-open range includes the terminating newline when present. */
-typedef struct {
- _PyTok_Off start;
- _PyTok_Off end;
- unsigned implicit_newline : 1;
- unsigned contains_nul : 1;
-} _PyTok_Line;
-
typedef struct {
char *bytes;
_PyTok_Off base_offset;
_PyTok_Off len;
_PyTok_Off cap;
- _PyTok_Off *line_checkpoints;
unsigned char *implicit_lines;
int nlines;
- int checkpoints_cap;
Py_ssize_t implicit_cap;
} _PyTok_SourceText;
@@ -50,9 +35,9 @@ _PyTok_SourceData(const _PyTok_SourceText *source)
}
PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
-/* Clear invalidates all cursors, spans, and views for the source. */
+/* Clear invalidates all spans and views for the source. */
PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
-/* Discard the retained window and invalidate its cursors, spans, and views.
+/* Discard the retained window and invalidate its spans and views.
Keep its allocation and advance the logical base to the end of the window. */
PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *);
/* Append one nonempty logical line and return its start offset. The input may
@@ -62,20 +47,15 @@ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *);
PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
_PyTok_SourceText *source, const char *bytes, Py_ssize_t len,
int implicit_newline);
-/* The returned view is invalidated by SourceAppendLine and SourceClear. */
-PyAPI_FUNC(const char *) _PyTok_SourceSpanView(
- const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *);
-/* Look up a 1-based line in the retained window. Empty and newline-terminated
- sources have an empty virtual line at EOF. */
-PyAPI_FUNC(int) _PyTok_SourceLine(
- const _PyTok_SourceText *, int, _PyTok_Line *);
+/* Return borrowed bytes excluding '\n', writing the byte length to *len.
+ Line numbers are 1-based and clamp to the first or final line; a trailing
+ '\n' adds an empty final line. The view need not be NUL-terminated.
+ This does not set an exception. Append, discard, and clear invalidate the view. */
+PyAPI_FUNC(const char *) _PyTok_SourceLineView(
+ const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len);
/* Return false for invalid line numbers and the virtual EOF line. */
PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit(
const _PyTok_SourceText *, int);
-/* At a line boundary, left affinity selects the preceding line at its end;
- right affinity selects the following line at byte column zero. */
-PyAPI_FUNC(int) _PyTok_SourceLocation(
- const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *);
static inline _PyTok_Span
_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end)
@@ -89,24 +69,4 @@ _PyTok_SpanIsValid(_PyTok_Span span)
return span.start >= 0 && span.end >= span.start;
}
-static inline _PyTok_Off
-_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start)
-{
- if (source->bytes == NULL || start < source->base_offset ||
- start - source->base_offset >= source->len) {
- PyErr_SetString(PyExc_SystemError,
- "corrupt tokenizer source line index");
- return -1;
- }
- _PyTok_Off relative_start = start - source->base_offset;
- const char *newline = memchr(
- source->bytes + relative_start, '\n', source->len - relative_start);
- if (newline == NULL) {
- PyErr_SetString(PyExc_SystemError,
- "corrupt tokenizer source line index");
- return -1;
- }
- return source->base_offset + (newline - source->bytes) + 1;
-}
-
#endif
diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h
index d8c889115cfd73..6bff1311046f20 100644
--- a/Parser/tokenizer/tokenizer.h
+++ b/Parser/tokenizer/tokenizer.h
@@ -3,6 +3,12 @@
#include "Python.h"
+struct tok_state;
+
+/* Return NUL-terminated retained input, or NULL without setting an exception
+ for streaming input or interactive input before its first line. */
+const char *_PyTokenizer_RetainedSource(const struct tok_state *);
+
struct tok_state *_PyTokenizer_FromString(const char *, int, int);
struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int);
struct tok_state *_PyTokenizer_FromReadline(PyObject *, const char *);
diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py
index bfd8e43c6912e8..a20b78271cf3a4 100644
--- a/Tools/peg_generator/pegen/build.py
+++ b/Tools/peg_generator/pegen/build.py
@@ -125,10 +125,10 @@ def compile_c_extension(
str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"),
str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"),
+ str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "layout.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "buffer.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "decoder.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "reader.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c"),