gh-153569: remove the unused tokenizer cursor and line index

Span-backed tokens removed the final production user of the tokenizer cursor.
The source location lookup and its per-line checkpoints now exist only for that
dead API.

Delete the cursor, its tests and build entries, then remove the source lookup
code and checkpoint storage with it.

Author: pablogsal

Lib/test/test_capi/test_tokenizer.py

@@ -12,9 +12,6 @@ def test_source(self):
     def test_source_discard(self):
         _testinternalcapi.test_tokenizer_source_discard()
 
-    def test_cursor(self):
-        _testinternalcapi.test_tokenizer_cursor()
-
 
 if __name__ == "__main__":
     unittest.main()

Makefile.pre.in

@@ -399,7 +399,6 @@ TOKENIZER_OBJS=		\
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
 		Parser/lexer/string.o \
-		Parser/tokenizer/cursor.o \
 		Parser/tokenizer/decoder.o \
 		Parser/tokenizer/reader.o \
 		Parser/tokenizer/source.o \
@@ -415,7 +414,6 @@ TOKENIZER_HEADERS= \
 		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \
-		Parser/tokenizer/cursor.h \
 		Parser/tokenizer/reader.h \
 		Parser/tokenizer/reader_internal.h \
 		Parser/tokenizer/source.h \
@@ -3463,7 +3461,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
 MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
 MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
 MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
 MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
 MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
 

Modules/_testinternalcapi/tokenizer.c

@@ -1,6 +1,6 @@
 #include "parts.h"
 
-#include "../../Parser/tokenizer/cursor.h"
+#include "../../Parser/tokenizer/source.h"
 
 static int
 check(int condition, const char *message)
@@ -23,97 +23,31 @@ check_system_error(int failed, const char *message)
     return 0;
 }
 
-static int
-same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right)
-{
-    return left->source == right->source &&
-        left->pos == right->pos &&
-        left->line_start == right->line_start &&
-        left->line_end == right->line_end &&
-        left->lineno == right->lineno;
-}
-
 static PyObject *
 test_tokenizer_source(PyObject *Py_UNUSED(module),
                       PyObject *Py_UNUSED(args))
 {
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
 
-    _PyTok_Loc loc;
-    _PyTok_Line line;
-    if (check(_PyTok_SourceLocation(
-                  &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0,
-              "cannot locate empty source") < 0 ||
-            check(loc.lineno == 1 && loc.byte_col == 0,
-                  "wrong empty source location") < 0 ||
-            check(_PyTok_SourceLine(&source, 1, &line) == 0,
-                  "cannot find empty source line") < 0 ||
-            check(line.start == 0 && line.end == 0,
-                  "wrong empty source line") < 0 ||
-            check_system_error(
-                _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
-                "accepted empty source line") < 0 ||
+    if (check_system_error(
+            _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
+            "accepted empty source line") < 0 ||
             check_system_error(
                 _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
                 "accepted multiple source lines") < 0 ||
             check_system_error(
                 _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
-                "accepted missing implicit newline") < 0) {
-        goto error;
-    }
-
-    if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0,
-              "wrong first source offset") < 0 ||
+                "accepted missing implicit newline") < 0 ||
+            check(_PyTok_SourceAppendLine(
+                      &source, "alpha\n", 6, 0) == 0,
+                  "wrong first source offset") < 0 ||
             check(_PyTok_SourceAppendLine(
                       &source, "\xce\xb2\n", 3, 1) == 6,
                   "wrong second source offset") < 0 ||
-            check(_PyTok_SourceAppendLine(
-                      &source, "nul\0x\n", 6, 0) == 9,
-                  "wrong third source offset") < 0) {
-        goto error;
-    }
-
-    int marker_line = 257;
-    int final_line = 300;
-    _PyTok_Off marker_start = -1;
-    for (int lineno = 4; lineno <= final_line; lineno++) {
-        const char *text = lineno == marker_line ? "marker\n" : "x\n";
-        Py_ssize_t len = (Py_ssize_t)strlen(text);
-        _PyTok_Off start = _PyTok_SourceAppendLine(
-            &source, text, len, lineno == final_line);
-        if (start < 0) {
-            goto error;
-        }
-        if (lineno == marker_line) {
-            marker_start = start;
-        }
-    }
-
-    if (check(source.nlines == final_line, "wrong source line count") < 0 ||
-            check(_PyTok_SourceLine(&source, marker_line, &line) == 0,
-                  "cannot find late source line") < 0 ||
-            check(line.start == marker_start &&
-                      line.end == marker_start + 7,
-                  "wrong late source line") < 0 ||
-            check(!line.implicit_newline && !line.contains_nul,
-                  "wrong late source flags") < 0 ||
-            check(_PyTok_SourceLine(&source, 2, &line) == 0,
-                  "cannot find second source line") < 0 ||
-            check(line.start == 6 && line.end == 9 &&
-                      line.implicit_newline && !line.contains_nul,
-                  "wrong second source line") < 0 ||
             check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
                       _PyTok_SourceLineIsImplicit(&source, 2),
-                  "wrong early implicit newline flags") < 0 ||
-            check(_PyTok_SourceLine(&source, 3, &line) == 0,
-                  "cannot find third source line") < 0 ||
-            check(line.contains_nul, "missing null byte flag") < 0 ||
-            check(_PyTok_SourceLine(&source, final_line, &line) == 0,
-                  "cannot find final source line") < 0 ||
-            check(line.implicit_newline &&
-                      _PyTok_SourceLineIsImplicit(&source, final_line),
-                  "missing late implicit newline flag") < 0) {
+                  "wrong implicit newline flags") < 0) {
         goto error;
     }
 
@@ -123,217 +57,19 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
     if (check(view != NULL && view_len == 2 &&
                   memcmp(view, "\xce\xb2", 2) == 0,
               "wrong source span view") < 0 ||
-            check(_PyTok_SourceLocation(
-                      &source, marker_start,
-                      _PYTOK_AFFINITY_LEFT, &loc) == 0,
-                  "cannot locate left line boundary") < 0 ||
-            check(loc.lineno == marker_line - 1 && loc.byte_col == 2,
-                  "wrong left boundary location") < 0 ||
-            check(_PyTok_SourceLocation(
-                      &source, marker_start,
-                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
-                  "cannot locate right line boundary") < 0 ||
-            check(loc.lineno == marker_line && loc.byte_col == 0,
-                  "wrong right boundary location") < 0 ||
-            check(_PyTok_SourceLocation(
-                      &source, marker_start + 1,
-                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
-                  "cannot locate late source byte") < 0 ||
-            check(loc.lineno == marker_line && loc.byte_col == 1,
-                  "wrong late source location") < 0) {
-        goto error;
-    }
-
-    if (check(_PyTok_SourceLocation(
-                  &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0,
-              "cannot locate left EOF") < 0 ||
-            check(loc.lineno == final_line && loc.byte_col == 2,
-                  "wrong left EOF location") < 0 ||
-            check(_PyTok_SourceLocation(
-                      &source, source.len,
-                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
-                  "cannot locate right EOF") < 0 ||
-            check(loc.lineno == final_line + 1 && loc.byte_col == 0,
-                  "wrong right EOF location") < 0 ||
-            check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0,
-                  "cannot find virtual EOF line") < 0 ||
-            check(line.start == source.len && line.end == source.len,
-                  "wrong virtual EOF line") < 0 ||
-            check(!_PyTok_SourceLineIsImplicit(&source, 0) &&
-                      !_PyTok_SourceLineIsImplicit(
-                          &source, final_line + 1),
-                  "virtual or invalid line is implicit") < 0) {
-        goto error;
-    }
-
-    view = _PyTok_SourceSpanView(
-        &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len);
-    if (check_system_error(view == NULL, "accepted invalid source span") < 0 ||
             check_system_error(
-                _PyTok_SourceLocation(
-                    &source, source.len + 1,
-                    _PYTOK_AFFINITY_RIGHT, &loc) < 0,
-                "accepted invalid source offset") < 0 ||
-            check_system_error(
-                _PyTok_SourceLine(&source, final_line + 2, &line) < 0,
-                "accepted invalid source line") < 0) {
+                _PyTok_SourceSpanView(
+                    &source, _PyTok_SpanFromBounds(0, source.len + 1),
+                    &view_len) == NULL,
+                "accepted invalid source span") < 0) {
         goto error;
     }
 
     _PyTok_SourceClear(&source);
-    _PyTok_SourceInit(&source);
     if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
             check_system_error(
                 _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
-                "appended after unterminated source line") < 0 ||
-            check(_PyTok_SourceLocation(
-                      &source, source.len,
-                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
-                  "cannot locate unterminated EOF") < 0 ||
-            check(loc.lineno == 1 && loc.byte_col == 4,
-                  "wrong unterminated EOF location") < 0) {
-        goto error;
-    }
-
-    _PyTok_SourceDiscard(&source);
-    if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4,
-              "wrong retained source offset") < 0 ||
-            _PyTok_SourceLine(&source, 1, &line) < 0 ||
-            check(line.start == 4 && line.end == 6,
-                  "wrong retained source line") < 0 ||
-            _PyTok_SourceLocation(
-                &source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 ||
-            check(loc.lineno == 1 && loc.byte_col == 0,
-                  "wrong retained source location") < 0) {
-        goto error;
-    }
-    view = _PyTok_SourceSpanView(
-        &source, _PyTok_SpanFromBounds(4, 5), &view_len);
-    if (check(view != NULL && view_len == 1 && view[0] == 'a',
-              "wrong retained source span") < 0 ||
-            check_system_error(_PyTok_SourceSpanView(
-                &source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL,
-                "accepted discarded source span") < 0) {
-        goto error;
-    }
-
-    _PyTok_SourceClear(&source);
-    Py_RETURN_NONE;
-
-error:
-    _PyTok_SourceClear(&source);
-    return NULL;
-}
-
-static PyObject *
-test_tokenizer_cursor(PyObject *Py_UNUSED(module),
-                      PyObject *Py_UNUSED(args))
-{
-    _PyTok_SourceText source;
-    _PyTok_SourceInit(&source);
-    if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
-            _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) {
-        goto error;
-    }
-
-    _PyTok_Cursor cursor;
-    _PyTok_CursorInit(&cursor, &source);
-    if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 ||
-            check(cursor.lineno == 3 && cursor.pos == source.len,
-                  "wrong cursor at virtual EOF") < 0 ||
-            _PyTok_CursorSetLine(&cursor, 1) < 0) {
-        goto error;
-    }
-
-    char large[BUFSIZ + 1];
-    memset(large, 'z', sizeof(large));
-    large[sizeof(large) - 1] = '\n';
-    if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) {
-        goto error;
-    }
-
-    if (check(_PyTok_CursorPeek(&cursor, 0) == 'a',
-              "wrong cursor peek after relocation") < 0 ||
-            check(_PyTok_CursorPeek(&cursor, 1) == 'b',
-                  "wrong distant cursor peek") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == 'a',
-                  "wrong first cursor byte") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == 'b',
-                  "wrong second cursor byte") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == '\n',
-                  "wrong final cursor byte") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == EOF,
-                  "cursor advanced past line") < 0 ||
-            check(_PyTok_CursorSetOffset(&cursor, 2) == 0,
-                  "cannot seek cursor offset") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == '\n',
-                  "wrong cursor byte after seek") < 0 ||
-            check(_PyTok_CursorSetOffset(&cursor, 3) == 0,
-                  "cannot seek line boundary") < 0 ||
-            check(cursor.lineno == 2 && cursor.line_start == 3 &&
-                      _PyTok_CursorAdvance(&cursor) == 'c',
-                  "wrong cursor at line boundary") < 0 ||
-            check(_PyTok_CursorSetLine(&cursor, 3) == 0,
-                  "cannot advance cursor to final line") < 0 ||
-            check(cursor.line_start == 6 &&
-                      _PyTok_CursorAdvance(&cursor) == 'z',
-                  "wrong cursor byte on final line") < 0) {
-        goto error;
-    }
-
-    _PyTok_Cursor saved = cursor;
-    if (check_system_error(
-            _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0,
-            "accepted invalid cursor offset") < 0 ||
-            check(same_cursor(&cursor, &saved),
-                  "invalid offset changed cursor") < 0 ||
-            check_system_error(
-                _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0,
-                "accepted invalid cursor line") < 0 ||
-            check(same_cursor(&cursor, &saved),
-                  "invalid line changed cursor") < 0 ||
-            check(_PyTok_CursorSetOffset(&cursor, source.len) == 0,
-                  "cannot set cursor to EOF") < 0 ||
-            check(cursor.lineno == 4 && cursor.pos == source.len,
-                  "wrong cursor at EOF") < 0) {
-        goto error;
-    }
-
-#if SIZEOF_VOID_P > 4
-    char byte = 0;
-    _PyTok_SourceText huge_source = {
-        .bytes = &byte,
-        .len = (_PyTok_Off)INT_MAX + 1,
-    };
-    _PyTok_Cursor huge_cursor = {
-        .source = &huge_source,
-        .pos = INT_MAX,
-        .line_end = (_PyTok_Off)INT_MAX + 1,
-        .lineno = 1,
-    };
-    if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF &&
-                  huge_cursor.pos == INT_MAX,
-              "cursor advanced past maximum column") < 0) {
-        goto error;
-    }
-#endif
-
-    _PyTok_Off base = source.len;
-    _PyTok_SourceDiscard(&source);
-    if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
-            _PyTok_SourceAppendLine(&source, "cd", 2, 0) < 0) {
-        goto error;
-    }
-    _PyTok_CursorInit(&cursor, &source);
-    if (_PyTok_CursorSetLine(&cursor, 1) < 0 ||
-            check(cursor.pos == base && _PyTok_CursorPeek(&cursor, 1) == 'b',
-                  "wrong retained cursor line") < 0 ||
-            _PyTok_CursorSetLine(&cursor, 2) < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == 'c',
-                  "wrong retained cursor byte") < 0 ||
-            _PyTok_CursorSetOffset(&cursor, base + 5) < 0 ||
-            check(cursor.lineno == 2 && _PyTok_CursorAdvance(&cursor) == EOF,
-                  "wrong retained cursor EOF") < 0) {
+                "appended after unterminated source line") < 0) {
         goto error;
     }
 
@@ -410,7 +146,6 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
 
 static PyMethodDef test_methods[] = {
     {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS},
-    {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS},
     {"test_tokenizer_source_discard", test_tokenizer_source_discard, METH_NOARGS},
     {NULL},
 };

PCbuild/pythoncore.vcxproj

@@ -424,7 +424,6 @@
     <ClInclude Include="..\Parser\lexer\lexer.h" />
     <ClInclude Include="..\Parser\lexer\lexer_internal.h" />
     <ClInclude Include="..\Parser\lexer\buffer.h" />
-    <ClInclude Include="..\Parser\tokenizer\cursor.h" />
     <ClInclude Include="..\Parser\tokenizer\reader.h" />
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
     <ClInclude Include="..\Parser\tokenizer\source.h" />
@@ -594,7 +593,6 @@
     <ClCompile Include="..\Parser\lexer\number.c" />
     <ClCompile Include="..\Parser\lexer\string.c" />
     <ClCompile Include="..\Parser\lexer\buffer.c" />
-    <ClCompile Include="..\Parser\tokenizer\cursor.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />
     <ClCompile Include="..\Parser\tokenizer\reader.c" />