gh-153569: borrow diagnostic lines through the source API

Author: pablogsal

Modules/_testinternalcapi/tokenizer.c

@@ -23,6 +23,17 @@ check_system_error(int failed, const char *message)
     return 0;
 }
 
+static int
+check_line_view(const _PyTok_SourceText *source, Py_ssize_t lineno,
+                const char *expected)
+{
+    Py_ssize_t len;
+    const char *line = _PyTok_SourceLineView(source, lineno, &len);
+    return check(len == (Py_ssize_t)strlen(expected) &&
+                 memcmp(line, expected, len) == 0,
+                 "wrong source line view");
+}
+
 static int
 same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right)
 {
@@ -40,6 +51,10 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
 
+    if (check_line_view(&source, 1, "") < 0) {
+        goto error;
+    }
+
     _PyTok_Loc loc;
     _PyTok_Line line;
     if (check(_PyTok_SourceLocation(
@@ -67,10 +82,20 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
               "wrong first source offset") < 0 ||
             check(_PyTok_SourceAppendLine(
                       &source, "\xce\xb2\n", 3, 1) == 6,
-                  "wrong second source offset") < 0 ||
-            check(_PyTok_SourceAppendLine(
-                      &source, "nul\0x\n", 6, 0) == 9,
-                  "wrong third source offset") < 0) {
+                  "wrong second source offset") < 0) {
+        goto error;
+    }
+
+    if (check_line_view(&source, PY_SSIZE_T_MIN, "alpha") < 0 ||
+            check_line_view(&source, 1, "alpha") < 0 ||
+            check_line_view(&source, 2, "\xce\xb2") < 0 ||
+            check_line_view(&source, 3, "") < 0 ||
+            check_line_view(&source, PY_SSIZE_T_MAX, "") < 0) {
+        goto error;
+    }
+
+    if (check(_PyTok_SourceAppendLine(&source, "nul\0x\n", 6, 0) == 9,
+              "wrong third source offset") < 0) {
         goto error;
     }
 
@@ -195,6 +220,11 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
         goto error;
     }
 
+    if (check_line_view(&source, 1, "tail") < 0 ||
+            check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) {
+        goto error;
+    }
+
     _PyTok_SourceDiscard(&source);
     if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4,
               "wrong retained source offset") < 0 ||

Parser/tokenizer/source.c

@@ -190,6 +190,26 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
     return source->base_offset + start;
 }
 
+const char *
+_PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno,
+                      Py_ssize_t *len)
+{
+    assert(len != NULL);
+    const char *line = _PyTok_SourceData(source);
+    const char *end = line + source->len;
+    while (lineno > 1) {
+        const char *newline = memchr(line, '\n', end - line);
+        if (newline == NULL) {
+            break;
+        }
+        line = newline + 1;
+        lineno--;
+    }
+    const char *newline = memchr(line, '\n', end - line);
+    *len = (newline != NULL ? newline : end) - line;
+    return line;
+}
+
 const char *
 _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
                       Py_ssize_t *len)

Parser/tokenizer/source.h

@@ -49,6 +49,12 @@ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *);
 PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
     _PyTok_SourceText *source, const char *bytes, Py_ssize_t len,
     int implicit_newline);
+/* Return borrowed bytes excluding '\n', writing the byte length to *len.
+   Line numbers are 1-based and clamp to the first or final line; a trailing
+   '\n' adds an empty final line. The view need not be NUL-terminated.
+   This does not set an exception. Append, discard, and clear invalidate the view. */
+PyAPI_FUNC(const char *) _PyTok_SourceLineView(
+    const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len);
 /* The returned view is invalidated by SourceAppendLine and SourceClear. */
 PyAPI_FUNC(const char *) _PyTok_SourceSpanView(
     const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *);