From 1a703ab9e6a050b40c469788ec7758713c6bf62b Mon Sep 17 00:00:00 2001 From: Pengyu Lee Date: Sun, 13 Sep 2026 08:09:22 +0800 Subject: [PATCH 1/6] gh-154719: Preserve trailing whitespace in t-string interpolation expressions (#154762) --- Lib/test/test_annotationlib.py | 2 +- Lib/test/test_fstring.py | 8 ++ Lib/test/test_tstring.py | 78 ++++++++++++++++++- Lib/test/test_unparse.py | 9 +++ ...-07-27-17-20-49.gh-issue-154719.eI88Gs.rst | 5 ++ Parser/action_helpers.c | 33 ++++++-- 6 files changed, 124 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 7bdb4a9993f4e33..a7d3bca7402bf22 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1869,7 +1869,7 @@ def nested(): self.assertEqual(type_repr(t'''{ 0 & 1 | 2 - }'''), 't"""{ 0\n & 1\n | 2}"""') + }'''), 't"""{ 0\n & 1\n | 2\n }"""') self.assertEqual( type_repr(Template("hi", Interpolation(42, "42"))), "t'hi{42}'" ) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 201ae6e24794933..2d6320549b03f62 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1695,6 +1695,14 @@ def __repr__(self): self.assertEqual(f'''{f"{d["a#b"]}"=}''', 'f"{d["a#b"]}"=\'42\'') + result = f'''{( + 1, # Force lexer metadata reconstruction. + "\"#")=}''' + self.assertEqual( + result, + '(\n 1, \n "\\"#")=(1, \'"#\')', + ) + self.assertEqual(f'{ # some comment goes here """hello"""=}', ' \n """hello"""=\'hello\'') self.assertEqual(f'{"""# this is not a comment diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index 854860b5ea43065..67a8e0fc6bcffb1 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -136,10 +136,47 @@ def test_debug_specifier(self): # Test white space in debug specifier t = t"Value: {value = }" self.assertTStringEqual( - t, ("Value: value = ", ""), [(value, "value", "r")] + t, ("Value: value = ", ""), [(value, "value ", "r")] ) self.assertEqual(fstring(t), "Value: value = 42") + # Explicit line continuations after the debug marker are part of + # the debug text, not the interpolation expression. + for template, strings, interpolation, rendered in ( + ( + t"""Value: {value =\ +}""", + ("Value: value =\\\n", ""), + (value, "value ", "r"), + "Value: value =\\\n42", + ), + ( + t"""Value: {value =\ +!r}""", + ("Value: value =\\\n", ""), + (value, "value ", "r"), + "Value: value =\\\n42", + ), + ( + t"""Value: {value =\ +:04}""", + ("Value: value =\\\n", ""), + (value, "value ", None, "04"), + "Value: value =\\\n0042", + ), + ( + t"""Value: {value =\ +\ +}""", + ("Value: value =\\\n\\\n", ""), + (value, "value ", "r"), + "Value: value =\\\n\\\n42", + ), + ): + with self.subTest(template=template): + self.assertTStringEqual(template, strings, [interpolation]) + self.assertEqual(fstring(template), rendered) + class C: def __format__(self, spec): return f"FORMAT-{spec}" @@ -149,6 +186,41 @@ def __format__(self, spec): self.assertEqual(t.interpolations[0].format_spec, "FORMAT-value=42") + def test_interpolation_expression_whitespace(self): + x = 42 + for template, expected in ( + (t"{x}", "x"), + (t"{x }", "x "), + (t"{ x}", " x"), + (t"{ x }", " x "), + (t"{ x }", " x "), + (t"""{ + x +}""", "\n x\n"), + (t"{ x !r}", " x "), + (t"{ x :.2f}", " x "), + (t"{ x = }", " x "), + (t"{ x = !r}", " x "), + (t"{ x = :.2f}", " x "), + (t"{x == 42 = }", "x == 42 "), + ): + with self.subTest(template=template): + self.assertEqual( + template.interpolations[0].expression, + expected, + ) + + def test_interpolation_expression_with_reconstructed_metadata(self): + regular = t'''{( + 1, # Force lexer metadata reconstruction. + "\"#")}''' + debug = t'''{( + 1, # Force lexer metadata reconstruction. + "\"#")=}''' + expected = '(\n 1, \n "\\"#")' + self.assertEqual(regular.interpolations[0].expression, expected) + self.assertEqual(debug.interpolations[0].expression, expected) + def test_raw_tstrings(self): path = r"C:\Users" t = rt"{path}\Documents" @@ -314,12 +386,12 @@ def test_triple_quoted(self): t = t'{"""a""""#" # outside }' - self.assertEqual(t.interpolations[0].expression, '"""a""""#"') + self.assertEqual(t.interpolations[0].expression, '"""a""""#" \n') x, y = 1, 2 t = t'{x != y # outside }' - self.assertEqual(t.interpolations[0].expression, 'x != y') + self.assertEqual(t.interpolations[0].expression, 'x != y \n') d = {'a#b': 42} t = t'''{f"{d["a#b"]}"}''' diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index dcaad49ffab5d26..28faede2e2a1a5c 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -216,6 +216,15 @@ def test_tstrings(self): self.check_ast_roundtrip('t""') self.check_ast_roundtrip("t'{(lambda x: x)}'") self.check_ast_roundtrip("t'{t'{x}'}'") + self.check_ast_roundtrip( + r"""t'''{( + 1, # Force lexer metadata reconstruction. + "\"#")}'''""" + ) + self.check_ast_roundtrip( + r'''t"""Value: {value =\ +}"""''' + ) def test_tstring_with_nonsensical_str_field(self): # `value` suggests that the original code is `t'{test1}`, but `str` suggests otherwise diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst new file mode 100644 index 000000000000000..d482a5214bd751f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst @@ -0,0 +1,5 @@ +Trailing whitespace in a t-string interpolation expression is now preserved +in :attr:`string.templatelib.Interpolation.expression`, up to the closing ``}`` +or the conversion (``!``), format (``:``), or debug (``=``) delimiter. +Explicit line continuations following a debug ``=`` remain part of the debug +text and are excluded from :attr:`~string.templatelib.Interpolation.expression`. diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 3e4d463b36ab872..3d2c53f631b8b40 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1555,21 +1555,38 @@ _get_interpolation_conversion(Parser *p, Token *debug, ResultTokenWithMetadata * } static PyObject * -_strip_interpolation_expr(PyObject *exprstr) +_strip_interpolation_debug_expr(PyObject *exprstr) { Py_ssize_t len = PyUnicode_GET_LENGTH(exprstr); - for (Py_ssize_t i = len - 1; i >= 0; i--) { - Py_UCS4 c = PyUnicode_READ_CHAR(exprstr, i); - if (_PyUnicode_IsWhitespace(c) || c == '=') { + /* Discard whitespace and explicit line continuations after the debug "=" + but preserve whitespace before it. */ + while (len > 0) { + int has_newline = 0; + while (len > 0) { + Py_UCS4 c = PyUnicode_READ_CHAR(exprstr, len - 1); + if (!_PyUnicode_IsWhitespace(c)) { + break; + } + if (c == '\r' || c == '\n') { + has_newline = 1; + } len--; } - else { + if (!has_newline || len == 0 || + PyUnicode_READ_CHAR(exprstr, len - 1) != '\\') + { break; } + len--; + } + + /* Preserve unexpected metadata instead of dropping source text. */ + if (len == 0 || PyUnicode_READ_CHAR(exprstr, len - 1) != '=') { + return Py_NewRef(exprstr); } - return PyUnicode_Substring(exprstr, 0, len); + return PyUnicode_Substring(exprstr, 0, len - 1); } expr_ty _PyPegen_interpolation(Parser *p, expr_ty expression, Token *debug, ResultTokenWithMetadata *conversion, @@ -1600,7 +1617,9 @@ expr_ty _PyPegen_interpolation(Parser *p, expr_ty expression, Token *debug, Resu } assert(exprstr != NULL); - PyObject *final_exprstr = _strip_interpolation_expr(exprstr); + PyObject *final_exprstr = debug + ? _strip_interpolation_debug_expr(exprstr) + : Py_NewRef(exprstr); if (!final_exprstr || _PyArena_AddPyObject(arena, final_exprstr) < 0) { Py_XDECREF(final_exprstr); return NULL; From 12a1de1a4e22732700cf7d8fd75ebff4b6e0513d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Sep 2026 02:28:39 +0200 Subject: [PATCH 2/6] gh-156939: Detect buffer overflow in PyBytesWriter in debug mode (#156943) Reserve one byte in PyBytesWriter used as a canary byte: set it to a special value. PyBytesWriter_Finish() checks if the canary byte has been overriden to detect a buffer overflow. Add tests for this feature. Update buffer overflow check in fcntl: allocate extra guard bytes in the writer and then truncate these bytes. --- Lib/test/test_capi/test_bytes.py | 145 +++++++++++++++--- ...-09-04-16-41-07.gh-issue-156939.bKaQuE.rst | 2 + Modules/_testcapi/bytes.c | 31 ++-- Modules/fcntlmodule.c | 16 +- Objects/bytesobject.c | 80 ++++++++-- Objects/unicodeobject.c | 15 +- 6 files changed, 230 insertions(+), 59 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-09-04-16-41-07.gh-issue-156939.bKaQuE.rst diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 6c19ad14b7e6c59..a500f2c702db0fb 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -1,7 +1,9 @@ import sys +import textwrap import unittest from test import support from test.support import import_helper +from test.support.script_helper import assert_python_failure _testlimitedcapi = import_helper.import_module('_testlimitedcapi') _testcapi = import_helper.import_module('_testcapi') @@ -316,12 +318,18 @@ def test_join(self): bytes_join(b'', NULL) +def get_data_canary(writer): + size = writer.get_size() + 1 + return writer.get_data(size) + + class BaseWriterTest: RESULT_TYPE = NotImplementedError SMALL_BUFFER = 11 # bytes assert SMALL_BUFFER < _testcapi.PyBytesWriter_small_buffer LARGE_BUFFER = _testcapi.PyBytesWriter_small_buffer + 17 # bytes NEW_BYTE = b'\xff' + CANARY_BYTE = b'\xdd' def create_writer(self, alloc=0, string=b''): raise NotImplementedError @@ -344,6 +352,7 @@ def test_get_data(self): # Test PyBytesWriter_GetData() writer = self.create_writer(6) NEW_BYTE = self.NEW_BYTE + CANARY_BYTE = self.CANARY_BYTE self.assertEqual(writer.get_data(), NEW_BYTE * 6) writer.write(0, b'abc') self.assertEqual(writer.get_data(), b'abc' + NEW_BYTE * 3) @@ -357,7 +366,7 @@ def test_get_data(self): writer.write(0, b's' * small) self.assertEqual(writer.get_data(), b's' * small) writer.resize(large) - self.assertEqual(writer.get_data(), b's' * small + NEW_BYTE * (large - small)) + self.assertEqual(writer.get_data(), b's' * small + CANARY_BYTE + NEW_BYTE * (large - small - 1)) writer.write(small, b'L' * (large - small)) self.assertEqual(writer.get_data(), b's' * small + b'L' * (large - small)) @@ -443,6 +452,47 @@ def test_resize(self): writer.resize(_testcapi.PY_SSIZE_T_MAX) self.assertEqual(writer.finish(), b'x' * size) + @unittest.skipUnless(support.Py_DEBUG, 'need debug build') + def test_resize_canary(self): + CANARY_BYTE = self.CANARY_BYTE + for size in (self.SMALL_BUFFER, self.LARGE_BUFFER): + with self.subTest(size=size): + # Truncate the last byte + data = b'x' * size + writer = self.create_writer(size) + writer.write(0, data) + self.assertEqual(get_data_canary(writer), data + CANARY_BYTE) + writer.resize(size - 1) + self.assertEqual(get_data_canary(writer), data[:-1] + CANARY_BYTE) + self.assertEqual(writer.finish(), data[:-1]) + + # Make the buffer empty + writer = self.create_writer(size) + writer.write(0, data) + writer.resize(0) + self.assertEqual(writer.get_data(), b'') + self.assertEqual(writer.finish(), b'') + + @support.nomemtest + def test_resize_error(self): + # Test PyBytesWriter_Resize() error + init = b'x' * self.LARGE_BUFFER + writer = self.create_writer(len(init)) + writer.write(0, init) + size = len(init) + 100 + try: + with self.assertRaises(MemoryError): + _testcapi.set_nomemory(0) + writer.resize(size) + finally: + _testcapi.remove_mem_hooks() + suffix = b'still working' + writer.write_bytes(suffix, -1) + self.assertEqual(writer.finish(), init + suffix) + + # Note: PyBytesWriter_Resize() leaves the buffer unchanged (no resize) + # if the new size is smaller than the allocated size + def test_grow(self): # Test PyBytesWriter_Grow() writer = self.create_writer(0) @@ -461,24 +511,6 @@ def test_grow(self): writer.grow(0) # noop self.assertEqual(writer.finish(), b'number=123') - for size in (self.SMALL_BUFFER, self.LARGE_BUFFER): - with self.subTest(size=size): - # Truncate the last byte - data = b'x' * size - writer = self.create_writer(size) - writer.write(0, data) - self.assertEqual(writer.get_data(), data) - writer.grow(-1) - self.assertEqual(writer.get_data(), data[:-1]) - self.assertEqual(writer.finish(), data[:-1]) - - # Make the buffer empty - writer = self.create_writer(size) - writer.write(0, data) - writer.grow(-size) - self.assertEqual(writer.get_data(), b'') - self.assertEqual(writer.finish(), b'') - # Switch from small buffer to large buffer writer = self.create_writer() small, large = self.SMALL_BUFFER, self.LARGE_BUFFER @@ -500,25 +532,45 @@ def test_grow(self): writer.grow(_testcapi.PY_SSIZE_T_MAX) self.assertEqual(writer.finish(), b'x' * size) + @unittest.skipUnless(support.Py_DEBUG, 'need debug build') + def test_grow_canary(self): + CANARY_BYTE = self.CANARY_BYTE + for size in (self.SMALL_BUFFER, self.LARGE_BUFFER): + with self.subTest(size=size): + # Truncate the last byte + data = b'x' * size + writer = self.create_writer(size) + writer.write(0, data) + self.assertEqual(get_data_canary(writer), data + CANARY_BYTE) + writer.grow(-1) + self.assertEqual(get_data_canary(writer), data[:-1] + CANARY_BYTE) + self.assertEqual(writer.finish(), data[:-1]) + + # Make the buffer empty + writer = self.create_writer(size) + writer.write(0, data) + writer.grow(-size) + self.assertEqual(writer.get_data(), b'') + self.assertEqual(writer.finish(), b'') + @support.nomemtest - def test_resize_error(self): - # Test PyBytesWriter_Resize() error + def test_grow_error(self): + # Test PyBytesWriter_Grow() error init = b'x' * self.LARGE_BUFFER writer = self.create_writer(len(init)) writer.write(0, init) - size = len(init) + 100 try: with self.assertRaises(MemoryError): _testcapi.set_nomemory(0) - writer.resize(size) + writer.grow(100) finally: _testcapi.remove_mem_hooks() suffix = b'still working' writer.write_bytes(suffix, -1) self.assertEqual(writer.finish(), init + suffix) - # Note: PyBytesWriter_Resize() leaves the buffer unchanged (no resize) - # if the new size is smaller than the allocated size + # Note: PyBytesWriter_Grow() leaves the buffer unchanged (no resize) + # if grow is negative. def test_format_i(self): # Test PyBytesWriter_Format() @@ -531,6 +583,49 @@ def test_format_i(self): writer.format_i(b'y=%i', 456) self.assertEqual(writer.finish(), b'x=123, y=456') + @unittest.skipUnless(support.Py_DEBUG, 'need a Python debug build') + def test_canary_byte(self): + small_buffer = _testcapi.PyBytesWriter_small_buffer + large_size = small_buffer * 10 + use_bytearray = (self.RESULT_TYPE == bytearray) + + # Test small buffer and large buffer + for size in (0, self.SMALL_BUFFER, self.LARGE_BUFFER): + with self.subTest(size=size): + code = textwrap.dedent(f""" + from test.support import SuppressCrashReport + import _testcapi + size = {size} + # Add an extra '#' byte to trigger a buffer overflow + data = b'x' * size + b'#' + use_bytearray = {use_bytearray} + writer = _testcapi.PyBytesWriter(size, use_bytearray) + with SuppressCrashReport(): + writer.write(0, data, check=False) + writer.finish() + """) + proc = assert_python_failure('-c', code) + self.assertIn(b'Buffer overflow detected in PyBytesWriter', + proc.err) + self.assertIn(f'at position {size}'.encode(), + proc.err) + + @unittest.skipUnless(support.Py_DEBUG, 'need debug build') + def test_get_data_canary(self): + # Test PyBytesWriter_GetData() + NEW_BYTE = self.NEW_BYTE + CANARY_BYTE = self.CANARY_BYTE + + writer = self.create_writer(6) + self.assertEqual(get_data_canary(writer), + NEW_BYTE * 6 + CANARY_BYTE) + writer.write(0, b'abc') + self.assertEqual(get_data_canary(writer), + b'abc' + NEW_BYTE * 3 + CANARY_BYTE) + writer.write(3, b'123') + self.assertEqual(get_data_canary(writer), + b'abc123' + CANARY_BYTE) + class BytesWriterTest(BaseWriterTest, unittest.TestCase): RESULT_TYPE = bytes diff --git a/Misc/NEWS.d/next/C_API/2026-09-04-16-41-07.gh-issue-156939.bKaQuE.rst b/Misc/NEWS.d/next/C_API/2026-09-04-16-41-07.gh-issue-156939.bKaQuE.rst new file mode 100644 index 000000000000000..57c9a0ab46e9046 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-04-16-41-07.gh-issue-156939.bKaQuE.rst @@ -0,0 +1,2 @@ +When Python is built in debug mode, :c:type:`PyBytesWriter` now detects +buffer overflow. Patch by Victor Stinner. diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index 83249a21c5a3f22..b4468dff0d0ba0d 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -135,22 +135,29 @@ writer_check(WriterObject *self) static PyObject* -writer_write(PyObject *self_raw, PyObject *args) +writer_write(PyObject *self_raw, PyObject *args, PyObject *kwargs) { WriterObject *self = (WriterObject *)self_raw; if (writer_check(self) < 0) { return NULL; } + static char *kwlist[] = {"pos", "str", "check", NULL}; Py_ssize_t pos, size; char *str; - if (!PyArg_ParseTuple(args, "ny#", &pos, &str, &size)) { + int check = 1; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "ny#|i", kwlist, + &pos, &str, &size, &check)) { return NULL; } - if (pos < 0 || (pos + size) > PyBytesWriter_GetSize(self->writer)) { - PyErr_SetString(PyExc_ValueError, "invalid position or size"); - return NULL; + // Use check=0 to trigger a buffer overflow for example + if (check) { + if (pos < 0 || (pos + size) > PyBytesWriter_GetSize(self->writer)) { + PyErr_SetString(PyExc_ValueError, "invalid position or size"); + return NULL; + } } char *data = PyBytesWriter_GetData(self->writer); @@ -168,7 +175,7 @@ writer_write_bytes(PyObject *self_raw, PyObject *args) return NULL; } - char *bytes; + const char *bytes; Py_ssize_t unused_size, size; if (!PyArg_ParseTuple(args, "y#n", &bytes, &unused_size, &size)) { return NULL; @@ -245,15 +252,19 @@ writer_grow(PyObject *self_raw, PyObject *args) static PyObject* -writer_get_data(PyObject *self_raw, PyObject *Py_UNUSED(args)) +writer_get_data(PyObject *self_raw, PyObject *args) { WriterObject *self = (WriterObject *)self_raw; if (writer_check(self) < 0) { return NULL; } - const char *data = PyBytesWriter_GetData(self->writer); Py_ssize_t size = PyBytesWriter_GetSize(self->writer); + if (!PyArg_ParseTuple(args, "|n", &size)) { + return NULL; + } + + const char *data = PyBytesWriter_GetData(self->writer); return PyBytes_FromStringAndSize(data, size); } @@ -305,12 +316,12 @@ writer_finish_with_size(PyObject *self_raw, PyObject *args) static PyMethodDef writer_methods[] = { - {"write", _PyCFunction_CAST(writer_write), METH_VARARGS}, + {"write", _PyCFunction_CAST(writer_write), METH_VARARGS | METH_KEYWORDS}, {"write_bytes", _PyCFunction_CAST(writer_write_bytes), METH_VARARGS}, {"format_i", _PyCFunction_CAST(writer_format_i), METH_VARARGS}, {"resize", _PyCFunction_CAST(writer_resize), METH_VARARGS}, {"grow", _PyCFunction_CAST(writer_grow), METH_VARARGS}, - {"get_data", _PyCFunction_CAST(writer_get_data), METH_NOARGS}, + {"get_data", _PyCFunction_CAST(writer_get_data), METH_VARARGS}, {"get_size", _PyCFunction_CAST(writer_get_size), METH_NOARGS}, {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, {"finish_with_size", _PyCFunction_CAST(writer_finish_with_size), METH_VARARGS}, diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index e6a40ffc5a26144..5dd3df9bb408f0c 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -121,13 +121,14 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) return PyBytes_FromStringAndSize(buf, len); } else { - PyBytesWriter *writer = PyBytesWriter_Create(len); + PyBytesWriter *writer = PyBytesWriter_Create(len + GUARDSZ); if (writer == NULL) { PyBuffer_Release(&view); return NULL; } char *ptr = PyBytesWriter_GetData(writer); memcpy(ptr, view.buf, len); + memcpy(ptr + len, guard, GUARDSZ); PyBuffer_Release(&view); do { @@ -142,7 +143,7 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) PyBytesWriter_Discard(writer); return NULL; } - if (ptr[len] != '\0') { + if (memcmp(ptr + len, guard, GUARDSZ) != 0) { PyErr_SetString(PyExc_SystemError, "Memory corruption in fcntl() due to " "buffer overflow. " @@ -151,7 +152,8 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) PyBytesWriter_Discard(writer); return NULL; } - return PyBytesWriter_Finish(writer); + // Truncate the trailing guard bytes + return PyBytesWriter_FinishWithSize(writer, len); } #undef FCNTL_BUFSZ } @@ -316,13 +318,14 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, return PyBytes_FromStringAndSize(buf, len); } else { - PyBytesWriter *writer = PyBytesWriter_Create(len); + PyBytesWriter *writer = PyBytesWriter_Create(len + GUARDSZ); if (writer == NULL) { PyBuffer_Release(&view); return NULL; } char *ptr = PyBytesWriter_GetData(writer); memcpy(ptr, view.buf, len); + memcpy(ptr + len, guard, GUARDSZ); PyBuffer_Release(&view); do { @@ -337,7 +340,7 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, PyBytesWriter_Discard(writer); return NULL; } - if (ptr[len] != '\0') { + if (memcmp(ptr + len, guard, GUARDSZ) != 0) { PyErr_SetString(PyExc_SystemError, "Memory corruption in ioctl() due to " "buffer overflow. " @@ -346,7 +349,8 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, PyBytesWriter_Discard(writer); return NULL; } - return PyBytesWriter_Finish(writer); + // Truncate the trailing guard bytes + return PyBytesWriter_FinishWithSize(writer, len); } #undef IOCTL_BUFSZ } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 080b53e088796f0..cf6b66d4dcd572a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3614,8 +3614,13 @@ _PyBytes_RepeatBuffer(char* dest, Py_ssize_t len_dest, // --- PyBytesWriter API ----------------------------------------------------- +// Byte pattern to fill newly allocated bytes #define PyBytesWrite_NEW_BYTE 0xff +// Use a value different than NUL (0) to be able to detect overflow writing +// one extra NUL byte which is a common error. +#define PyBytesWriter_CANARY_BYTE PYMEM_DEADBYTE + static inline char* byteswriter_data(PyBytesWriter *writer) { @@ -3627,7 +3632,8 @@ static inline Py_ssize_t byteswriter_allocated(PyBytesWriter *writer) { if (writer->obj == NULL) { - return sizeof(writer->small_buffer); + // Reserve the last byte for the canary byte + return sizeof(writer->small_buffer) - 1; } else if (writer->use_bytearray) { return PyByteArray_GET_SIZE(writer->obj); @@ -3638,6 +3644,30 @@ byteswriter_allocated(PyBytesWriter *writer) } +#ifdef Py_DEBUG +static void +byteswriter_check_canary_byte(PyBytesWriter *writer) +{ + const unsigned char *data = (const unsigned char*)byteswriter_data(writer); + unsigned char canary = data[writer->size]; + if (canary != PyBytesWriter_CANARY_BYTE) { + _Py_FatalErrorFormat(__func__, + "Buffer overflow detected in PyBytesWriter %p " + "at position %zd", + writer, writer->size); + } +} + + +static void +byteswriter_write_canary_byte(PyBytesWriter *writer) +{ + unsigned char *data = (unsigned char*)byteswriter_data(writer); + data[writer->size] = PyBytesWriter_CANARY_BYTE; +} +#endif + + #ifdef MS_WINDOWS /* On Windows, overallocate by 50% is the best factor */ # define OVERALLOCATE_FACTOR 2 @@ -3748,6 +3778,7 @@ byteswriter_create(Py_ssize_t size, int use_bytearray) #ifdef Py_DEBUG memset(byteswriter_data(writer), PyBytesWrite_NEW_BYTE, byteswriter_allocated(writer)); + byteswriter_write_canary_byte(writer); #endif return writer; } @@ -3793,6 +3824,19 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) goto error; } +#ifdef Py_DEBUG + // Check for buffer overflow + byteswriter_check_canary_byte(writer); + + if (writer->obj != NULL) { + // byteswriter_write_canary_byte() can override the trailing NUL byte. + // So reset the trailing NUL byte to NUL. + Py_ssize_t allocated = byteswriter_allocated(writer); + char *data = byteswriter_data(writer); + data[allocated] = '\0'; + } +#endif + PyObject *result; if (size == 0) { result = bytes_get_empty(); @@ -3869,21 +3913,24 @@ PyBytesWriter_GetSize(PyBytesWriter *writer) int -PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) +PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t new_size) { - if (size < 0) { + if (new_size < 0) { PyErr_SetString(PyExc_ValueError, "size must be >= 0"); return -1; } - if (writer->size < size) { - if (byteswriter_resize(writer, size, 1) < 0) { + if (writer->size < new_size) { + if (byteswriter_resize(writer, new_size, 1) < 0) { return -1; } } else { // The buffer is already large enough. Never shrink the buffer. } - writer->size = size; + writer->size = new_size; +#ifdef Py_DEBUG + byteswriter_write_canary_byte(writer); +#endif return 0; } @@ -3908,24 +3955,30 @@ PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) return 0; } - if (grow >= 0) { + if (grow > 0) { if (grow > PY_SSIZE_T_MAX - writer->size) { PyErr_NoMemory(); return -1; } + Py_ssize_t new_size = writer->size + grow; + + if (byteswriter_resize(writer, new_size, 1) < 0) { + return -1; + } + writer->size = new_size; } else { if (writer->size + grow < 0) { PyErr_SetString(PyExc_ValueError, "invalid size"); return -1; } + // The buffer is already large enough. Never shrink the buffer. + writer->size = writer->size + grow; } - Py_ssize_t size = writer->size + grow; - if (byteswriter_resize(writer, size, 1) < 0) { - return -1; - } - writer->size = size; +#ifdef Py_DEBUG + byteswriter_write_canary_byte(writer); +#endif return 0; } @@ -3991,5 +4044,8 @@ _PyBytesWriter_ResizeToAllocated(PyBytesWriter *writer) { Py_ssize_t allocated = byteswriter_allocated(writer); writer->size = allocated; +#ifdef Py_DEBUG + byteswriter_write_canary_byte(writer); +#endif return allocated; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e86291347c75be0..86b9baadd0d8aa9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -796,6 +796,8 @@ backslashreplace(PyBytesWriter *writer, char *str, } size += incr; } + /* subtract preallocated bytes */ + size -= (collend - collstart); str = PyBytesWriter_GrowAndUpdatePointer(writer, size, str); if (str == NULL) { @@ -871,6 +873,8 @@ xmlcharrefreplace(PyBytesWriter *writer, char *str, } size += incr; } + /* subtract preallocated bytes */ + size -= (collend - collstart); str = PyBytesWriter_GrowAndUpdatePointer(writer, size, str); if (str == NULL) { @@ -7262,8 +7266,6 @@ unicode_encode_ucs1(PyObject *unicode, break; case _Py_ERROR_BACKSLASHREPLACE: - /* subtract preallocated bytes */ - writer->size -= (collend - collstart); str = backslashreplace(writer, str, unicode, collstart, collend); if (str == NULL) @@ -7272,8 +7274,6 @@ unicode_encode_ucs1(PyObject *unicode, break; case _Py_ERROR_XMLCHARREFREPLACE: - /* subtract preallocated bytes */ - writer->size -= (collend - collstart); str = xmlcharrefreplace(writer, str, unicode, collstart, collend); if (str == NULL) @@ -7314,10 +7314,13 @@ unicode_encode_ucs1(PyObject *unicode, } } else { - /* subtract preallocated bytes */ - writer->size -= newpos - collstart; /* Only overallocate the buffer if it's not the last write */ writer->overallocate = (newpos < size); + + /* subtract preallocated bytes */ + if (PyBytesWriter_Grow(writer, -(newpos - collstart)) < 0) { + goto onError; + } } const char *rep_str; From 2cd6d4bc5b3804068ba5c93fefb72601d86b1d5e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Sep 2026 03:14:17 +0200 Subject: [PATCH 3/6] gh-156939: Update UTF-8 encoder for PyBytesWriter changes (#157383) Don't modify directly the writer size, call PyBytesWriter_GrowAndUpdatePointer() instead. --- Objects/stringlib/codecs.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 6ea7ef1c9a92bf4..aa3de5e34eaaae0 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -350,8 +350,6 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, break; case _Py_ERROR_BACKSLASHREPLACE: - /* subtract preallocated bytes */ - writer->size -= max_char_size * (endpos - startpos); p = backslashreplace(writer, p, unicode, startpos, endpos); if (p == NULL) @@ -360,8 +358,6 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, break; case _Py_ERROR_XMLCHARREFREPLACE: - /* subtract preallocated bytes */ - writer->size -= max_char_size * (endpos - startpos); p = xmlcharrefreplace(writer, p, unicode, startpos, endpos); if (p == NULL) @@ -400,10 +396,15 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, } } else { - /* subtract preallocated bytes */ - writer->size -= max_char_size * (newpos - startpos); /* Only overallocate the buffer if it's not the last write */ writer->overallocate = (newpos < size); + + /* subtract preallocated bytes */ + Py_ssize_t prealloc = max_char_size * (newpos - startpos); + p = PyBytesWriter_GrowAndUpdatePointer(writer, -prealloc, p); + if (p == NULL) { + goto error; + } } const char *rep_str; From 58cdff72de89d92c27c53146bf51cb8ad2558b02 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Sep 2026 05:47:24 +0200 Subject: [PATCH 4/6] gh-156939: Detect PyBytesWriter buffer overflow earlier (#157385) Check the canary byte in all PyBytesWriter methods, not only in PyBytesWriter_Finish(). Add a discard() method to the _testcapi wrapper. --- Lib/test/test_capi/test_bytes.py | 54 +++++++++++++++++++++----------- Modules/_testcapi/bytes.c | 15 +++++++++ Objects/bytesobject.c | 41 ++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index a500f2c702db0fb..a0006ea35e21fe7 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -591,24 +591,42 @@ def test_canary_byte(self): # Test small buffer and large buffer for size in (0, self.SMALL_BUFFER, self.LARGE_BUFFER): - with self.subTest(size=size): - code = textwrap.dedent(f""" - from test.support import SuppressCrashReport - import _testcapi - size = {size} - # Add an extra '#' byte to trigger a buffer overflow - data = b'x' * size + b'#' - use_bytearray = {use_bytearray} - writer = _testcapi.PyBytesWriter(size, use_bytearray) - with SuppressCrashReport(): - writer.write(0, data, check=False) - writer.finish() - """) - proc = assert_python_failure('-c', code) - self.assertIn(b'Buffer overflow detected in PyBytesWriter', - proc.err) - self.assertIn(f'at position {size}'.encode(), - proc.err) + for operation in ( + 'writer.get_data()', + 'writer.get_size()', + f'writer.resize({size} * 2)', + f'writer.grow({size})', + 'writer.discard()', + 'writer.finish()', + ): + with self.subTest(size=size, operation=operation): + code = textwrap.dedent(f""" + from test.support import SuppressCrashReport + import os + import _testcapi + size = {size} + # Add an extra '#' byte to trigger a buffer overflow + data = b'x' * size + b'#' + use_bytearray = {use_bytearray} + writer = _testcapi.PyBytesWriter(size, use_bytearray) + with SuppressCrashReport(): + writer.write(0, data, check=False) + try: + {operation} + except: + # Ignore all exceptions + pass + # If we reached this line, the operation didn't + # detect the overflow. Exit immediatetly without + # calling the writer destructor since it can detect + # the overflow. + os._exit(0) + """) + proc = assert_python_failure('-c', code) + self.assertIn(b'Buffer overflow detected in PyBytesWriter', + proc.err) + self.assertIn(f'at position {size}'.encode(), + proc.err) @unittest.skipUnless(support.Py_DEBUG, 'need debug build') def test_get_data_canary(self): diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index b4468dff0d0ba0d..79effcad40090e0 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -315,6 +315,20 @@ writer_finish_with_size(PyObject *self_raw, PyObject *args) } +static PyObject* +writer_discard(PyObject *self_raw, PyObject *Py_UNUSED(args)) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyBytesWriter_Discard(self->writer); + self->writer = NULL; + Py_RETURN_NONE; +} + + static PyMethodDef writer_methods[] = { {"write", _PyCFunction_CAST(writer_write), METH_VARARGS | METH_KEYWORDS}, {"write_bytes", _PyCFunction_CAST(writer_write_bytes), METH_VARARGS}, @@ -325,6 +339,7 @@ static PyMethodDef writer_methods[] = { {"get_size", _PyCFunction_CAST(writer_get_size), METH_NOARGS}, {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, {"finish_with_size", _PyCFunction_CAST(writer_finish_with_size), METH_VARARGS}, + {"discard", _PyCFunction_CAST(writer_discard), METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index cf6b66d4dcd572a..117d8b56017b64a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3696,6 +3696,10 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) if (writer->obj != NULL) { if (writer->use_bytearray) { if (PyByteArray_Resize(writer->obj, size)) { +#ifdef Py_DEBUG + // bytearray can override the canary byte on error + byteswriter_write_canary_byte(writer); +#endif return -1; } } @@ -3770,6 +3774,11 @@ byteswriter_create(Py_ssize_t size, int use_bytearray) if (size >= 1) { if (byteswriter_resize(writer, size, 0) < 0) { +#ifdef Py_DEBUG + // Write the canary byte so byteswriter_check_canary_byte() + // doesn't fail in PyBytesWriter_Discard() + byteswriter_write_canary_byte(writer); +#endif PyBytesWriter_Discard(writer); return NULL; } @@ -3803,6 +3812,10 @@ PyBytesWriter_Discard(PyBytesWriter *writer) return; } +#ifdef Py_DEBUG + byteswriter_check_canary_byte(writer); +#endif + Py_XDECREF(writer->obj); _Py_FREELIST_FREE(bytes_writers, writer, PyMem_Free); } @@ -3875,6 +3888,14 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) // The function returns single byte singleton if size equals 1 result = PyBytes_FromStringAndSize(writer->small_buffer, size); } + +#ifdef Py_DEBUG + // Reset the writer, so byteswriter_check_canary_byte() doesn't fail + // in PyBytesWriter_Discard(). + writer->size = 0; + byteswriter_write_canary_byte(writer); +#endif + PyBytesWriter_Discard(writer); return result; @@ -3901,6 +3922,10 @@ PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) void* PyBytesWriter_GetData(PyBytesWriter *writer) { +#ifdef Py_DEBUG + byteswriter_check_canary_byte(writer); +#endif + return byteswriter_data(writer); } @@ -3908,6 +3933,10 @@ PyBytesWriter_GetData(PyBytesWriter *writer) Py_ssize_t PyBytesWriter_GetSize(PyBytesWriter *writer) { +#ifdef Py_DEBUG + byteswriter_check_canary_byte(writer); +#endif + return _PyBytesWriter_GetSize(writer); } @@ -3915,6 +3944,10 @@ PyBytesWriter_GetSize(PyBytesWriter *writer) int PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t new_size) { +#ifdef Py_DEBUG + byteswriter_check_canary_byte(writer); +#endif + if (new_size < 0) { PyErr_SetString(PyExc_ValueError, "size must be >= 0"); return -1; @@ -3950,6 +3983,10 @@ _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) { +#ifdef Py_DEBUG + byteswriter_check_canary_byte(writer); +#endif + if (grow == 0) { // Nothing to do return 0; @@ -4042,6 +4079,10 @@ PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) static Py_ssize_t _PyBytesWriter_ResizeToAllocated(PyBytesWriter *writer) { +#ifdef Py_DEBUG + byteswriter_check_canary_byte(writer); +#endif + Py_ssize_t allocated = byteswriter_allocated(writer); writer->size = allocated; #ifdef Py_DEBUG From 8a2db991dabe7ee5afd8385ef569f6e1f0120d6f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 13 Sep 2026 07:20:58 +0300 Subject: [PATCH 5/6] gh-90304: Fix IDLE startup failure with a broken font (#152419) On some systems a font reports a zero width for the '0' character (seen with a broken font on openSUSE). EditorWindow.set_width() divides the Text widget's pixel width by that value, so the ZeroDivisionError prevented IDLE from starting at all. set_width() now falls back to the configured width when the measured width is zero. self.width is consumed by the === RESTART === separator in the shell and by the squeezer's line wrapping. Co-authored-by: Claude Opus 4.8 --- Lib/idlelib/editor.py | 4 +++- Lib/idlelib/idle_test/test_editor.py | 13 +++++++++++++ .../2026-06-27-17-27-21.gh-issue-90304.464d22.rst | 4 ++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/IDLE/2026-06-27-17-27-21.gh-issue-90304.464d22.rst diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 2b4b95e053ba10b..5e9f6aa86e81925 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -325,7 +325,9 @@ def set_width(self): # http://www.tcl.tk/man/tcl8.6/TkCmd/text.htm#M21 zero_char_width = \ Font(text, font=text.cget('font')).measure('0') - self.width = pixel_width // zero_char_width + # Some fonts report a zero width for '0' (gh-90304). + self.width = (pixel_width // zero_char_width if zero_char_width + else text.tk.getint(text.cget('width'))) def new_callback(self, event): dirname, basename = self.io.defaultfilename() diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index 873637f67defa37..e32981091b72a6e 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -3,6 +3,7 @@ from idlelib import editor import unittest from collections import namedtuple +from unittest import mock from test.support import requires from tkinter import Tk, Text @@ -30,6 +31,18 @@ def test_init(self): self.assertEqual(e.root, self.root) e._close() + def test_set_width_zero_char_width(self): + # A zero-width '0' must not raise ZeroDivisionError (gh-90304). + e = Editor(root=self.root) + try: + with mock.patch.object(editor, 'Font') as MockFont: + MockFont.return_value.measure.return_value = 0 + e.set_width() + self.assertEqual(e.width, + e.text.tk.getint(e.text.cget('width'))) + finally: + e._close() + class GetLineIndentTest(unittest.TestCase): def test_empty_lines(self): diff --git a/Misc/NEWS.d/next/IDLE/2026-06-27-17-27-21.gh-issue-90304.464d22.rst b/Misc/NEWS.d/next/IDLE/2026-06-27-17-27-21.gh-issue-90304.464d22.rst new file mode 100644 index 000000000000000..c92c1f75b9d7a7d --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-06-27-17-27-21.gh-issue-90304.464d22.rst @@ -0,0 +1,4 @@ +Prevent IDLE from failing to start when a font reports a zero width for +the ``'0'`` character. Such a broken font caused a +:exc:`ZeroDivisionError`; the editor now falls back to the configured +width. From c947a4649a1fbadad1ca5423ed3bc5043d3cc4e4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Sep 2026 07:21:18 +0200 Subject: [PATCH 6/6] gh-155742: Use PyBytesWriter in winconsoleio.c (#157391) Replace soft deprecated _PyBytes_Resize() with PyBytesWriter. --- Modules/_io/winconsoleio.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index e96c5bd738fe0ad..bc375e3dfe7de81 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -1009,7 +1009,6 @@ _io__WindowsConsoleIO_read_impl(winconsoleio *self, PyTypeObject *cls, Py_ssize_t size) /*[clinic end generated code: output=7e569a586537c0ae input=a14570a5da273365]*/ { - PyObject *bytes; Py_ssize_t bytes_size; if (self->fd == -1) @@ -1026,26 +1025,20 @@ _io__WindowsConsoleIO_read_impl(winconsoleio *self, PyTypeObject *cls, return NULL; } - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; + } _PyIO_State *state = get_io_state_by_cls(cls); - bytes_size = readinto(state, self, PyBytes_AS_STRING(bytes), - PyBytes_GET_SIZE(bytes)); + bytes_size = readinto(state, self, PyBytesWriter_GetData(writer), + PyBytesWriter_GetSize(writer)); if (bytes_size < 0) { - Py_CLEAR(bytes); + PyBytesWriter_Discard(writer); return NULL; } - if (bytes_size < PyBytes_GET_SIZE(bytes)) { - if (_PyBytes_Resize(&bytes, bytes_size) < 0) { - Py_CLEAR(bytes); - return NULL; - } - } - - return bytes; + return PyBytesWriter_FinishWithSize(writer, bytes_size); } /*[clinic input]