diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4d9e73db48ae57c..d12eaee2adb3963 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -630,7 +630,7 @@ Lib/test/test_unittest/testmock/ @cjw296 Doc/library/zlib.rst @StanFromIreland Lib/compression/zlib.py @StanFromIreland Lib/test/test_zlib.py @StanFromIreland -Modules/_zlibmodule.c @StanFromIreland +Modules/zlibmodule.c @StanFromIreland # Zipfile.Path Lib/test/test_zipfile/_path/ @jaraco diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 199e0fd8d181f0b..c7f7663e3162ba2 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -86,6 +86,9 @@ jobs: --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit + - name: 'Build list of changes' + run: | + make -C Doc/ PYTHON=../python changes - name: 'Collect HTML IDs' if: github.event_name == 'pull_request' run: python Doc/tools/check-html-ids.py collect Doc/build/html -o Doc/build/html-ids-head.json.gz diff --git a/Doc/tools/extensions/changes.py b/Doc/tools/extensions/changes.py index 02dc51b3a76943a..e6a912cef8810ea 100644 --- a/Doc/tools/extensions/changes.py +++ b/Doc/tools/extensions/changes.py @@ -6,6 +6,7 @@ from docutils import nodes from sphinx import addnodes +from sphinx.builders.changes import ChangesBuilder from sphinx.domains.changeset import ( VersionChange, versionlabel_classes, @@ -17,6 +18,7 @@ if TYPE_CHECKING: from docutils.nodes import Node from sphinx.application import Sphinx + from sphinx.environment import BuildEnvironment from sphinx.util.typing import ExtensionMetadata @@ -146,6 +148,32 @@ def _add_glossary_link(cls, inline: nodes.inline) -> None: break +def _fixup_changesets(app: Sphinx, env: BuildEnvironment) -> None: + changesets = env.get_domain("changeset").changesets + + # The changeset domain records each entry's plain text before SoftDeprecated + # replaces the :term:, so strip the markup before the changes builder renders it. + for entries in changesets.values(): + for i, entry in enumerate(entries): + if entry.type == "soft-deprecated": + entries[i] = entry._replace( + content=SoftDeprecated._TERM_RE.sub(r"\1", entry.content) + ) + + # DeprecatedRemoved entries are recorded under their (deprecated, + # removed) version tuple, which the changes builder ignores. + # Re-file them under both versions. + for versions in [v for v in changesets if isinstance(v, tuple)]: + deprecated, removed = versions + for entry in changesets.pop(versions): + changesets.setdefault(deprecated, []).append( + entry._replace(type="deprecated") + ) + changesets.setdefault(removed, []).append( + entry._replace(type="versionremoved") + ) + + def setup(app: Sphinx) -> ExtensionMetadata: # Override Sphinx's directives with support for 'next' app.add_directive("versionadded", PyVersionChange, override=True) @@ -155,9 +183,15 @@ def setup(app: Sphinx) -> ExtensionMetadata: # Register the ``.. deprecated-removed::`` directive app.add_directive("deprecated-removed", DeprecatedRemoved) + # _fixup_changesets() changes these entries to 'deprecated'/'versionremoved' + ChangesBuilder.typemap["deprecated-removed"] = "deprecated-removed" # Register the ``.. soft-deprecated::`` directive app.add_directive("soft-deprecated", SoftDeprecated) + ChangesBuilder.typemap["soft-deprecated"] = "soft deprecated" + + # Repair the recorded changesets for the couple of custom directives above + app.connect("env-updated", _fixup_changesets) return { "version": "1.0", diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 799a17617be1a3f..1500faa7f71344b 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -317,7 +317,11 @@ def test_join(self): class BaseWriterTest: - result_type = NotImplementedError + RESULT_TYPE = NotImplementedError + SMALL_BUFFER = 11 # bytes + assert SMALL_BUFFER < _testcapi.PyBytesWriter_small_buffer + LARGE_BUFFER = _testcapi.PyBytesWriter_small_buffer + 17 # bytes + NEW_BYTE = b'\xff' def create_writer(self, alloc=0, string=b''): raise NotImplementedError @@ -326,85 +330,174 @@ def test_create(self): # Test PyBytesWriter_Create() writer = self.create_writer() self.assertEqual(writer.get_size(), 0) - self.assertEqual(writer.finish(), self.result_type(b'')) + self.assertEqual(writer.finish(), b'') - writer = self.create_writer(3, b'abc') + writer = self.create_writer(3) + writer.write(0, b'abc') self.assertEqual(writer.get_size(), 3) - self.assertEqual(writer.finish(), self.result_type(b'abc')) + result = writer.finish() + self.assertEqual(result, b'abc') + self.assertEqual(type(result), self.RESULT_TYPE) + + @unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') + def test_get_data(self): + # Test PyBytesWriter_GetData() + writer = self.create_writer(6) + NEW_BYTE = self.NEW_BYTE + self.assertEqual(writer.get_data(), NEW_BYTE * 6) + writer.write(0, b'abc') + self.assertEqual(writer.get_data(), b'abc' + NEW_BYTE * 3) + writer.write(3, b'123') + self.assertEqual(writer.get_data(), b'abc123') + + # Switch from small buffer to large buffer + small, large = self.SMALL_BUFFER, self.LARGE_BUFFER + writer = self.create_writer(small) + self.assertEqual(writer.get_data(), NEW_BYTE * small) + writer.write(0, b's' * small) + self.assertEqual(writer.get_data(), b's' * small) + writer.resize(large) + self.assertEqual(writer.get_data(), b's' * small + NEW_BYTE * (large - small)) + writer.write(small, b'L' * (large - small)) + self.assertEqual(writer.get_data(), b's' * small + b'L' * (large - small)) + + # Resize large buffer + small, large = self.SMALL_BUFFER, self.LARGE_BUFFER + writer = self.create_writer(large) + self.assertEqual(writer.get_data(), NEW_BYTE * large) + writer.write(0, b'L' * large) + self.assertEqual(writer.get_data(), b'L' * large) + writer.resize(large + 10) + self.assertEqual(writer.get_data(), b'L' * large + NEW_BYTE * 10) + writer.write(large, b'#' * 10) + self.assertEqual(writer.get_data(), b'L' * large + b'#' * 10) def test_finish_with_size(self): # Test PyBytesWriter_FinishWithSize() - writer = self.create_writer(10, b'abcdef') + writer = self.create_writer(10) + writer.write(0, b'abc123') self.assertEqual(writer.get_size(), 10) - self.assertEqual(writer.finish_with_size(3), self.result_type(b'abc')) + result = writer.finish_with_size(3) + self.assertEqual(result, b'abc') + self.assertEqual(type(result), self.RESULT_TYPE) # Error if the size is negative - writer = self.create_writer(3, b'abc') + writer = self.create_writer(3, ) + writer.write(0, b'abc') with self.assertRaises(ValueError): writer.finish_with_size(-3) # Error if the requested size is larger than the allocated size - writer = self.create_writer(3, b'abc') + writer = self.create_writer(3) + writer.write(0, b'abc') with self.assertRaises(ValueError): writer.finish_with_size(4) def test_write_bytes(self): - # Test PyBytesWriter_WriteBytes() - writer = self.create_writer() - writer.write_bytes(b'Hello World!', -1) - self.assertEqual(writer.finish(), self.result_type(b'Hello World!')) + # Test PyBytesWriter_WriteBytes() + writer = self.create_writer() + writer.write_bytes(b'Hello World!', -1) + self.assertEqual(writer.finish(), b'Hello World!') - writer = self.create_writer() - writer.write_bytes(b'Hello ', -1) - writer.write_bytes(b'World! ', 6) - self.assertEqual(writer.finish(), self.result_type(b'Hello World!')) + writer = self.create_writer() + writer.write_bytes(b'Hello ', -1) + writer.write_bytes(b'World! ', 6) + self.assertEqual(writer.finish(), b'Hello World!') def test_resize(self): # Test PyBytesWriter_Resize() writer = self.create_writer() - writer.resize(len(b'number=123456'), b'number=123456') - writer.resize(len(b'number=123456'), b'') - self.assertEqual(writer.get_size(), len(b'number=123456')) - self.assertEqual(writer.finish(), self.result_type(b'number=123456')) + writer.resize(len(b'hello')) + writer.write(0, b'hello') + self.assertEqual(writer.finish(), b'hello') writer = self.create_writer() - writer.resize(0, b'') - writer.resize(len(b'number=123456'), b'number=123456') - self.assertEqual(writer.finish(), self.result_type(b'number=123456')) - + writer.resize(0) # noop + writer.resize(len(b'number')) + writer.write(0, b'number') + writer.resize(len(b'number=')) + writer.write(len(b'number'), b'=') + writer.resize(len(b'number=123'), ) + writer.write(len(b'number='), b'123') + writer.resize(len(b'number=123')) # noop + self.assertEqual(writer.finish(), b'number=123') + + # Switch from small buffer to large buffer writer = self.create_writer() - writer.resize(len(b'number='), b'number=') - writer.resize(len(b'number=123456'), b'123456') - self.assertEqual(writer.finish(), self.result_type(b'number=123456')) + small, large = self.SMALL_BUFFER, self.LARGE_BUFFER + writer.resize(small) + writer.write(0, b's' * small) + writer.resize(large) + writer.write(small, b'L' * (large - small)) + self.assertEqual(writer.finish(), + b's' * small + b'L' * (large - small)) + + # invalid size + for size in (self.SMALL_BUFFER, self.LARGE_BUFFER): + with self.subTest(size=size): + writer = self.create_writer() + writer.write_bytes(b'x' * size, -1) + with self.assertRaisesRegex(ValueError, 'size must be >= 0'): + writer.resize(-1) + with self.assertRaises((MemoryError, OverflowError)): + writer.resize(_testcapi.PY_SSIZE_T_MAX) + self.assertEqual(writer.finish(), b'x' * size) + + def test_grow(self): + # Test PyBytesWriter_Grow() + writer = self.create_writer(0) + writer.grow(len(b'number=123')) + writer.write(0, b'number=123') + self.assertEqual(writer.finish(), b'number=123') writer = self.create_writer() - writer.resize(len(b'number='), b'number=') - writer.resize(len(b'number='), b'') - writer.resize(len(b'number=123456'), b'123456') - self.assertEqual(writer.finish(), self.result_type(b'number=123456')) - + writer.grow(0) # noop + writer.grow(len(b'number')) + writer.write(0, b'number') + writer.grow(len(b'=')) + writer.write(len(b'number'), b'=') + writer.grow(len(b'123'), ) + writer.write(len(b'number='), b'123') + writer.grow(0) # noop + self.assertEqual(writer.finish(), b'number=123') + + # Switch from small buffer to large buffer writer = self.create_writer() - writer.resize(len(b'number'), b'number') - writer.resize(len(b'number='), b'=') - writer.resize(len(b'number=123'), b'123') - writer.resize(len(b'number=123456'), b'456') - self.assertEqual(writer.finish(), self.result_type(b'number=123456')) + small, large = self.SMALL_BUFFER, self.LARGE_BUFFER + writer.grow(small) + writer.write(0, b's' * small) + writer.grow(large - small) + writer.write(small, b'L' * (large - small)) + self.assertEqual(writer.finish(), + b's' * small + b'L' * (large - small)) + + # invalid size + for size in (self.SMALL_BUFFER, self.LARGE_BUFFER): + with self.subTest(size=size): + writer = self.create_writer() + writer.write_bytes(b'x' * size, -1) + with self.assertRaisesRegex(ValueError, 'size must be >= 0'): + writer.grow(-1) + with self.assertRaises(MemoryError): + writer.grow(_testcapi.PY_SSIZE_T_MAX) + self.assertEqual(writer.finish(), b'x' * size) @support.nomemtest def test_resize_error(self): - small_buffer = _testcapi.PyBytesWriter_small_buffer - init = b'x' * (small_buffer * 2) - writer = self.create_writer(len(init), init) + # Test PyBytesWriter_Resize() error + init = b'x' * self.LARGE_BUFFER + writer = self.create_writer(len(init)) + writer.write(0, init) size = len(init) + 100 try: with self.assertRaises(MemoryError): _testcapi.set_nomemory(0) - writer.resize(size, b'') + writer.resize(size) finally: _testcapi.remove_mem_hooks() suffix = b'still working' writer.write_bytes(suffix, -1) - self.assertEqual(writer.finish(), self.result_type(init + suffix)) + self.assertEqual(writer.finish(), init + suffix) # Note: PyBytesWriter_Resize() leaves the buffer unchanged (no resize) # if the new size is smaller than the allocated size @@ -413,48 +506,54 @@ def test_format_i(self): # Test PyBytesWriter_Format() writer = self.create_writer() writer.format_i(b'x=%i', 123456) - self.assertEqual(writer.finish(), self.result_type(b'x=123456')) + self.assertEqual(writer.finish(), b'x=123456') writer = self.create_writer() writer.format_i(b'x=%i, ', 123) writer.format_i(b'y=%i', 456) - self.assertEqual(writer.finish(), self.result_type(b'x=123, y=456')) + self.assertEqual(writer.finish(), b'x=123, y=456') class BytesWriterTest(BaseWriterTest, unittest.TestCase): - result_type = bytes + RESULT_TYPE = bytes - def create_writer(self, alloc=0, string=b''): + def create_writer(self, size=0): # Test PyBytesWriter_Create() - return _testcapi.PyBytesWriter(alloc, string, 0) + return _testcapi.PyBytesWriter(size, 0) # Only PyBytesWriter_Create() returns singletons def test_singletons(self): empty = b'' singletons = {ch: bytes((ch,)) for ch in range(256)} - small_buffer = _testcapi.PyBytesWriter_small_buffer writer = self.create_writer() self.assertIs(writer.finish(), empty) - # Test writer larger than small_buffer + # Large buffer writer = self.create_writer() - unused_text = b'x' * (small_buffer * 2) + unused_text = b'x' * self.LARGE_BUFFER writer.write_bytes(unused_text, len(unused_text)) self.assertIs(writer.finish_with_size(0), empty) + # Large buffer with resize + writer = self.create_writer() + unused_text = b'x' * self.LARGE_BUFFER + writer.write_bytes(unused_text, len(unused_text)) + writer.resize(0) + self.assertIs(writer.finish(), empty) + for ch in range(256): - text = bytes((ch,)) + byte = bytes((ch,)) writer = self.create_writer() - writer.write_bytes(text, 1) + writer.write_bytes(byte, len(byte)) self.assertIs(writer.finish(), singletons[ch]) - # Test writer larger than small_buffer + # Large buffer writer = self.create_writer() - writer.write_bytes(text, 1) - unused_text = b'x' * (small_buffer * 2) - writer.write_bytes(unused_text, len(unused_text)) + unused_text = b'x' * self.LARGE_BUFFER + writer.write_bytes(byte + unused_text, + len(byte) + len(unused_text)) self.assertIs(writer.finish_with_size(1), singletons[ch]) def test_example_abc(self): @@ -468,11 +567,11 @@ def test_example_highlevel(self): class ByteArrayWriterTest(BaseWriterTest, unittest.TestCase): - result_type = bytearray + RESULT_TYPE = bytearray - def create_writer(self, alloc=0, string=b''): + def create_writer(self, size=0): # Test private _PyBytesWriter_CreateByteArray() - return _testcapi.PyBytesWriter(alloc, string, 1) + return _testcapi.PyBytesWriter(size, 1) if __name__ == "__main__": diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 1253f124108bdbf..b6d10c8568feb68 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -571,20 +571,19 @@ _io__IOBase_readline_impl(PyObject *self, Py_ssize_t limit) { /* For backwards compatibility, a (slowish) readline(). */ - PyObject *peek, *buffer, *result; - Py_ssize_t old_size = -1; + PyObject *peek; if (PyObject_GetOptionalAttr(self, &_Py_ID(peek), &peek) < 0) { return NULL; } - buffer = PyByteArray_FromStringAndSize(NULL, 0); - if (buffer == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(0); + if (writer == NULL) { Py_XDECREF(peek); return NULL; } - while (limit < 0 || PyByteArray_GET_SIZE(buffer) < limit) { + while (limit < 0 || PyBytesWriter_GetSize(writer) < limit) { Py_ssize_t nreadahead = 1; PyObject *b; @@ -650,28 +649,25 @@ _io__IOBase_readline_impl(PyObject *self, Py_ssize_t limit) break; } - old_size = PyByteArray_GET_SIZE(buffer); - if (PyByteArray_Resize(buffer, old_size + PyBytes_GET_SIZE(b)) < 0) { + if (PyBytesWriter_WriteBytes(writer, + PyBytes_AS_STRING(b), + PyBytes_GET_SIZE(b)) < 0) { Py_DECREF(b); goto fail; } - memcpy(PyByteArray_AS_STRING(buffer) + old_size, - PyBytes_AS_STRING(b), PyBytes_GET_SIZE(b)); Py_DECREF(b); - if (PyByteArray_AS_STRING(buffer)[PyByteArray_GET_SIZE(buffer) - 1] == '\n') + char *data = PyBytesWriter_GetData(writer); + if (data[PyBytesWriter_GetSize(writer) - 1] == '\n') break; } - result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(buffer), - PyByteArray_GET_SIZE(buffer)); Py_XDECREF(peek); - Py_DECREF(buffer); - return result; + return PyBytesWriter_Finish(writer); fail: Py_XDECREF(peek); - Py_DECREF(buffer); + PyBytesWriter_Discard(writer); return NULL; } diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index 4830cc8b54bd837..83249a21c5a3f22 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -87,34 +87,25 @@ writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs) return -1; } - Py_ssize_t alloc; - char *str; - Py_ssize_t str_size; - int use_bytearray; - if (!PyArg_ParseTuple(args, "ny#i", - &alloc, &str, &str_size, &use_bytearray)) { + Py_ssize_t size; + int use_bytearray = 0; + if (!PyArg_ParseTuple(args, "n|i", &size, &use_bytearray)) { return -1; } WriterObject *self = (WriterObject *)self_raw; - if (self->writer) { - PyBytesWriter_Discard(self->writer); - } + PyBytesWriter_Discard(self->writer); + if (use_bytearray) { - self->writer = _PyBytesWriter_CreateByteArray(alloc); + self->writer = _PyBytesWriter_CreateByteArray(size); } else { - self->writer = PyBytesWriter_Create(alloc); + self->writer = PyBytesWriter_Create(size); } if (self->writer == NULL) { return -1; } - if (str_size) { - char *buf = PyBytesWriter_GetData(self->writer); - memcpy(buf, str, str_size); - } - return 0; } @@ -143,6 +134,32 @@ writer_check(WriterObject *self) } +static PyObject* +writer_write(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + Py_ssize_t pos, size; + char *str; + if (!PyArg_ParseTuple(args, "ny#", &pos, &str, &size)) { + return NULL; + } + + if (pos < 0 || (pos + size) > PyBytesWriter_GetSize(self->writer)) { + PyErr_SetString(PyExc_ValueError, "invalid position or size"); + return NULL; + } + + char *data = PyBytesWriter_GetData(self->writer); + memcpy(data + pos, str, size); + + Py_RETURN_NONE; +} + + static PyObject* writer_write_bytes(PyObject *self_raw, PyObject *args) { @@ -185,6 +202,7 @@ writer_format_i(PyObject *self_raw, PyObject *args) } +// PyBytesWriter_Resize static PyObject* writer_resize(PyObject *self_raw, PyObject *args) { @@ -194,27 +212,52 @@ writer_resize(PyObject *self_raw, PyObject *args) } Py_ssize_t size; - char *str; - Py_ssize_t str_size; - if (!PyArg_ParseTuple(args, - "ny#", - &size, &str, &str_size)) { + if (!PyArg_ParseTuple(args, "n", &size)) { return NULL; } - assert(size >= str_size); - Py_ssize_t pos = PyBytesWriter_GetSize(self->writer); if (PyBytesWriter_Resize(self->writer, size) < 0) { return NULL; } + Py_RETURN_NONE; +} + - char *buf = PyBytesWriter_GetData(self->writer); - memcpy(buf + pos, str, str_size); +// Test PyBytesWriter_Grow() +static PyObject* +writer_grow(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "n", &size)) { + return NULL; + } + + if (PyBytesWriter_Grow(self->writer, size) < 0) { + return NULL; + } Py_RETURN_NONE; } +static PyObject* +writer_get_data(PyObject *self_raw, PyObject *Py_UNUSED(args)) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + const char *data = PyBytesWriter_GetData(self->writer); + Py_ssize_t size = PyBytesWriter_GetSize(self->writer); + return PyBytes_FromStringAndSize(data, size); +} + + static PyObject* writer_get_size(PyObject *self_raw, PyObject *Py_UNUSED(args)) { @@ -223,8 +266,8 @@ writer_get_size(PyObject *self_raw, PyObject *Py_UNUSED(args)) return NULL; } - Py_ssize_t alloc = PyBytesWriter_GetSize(self->writer); - return PyLong_FromSsize_t(alloc); + Py_ssize_t size = PyBytesWriter_GetSize(self->writer); + return PyLong_FromSsize_t(size); } @@ -262,9 +305,12 @@ writer_finish_with_size(PyObject *self_raw, PyObject *args) static PyMethodDef writer_methods[] = { + {"write", _PyCFunction_CAST(writer_write), METH_VARARGS}, {"write_bytes", _PyCFunction_CAST(writer_write_bytes), METH_VARARGS}, {"format_i", _PyCFunction_CAST(writer_format_i), METH_VARARGS}, {"resize", _PyCFunction_CAST(writer_resize), METH_VARARGS}, + {"grow", _PyCFunction_CAST(writer_grow), METH_VARARGS}, + {"get_data", _PyCFunction_CAST(writer_get_data), METH_NOARGS}, {"get_size", _PyCFunction_CAST(writer_get_size), METH_NOARGS}, {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, {"finish_with_size", _PyCFunction_CAST(writer_finish_with_size), METH_VARARGS}, @@ -313,8 +359,9 @@ byteswriter_resize(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) char *buf = PyBytesWriter_GetData(writer); // Write some bytes - memcpy(buf, "Hello ", strlen("Hello ")); - buf += strlen("Hello "); + const char *hello = "Hello "; + memcpy(buf, hello, strlen(hello)); + buf += strlen(hello); // Allocate 10 more bytes buf = PyBytesWriter_GrowAndUpdatePointer(writer, 10, buf); @@ -324,8 +371,9 @@ byteswriter_resize(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } // Write more bytes - memcpy(buf, "World", strlen("World")); - buf += strlen("World"); + const char *world = "World"; + memcpy(buf, world, strlen(world)); + buf += strlen(world); // Truncate to the exact size and create a bytes object return PyBytesWriter_FinishWithPointer(writer, buf); diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index b84fdcd0ecc0153..a5ec05764b2b700 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3356,6 +3356,12 @@ _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) } assert(v != bytes_get_empty()); + if ((size_t)newsize > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { + PyErr_SetString(PyExc_OverflowError, + "byte string is too large"); + return -1; + } + #ifdef Py_TRACE_REFS _Py_ForgetReference(v); #endif @@ -3580,6 +3586,8 @@ _PyBytes_RepeatBuffer(char* dest, Py_ssize_t len_dest, // --- PyBytesWriter API ----------------------------------------------------- +#define PyBytesWrite_NEW_BYTE 0xff + static inline char* byteswriter_data(PyBytesWriter *writer) { @@ -3617,6 +3625,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) Py_ssize_t old_allocated = byteswriter_allocated(writer); if (size <= old_allocated) { + // Do not shrink the buffer before PyBytesWriter_FinishWithSize() return 0; } @@ -3633,6 +3642,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) } } else { + // Can raise MemoryError or OverflowError if (_PyBytes_ResizeKeepOnError(&writer->obj, size)) { assert(writer->obj != NULL); return -1; @@ -3668,7 +3678,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) #ifdef Py_DEBUG Py_ssize_t allocated = byteswriter_allocated(writer); if (resize && allocated > old_allocated) { - memset(byteswriter_data(writer) + old_allocated, 0xff, + memset(byteswriter_data(writer) + old_allocated, PyBytesWrite_NEW_BYTE, allocated - old_allocated); } #endif @@ -3706,7 +3716,8 @@ byteswriter_create(Py_ssize_t size, int use_bytearray) writer->size = size; } #ifdef Py_DEBUG - memset(byteswriter_data(writer), 0xff, byteswriter_allocated(writer)); + memset(byteswriter_data(writer), PyBytesWrite_NEW_BYTE, + byteswriter_allocated(writer)); #endif return writer; } @@ -3857,10 +3868,15 @@ _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t size) { - if (size < 0 && writer->size + size < 0) { - PyErr_SetString(PyExc_ValueError, "invalid size"); + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must be >= 0"); return -1; } + if (size == 0) { + // Nothing to do + return 0; + } + if (size > PY_SSIZE_T_MAX - writer->size) { PyErr_NoMemory(); return -1;