diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index ff68ecafcda4d08..60a90b3f096912c 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -231,6 +231,7 @@ called with a non-bytes parameter. Resize a bytes object. *newsize* will be the new length of the bytes object. You can think of it as creating a new bytes object and destroying the old one, only more efficiently. + Pass the address of an existing bytes object as an lvalue (it may be written into), and the new size desired. On success, *\*bytes* holds the resized bytes object and ``0`` is @@ -239,6 +240,11 @@ called with a non-bytes parameter. *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. + While bytes objects are usually immutable in Python, this special C API + allows mutating a bytes object in-place. The returned bytes object can still + be mutated using :c:func:`PyBytesWriter_GetData`; except if *newsize* is + zero in which case it returns the immutable empty bytes string. + .. soft-deprecated:: 3.15 Use the :c:type:`PyBytesWriter` API instead. @@ -290,10 +296,10 @@ object. .. c:type:: PyBytesWriter - A bytes writer instance. + A bytes writer object. - The API is **not thread safe**: a writer should only be used by a single - thread at the same time. + The API is **not thread safe**. A :c:type:`PyBytesWriter` object must only + be used by a single thread, it must not be shared between threads. The instance must be destroyed by :c:func:`PyBytesWriter_Finish` on success, or :c:func:`PyBytesWriter_Discard` on error. @@ -429,7 +435,7 @@ Low-level API On success, return ``0``. On error, set an exception and return ``-1``. - *size* can be negative to shrink the writer. + *grow* can be negative to shrink the writer. .. c:function:: void* PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, void *buf) diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 32da177c637c268..443bdb26ff8738c 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -77,6 +77,10 @@ PyAPI_FUNC(PyObject *) _PyBytes_Repeat(PyObject *self, Py_ssize_t n); extern int _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize); +#ifndef NDEBUG +extern int _PyBytes_IsMutable(PyObject *obj); +#endif + /* --- PyBytesWriter ------------------------------------------------------ */ struct PyBytesWriter { diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 4b2bed9a0a56446..3297a53dceff005 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1130,13 +1130,14 @@ def test_translate(self): self.assertRaises(ValueError, b.translate, bytes(range(255))) c = b.translate(rosetta, b'hello') - self.assertEqual(b, b'hello') - self.assertIsInstance(c, self.type2test) + self.assertEqual(c, b'') + self.assertEqual(type(c), self.type2test) c = b.translate(rosetta) d = b.translate(rosetta, b'') - self.assertEqual(c, d) self.assertEqual(c, b'helle') + self.assertEqual(type(c), self.type2test) + self.assertEqual(d, b'helle') c = b.translate(rosetta, b'l') self.assertEqual(c, b'hee') diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 1500faa7f71344b..6c19ad14b7e6c59 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -461,6 +461,24 @@ def test_grow(self): writer.grow(0) # noop self.assertEqual(writer.finish(), b'number=123') + for size in (self.SMALL_BUFFER, self.LARGE_BUFFER): + with self.subTest(size=size): + # Truncate the last byte + data = b'x' * size + writer = self.create_writer(size) + writer.write(0, data) + self.assertEqual(writer.get_data(), data) + writer.grow(-1) + self.assertEqual(writer.get_data(), data[:-1]) + self.assertEqual(writer.finish(), data[:-1]) + + # Make the buffer empty + writer = self.create_writer(size) + writer.write(0, data) + writer.grow(-size) + self.assertEqual(writer.get_data(), b'') + self.assertEqual(writer.finish(), b'') + # Switch from small buffer to large buffer writer = self.create_writer() small, large = self.SMALL_BUFFER, self.LARGE_BUFFER @@ -476,8 +494,8 @@ def test_grow(self): with self.subTest(size=size): writer = self.create_writer() writer.write_bytes(b'x' * size, -1) - with self.assertRaisesRegex(ValueError, 'size must be >= 0'): - writer.grow(-1) + with self.assertRaisesRegex(ValueError, 'invalid size'): + writer.grow(-size - 1) with self.assertRaises(MemoryError): writer.grow(_testcapi.PY_SSIZE_T_MAX) self.assertEqual(writer.finish(), b'x' * size) diff --git a/Misc/NEWS.d/next/Library/2026-09-12-15-50-23.gh-issue-157366.djwD2M.rst b/Misc/NEWS.d/next/Library/2026-09-12-15-50-23.gh-issue-157366.djwD2M.rst new file mode 100644 index 000000000000000..c29b132e4b3b198 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-12-15-50-23.gh-issue-157366.djwD2M.rst @@ -0,0 +1,7 @@ +Remove the accidental acceptance of :class:`bytes` in the C implementation of +:mod:`xml.etree.ElementTree`, a leftover of the Python 2 to Python 3 migration, +for paths of :meth:`~xml.etree.ElementTree.Element.find` +and similar methods, for the tag of :meth:`~xml.etree.ElementTree.Element.iter`, +and for the names of events of :class:`~xml.etree.ElementTree.XMLPullParser` +and :func:`~xml.etree.ElementTree.iterparse`. It now raises the same +exceptions as the Python implementation. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f6dedeed981c40d..6f51f10b2b22759 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1230,24 +1230,6 @@ checkpath(PyObject* tag) } return 0; } - if (PyBytes_Check(tag)) { - const char *p = PyBytes_AS_STRING(tag); - const Py_ssize_t len = PyBytes_GET_SIZE(tag); - if (len >= 3 && p[0] == '{' && ( - p[1] == '}' || (p[1] == '*' && p[2] == '}'))) { - /* wildcard: '{}tag' or '{*}tag' */ - return 1; - } - for (i = 0; i < len; i++) { - if (p[i] == '{') - check = 0; - else if (p[i] == '}') - check = 1; - else if (check && PATHCHAR(p[i])) - return 1; - } - return 0; - } return 1; /* unknown type; might be path expression */ } @@ -1552,10 +1534,6 @@ _elementtree_Element_iter_impl(ElementObject *self, PyTypeObject *cls, if (PyUnicode_GET_LENGTH(tag) == 1 && PyUnicode_READ_CHAR(tag, 0) == '*') tag = Py_None; } - else if (PyBytes_Check(tag)) { - if (PyBytes_GET_SIZE(tag) == 1 && *PyBytes_AS_STRING(tag) == '*') - tag = Py_None; - } elementtreestate *st = get_elementtree_state_by_cls(cls); return create_elementiter(st, self, tag, 0); @@ -2935,17 +2913,7 @@ treebuilder_handle_data(TreeBuilderObject* self, PyObject* data) self->data = Py_NewRef(data); } else { /* more than one item; use a list to collect items */ - if (PyBytes_CheckExact(self->data) - && _PyObject_IsUniquelyReferenced(self->data) - && PyBytes_CheckExact(data) && PyBytes_GET_SIZE(data) == 1) { - /* XXX this code path unused in Python 3? */ - /* expat often generates single character data sections; handle - the most common case by resizing the existing string... */ - Py_ssize_t size = PyBytes_GET_SIZE(self->data); - if (_PyBytes_Resize(&self->data, size + 1) < 0) - return NULL; - PyBytes_AS_STRING(self->data)[size] = PyBytes_AS_STRING(data)[0]; - } else if (PyList_CheckExact(self->data)) { + if (PyList_CheckExact(self->data)) { if (PyList_Append(self->data, data) < 0) return NULL; } else { @@ -4363,18 +4331,14 @@ _elementtree_XMLParser__setevents_impl(XMLParserObject *self, for (i = 0; i < PySequence_Fast_GET_SIZE(events_seq); ++i) { PyObject *event_name_obj = PySequence_Fast_GET_ITEM(events_seq, i); - const char *event_name = NULL; - if (PyUnicode_Check(event_name_obj)) { - event_name = PyUnicode_AsUTF8(event_name_obj); - } else if (PyBytes_Check(event_name_obj)) { - event_name = PyBytes_AS_STRING(event_name_obj); + if (!PyUnicode_Check(event_name_obj)) { + goto unknown_event; } + const char *event_name = PyUnicode_AsUTF8(event_name_obj); if (event_name == NULL) { Py_DECREF(events_seq); - PyErr_Format(PyExc_ValueError, "invalid events sequence"); return NULL; } - if (strcmp(event_name, "start") == 0) { Py_XSETREF(target->start_event_obj, Py_NewRef(event_name_obj)); } else if (strcmp(event_name, "end") == 0) { @@ -4406,7 +4370,8 @@ _elementtree_XMLParser__setevents_impl(XMLParserObject *self, (XML_ProcessingInstructionHandler) expat_pi_handler ); } else { - PyErr_Format(PyExc_ValueError, "unknown event '%s'", event_name); +unknown_event: + PyErr_Format(PyExc_ValueError, "unknown event %R", event_name_obj); Py_DECREF(events_seq); return NULL; } diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 05e1b27dc82558a..de30c6118ba176b 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -256,6 +256,7 @@ bytearray_resize_storage(PyByteArrayObject *self, bytearray_write_trailing_null_byte(self); return -1; } + assert(_PyBytes_IsMutable(self->ob_bytes_object)); return 0; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index a5ec05764b2b700..080b53e088796f0 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -37,7 +37,7 @@ static Py_ssize_t _PyBytesWriter_ResizeToAllocated(PyBytesWriter *writer); #define CHARACTERS _Py_SINGLETON(bytes_characters) #define CHARACTER(ch) \ - ((PyBytesObject *)&(CHARACTERS[ch])); + ((PyBytesObject *)&(CHARACTERS[ch])) #define EMPTY (&_Py_SINGLETON(bytes_empty)) @@ -2276,7 +2276,6 @@ bytes_translate_impl(PyBytesObject *self, PyObject *table, PyObject *input_obj = (PyObject*)self; const char *output_start, *del_table_chars=NULL; Py_ssize_t inlen, tablen, dellen = 0; - PyObject *result; int trans_table[256]; if (PyBytes_Check(table)) { @@ -2321,13 +2320,13 @@ bytes_translate_impl(PyBytesObject *self, PyObject *table, } inlen = PyBytes_GET_SIZE(input_obj); - result = PyBytes_FromStringAndSize((char *)NULL, inlen); - if (result == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(inlen); + if (writer == NULL) { PyBuffer_Release(&del_table_view); PyBuffer_Release(&table_view); return NULL; } - output_start = output = PyBytes_AS_STRING(result); + output_start = output = PyBytesWriter_GetData(writer); input = PyBytes_AS_STRING(input_obj); if (dellen == 0 && table_chars != NULL) { @@ -2336,14 +2335,17 @@ bytes_translate_impl(PyBytesObject *self, PyObject *table, c = Py_CHARMASK(*input++); *output++ = table_chars[c]; } + PyObject *result = PyBytesWriter_Finish(writer); + /* Check if anything changed (for returning original object) */ /* We save this check until the end so that the compiler will */ /* unroll the loop above leading to MUCH faster code. */ - if (PyBytes_CheckExact(input_obj)) { + if (result != NULL && PyBytes_CheckExact(input_obj)) { if (memcmp(PyBytes_AS_STRING(input_obj), output_start, inlen) == 0) { Py_SETREF(result, Py_NewRef(input_obj)); } } + PyBuffer_Release(&del_table_view); PyBuffer_Release(&table_view); return result; @@ -2370,13 +2372,11 @@ bytes_translate_impl(PyBytesObject *self, PyObject *table, changed = 1; } if (!changed && PyBytes_CheckExact(input_obj)) { - Py_DECREF(result); + PyBytesWriter_Discard(writer); return Py_NewRef(input_obj); } /* Fix the size of the resulting byte string */ - if (inlen > 0) - _PyBytes_Resize(&result, output - output_start); - return result; + return PyBytesWriter_FinishWithPointer(writer, output); } @@ -3294,6 +3294,29 @@ PyBytes_ConcatAndDel(PyObject **pv, PyObject *w) } +#ifndef NDEBUG +// Make sure that a bytes object can still be mutated. +// +// Usage: assert(_PyBytes_IsMutable(obj)). +int +_PyBytes_IsMutable(PyObject *v) +{ + // Singleton objects must never be modified + assert(!_Py_IsImmortal(v)); + + Py_ssize_t size = PyBytes_GET_SIZE(v); + if (size == 0) { + assert(v != bytes_get_empty()); + } + else if (size == 1) { + unsigned char ch = PyBytes_AS_STRING(v)[0]; + assert(v != (PyObject*)CHARACTER(ch)); + } + return 1; +} +#endif + + /* The following function breaks the notion that bytes are immutable: it changes the size of a bytes object. You can think of it as creating a new bytes object and destroying the old one, only @@ -3331,6 +3354,7 @@ _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) } *pv = result; Py_DECREF(v); + assert(_PyBytes_IsMutable(*pv)); return 0; } @@ -3352,9 +3376,12 @@ _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) Py_MIN(oldsize, newsize)); *pv = result; Py_DECREF(v); + assert(_PyBytes_IsMutable(*pv)); return 0; } - assert(v != bytes_get_empty()); + + // Only mutable bytes can be resized in-place + assert(_PyBytes_IsMutable(v)); if ((size_t)newsize > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { PyErr_SetString(PyExc_OverflowError, @@ -3385,6 +3412,7 @@ _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) Py_SET_SIZE(sv, newsize); sv->ob_sval[newsize] = '\0'; set_ob_shash(sv, -1); /* invalidate cached hash value */ + assert(_PyBytes_IsMutable(*pv)); return 0; } @@ -3647,6 +3675,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) assert(writer->obj != NULL); return -1; } + assert(_PyBytes_IsMutable(writer->obj)); } assert(writer->obj != NULL); } @@ -3673,6 +3702,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) writer->small_buffer, sizeof(writer->small_buffer)); } + assert(_PyBytes_IsMutable(writer->obj)); } #ifdef Py_DEBUG @@ -3845,8 +3875,13 @@ PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) PyErr_SetString(PyExc_ValueError, "size must be >= 0"); return -1; } - if (byteswriter_resize(writer, size, 1) < 0) { - return -1; + if (writer->size < size) { + if (byteswriter_resize(writer, size, 1) < 0) { + return -1; + } + } + else { + // The buffer is already large enough. Never shrink the buffer. } writer->size = size; return 0; @@ -3866,22 +3901,26 @@ _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, int -PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t size) +PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) { - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "size must be >= 0"); - return -1; - } - if (size == 0) { + if (grow == 0) { // Nothing to do return 0; } - if (size > PY_SSIZE_T_MAX - writer->size) { - PyErr_NoMemory(); - return -1; + if (grow >= 0) { + if (grow > PY_SSIZE_T_MAX - writer->size) { + PyErr_NoMemory(); + return -1; + } + } + else { + if (writer->size + grow < 0) { + PyErr_SetString(PyExc_ValueError, "invalid size"); + return -1; + } } - size = writer->size + size; + Py_ssize_t size = writer->size + grow; if (byteswriter_resize(writer, size, 1) < 0) { return -1;