diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 4000f8aa1d4c0c7..72d8f97b67e4d05 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1869,9 +1869,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) PyObject *tp_iternext(PyObject *self); - When the iterator is exhausted, it must return ``NULL``; a :exc:`StopIteration` - exception may or may not be set. When another error occurs, it must return - ``NULL`` too. Its presence signals that the instances of this type are + When the iterator is :term:`exhausted`, the ``tp_iternext`` function must + return ``NULL``; a :exc:`StopIteration` exception may or may not be set. + When another error occurs, it must return ``NULL`` too. + The presence of ``tp_iternext`` signals that the instances of this type are iterators. Iterator types should also define the :c:member:`~PyTypeObject.tp_iter` function, and that diff --git a/Doc/glossary.rst b/Doc/glossary.rst index cd9d38b2fe4af29..68bc0560911161f 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -505,6 +505,14 @@ Glossary of an object, such as the value of type aliases created with the :keyword:`type` statement. + exhausted + An :term:`iterator` that has produced all of its values is said to be + :dfn:`exhausted`. + Further attempts to get the next value (for example, calls to + :func:`next`) raise :exc:`StopIteration` + (or :exc:`StopAsyncIteration` in the case of an :term:`asynchronous + iterator`). + expression A piece of syntax which can be evaluated to some value. In other words, an expression is an accumulation of expression elements like literals, @@ -869,7 +877,7 @@ Glossary :meth:`~iterator.__next__` method (or passing it to the built-in function :func:`next`) return successive items in the stream. When no more data are available a :exc:`StopIteration` exception is raised instead. At this - point, the iterator object is exhausted and any further calls to its + point, the iterator object is :term:`exhausted` and any further calls to its :meth:`!__next__` method just raise :exc:`StopIteration` again. Iterators are required to have an :meth:`~iterator.__iter__` method that returns the iterator object itself so every iterator is also iterable and may be used in most diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index a61fdaee27f6b18..f2ff5edeb733d7f 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -720,9 +720,10 @@ returns them in a tuple:: zip(['a', 'b', 'c'], (1, 2, 3)) => ('a', 1), ('b', 2), ('c', 3) -It doesn't construct an in-memory list and exhaust all the input iterators -before returning; instead tuples are constructed and returned only if they're -requested. (The technical term for this behaviour is `lazy evaluation +It doesn't construct an in-memory list and :term:`exhaust ` all +the input iterators before returning; instead tuples are constructed and +returned only if they're requested. +(The technical term for this behaviour is `lazy evaluation `__.) This iterator is intended to be used with iterables that are all of the same @@ -783,7 +784,7 @@ element *n* times, or returns the element endlessly if *n* is not provided. :: :func:`itertools.chain(iterA, iterB, ...) ` takes an arbitrary number of iterables as input, and returns all the elements of the first iterator, then all the elements of the second, and so on, until all of the -iterables have been exhausted. :: +iterables have been :term:`exhausted`. :: itertools.chain(['a', 'b', 'c'], (1, 2, 3)) => a, b, c, 1, 2, 3 @@ -878,7 +879,7 @@ iterable's results. :: :func:`itertools.compress(data, selectors) ` takes two iterators and returns only those elements of *data* for which the corresponding -element of *selectors* is true, stopping whenever either one is exhausted:: +element of *selectors* is true, stopping whenever either one is :term:`exhausted`:: itertools.compress([1, 2, 3, 4, 5], [True, True, False, False, True]) => 1, 2, 5 @@ -1028,7 +1029,7 @@ that takes two elements and returns a single value. :func:`functools.reduce` takes the first two elements A and B returned by the iterator and calculates ``func(A, B)``. It then requests the third element, C, calculates ``func(func(A, B), C)``, combines this result with the fourth element returned, -and continues until the iterable is exhausted. If the iterable returns no +and continues until the iterable is :term:`exhausted`. If the iterable returns no values at all, a :exc:`TypeError` exception is raised. If the initial value is supplied, it's used as a starting point and ``func(initial_value, A)`` is the first calculation. :: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 599a898eb2cef83..40f780c7d049d4e 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -699,7 +699,7 @@ added elements by appending to the right and popping to the left:: A `round-robin scheduler `_ can be implemented with input iterators stored in a :class:`deque`. Values are yielded from the active -iterator in position zero. If that iterator is exhausted, it can be removed +iterator in position zero. If that iterator is :term:`exhausted`, it can be removed with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with the :meth:`~deque.rotate` method:: diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index d76c449626fab4e..169459881328ab6 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -708,7 +708,7 @@ Specifying function pointers using type annotations @wrap_dll_function(dll_to_wrap) def function_ptr_name(arg_name: ctypes_type, ...) -> ctypes_type: - """Optional docstring. There should be no function body.""" + """Optional docstring. There should be no function body.""" The body of the decorated function is ignored, and any parameters that are missing type annotations are skipped. The names of the parameters are ignored @@ -728,7 +728,7 @@ Specifying function pointers using type annotations @wrap_dll_function(ctypes.pythonapi) def PyObject_GetAttrString(op: ctypes.py_object, attr: ctypes.c_char_p) -> ctypes.py_object: - pass + pass PyObject_GetAttrString(42, b"real") @@ -3207,7 +3207,7 @@ fields, or any other data types containing pointer type fields. that should be merged into a containing structure or union. -.. decorator:: struct(*, align=None, layout, endian='native', pack=None) +.. decorator:: struct(*, align=None, layout=None, endian='native', pack=None) :module: ctypes.util A :term:`decorator` that allows generating structure types using an @@ -3244,14 +3244,18 @@ fields, or any other data types containing pointer type fields. .. code-block:: python + from typing import Annotated + from ctypes import c_ssize_t, c_void_p + from ctypes.util import struct, CFieldInfo + @struct class PyObject: - ob_refcnt: c_ssize_t - ob_type: c_void_p + ob_refcnt: c_ssize_t + ob_type: c_void_p @struct class PyHovercraftObject: - ob_base: Annotated[PyObject, CFieldInfo(anonymous=True)] + ob_base: Annotated[PyObject, CFieldInfo(anonymous=True)] .. versionadded:: next diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index af654f7f82323af..ced19268dea2364 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1427,7 +1427,7 @@ iterations of the loop. ``STACK[-1]`` is an :term:`iterator`. Call its :meth:`~iterator.__next__` method. If this yields a new value, push it on the stack (leaving the iterator below - it). If the iterator indicates it is exhausted then the byte code counter is + it). If the iterator indicates it is :term:`exhausted` then the byte code counter is incremented by *delta*. .. versionchanged:: 3.12 diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 013150535cb089c..67893e670fdba7c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -129,7 +129,7 @@ are always available. They are listed here in alphabetical order. anext(async_iterator, default, /) When awaited, return the next item from the given :term:`asynchronous - iterator`, or *default* if given and the iterator is exhausted. + iterator`, or *default* if given and the iterator is :term:`exhausted`. This is the async variant of the :func:`next` builtin, and behaves similarly. @@ -1223,7 +1223,7 @@ are always available. They are listed here in alphabetical order. process_block(block) *stop_exception* is useful for callables - which report exhaustion by raising an exception + which report :term:`exhaustion ` by raising an exception instead of returning a special value. For example, draining a queue:: @@ -1315,7 +1315,7 @@ are always available. They are listed here in alphabetical order. yielding the results. If additional *iterables* arguments are passed, *function* must take that many arguments and is applied to the items from all iterables in parallel. With multiple iterables, the iterator stops when the - shortest iterable is exhausted. If *strict* is ``True`` and one of the + shortest iterable is :term:`exhausted`. If *strict* is ``True`` and one of the iterables is exhausted before the others, a :exc:`ValueError` is raised. For cases where the function inputs are already arranged into argument tuples, see :func:`itertools.starmap`. @@ -1397,7 +1397,7 @@ are always available. They are listed here in alphabetical order. Retrieve the next item from the :term:`iterator` by calling its :meth:`~iterator.__next__` method. If *default* is given, it is returned - if the iterator is exhausted, otherwise :exc:`StopIteration` is raised. + if the iterator is :term:`exhausted`, otherwise :exc:`StopIteration` is raised. .. class:: object() @@ -2312,7 +2312,7 @@ are always available. They are listed here in alphabetical order. the code that prepared these iterables. Python offers three different approaches to dealing with this issue: - * By default, :func:`zip` stops when the shortest iterable is exhausted. + * By default, :func:`zip` stops when the shortest iterable is :term:`exhausted`. It will ignore the remaining items in the longer iterables, cutting off the result to the length of the shortest iterable:: @@ -2327,7 +2327,7 @@ are always available. They are listed here in alphabetical order. [('a', 1), ('b', 2), ('c', 3)] Unlike the default behavior, it raises a :exc:`ValueError` if one iterable - is exhausted before the others: + is :term:`exhausted` before the others: >>> for item in zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True): # doctest: +SKIP ... print(item) diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 98ea09d8f72d8b6..4002fc697ceaa56 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -277,7 +277,7 @@ HTTPConnection Objects instance of :class:`io.TextIOBase`, the data returned by the ``read()`` method will be encoded as ISO-8859-1, otherwise the data returned by ``read()`` is sent as is. If *body* is an iterable, the elements of the - iterable are sent as is until the iterable is exhausted. + iterable are sent as is until the iterable is :term:`exhausted`. The *headers* argument should be a mapping of extra HTTP headers to send with the request. A :rfc:`Host header <2616#section-14.23>` diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index e1730608887b3dd..1bc3158973930dc 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -158,7 +158,7 @@ loops that truncate the stream. Loops over the input iterable and accumulates data into tuples up to size *n*. The input is consumed lazily, just enough to fill a batch. The result is yielded as soon as the batch is full or when the input - iterable is exhausted: + iterable is :term:`exhausted`: .. doctest:: @@ -188,7 +188,7 @@ loops that truncate the stream. .. function:: chain(*iterables) Make an iterator that returns elements from the first iterable until - it is exhausted, then proceeds to the next iterable, until all of the + it is :term:`exhausted`, then proceeds to the next iterable, until all of the iterables are exhausted. This combines multiple data sources into a single iterator. Roughly equivalent to:: @@ -305,7 +305,7 @@ loops that truncate the stream. Make an iterator that returns elements from *data* where the corresponding element in *selectors* is true. Stops when either the - *data* or *selectors* iterables have been exhausted. Roughly + *data* or *selectors* iterables have been :term:`exhausted`. Roughly equivalent to:: def compress(data, selectors): @@ -341,7 +341,7 @@ loops that truncate the stream. .. function:: cycle(iterable) Make an iterator returning elements from the *iterable* and saving a - copy of each. When the iterable is exhausted, return elements from + copy of each. When the iterable is :term:`exhausted`, return elements from the saved copy. Repeats indefinitely. Roughly equivalent to:: def cycle(iterable): @@ -472,7 +472,7 @@ loops that truncate the stream. elements from the iterable are skipped until *start* is reached. If *stop* is ``None``, iteration continues until the input is - exhausted, if at all. Otherwise, it stops at the specified position. + :term:`exhausted`, if at all. Otherwise, it stops at the specified position. If *step* is ``None``, the step defaults to one. Elements are returned consecutively unless *step* is set higher than one which results in @@ -677,9 +677,9 @@ loops that truncate the stream. Note, the element that first fails the predicate condition is consumed from the input iterator and there is no way to access it. This could be an issue if an application wants to further consume the - input iterator after *takewhile* has been run to exhaustion. To work - around this problem, consider using `more-itertools before_and_after() - `_ + input iterator after *takewhile* has been run to :term:`exhaustion `. + To work around this problem, consider using `more-itertools before_and_after() + `__ instead. @@ -766,7 +766,7 @@ loops that truncate the stream. If the iterables are of uneven length, missing values are filled-in with *fillvalue*. If not specified, *fillvalue* defaults to ``None``. - Iteration continues until the longest iterable is exhausted. + Iteration continues until the longest iterable is :term:`exhausted`. Roughly equivalent to:: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index bedea46cb16d60d..43fad57139057cd 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2538,7 +2538,7 @@ with the :class:`Pool` class. .. method:: imap(func, iterable, chunksize=1, *, buffersize=None) - A lazier version of :meth:`.map`. + An iterator-based version of :meth:`.map`. The *chunksize* argument is the same as the one used by the :meth:`.map` method. For very long iterables using a large value for *chunksize* can diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 6e0e67d2a613c85..9eba3bafae0cb6e 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3002,7 +3002,7 @@ features: Close the iterator and free acquired resources. - This is called automatically when the iterator is exhausted or garbage + This is called automatically when the iterator is :term:`exhausted` or garbage collected, or when an error happens during iterating. However it is advisable to call it explicitly or use the :keyword:`with` statement. diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 38a77119769d724..9e928e6dd3637b0 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -354,6 +354,10 @@ Standard names are defined for the following types: .. seealso:: :pep:`810` + .. method:: resolve() + + Reify the lazy import and return the "real" object being imported. + .. class:: GetSetDescriptorType diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index b9e6f65cb917df0..0b33240a4504ff9 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -922,7 +922,7 @@ object:: exception, - if ``side_effect`` is an iterable, the async function will return the next value of the iterable, however, if the sequence of result is - exhausted, ``StopAsyncIteration`` is raised immediately, + :term:`exhausted`, ``StopAsyncIteration`` is raised immediately, - if ``side_effect`` is not defined, the async function will return the value defined by ``return_value``, hence, by default, the async function returns a new :class:`AsyncMock` object. @@ -1272,7 +1272,7 @@ To remove a :attr:`~Mock.side_effect`, and return to the default behaviour, set 6 The :attr:`~Mock.side_effect` can also be any iterable object. Repeated calls to the mock -will return values from the iterable (until the iterable is exhausted and +will return values from the iterable (until the iterable is :term:`exhausted` and a :exc:`StopIteration` is raised): >>> m = MagicMock(side_effect=[1, 2, 3]) @@ -2949,7 +2949,7 @@ precedence remains the same: >>> order_mock.get_value() 'third' -If :attr:`~Mock.side_effect` is exhausted, the order of precedence will not +If :attr:`~Mock.side_effect` is :term:`exhausted`, the order of precedence will not cause a value to be obtained from the successors. Instead, ``StopIteration`` exception is raised. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 28850ba801138f0..c13860eb3e9319a 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -162,7 +162,7 @@ once; it should yield an :term:`iterable` object. An :term:`iterator` is created for that iterable. The first item provided by the iterator is then assigned to the target list using the standard rules for assignments (see :ref:`assignment`), and the suite is executed. This repeats for each -item provided by the iterator. When the iterator is exhausted, +item provided by the iterator. When the iterator is :term:`exhausted`, the suite in the :keyword:`!else` clause, if present, is executed, and the loop terminates. diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 8bac8df4368c00a..c397ed4f218b6a3 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -147,7 +147,7 @@ the list, thus saving space. We say such an object is :term:`iterable`, that is, suitable as a target for functions and constructs that expect something from which they can -obtain successive items until the supply is exhausted. We have seen that +obtain successive items until the supply is :term:`exhausted`. We have seen that the :keyword:`for` statement is such a construct, while an example of a function that takes an iterable is :func:`sum`:: diff --git a/Include/pyport.h b/Include/pyport.h index 73a3e6cdaf09200..03f9b869bc82b55 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -538,12 +538,15 @@ extern "C" { // // Example: _Py_TYPEOF(x) x_copy = (x); // -// On C23, use typeof(). Otherwise, the macro is only defined -// if GCC or clang compiler is used. +// On C23, use typeof(). Otherwise __typeof__() if on GCC, clang or +// MSVC 17.9 and newer. Else if on C++11 or newer, decltype() is used. #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 202311L # define _Py_TYPEOF(expr) typeof(expr) -#elif defined(__GNUC__) || defined(__clang__) +#elif defined(__GNUC__) || defined(__clang__) || \ + (defined(_MSC_VER) && _MSC_VER >= 1939) # define _Py_TYPEOF(expr) __typeof__(expr) +#elif defined(__cplusplus) && __cplusplus >= 201103L +# define _Py_TYPEOF(expr) decltype(expr) #endif diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index ab09913de6812dd..c82d3a21981d0f0 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -873,7 +873,8 @@ def wrapper(*args, **kwargs): _DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" -def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): +def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info, + stacklevel=3): """Warn that *name* is deprecated or should be removed. RuntimeError is raised if *remove* specifies a major/minor tuple older than @@ -889,7 +890,7 @@ def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_i raise RuntimeError(msg) else: msg = message.format(name=name, remove=remove_formatted) - _wm.warn(msg, DeprecationWarning, stacklevel=3) + _wm.warn(msg, DeprecationWarning, stacklevel=stacklevel) # Private utility function called by _PyErr_WarnUnawaitedCoroutine diff --git a/Lib/linecache.py b/Lib/linecache.py index b5bf9dbdd3cbc7e..d1391510473ae57 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -144,6 +144,7 @@ def updatecache(filename, module_globals=None): lazy_entry = entry if entry is not None and len(entry) == 1 else None if lazy_entry is None: lazy_entry = _make_lazycache_entry(filename, module_globals) + data = None if lazy_entry is not None: try: data = lazy_entry[0]() @@ -154,14 +155,23 @@ def updatecache(filename, module_globals=None): # No luck, the PEP302 loader cannot find the source # for this module. return [] - entry = ( - len(data), - None, - [line + '\n' for line in data.splitlines()], - fullname - ) - cache[filename] = entry - return entry[2] + if data is None: + # The file may be inside an archive on the module search path, + # such as a zip file. + try: + data = _read_from_archive(fullname) + except ImportError: + # Can happen if the interpreter is shutting down. + return [] + if data is not None: + entry = ( + len(data), + None, + [line + '\n' for line in data.splitlines()], + fullname + ) + cache[filename] = entry + return entry[2] # Try looking through the module search path, which is only useful # when handling a relative filename. @@ -197,6 +207,42 @@ def updatecache(filename, module_globals=None): return lines +def _read_from_archive(filename): + """Return the decoded contents of a file inside an archive on sys.path. + + Path entry finders for archives, such as zipimport.zipimporter, have a + get_data() method that reads files by their path below the archive, + which is what __file__ and co_filename contain for modules imported + from it. The archive is one of the parent directories of the file, so + look for a finder registered for one of them. Return None if the file + is not in such an archive. + """ + import os + import sys + importers = sys.path_importer_cache + if importers is None: + # Cleared while the interpreter is shutting down. + return None + path = filename + while True: + parent = os.path.dirname(path) + if parent == path: + return None + path = parent + get_data = getattr(importers.get(path), 'get_data', None) + if get_data is None: + continue + try: + data = get_data(filename) + except (ImportError, OSError): + continue + import importlib.util + try: + return importlib.util.decode_source(data) + except (UnicodeDecodeError, SyntaxError): + return None + + def lazycache(filename, module_globals): """Seed the cache for filename with module_globals. diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index d07447d66571e52..9145ab1ee26e6f8 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -644,7 +644,9 @@ def test_directive_output_invalid_command(self): - 'methoddef_define' - 'impl_prototype' - 'parser_prototype' + - 'parser_helper' - 'parser_definition' + - 'vectorcall_definition' - 'cpp_endif' - 'methoddef_ifndef' - 'impl_definition' @@ -2887,6 +2889,112 @@ def test_duplicate_coexist(self): """ self.expect_failure(block, err, lineno=2) + def test_duplicate_vectorcall(self): + err = "Called @vectorcall twice" + block = """ + module m + class Foo "FooObject *" "" + @vectorcall + @vectorcall + Foo.__init__ + """ + self.expect_failure(block, err, lineno=3) + + def test_vectorcall_on_regular_method(self): + err = "@vectorcall can only be used with __init__ and __new__ methods" + block = """ + module m + class Foo "FooObject *" "" + @vectorcall + Foo.some_method + """ + self.expect_failure(block, err, lineno=3) + + def test_vectorcall_on_module_function(self): + err = "@vectorcall can only be used with __init__ and __new__ methods" + block = """ + module m + @vectorcall + m.fn + """ + self.expect_failure(block, err, lineno=2) + + def test_vectorcall_on_init(self): + block = """ + module m + class Foo "FooObject *" "Foo_Type" + @vectorcall + Foo.__init__ + iterable: object = NULL + / + """ + func = self.parse_function(block, signatures_in_block=3, + function_index=2) + self.assertTrue(func.vectorcall) + + def test_vectorcall_on_new(self): + block = """ + module m + class Foo "FooObject *" "Foo_Type" + @classmethod + @vectorcall + Foo.__new__ + x: object = NULL + / + """ + func = self.parse_function(block, signatures_in_block=3, + function_index=2) + self.assertTrue(func.vectorcall) + + def test_vectorcall_takes_no_arguments(self): + err = "at_vectorcall() takes 1 positional argument but 2 were given" + block = """ + module m + class Foo "FooObject *" "Foo_Type" + @vectorcall bogus=True + Foo.__init__ + """ + self.expect_failure(block, err, lineno=2) + + def test_vectorcall_without_type_object(self): + err = "@vectorcall requires the type object of 'Foo'" + block = """ + module m + class Foo "FooObject *" "" + @vectorcall + Foo.__init__ + """ + self.expect_failure(block, err, lineno=3) + + def test_vectorcall_unsupported_converter(self): + # str(encoding=...) has no parse_arg() implementation. + err = ("@vectorcall requires all converters to support " + "parse_arg(); parameter 's' does not") + block = """ + module m + class Foo "FooObject *" "Foo_Type" + @classmethod + @vectorcall + Foo.__new__ + s: str(encoding="utf-8") + / + """ + self.expect_failure(block, err, lineno=6) + + def test_vectorcall_with_option_groups(self): + err = "@vectorcall does not support optional groups" + block = """ + module m + class Foo "FooObject *" "Foo_Type" + @vectorcall + Foo.__init__ + [ + a: object + ] + / + """ + self.expect_failure(block, err, lineno=7) + def test_unused_param(self): block = self.parse(""" module foo @@ -5020,6 +5128,105 @@ def test_kwds_with_pos_only_and_stararg(self): self.assertEqual(ac_tester.kwds_with_pos_only_and_stararg(1, 2, *args, **kwds), (1, 2, args, kwds)) +@unittest.skipIf(ac_tester is None, "_testclinic is missing") +class VectorcallFunctionalTest(unittest.TestCase): + """Runtime tests for @vectorcall exemplar types.""" + + def test_vc_new(self): + self.assertIsInstance(ac_tester.VcNew(), ac_tester.VcNew) + self.assertIsInstance(ac_tester.VcNew(1), ac_tester.VcNew) + self.assertIsInstance(ac_tester.VcNew(a=1), ac_tester.VcNew) + + def test_vc_new_rejects_extra_args(self): + with self.assertRaises(TypeError): + ac_tester.VcNew(1, 2) + + def test_vc_init(self): + self.assertIsInstance(ac_tester.VcInit(1), ac_tester.VcInit) + self.assertIsInstance(ac_tester.VcInit(1, 2), ac_tester.VcInit) + self.assertIsInstance(ac_tester.VcInit(1, b=2), ac_tester.VcInit) + + def test_vc_init_missing_required(self): + with self.assertRaises(TypeError): + ac_tester.VcInit() + + def test_vc_init_rejects_a_as_keyword(self): + # 'a' is positional-only + with self.assertRaises(TypeError): + ac_tester.VcInit(a=1) + + def test_vc_new_base(self): + self.assertIsInstance(ac_tester.VcNewBase(1), ac_tester.VcNewBase) + self.assertIsInstance(ac_tester.VcNewBase(1, 2), ac_tester.VcNewBase) + self.assertIsInstance(ac_tester.VcNewBase(1, b=2), ac_tester.VcNewBase) + + def test_vc_new_base_missing_required(self): + with self.assertRaises(TypeError): + ac_tester.VcNewBase() + + def test_vc_new_base_subclass(self): + # tp_vectorcall is not inherited, so the subclass is constructed + # through tp_new. The generated vectorcall asserts on that, so a + # debug build aborts here if that ever stops holding. + Sub = type('Sub', (ac_tester.VcNewBase,), {}) + obj = Sub(1) + self.assertIsInstance(obj, Sub) + self.assertIsInstance(obj, ac_tester.VcNewBase) + + def test_vc_kwonly(self): + # keyword-only 'b': vectorcall has no kwnames==NULL fast path, + # so every call goes through the helper. + self.assertIsInstance(ac_tester.VcKwOnly(1), ac_tester.VcKwOnly) + self.assertIsInstance(ac_tester.VcKwOnly(1, b=2), ac_tester.VcKwOnly) + self.assertIsInstance(ac_tester.VcKwOnly(a=1, b=2), ac_tester.VcKwOnly) + + def test_vc_kwonly_b_as_positional(self): + with self.assertRaises(TypeError): + ac_tester.VcKwOnly(1, 2) + + def test_vc_kwonly_missing_required(self): + with self.assertRaises(TypeError): + ac_tester.VcKwOnly() + + def test_parse_errors_match_slot(self): + # tp_vectorcall and tp_new/tp_init slot should match in argument parsing + # error messages. Explicit calls to __new__ and __init__, as well as + # subtype calls, will not hit the vectorcall slot. Test errors match. + def error(func, args, kwargs): + try: + func(*args, **kwargs) + except TypeError as exc: + return str(exc) + return None + + def through_new(cls): + return cls, partial(cls.__new__, cls) + + def through_init(cls): + # Not subclassable, and tp_new is PyType_GenericNew, so reach + # tp_init through the __init__ slot wrapper on an instance. + return cls, partial(cls.__init__, cls(1)) + + entry_points = [ + through_new(enumerate), # the only non-test @vectorcall function + through_new(ac_tester.VcNew), + through_new(ac_tester.VcNewBase), + through_new(ac_tester.VcKwOnly), + through_init(ac_tester.VcInit), + ] + invalid_calls = [ + ((), {}), # too few positional arguments + ((1, 2, 3), {}), # too many positional arguments + ((), {'zz': 1}), # unknown keyword argument + ] + + for direct, slot in entry_points: + for args, kwargs in invalid_calls: + with self.subTest(cls=direct, args=args, kwargs=kwargs): + self.assertEqual(error(direct, args, kwargs), + error(slot, args, kwargs)) + + class LimitedCAPIOutputTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index fcd94edc611fac3..202e30e6f6c07ac 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -1,13 +1,18 @@ """ Tests for the linecache module """ +import importlib import linecache import unittest import os.path +import sys import tempfile import threading import tokenize +import zipfile +import zipimport from importlib.machinery import ModuleSpec from test import support +from test.support import import_helper from test.support import os_helper from test.support import threading_helper from test.support.script_helper import assert_python_ok @@ -356,6 +361,17 @@ def test_linecache_python_string(self): self.assertEqual(stdout, b'') self.assertEqual(stderr, b'') + def test_path_importer_cache_None(self): + # sys.path_importer_cache is set to None while the interpreter is + # shutting down, before objects with a __del__ that may end up here + # are released. + filename = os.path.abspath(os_helper.TESTFN + '.py') + with support.swap_attr(sys, 'path_importer_cache', None): + self.assertEqual(linecache.getlines(filename), []) + self.assertEqual(linecache.getline(filename, 1), '') + self.assertNotIn(filename, linecache.cache) + + class LineCacheInvalidationTests(unittest.TestCase): def setUp(self): super().setUp() @@ -398,6 +414,98 @@ def test_checkcache_with_no_parameter(self): self.assertIn(self.unchanged_file, linecache.cache) +class ZipArchiveTests(unittest.TestCase): + """Sources of modules imported from a zip archive on sys.path.""" + + MODULE_SOURCE = ( + '"""A module inside a zip archive."""\n' + '\n' + 'def f():\n' + ' return "from the zip"\n' + ) + PACKAGE_SOURCE = 'value = 42\n' + LATIN1_SOURCE = ( + '# -*- coding: latin-1 -*-\n' + 'value = "caf\xe9"\n' + ) + + def setUp(self): + linecache.clearcache() + self.addCleanup(linecache.clearcache) + tmpdir = self.enterContext(os_helper.temp_dir()) + self.zip_name = os.path.join(tmpdir, 'sources.zip') + with zipfile.ZipFile(self.zip_name, 'w') as zf: + zf.writestr('zipmod.py', self.MODULE_SOURCE) + zf.writestr('zippkg/__init__.py', self.PACKAGE_SOURCE) + zf.writestr('ziplatin1.py', self.LATIN1_SOURCE.encode('latin-1')) + self.enterContext(import_helper.DirsOnSysPath(self.zip_name)) + for name in 'zipmod', 'zippkg', 'ziplatin1': + self.addCleanup(import_helper.unload, name) + self.addCleanup(sys.path_importer_cache.pop, self.zip_name, None) + self.addCleanup(zipimport._zip_directory_cache.pop, + self.zip_name, None) + self.zipmod = importlib.import_module('zipmod') + + def test_getlines_without_module_globals(self): + filename = self.zipmod.__file__ + self.assertEqual(filename, os.path.join(self.zip_name, 'zipmod.py')) + self.assertFalse(os.path.exists(filename)) + lines = self.MODULE_SOURCE.splitlines(keepends=True) + self.assertEqual(linecache.getlines(filename), lines) + self.assertEqual(linecache.getline(filename, 4), + ' return "from the zip"\n') + self.assertEqual(linecache.getline(filename, 5), '') + code = self.zipmod.f.__code__ + self.assertEqual(code.co_filename, filename) + self.assertEqual(linecache.getline(filename, code.co_firstlineno), + 'def f():\n') + + def test_relative_archive_path(self): + # A relative sys.path entry gives its modules a relative __file__. + tmpdir, zip_base = os.path.split(self.zip_name) + self.addCleanup(sys.path_importer_cache.pop, zip_base, None) + self.addCleanup(zipimport._zip_directory_cache.pop, zip_base, None) + sys.path.insert(0, zip_base) + self.addCleanup(sys.path.remove, zip_base) + with os_helper.change_cwd(tmpdir): + zippkg = importlib.import_module('zippkg') + self.assertEqual(zippkg.__file__, + os.path.join(zip_base, 'zippkg', '__init__.py')) + self.assertEqual(linecache.getlines(zippkg.__file__), + ['value = 42\n']) + + def test_package(self): + zippkg = importlib.import_module('zippkg') + self.assertEqual(linecache.getlines(zippkg.__file__), + ['value = 42\n']) + + def test_encoding_declaration(self): + ziplatin1 = importlib.import_module('ziplatin1') + self.assertEqual(linecache.getlines(ziplatin1.__file__), + self.LATIN1_SOURCE.splitlines(keepends=True)) + + def test_missing_file(self): + filename = os.path.join(self.zip_name, 'missing.py') + self.assertEqual(linecache.getlines(filename), []) + self.assertEqual(linecache.getline(filename, 1), '') + self.assertNotIn(filename, linecache.cache) + + def test_checkcache_and_clearcache(self): + filename = self.zipmod.__file__ + lines = linecache.getlines(filename) + self.assertIn(filename, linecache.cache) + # A file inside an archive has no mtime of its own, so checkcache() + # keeps the entry, as it does for entries loaded through a loader. + self.assertIsNone(linecache.cache[filename][1]) + linecache.checkcache(filename) + linecache.checkcache() + self.assertIn(filename, linecache.cache) + self.assertEqual(linecache.getlines(filename), lines) + linecache.clearcache() + self.assertNotIn(filename, linecache.cache) + self.assertEqual(linecache.getlines(filename), lines) + + class MultiThreadingTest(unittest.TestCase): @threading_helper.reap_threads @threading_helper.requires_working_threading() diff --git a/Lib/test/test_tuple.py b/Lib/test/test_tuple.py index e533392b8cae94a..7e5e57f09449c0f 100644 --- a/Lib/test/test_tuple.py +++ b/Lib/test/test_tuple.py @@ -38,6 +38,10 @@ def test_constructors(self): self.assertEqual(tuple(x for x in range(10) if x % 2), (1, 3, 5, 7, 9)) + def test_too_many_args(self): + with self.assertRaises(TypeError): + tuple([1, 2], 3) + def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): tuple(sequence=()) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index fdf2cd26f8c7c64..708db4d4387df5e 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -33,7 +33,9 @@ with_source_date_epoch, without_source_date_epoch, ) from test.support.import_helper import ensure_lazy_imports -from test.support.warnings_helper import check_no_resource_warning +from test.support.warnings_helper import ( + check_no_resource_warning, ignore_warnings, +) TESTFN2 = TESTFN + "2" @@ -4916,6 +4918,75 @@ class ZstdBoundedDecompressTests(AbstractBoundedDecompressTests, compression = zipfile.ZIP_ZSTANDARD +class MonkeypatchedDecompressorTests(unittest.TestCase): + # Some third-party projects monkey-patch _get_decompressor() to add + # additional compression schemes. This can break at any time as the + # internal compressor objects change. + # To protect users, we try to keep this case working. + # See also: GH-156002 and GH-113767. + COMPRESSION = 99 + + class Compressor: + """Compressor with only the original BZ2Compressor API""" + def compress(self, data): + return data.swapcase() + + def flush(self): + return b'' + + class Decompressor: + """Decompressor with only the 3.3+ BZ2Decompressor API""" + eof = False + + def decompress(self, data): + return data.swapcase() + + def setUp(self): + orig_check_compression = zipfile._check_compression + orig_get_compressor = zipfile._get_compressor + orig_get_decompressor = zipfile._get_decompressor + + def check_compression(compression): + if compression != self.COMPRESSION: + orig_check_compression(compression) + + def get_compressor(compress_type, compresslevel=None): + if compress_type == self.COMPRESSION: + return self.Compressor() + return orig_get_compressor(compress_type, compresslevel) + + def get_decompressor(compress_type): + if compress_type == self.COMPRESSION: + return self.Decompressor() + return orig_get_decompressor(compress_type) + + self.enterContext(mock.patch.object( + zipfile, '_check_compression', check_compression)) + self.enterContext(mock.patch.object( + zipfile, '_get_compressor', get_compressor)) + self.enterContext(mock.patch.object( + zipfile, '_get_decompressor', get_decompressor)) + + def test_roundtrip_monkeypatched_decompressor(self): + data = bytes(range(256)) * 8 + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=self.COMPRESSION) as zf: + zf.writestr("member", data) + self.assertIn(data.swapcase(), buf.getvalue()) + with (ignore_warnings(category=DeprecationWarning, + message='.*two arguments.*'), + zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf): + self.assertEqual(zf.read("member"), data) + with zf.open("member") as f: + self.assertEqual(f.read(100), data[:100]) + self.assertEqual(f.read1(100), data[100:200]) + f.seek(-100, os.SEEK_END) + self.assertEqual(f.read(), data[-100:]) + # Rewinding past the read buffer re-creates the decompressor. + f.seek(0) + self.assertEqual(f.read(), data) + + class AbstractBadCrcTests: def test_testzip_with_bad_crc(self): """Tests that files with bad CRCs return their name from testzip.""" diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 0accf324c90e3fd..d817bdd8769e7d7 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -802,7 +802,7 @@ def unused_data(self): return b'' @property - def _needs_input(self): + def needs_input(self): # While the LZMA properties header is still being buffered, more input # is required; afterwards defer to the wrapped decompressor so a bounded # decompress() call can be drained across reads. @@ -893,13 +893,6 @@ def _get_compressor(compress_type, compresslevel=None): return None -def _decompressor_needs_input(decompressor): - # bz2/zstd expose the stdlib decompressor's public needs_input; the LZMA - # wrapper keeps it private (_needs_input) to avoid adding public API. - needs_input = getattr(decompressor, "needs_input", None) - return decompressor._needs_input if needs_input is None else needs_input - - def _get_decompressor(compress_type): _check_compression(compress_type) if compress_type == ZIP_STORED: @@ -1207,7 +1200,7 @@ def _read1(self, n): else: # bzip2/lzma/zstd: a bounded decompress() call may leave input # buffered inside the decompressor; drain that before reading more. - if _decompressor_needs_input(self._decompressor): + if getattr(self._decompressor, "needs_input", True): data = self._read2(n) else: data = b'' @@ -1226,10 +1219,23 @@ def _read1(self, n): # Bound the output of a single decompress() call (mirroring the # DEFLATE path above) so that a small compressed member cannot # expand into one unbounded read. - data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) + try: + data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) + except TypeError: + # See MonkeypatchedDecompressorTests in test_core.py + warnings._deprecated( + 'one-argument decompress()', + 'The decompress() method of ' + + type(self._decompressor).__name__ + + ' should take two arguments, data and max_length.' + + ' One-argument calls will stop working before' + + ' Python 3.21.', + remove=(3, 21), + stacklevel=4) + data = self._decompressor.decompress(data) self._eof = (self._decompressor.eof or self._compress_left <= 0 and - _decompressor_needs_input(self._decompressor)) + getattr(self._decompressor, "needs_input", True)) data = data[:self._left] self._left -= len(data) diff --git a/Misc/NEWS.d/next/Library/2026-09-07-23-40-00.gh-issue-157146.lczip1.rst b/Misc/NEWS.d/next/Library/2026-09-07-23-40-00.gh-issue-157146.lczip1.rst new file mode 100644 index 000000000000000..d8b81798bd64bef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-07-23-40-00.gh-issue-157146.lczip1.rst @@ -0,0 +1,3 @@ +:mod:`linecache` can now read the source of a module that was imported from +a zip archive on :data:`sys.path` when given only the file name, as +:mod:`pdb`, :mod:`warnings` and :mod:`doctest` do. diff --git a/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst new file mode 100644 index 000000000000000..3fc3b6d4c2279be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst @@ -0,0 +1,6 @@ +:mod:`zipfile` again reads members through a third-party decompressor +installed by monkey-patching the private ``_get_decompressor()`` to return an +object that only implements old BZ2Decompressor API from Python 3.3. +Calling decompress() with one argument is deprecated. +Note that decompressors without ``needs_input`` and two-argument +``decompress()`` are vulnerable to :cve:`2026-15310`. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-02-28-20-35-47.gh-issue-87613.Nwzu6U.rst b/Misc/NEWS.d/next/Tools-Demos/2026-02-28-20-35-47.gh-issue-87613.Nwzu6U.rst new file mode 100644 index 000000000000000..0e1deced80712e1 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-02-28-20-35-47.gh-issue-87613.Nwzu6U.rst @@ -0,0 +1,2 @@ +Add a ``@vectorcall`` decorator to Argument Clinic to generate :ref:`vectorcall` +parsing code for :func:`object.__init__` and :func:`object.__new__`. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 95209cf81270413..9cbacbd14f86a8a 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -21,6 +21,12 @@ custom_converter(PyObject *obj, custom_t *val) } +/* Forward declarations for vectorcall types, needed because + * clinic/_testclinic.c.h is included before the type definitions. */ +static PyTypeObject VcNew_Type; +static PyTypeObject VcInit_Type; +static PyTypeObject VcNewBase_Type; +static PyTypeObject VcKwOnly_Type; #include "clinic/_testclinic.c.h" @@ -2431,6 +2437,131 @@ output pop /*[clinic end generated code: output=da39a3ee5e6b4b0d input=e7c7c42daced52b0]*/ +/* @vectorcall test types. Multiple types as tp_vectorcall is a single slot. */ + +/* VcNew: __new__ with one optional positional-or-keyword arg */ + +/*[clinic input] +class _testclinic.VcNew "PyObject *" "&VcNew_Type" +@classmethod +@vectorcall +_testclinic.VcNew.__new__ as vc_plain_new + a: object = None +[clinic start generated code]*/ + +static PyObject * +vc_plain_new_impl(PyTypeObject *type, PyObject *a) +/*[clinic end generated code: output=55b273e9797a3013 input=e15d88606280badc]*/ +{ + return type->tp_alloc(type, 0); +} + +static PyTypeObject VcNew_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.VcNew", + .tp_basicsize = sizeof(PyObject), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = vc_plain_new, + .tp_vectorcall = vc_plain_vectorcall, +}; + + +/* VcInit: __init__ with one required positional-only and one optional keyword + * arg. Uses @critical_section to exercise the {lock}/impl/{unlock} placement + * in both the helper body and the vectorcall fast-path inner block. */ + +/*[clinic input] +class _testclinic.VcInit "PyObject *" "&VcInit_Type" +@vectorcall +@critical_section +_testclinic.VcInit.__init__ as vc_posorkw_init + a: object + / + b: object = None +[clinic start generated code]*/ + +static int +vc_posorkw_init_impl(PyObject *self, PyObject *a, PyObject *b) +/*[clinic end generated code: output=6018424ba9fb0744 input=7a4513f78dd42b57]*/ +{ + return 0; +} + +static PyTypeObject VcInit_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.VcInit", + .tp_basicsize = sizeof(PyObject), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = PyType_GenericNew, + .tp_init = vc_posorkw_init, + .tp_vectorcall = vc_posorkw_vectorcall, +}; + + +/* VcNewBase: __new__ with a required positional-only argument, and the one + * subclassable vectorcall type. tp_vectorcall is not inherited, so a subclass + * is constructed through tp_new, never reaching vc_base_vectorcall. */ + +/*[clinic input] +class _testclinic.VcNewBase "PyObject *" "&VcNewBase_Type" +@classmethod +@vectorcall +_testclinic.VcNewBase.__new__ as vc_base_new + a: object + / + b: object = None +[clinic start generated code]*/ + +static PyObject * +vc_base_new_impl(PyTypeObject *type, PyObject *a, PyObject *b) +/*[clinic end generated code: output=e4ca5a11e7fb1148 input=c204ca773dc608bf]*/ +{ + return type->tp_alloc(type, 0); +} + +static PyTypeObject VcNewBase_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.VcNewBase", + .tp_basicsize = sizeof(PyObject), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_new = vc_base_new, + .tp_vectorcall = vc_base_vectorcall, +}; + + +/* VcKwOnly: @vectorcall + keyword-only arg. + * Exercises the no-kwnames==NULL-fast-path branch of the vectorcall codegen: + * the vectorcall function delegates unconditionally to the helper because the + * keyword-only parameter rules out the positional-only fast path. */ + +/*[clinic input] +class _testclinic.VcKwOnly "PyObject *" "&VcKwOnly_Type" +@classmethod +@vectorcall +_testclinic.VcKwOnly.__new__ as vc_kwonly_new + a: object + * + b: object = None +[clinic start generated code]*/ + +static PyObject * +vc_kwonly_new_impl(PyTypeObject *type, PyObject *a, PyObject *b) +/*[clinic end generated code: output=00417079caa234dc input=68c863b55575a9e1]*/ +{ + return type->tp_alloc(type, 0); +} + +static PyTypeObject VcKwOnly_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.VcKwOnly", + .tp_basicsize = sizeof(PyObject), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = vc_kwonly_new, + .tp_vectorcall = vc_kwonly_vectorcall, +}; + + + /*[clinic input] output push destination kwarg new file '{dirname}/clinic/_testclinic_kwds.c.h' @@ -2673,6 +2804,18 @@ PyInit__testclinic(void) if (PyModule_AddType(m, &DeprKwdInitNoInline) < 0) { goto error; } + if (PyModule_AddType(m, &VcNew_Type) < 0) { + goto error; + } + if (PyModule_AddType(m, &VcInit_Type) < 0) { + goto error; + } + if (PyModule_AddType(m, &VcNewBase_Type) < 0) { + goto error; + } + if (PyModule_AddType(m, &VcKwOnly_Type) < 0) { + goto error; + } return m; error: diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 088e7d103504ead..9eee8c15fdedf9c 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() @@ -4803,4 +4804,397 @@ _testclinic_TestClass_posonly_poskw_varpos_array_no_fastcall(PyObject *type, PyO exit: return return_value; } -/*[clinic end generated code: output=10c3b999199d7bbb input=a9049054013a1b77]*/ + +static PyObject * +vc_plain_new_impl(PyTypeObject *type, PyObject *a); + +static PyObject * +vc_plain_new_helper(PyTypeObject *type, PyObject *const *args, + Py_ssize_t nargs, Py_ssize_t nkw, PyObject *kwargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { _Py_LATIN1_CHR('a'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "VcNew", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t noptargs = nargs + nkw - 0; + PyObject *a = Py_None; + + fastargs = _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + a = fastargs[0]; +skip_optional_pos: + return_value = vc_plain_new_impl(type, a); + +exit: + return return_value; +} + +static PyObject * +vc_plain_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + return vc_plain_new_helper(type, _PyTuple_CAST(args)->ob_item, + PyTuple_GET_SIZE(args), + kwargs ? PyDict_GET_SIZE(kwargs) : 0, + kwargs, NULL); +} + +static PyObject * +vc_plain_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + PyObject *return_value = NULL; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *a = Py_None; + + assert(Py_Is(_PyType_CAST(type), &VcNew_Type)); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + if (kwnames != NULL || nargs > 1) { + return vc_plain_new_helper(_PyType_CAST(type), args, nargs, + kwnames ? PyTuple_GET_SIZE(kwnames) : 0, + NULL, kwnames); + } + if (nargs < 1) { + goto skip_optional; + } + a = args[0]; +skip_optional: + return_value = vc_plain_new_impl(_PyType_CAST(type), a); + + return return_value; +} + +static int +vc_posorkw_init_impl(PyObject *self, PyObject *a, PyObject *b); + +static int +vc_posorkw_init_helper(PyObject *self, PyObject *const *args, + Py_ssize_t nargs, Py_ssize_t nkw, PyObject *kwargs, PyObject *kwnames) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { _Py_LATIN1_CHR('b'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "VcInit", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t noptargs = nargs + nkw - 1; + PyObject *a; + PyObject *b = Py_None; + + fastargs = _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + if (!noptargs) { + goto skip_optional_pos; + } + b = fastargs[1]; +skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = vc_posorkw_init_impl(self, a, b); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +static int +vc_posorkw_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + return vc_posorkw_init_helper(self, _PyTuple_CAST(args)->ob_item, + PyTuple_GET_SIZE(args), + kwargs ? PyDict_GET_SIZE(kwargs) : 0, + kwargs, NULL); +} + +static PyObject * +vc_posorkw_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + PyObject *return_value = NULL; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; + int _result; + PyObject *a; + PyObject *b = Py_None; + + assert(Py_Is(_PyType_CAST(type), &VcInit_Type)); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + if (kwnames != NULL || nargs < 1 || nargs > 2) { + self = _PyType_CAST(type)->tp_new(_PyType_CAST(type), + (PyObject *)&_Py_SINGLETON(tuple_empty), NULL); + if (self == NULL) { + return NULL; + } + _result = vc_posorkw_init_helper(self, args, nargs, + kwnames ? PyTuple_GET_SIZE(kwnames) : 0, + NULL, kwnames); + if (_result != 0) { + Py_DECREF(self); + return NULL; + } + return self; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional; + } + b = args[1]; +skip_optional: + self = _PyType_CAST(type)->tp_new(_PyType_CAST(type), + (PyObject *)&_Py_SINGLETON(tuple_empty), NULL); + if (self == NULL) { + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + _result = vc_posorkw_init_impl((PyObject *)self, a, b); + Py_END_CRITICAL_SECTION(); + if (_result != 0) { + Py_DECREF(self); + goto exit; + } + return_value = self; + +exit: + return return_value; +} + +static PyObject * +vc_base_new_impl(PyTypeObject *type, PyObject *a, PyObject *b); + +static PyObject * +vc_base_new_helper(PyTypeObject *type, PyObject *const *args, + Py_ssize_t nargs, Py_ssize_t nkw, PyObject *kwargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { _Py_LATIN1_CHR('b'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "VcNewBase", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t noptargs = nargs + nkw - 1; + PyObject *a; + PyObject *b = Py_None; + + fastargs = _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + if (!noptargs) { + goto skip_optional_pos; + } + b = fastargs[1]; +skip_optional_pos: + return_value = vc_base_new_impl(type, a, b); + +exit: + return return_value; +} + +static PyObject * +vc_base_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + return vc_base_new_helper(type, _PyTuple_CAST(args)->ob_item, + PyTuple_GET_SIZE(args), + kwargs ? PyDict_GET_SIZE(kwargs) : 0, + kwargs, NULL); +} + +static PyObject * +vc_base_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + PyObject *return_value = NULL; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *a; + PyObject *b = Py_None; + + assert(Py_Is(_PyType_CAST(type), &VcNewBase_Type)); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + if (kwnames != NULL || nargs < 1 || nargs > 2) { + return vc_base_new_helper(_PyType_CAST(type), args, nargs, + kwnames ? PyTuple_GET_SIZE(kwnames) : 0, + NULL, kwnames); + } + a = args[0]; + if (nargs < 2) { + goto skip_optional; + } + b = args[1]; +skip_optional: + return_value = vc_base_new_impl(_PyType_CAST(type), a, b); + + return return_value; +} + +static PyObject * +vc_kwonly_new_impl(PyTypeObject *type, PyObject *a, PyObject *b); + +static PyObject * +vc_kwonly_new_helper(PyTypeObject *type, PyObject *const *args, + Py_ssize_t nargs, Py_ssize_t nkw, PyObject *kwargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "VcKwOnly", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t noptargs = nargs + nkw - 1; + PyObject *a; + PyObject *b = Py_None; + + fastargs = _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + b = fastargs[1]; +skip_optional_kwonly: + return_value = vc_kwonly_new_impl(type, a, b); + +exit: + return return_value; +} + +static PyObject * +vc_kwonly_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + return vc_kwonly_new_helper(type, _PyTuple_CAST(args)->ob_item, + PyTuple_GET_SIZE(args), + kwargs ? PyDict_GET_SIZE(kwargs) : 0, + kwargs, NULL); +} + +static PyObject * +vc_kwonly_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + + assert(Py_Is(_PyType_CAST(type), &VcKwOnly_Type)); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + return vc_kwonly_new_helper(_PyType_CAST(type), args, nargs, + kwnames ? PyTuple_GET_SIZE(kwnames) : 0, + NULL, kwnames); +} +/*[clinic end generated code: output=10fcd30a5d85ce11 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index e2db4fd87ed26b7..35f0394e2d3da14 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() @@ -2474,4 +2475,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=2231bec0ed196830 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9429e9340f69c4b7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_kwds.c.h b/Modules/clinic/_testclinic_kwds.c.h index 475bb12120c8f8a..ce4ee7a850f45b9 100644 --- a/Modules/clinic/_testclinic_kwds.c.h +++ b/Modules/clinic/_testclinic_kwds.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() @@ -228,4 +229,4 @@ kwds_with_pos_only_and_stararg(PyObject *module, PyObject *args, PyObject *kwarg return return_value; } -/*[clinic end generated code: output=d4e257c529010ae1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=62804c41a11bbd73 input=a9049054013a1b77]*/ diff --git a/Objects/cellobject.c b/Objects/cellobject.c index ec2eeb1a855b633..823f3c91044c780 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -29,7 +29,7 @@ PyDoc_STRVAR(cell_new_doc, "\n" " contents\n" " the contents of the cell. If not specified, the cell will be empty,\n" -" and \n further attempts to access its cell_contents attribute will\n" +" and further attempts to access its cell_contents attribute will\n" " raise a ValueError."); diff --git a/Objects/clinic/enumobject.c.h b/Objects/clinic/enumobject.c.h index 1bda482f4955aea..26b1801cb7312ed 100644 --- a/Objects/clinic/enumobject.c.h +++ b/Objects/clinic/enumobject.c.h @@ -27,7 +27,8 @@ static PyObject * enum_new_impl(PyTypeObject *type, PyObject *iterable, PyObject *start); static PyObject * -enum_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +enum_new_helper(PyTypeObject *type, PyObject *const *args, + Py_ssize_t nargs, Py_ssize_t nkw, PyObject *kwargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -59,12 +60,11 @@ enum_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) #undef KWTUPLE PyObject *argsbuf[2]; PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; + Py_ssize_t noptargs = nargs + nkw - 1; PyObject *iterable; PyObject *start = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + fastargs = _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; @@ -81,6 +81,44 @@ enum_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +static PyObject * +enum_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + return enum_new_helper(type, _PyTuple_CAST(args)->ob_item, + PyTuple_GET_SIZE(args), + kwargs ? PyDict_GET_SIZE(kwargs) : 0, + kwargs, NULL); +} + +static PyObject * +enum_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + PyObject *return_value = NULL; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *iterable; + PyObject *start = 0; + + assert(Py_Is(_PyType_CAST(type), &PyEnum_Type)); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + if (kwnames != NULL || nargs < 1 || nargs > 2) { + return enum_new_helper(_PyType_CAST(type), args, nargs, + kwnames ? PyTuple_GET_SIZE(kwnames) : 0, + NULL, kwnames); + } + iterable = args[0]; + if (nargs < 2) { + goto skip_optional; + } + start = args[1]; +skip_optional: + return_value = enum_new_impl(_PyType_CAST(type), iterable, start); + + return return_value; +} + PyDoc_STRVAR(reversed_new__doc__, "reversed(object, /)\n" "--\n" @@ -110,4 +148,29 @@ reversed_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=155cc9483d5f9eab input=a9049054013a1b77]*/ + +static PyObject * +reversed_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + PyObject *return_value = NULL; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *seq; + + assert(Py_Is(_PyType_CAST(type), &PyReversed_Type)); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + if (!_PyArg_NoKwnames("reversed", kwnames)) { + goto exit; + } + if (!_PyArg_CheckPositional("reversed", nargs, 1, 1)) { + goto exit; + } + seq = args[0]; + return_value = reversed_new_impl(_PyType_CAST(type), seq); + +exit: + return return_value; +} +/*[clinic end generated code: output=d0c066334eeb3b17 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/tupleobject.c.h b/Objects/clinic/tupleobject.c.h index 1c12706c0bb43bc..5e136b2d1cdfdf3 100644 --- a/Objects/clinic/tupleobject.c.h +++ b/Objects/clinic/tupleobject.c.h @@ -111,6 +111,35 @@ tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +static PyObject * +tuple_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + PyObject *return_value = NULL; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *iterable = NULL; + + assert(Py_Is(_PyType_CAST(type), &PyTuple_Type)); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + if (!_PyArg_NoKwnames("tuple", kwnames)) { + goto exit; + } + if (!_PyArg_CheckPositional("tuple", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + iterable = args[0]; +skip_optional: + return_value = tuple_new_impl(_PyType_CAST(type), iterable); + +exit: + return return_value; +} + PyDoc_STRVAR(tuple___getnewargs____doc__, "__getnewargs__($self, /)\n" "--\n" @@ -127,4 +156,4 @@ tuple___getnewargs__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return tuple___getnewargs___impl((PyTupleObject *)self); } -/*[clinic end generated code: output=bd11662d62d973c2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=69cab12f1ecb03e9 input=a9049054013a1b77]*/ diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 68aa594c5540cee..4353d7196f005b3 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -28,6 +28,7 @@ typedef struct { #define _enumobject_CAST(op) ((enumobject *)(op)) /*[clinic input] +@vectorcall @classmethod enumerate.__new__ as enum_new @@ -46,7 +47,7 @@ enumerate is useful for obtaining an indexed list: static PyObject * enum_new_impl(PyTypeObject *type, PyObject *iterable, PyObject *start) -/*[clinic end generated code: output=e95e6e439f812c10 input=782e4911efcb8acf]*/ +/*[clinic end generated code: output=e95e6e439f812c10 input=a139e88889360e8f]*/ { enumobject *en; @@ -87,71 +88,6 @@ enum_new_impl(PyTypeObject *type, PyObject *iterable, PyObject *start) return (PyObject *)en; } -static int check_keyword(PyObject *kwnames, int index, - const char *name) -{ - PyObject *kw = PyTuple_GET_ITEM(kwnames, index); - if (!_PyUnicode_EqualToASCIIString(kw, name)) { - PyErr_Format(PyExc_TypeError, - "'%S' is an invalid keyword argument for enumerate()", kw); - return 0; - } - return 1; -} - -// TODO: Use AC when bpo-43447 is supported -static PyObject * -enumerate_vectorcall(PyObject *type, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyTypeObject *tp = _PyType_CAST(type); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - Py_ssize_t nkwargs = 0; - if (kwnames != NULL) { - nkwargs = PyTuple_GET_SIZE(kwnames); - } - - // Manually implement enumerate(iterable, start=...) - if (nargs + nkwargs == 2) { - if (nkwargs == 1) { - if (!check_keyword(kwnames, 0, "start")) { - return NULL; - } - } else if (nkwargs == 2) { - PyObject *kw0 = PyTuple_GET_ITEM(kwnames, 0); - if (_PyUnicode_EqualToASCIIString(kw0, "start")) { - if (!check_keyword(kwnames, 1, "iterable")) { - return NULL; - } - return enum_new_impl(tp, args[1], args[0]); - } - if (!check_keyword(kwnames, 0, "iterable") || - !check_keyword(kwnames, 1, "start")) { - return NULL; - } - - } - return enum_new_impl(tp, args[0], args[1]); - } - - if (nargs + nkwargs == 1) { - if (nkwargs == 1 && !check_keyword(kwnames, 0, "iterable")) { - return NULL; - } - return enum_new_impl(tp, args[0], NULL); - } - - if (nargs == 0) { - PyErr_SetString(PyExc_TypeError, - "enumerate() missing required argument 'iterable'"); - return NULL; - } - - PyErr_Format(PyExc_TypeError, - "enumerate() takes at most 2 arguments (%zd given)", nargs + nkwargs); - return NULL; -} - static void enum_dealloc(PyObject *op) { @@ -339,7 +275,7 @@ PyTypeObject PyEnum_Type = { PyType_GenericAlloc, /* tp_alloc */ enum_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ - .tp_vectorcall = enumerate_vectorcall + .tp_vectorcall = enum_vectorcall }; /* Reversed Object ***************************************************************/ @@ -353,6 +289,7 @@ typedef struct { #define _reversedobject_CAST(op) ((reversedobject *)(op)) /*[clinic input] +@vectorcall @classmethod reversed.__new__ as reversed_new @@ -364,7 +301,7 @@ Return a reverse iterator over the values of the given sequence. static PyObject * reversed_new_impl(PyTypeObject *type, PyObject *seq) -/*[clinic end generated code: output=f7854cc1df26f570 input=4781869729e3ba50]*/ +/*[clinic end generated code: output=f7854cc1df26f570 input=7db568182ab28c59]*/ { Py_ssize_t n; PyObject *reversed_meth; @@ -406,22 +343,6 @@ reversed_new_impl(PyTypeObject *type, PyObject *seq) return (PyObject *)ro; } -static PyObject * -reversed_vectorcall(PyObject *type, PyObject * const*args, - size_t nargsf, PyObject *kwnames) -{ - if (!_PyArg_NoKwnames("reversed", kwnames)) { - return NULL; - } - - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - if (!_PyArg_CheckPositional("reversed", nargs, 1, 1)) { - return NULL; - } - - return reversed_new_impl(_PyType_CAST(type), args[0]); -} - static void reversed_dealloc(PyObject *op) { diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index bb5e18cb790acf0..599ffad4f7b1027 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -781,6 +781,7 @@ static PyObject * tuple_subtype_new(PyTypeObject *type, PyObject *iterable); /*[clinic input] +@vectorcall @classmethod tuple.__new__ as tuple_new iterable: object(c_default="NULL") = () @@ -796,7 +797,7 @@ If the argument is a tuple, the return value is the same object. static PyObject * tuple_new_impl(PyTypeObject *type, PyObject *iterable) -/*[clinic end generated code: output=4546d9f0d469bce7 input=86963bcde633b5a2]*/ +/*[clinic end generated code: output=4546d9f0d469bce7 input=8fdda913493ebe48]*/ { if (type != &PyTuple_Type) return tuple_subtype_new(type, iterable); @@ -809,27 +810,6 @@ tuple_new_impl(PyTypeObject *type, PyObject *iterable) } } -static PyObject * -tuple_vectorcall(PyObject *type, PyObject * const*args, - size_t nargsf, PyObject *kwnames) -{ - if (!_PyArg_NoKwnames("tuple", kwnames)) { - return NULL; - } - - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - if (!_PyArg_CheckPositional("tuple", nargs, 0, 1)) { - return NULL; - } - - if (nargs) { - return tuple_new_impl(_PyType_CAST(type), args[0]); - } - else { - return tuple_get_empty(); - } -} - static PyObject * tuple_subtype_new(PyTypeObject *type, PyObject *iterable) { diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 489043103aa9b5b..1d062c57a430134 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -345,7 +345,7 @@ def format_tsv_lines(lines): _abs('Modules/_ssl_data_300.h'): (80_000, 10_000), _abs('Modules/_ssl_data_111.h'): (80_000, 10_000), _abs('Modules/cjkcodecs/mappings_*.h'): (160_000, 2_000), - _abs('Modules/clinic/_testclinic.c.h'): (135_000, 5_500), + _abs('Modules/clinic/_testclinic.c.h'): (180_000, 5_500), _abs('Modules/unicodedata_db.h'): (180_000, 3_000), _abs('Modules/unicodename_db.h'): (1_200_000, 15_000), _abs('Objects/unicodetype_db.h'): (240_000, 3_000), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 148f6e68ab806e5..b8488899c4595de 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -357,6 +357,10 @@ Modules/_testclinic.c - DeprKwdInit - Modules/_testclinic.c - DeprKwdInitNoInline - Modules/_testclinic.c - DeprKwdNew - Modules/_testclinic.c - TestClass - +Modules/_testclinic.c - VcInit_Type - +Modules/_testclinic.c - VcKwOnly_Type - +Modules/_testclinic.c - VcNew_Type - +Modules/_testclinic.c - VcNewBase_Type - ################################## diff --git a/Tools/clinic/libclinic/app.py b/Tools/clinic/libclinic/app.py index d8de3687a35ce64..6768029be2a7dbe 100644 --- a/Tools/clinic/libclinic/app.py +++ b/Tools/clinic/libclinic/app.py @@ -122,7 +122,9 @@ def __init__( 'methoddef_define': d('file'), 'impl_prototype': d('file'), 'parser_prototype': d('suppress'), + 'parser_helper': d('file'), 'parser_definition': d('file'), + 'vectorcall_definition': d('file'), 'cpp_endif': d('file'), 'methoddef_ifndef': d('file', 1), 'impl_definition': d('block'), diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index d7b86a18680ae46..3747e752f8a7608 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -12,7 +12,7 @@ from libclinic.function import ( Module, Class, Function, Parameter, group_to_variable_name, - GETTER, METHOD_INIT, + GETTER, METHOD_INIT, METHOD_NEW, ACCESSORS, SETTERS) from libclinic.converters import self_converter from libclinic.parse_args import ParseArgsCodeGen @@ -352,6 +352,9 @@ def render_function( if f.kind not in SETTERS | {METHOD_INIT}: f.return_converter.render(f, data) template_dict['impl_return_type'] = f.return_converter.type + # tp_init returns int; every other parser returns an object. + template_dict['return_type'] = ( + 'int' if f.kind is METHOD_INIT else 'PyObject *') template_dict['declarations'] = libclinic.format_escape("\n".join(data.declarations)) template_dict['initializers'] = "\n\n".join(data.initializers) @@ -371,6 +374,21 @@ def render_function( template_dict['parser_parameters'] = ", ".join(data.impl_parameters[1:]) template_dict['impl_arguments'] = ", ".join(data.impl_arguments) + # First vectorcall argument depends on method. + if f.vectorcall and f.cls: + if f.kind is METHOD_INIT: + vc_first = f"({f.cls.typedef})self" + elif f.kind is METHOD_NEW: + vc_first = "_PyType_CAST(type)" + else: + raise AssertionError( + f"Unhandled function kind for vectorcall: {f.kind!r}" + ) + vc_impl_args = [vc_first] + data.impl_arguments[1:] + template_dict['vectorcall_impl_arguments'] = ", ".join(vc_impl_args) + else: + pass + template_dict['return_conversion'] = libclinic.format_escape("".join(data.return_conversion).rstrip()) template_dict['post_parsing'] = libclinic.format_escape("".join(data.post_parsing).rstrip()) template_dict['cleanup'] = libclinic.format_escape("".join(data.cleanup)) diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index b241f58711e68a4..0202d9d3daf8875 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -307,6 +307,7 @@ def reset(self) -> None: self.critical_section = False self.target_critical_section = [] self.disable_fastcall = False + self.vectorcall: bool = False self.permit_long_summary = False self.permit_long_docstring_body = False @@ -481,6 +482,11 @@ def at_staticmethod(self) -> None: fail("Can't set @staticmethod, function is not a normal callable") self.kind = STATIC_METHOD + def at_vectorcall(self) -> None: + if self.vectorcall: + fail("Called @vectorcall twice!") + self.vectorcall = True + def at_coexist(self) -> None: if self.coexist: fail("Called @coexist twice!") @@ -622,6 +628,17 @@ def normalize_function_kind(self, fullname: str) -> None: elif name == '__init__': self.kind = METHOD_INIT + # Validate @vectorcall usage. + if self.vectorcall: + if not self.kind.new_or_init: + fail("@vectorcall can only be used with __init__ and __new__ " + "methods currently") + # Guaranteed by the __new__ / __init__ checks above. + assert cls is not None + if not cls.type_object: + fail(f"@vectorcall requires the type object of {cls.name!r}, " + f"which was declared without one") + def resolve_return_converter( self, full_name: str, forced_converter: str ) -> CReturnConverter: @@ -750,6 +767,7 @@ def state_modulename_name(self, line: str) -> None: target_critical_section=self.target_critical_section, forced_text_signature=self.forced_text_signature, line_number=self.line_number, + vectorcall=self.vectorcall, ) self.add_function(func) @@ -1526,6 +1544,27 @@ def check_previous_star(self) -> None: fail(f"Function {self.function.name!r} uses '*' more than once.") + def check_vectorcall_parameters(self, lineno: int) -> None: + assert self.function is not None + if not self.function.vectorcall: + return + for i, p in enumerate(self.function.parameters.values()): + if p.group: + fail("@vectorcall does not support optional groups", + line_number=lineno) + if p.is_vararg() or p.is_var_keyword(): + continue + if isinstance(p.converter, (self_converter, + defining_class_converter)): + continue + parse_arg = p.converter.parse_arg(f'args[{i}]', + p.get_displayname(i), + limited_capi=False) + if parse_arg is None: + fail("@vectorcall requires all converters to support " + f"parse_arg(); parameter {p.name!r} does not", + line_number=lineno) + def do_post_block_processing_cleanup(self, lineno: int) -> None: """ Called when processing the block is done. @@ -1534,6 +1573,7 @@ def do_post_block_processing_cleanup(self, lineno: int) -> None: return self.check_remaining_star(lineno) + self.check_vectorcall_parameters(lineno) try: self.function.docstring = self.format_docstring() except ClinicError as exc: diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index d7625f972944929..d61cc136b7fefac 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -122,6 +122,7 @@ class Function: line_number: int | None = None # Line on which the docstring starts (`None` if there is no docstring). docstring_line_number: int | None = None + vectorcall: bool = False def __post_init__(self) -> None: self.parent = self.cls or self.module @@ -137,6 +138,21 @@ def displayname(self) -> str: else: return self.name + @functools.cached_property + def c_basename_vectorcall(self) -> str: + """C function name for vectorcall parser. + + Strips the __init__/__new__ suffix from c_basename and appends + _vectorcall. Respects 'as' renaming in clinic input, e.g. + 'str.__new__ as unicode_new' produces 'unicode_vectorcall'. + """ + name = self.c_basename + for suffix in ('___init__', '___new__', '_new', '_init'): + if name.endswith(suffix): + name = name.removesuffix(suffix) + break + return f'{name}_vectorcall' + @functools.cached_property def fulldisplayname(self) -> str: parent: Class | Module | Clinic | None diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index ee1850e67f84e01..4aa159010e8296a 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -6,7 +6,7 @@ from libclinic.function import ( Function, Parameter, ParamTuple, count_required, group_to_variable_name, permute_optional_groups, - GETTER, SETTER, METHOD_NEW, + GETTER, SETTER, METHOD_INIT, ACCESSORS, SETTERS) from libclinic.converter import CConverter from libclinic.converters import ( @@ -101,12 +101,13 @@ def declare_parser( NO_VARARG: Final[str] = "PY_SSIZE_T_MAX" PARSER_PROTOTYPE_KEYWORD: Final[str] = libclinic.normalize_snippet(""" - static PyObject * + static {return_type} {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) """) -PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = libclinic.normalize_snippet(""" - static int - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) +PARSER_PROTOTYPE_KEYWORD_HELPER: Final[str] = libclinic.normalize_snippet(""" + static {return_type} + {c_basename}_helper({self_type}{self_name}, PyObject *const *args, + Py_ssize_t nargs, Py_ssize_t nkw, PyObject *kwargs, PyObject *kwnames) """) PARSER_PROTOTYPE_VARARGS: Final[str] = libclinic.normalize_snippet(""" static PyObject * @@ -120,6 +121,11 @@ def declare_parser( static PyObject * {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) """) +PARSER_PROTOTYPE_VECTORCALL: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {vc_basename}(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +""") PARSER_PROTOTYPE_DEF_CLASS: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -205,6 +211,51 @@ def declare_parser( return -1; }} """, indent=4) +# Every parser body ends with this shape; parser_body() and +# _assemble_vectorcall() fill the assembly-time markers. +PARSER_FINALE_SKELETON: Final[str] = libclinic.normalize_snippet(""" + {modifications} + {self_alloc} + {lock} + {impl_call} + {unlock} + {init_result_check} + {return_conversion} + {post_parsing} + + {exit_label} + {cleanup} + return {parser_retval}; + }} +""") +VECTORCALL_FINALE_MARKERS_NEW: Final[dict[str, str]] = { + "init_declarations": "", + "self_alloc": "", + "impl_call": + "{return_value} = {c_basename}_impl({vectorcall_impl_arguments});", + "init_result_check": "", +} +# METHOD_INIT: Create self through tp_new. In vectorcall we have no tuple of +# args and want to void constructing one so pass the empty tuple. This is okay +# for PyType_GenericNew which ignores args. +VECTORCALL_FINALE_MARKERS_INIT: Final[dict[str, str]] = { + "init_declarations": "PyObject *self;\nint _result;", + "self_alloc": libclinic.normalize_snippet(""" + self = _PyType_CAST(type)->tp_new(_PyType_CAST(type), + (PyObject *)&_Py_SINGLETON(tuple_empty), NULL); + if (self == NULL) {{ + goto exit; + }} + """), + "impl_call": "_result = {c_basename}_impl({vectorcall_impl_arguments});", + "init_result_check": libclinic.normalize_snippet(""" + if (_result != 0) {{ + Py_DECREF(self); + goto exit; + }} + return_value = self; + """), +} class ParseArgsCodeGen: @@ -246,6 +297,7 @@ class ParseArgsCodeGen: methoddef_define: str parser_prototype: str parser_definition: str + parser_helper: str cpp_if: str cpp_endif: str methoddef_ifndef: str @@ -365,11 +417,7 @@ def init_limited_capi(self) -> None: warn(f"Function {self.func.full_name} cannot use limited C API") self.limited_capi = False - def parser_body( - self, - *fields: str, - declarations: str = '' - ) -> None: + def parser_body(self, *fields: str) -> None: lines = [self.parser_prototype] self.parser_body_fields = fields @@ -380,23 +428,15 @@ def parser_body( {declarations} {initializers} """) + "\n" - finale = libclinic.normalize_snippet(""" - {modifications} - {lock} - {return_value} = {c_basename}_impl({impl_arguments}); - {unlock} - {return_conversion} - {post_parsing} - - {exit_label} - {cleanup} - return {parser_retval}; - }} - """) + finale = PARSER_FINALE_SKELETON for field in preamble, *fields, finale: lines.append(field) - code = libclinic.linear_format("\n".join(lines), - parser_declarations=self.declarations) + code = libclinic.linear_format( + "\n".join(lines), + parser_declarations=self.declarations, + self_alloc="", + impl_call="{return_value} = {c_basename}_impl({impl_arguments});", + init_result_check="") self.parser_definition = code def parse_no_args(self) -> None: @@ -809,6 +849,67 @@ def _parse_kwarg(self) -> str: assert isinstance(c, libclinic.converters.VarKeywordCConverter) return c.parse_var_keyword() + def _check_positional(self, nargs: str, *, + indent: int = 4) -> list[str]: + """Emit an argument count check when needed. + + Varpos functions have no upper bound but still need a check when a + minimum number of positional arguments are required. + """ + max_args = NO_VARARG if self.varpos else self.max_pos + if not self.min_pos and max_args == NO_VARARG: + return [] + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_CheckPositional()') + return [libclinic.normalize_snippet(f""" + if (!_PyArg_CheckPositional("{{name}}", {nargs}, {self.min_pos}, {max_args})) {{{{ + goto exit; + }}}} + """, indent=indent)] + + def _parse_positional_args( + self, + *, + argname_fmt: str, + nargs: str, + limited_capi: bool, + ) -> list[str] | None: + """Emit per-parameter positional argument parsing. + + Shared by parse_pos_only() and the vectorcall paths. Returns the + code snippets, or None if a converter doesn't support parse_arg + (the caller must fall back to a tuple/stack parser). + """ + parser_code: list[str] = [] + for i, p in enumerate(self.parameters): + parsearg = p.converter.parse_arg(argname_fmt % i, + p.get_displayname(i + 1), + limited_capi=limited_capi) + if parsearg is None: + if self.varpos: + raise ValueError( + f"Using converter {p.converter} is not supported " + f"in function with var-positional parameter") + return None + if i >= self.min_pos: + # p and everything after it is optional. + parser_code.append(libclinic.normalize_snippet(f""" + if ({nargs} < {i + 1}) {{{{ + goto skip_optional; + }}}} + """, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + + if self.min_pos < len(self.parameters): + parser_code.append("skip_optional:") + if self.varpos: + parser_code.append(libclinic.normalize_snippet(self._parse_vararg(), + indent=4)) + elif self.var_keyword: + parser_code.append(libclinic.normalize_snippet(self._parse_kwarg(), + indent=4)) + return parser_code + def select_positional_convention(self) -> tuple[str, str]: """Select the calling convention of a positional-only function. @@ -884,46 +985,14 @@ def parse_pos_only(self) -> None: }}}} """, indent=4)) - elif self.min_pos or max_args != NO_VARARG: - self.codegen.add_include('pycore_modsupport.h', - '_PyArg_CheckPositional()') - parser_code.append(libclinic.normalize_snippet(f""" - if (!_PyArg_CheckPositional("{{name}}", {nargs}, {self.min_pos}, {max_args})) {{{{ - goto exit; - }}}} - """, indent=4)) - - has_optional = False - use_parser_code = True - for i, p in enumerate(self.parameters): - displayname = p.get_displayname(i+1) - argname = argname_fmt % i - parsearg: str | None - parsearg = p.converter.parse_arg(argname, displayname, limited_capi=self.limited_capi) - if parsearg is None: - if self.varpos: - raise ValueError( - f"Using converter {p.converter} is not supported " - f"in function with var-positional parameter") - use_parser_code = False - parser_code = [] - break - if has_optional or p.is_optional(): - has_optional = True - parser_code.append(libclinic.normalize_snippet(""" - if (%s < %d) {{ - goto skip_optional; - }} - """, indent=4) % (nargs, i + 1)) - parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + else: + parser_code.extend(self._check_positional(nargs)) - if use_parser_code: - if has_optional: - parser_code.append("skip_optional:") - if self.varpos: - parser_code.append(libclinic.normalize_snippet(self._parse_vararg(), indent=4)) - elif self.var_keyword: - parser_code.append(libclinic.normalize_snippet(self._parse_kwarg(), indent=4)) + pos_code = self._parse_positional_args( + argname_fmt=argname_fmt, nargs=nargs, + limited_capi=self.limited_capi) + if pos_code is not None: + parser_code.extend(pos_code) else: parse_call = self.render_parse_all_arguments() parser_code = [libclinic.normalize_snippet(""" @@ -940,7 +1009,6 @@ def parse_var_keyword(self) -> None: nargs = 'PyTuple_GET_SIZE(args)' parser_code = [] - max_args = NO_VARARG if self.varpos else self.max_pos if self.varpos is None and self.min_pos == self.max_pos == 0: self.codegen.add_include('pycore_modsupport.h', '_PyArg_NoPositional()') @@ -949,14 +1017,8 @@ def parse_var_keyword(self) -> None: goto exit; }} """, indent=4)) - elif self.min_pos or max_args != NO_VARARG: - self.codegen.add_include('pycore_modsupport.h', - '_PyArg_CheckPositional()') - parser_code.append(libclinic.normalize_snippet(f""" - if (!_PyArg_CheckPositional("{{name}}", {nargs}, {self.min_pos}, {max_args})) {{{{ - goto exit; - }}}} - """, indent=4)) + else: + parser_code.extend(self._check_positional(nargs)) has_optional = False for i, p in enumerate(self.parameters): @@ -984,7 +1046,6 @@ def parse_var_keyword(self) -> None: self.parser_body(*parser_code) def parse_general(self, clang: CLanguage) -> None: - parsearg: str | None deprecated_positionals: dict[int, Parameter] = {} deprecated_keywords: dict[int, Parameter] = {} for i, p in enumerate(self.parameters): @@ -1026,6 +1087,21 @@ def parse_general(self, clang: CLanguage) -> None: if has_optional_kw: self.declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, self.min_pos + self.min_kw_only) unpack_args = 'args, nargs, NULL, kwnames' + elif self.func.vectorcall: + # Emit parsing body as a helper that takes both vectorcall and + # fastcall calling conventions. + self.flags = "METH_VARARGS|METH_KEYWORDS" + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD_HELPER + argsname = 'fastargs' + argname_fmt = 'fastargs[%d]' + self.declarations = declare_parser(self.func, codegen=self.codegen) + self.declarations += "\nPyObject *argsbuf[%s];" % (len(self.converters) or 1) + self.declarations += "\nPyObject * const *fastargs;" + if has_optional_kw: + self.declarations += ( + "\nPy_ssize_t noptargs = %s + nkw - %d;" + % (nargs, self.min_pos + self.min_kw_only)) + unpack_args = 'args, nargs, kwargs, kwnames' else: # positional-or-keyword arguments self.flags = "METH_VARARGS|METH_KEYWORDS" @@ -1173,7 +1249,7 @@ def parse_general(self, clang: CLanguage) -> None: parser_code.insert(0, code) assert self.parser_prototype is not None - self.parser_body(*parser_code, declarations=self.declarations) + self.parser_body(*parser_code) def copy_includes(self) -> None: # Copy includes from parameters to Clinic after parse_arg() @@ -1191,15 +1267,41 @@ def copy_includes(self) -> None: def handle_new_or_init(self) -> None: self.methoddef_define = '' - if self.func.kind is METHOD_NEW: - self.parser_prototype = PARSER_PROTOTYPE_KEYWORD - else: + if self.func.kind is METHOD_INIT: self.return_value_declaration = "int {parser_retval} = -1;" - self.parser_prototype = PARSER_PROTOTYPE_KEYWORD___INIT__ + + if self.func.vectorcall and 'METH_KEYWORDS' in self.flags: + self._new_or_init_delegate_to_helper() + else: + self._new_or_init_parser_body() + + def _new_or_init_delegate_to_helper(self) -> None: + """Change the parser to a helper that call and vectorcall can use. + + The parsing code is almost identical with slightly different args so + share the parser body as a {c_basename}_helper helper and the slot + entry point is a thin wrapper around it. + """ + self.parser_helper = self.parser_definition + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD + self.parser_definition = '\n'.join([ + self.parser_prototype, + '{{', + ' return {c_basename}_helper({self_name}, ' + '_PyTuple_CAST(args)->ob_item,', + ' PyTuple_GET_SIZE(args),', + ' kwargs ? PyDict_GET_SIZE(kwargs) : 0,', + ' kwargs, NULL);', + '}}', + ]) + + def _new_or_init_parser_body(self) -> None: + """Rebuild the parser body with the checks tp_new / tp_init need.""" + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD fields: list[str] = list(self.parser_body_fields) - parses_positional = 'METH_NOARGS' not in self.flags parses_keywords = 'METH_KEYWORDS' in self.flags + parses_positional = 'METH_NOARGS' not in self.flags if parses_keywords: assert parses_positional @@ -1224,7 +1326,7 @@ def handle_new_or_init(self) -> None: }} """, indent=4)) - self.parser_body(*fields, declarations=self.declarations) + self.parser_body(*fields) def process_methoddef(self, clang: CLanguage) -> None: methoddef_cast_end = "" @@ -1273,6 +1375,9 @@ def finalize(self, clang: CLanguage) -> None: self.impl_prototype += ";" self.parser_definition = self.parser_definition.replace("{return_value_declaration}", self.return_value_declaration) + if self.parser_helper: + self.parser_helper = self.parser_helper.replace( + "{return_value_declaration}", self.return_value_declaration) compiler_warning = clang.compiler_deprecated_warning(self.func, self.parameters) if compiler_warning: @@ -1286,10 +1391,12 @@ def create_template_dict(self) -> dict[str, str]: "methoddef_define" : self.methoddef_define, "parser_prototype" : self.parser_prototype, "parser_definition" : self.parser_definition, + "parser_helper" : self.parser_helper, "impl_definition" : self.impl_definition, "cpp_if" : self.cpp_if, "cpp_endif" : self.cpp_endif, "methoddef_ifndef" : self.methoddef_ifndef, + "vectorcall_definition" : self.vectorcall_definition, } # make sure we didn't forget to assign something, @@ -1302,6 +1409,180 @@ def create_template_dict(self) -> dict[str, str]: d2[name] = value return d2 + def _vectorcall_type_check(self) -> list[str]: + """Assert `type` is the one type this vectorcall was generated for. + + The generated code is only correct for that type: __init__ calls + tp_new with no arguments, then the impl. tp_vectorcall is not + inherited, so subclasses never reach it; the assert catches C code + installing the function on a second type. + """ + func = self.func + # The DSL parser rejects @vectorcall without a class and type object. + assert func.cls is not None + assert func.cls.type_object + return [libclinic.normalize_snippet(f""" + assert(Py_Is(_PyType_CAST(type), {func.cls.type_object})); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + """, indent=4)] + + def _vectorcall_positional(self, *, + arity_checked: bool = False) -> list[str]: + """Positional argument parsing for vectorcall. + + arity_checked: Already have a number of arguments check. + """ + pos_code = self._parse_positional_args( + argname_fmt='args[%d]', nargs='nargs', limited_capi=False) + # Converter support was validated when @vectorcall was parsed. + assert pos_code is not None + if arity_checked: + return pos_code + return [*self._check_positional('nargs'), *pos_code] + + def _assemble_vectorcall(self, preamble: str, fields: tuple[str, ...], + finale: str) -> None: + """Wrap parser code in the vectorcall prototype.""" + prototype = PARSER_PROTOTYPE_VECTORCALL.replace( + "{vc_basename}", self.func.c_basename_vectorcall) + lines = [prototype, preamble, *fields, finale] + + if self.func.kind is METHOD_INIT: + markers = VECTORCALL_FINALE_MARKERS_INIT + self.codegen.add_include('pycore_runtime.h', '_Py_SINGLETON()') + else: + markers = VECTORCALL_FINALE_MARKERS_NEW + code = libclinic.linear_format("\n".join(lines), **markers) + self.vectorcall_definition = code + + def vectorcall_body(self, *fields: str) -> None: + """Assemble a vectorcall function that parses inline and calls the impl. + + The preamble declares return_value and the per-arg locals, and the + finale calls {c_basename}_impl before running the exit/cleanup + block. + """ + preamble = libclinic.normalize_snippet(""" + {{ + PyObject *return_value = NULL; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + {init_declarations} + {declarations} + {initializers} + """) + "\n" + self._assemble_vectorcall(preamble, fields, PARSER_FINALE_SKELETON) + + def parse_vectorcall_pos_only(self) -> None: + """All positional sometimes optional arguments.""" + parser_code = self._vectorcall_type_check() + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_NoKwnames()') + parser_code.append(libclinic.normalize_snippet(""" + if (!_PyArg_NoKwnames("{name}", kwnames)) {{ + goto exit; + }} + """, indent=4)) + + parser_code.extend(self._vectorcall_positional()) + self.vectorcall_body(*parser_code) + + def _vectorcall_guarded_delegate(self, condition: str, nkw: str) -> str: + """Emit `if (condition) { }`.""" + return libclinic.linear_format( + libclinic.normalize_snippet(f""" + if ({condition}) {{{{ + {{delegate}} + }}}} + """, indent=4), + delegate=self._vectorcall_delegate_to_helper(nkw)) + + def _vectorcall_delegate_to_helper(self, nkw: str) -> str: + """Hand off to the {c_basename}_helper helper and return. + + nkw: Number of keyword arguments. + """ + if self.func.kind is METHOD_INIT: + receiver = "self" + bind_result = "_result = " + prologue = libclinic.normalize_snippet(""" + self = _PyType_CAST(type)->tp_new(_PyType_CAST(type), + (PyObject *)&_Py_SINGLETON(tuple_empty), NULL); + if (self == NULL) {{ + return NULL; + }} + """, indent=4) + epilogue = libclinic.normalize_snippet(""" + if (_result != 0) {{ + Py_DECREF(self); + return NULL; + }} + return self; + """, indent=4) + else: + receiver = "_PyType_CAST(type)" + bind_result = "return " + prologue = epilogue = "" + helper_call = libclinic.normalize_snippet(f""" + {bind_result}{{c_basename}}_helper({receiver}, args, nargs, + {nkw}, + NULL, kwnames); + """, indent=4) + parts = [prologue, helper_call, epilogue] + return "\n".join(part for part in parts if part) + + def parse_vectorcall_kw_required(self) -> None: + """Required keyword arguemnts; always delegate to helper.""" + parser_code = self._vectorcall_type_check() + parser_code.append(self._vectorcall_delegate_to_helper( + 'kwnames ? PyTuple_GET_SIZE(kwnames) : 0')) + preamble = libclinic.normalize_snippet(""" + {{ + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + {init_declarations} + """) + "\n" + self._assemble_vectorcall(preamble, tuple(parser_code), "}}") + + def parse_vectorcall_pos_or_kw(self) -> None: + """Optional positional and keyword argument vectorcall. + + Delegate to the helper if keywords present or if position count is out + of range. Position count so the error messages match the non-vectorcall. + """ + assert not self.varpos + checks = ['kwnames != NULL'] + if self.min_pos: + checks.append(f"nargs < {self.min_pos}") + checks.append(f"nargs > {self.max_pos}") + + parser_code = self._vectorcall_type_check() + parser_code.append(self._vectorcall_guarded_delegate( + " || ".join(checks), 'kwnames ? PyTuple_GET_SIZE(kwnames) : 0')) + parser_code.extend(self._vectorcall_positional(arity_checked=True)) + self.vectorcall_body(*parser_code) + + def parse_vectorcall(self) -> None: + """Generate the vectorcall entry point for __new__ / __init__. + + Dispatch to specific parser-code builders based on parameter shape. + """ + # Branches ordered to mirror parse_args(). The DSL parser rejects + # @vectorcall with optional groups, and METH_O never applies to + # __new__/__init__. They always have arguments. + assert not self.has_option_groups() + assert not self.use_meth_o() + if not self.parameters and not self.varpos and not self.var_keyword: + raise NotImplementedError("No argument vectorcall") + elif self.var_keyword is not None: + self.parse_vectorcall_kw_required() + elif self.pos_only == len(self.parameters): + self.parse_vectorcall_pos_only() + elif any(p.is_keyword_only() for p in self.parameters) or self.varpos: + self.parse_vectorcall_kw_required() + else: + self.parse_vectorcall_pos_or_kw() + def parse_args(self, clang: CLanguage) -> dict[str, str]: self.select_prototypes() self.init_limited_capi() @@ -1310,8 +1591,10 @@ def parse_args(self, clang: CLanguage) -> dict[str, str]: self.declarations = "" self.parser_prototype = "" self.parser_definition = "" + self.parser_helper = "" self.impl_prototype = None self.impl_definition = IMPL_DEFINITION_PROTOTYPE + self.vectorcall_definition = "" # parser_body_fields remembers the fields passed in to the # previous call to parser_body. this is used for an awful hack. @@ -1337,4 +1620,8 @@ def parse_args(self, clang: CLanguage) -> dict[str, str]: self.process_methoddef(clang) self.finalize(clang) + # Generate vectorcall function if requested + if self.func.vectorcall: + self.parse_vectorcall() + return self.create_template_dict()