gh-146636: PEP 803: add Py_TARGET_ABI3T and .abi3t.so extension (GH-146637)

- Add Py_TARGET_ABI3T macro.
- Add ".abi3t.so" to importlib EXTENSION_SUFFIXES.
- Remove ".abi3.so" from importlib EXTENSION_SUFFIXES on Free Threading.
- Adjust tests

This is part of the implementation for PEP-803.
Detailed documentation to come later.

Co-authored-by: Nathan Goldbaum <nathan.goldbaum@gmail.com>

Author: encukou

Doc/c-api/stable.rst

@@ -88,6 +88,16 @@ Contents of the Limited API are :ref:`listed below <limited-api-list>`.
    You can also define ``Py_LIMITED_API`` to ``3``. This works the same as
    ``0x03020000`` (Python 3.2, the version that introduced Limited API).
 
+.. c:macro:: Py_TARGET_ABI3T
+
+   Define this macro before including ``Python.h`` to opt in to only use
+   the Limited API for :term:`free-threaded builds <free-threaded build>`,
+   and to select the Limited API version.
+
+   .. seealso:: :pep:`803`
+
+   .. versionadded:: next
+
 
 .. _stable-abi:
 

Include/Python.h

@@ -47,10 +47,6 @@
 #endif
 
 #if defined(Py_GIL_DISABLED)
-#  if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT)
-#    error "Py_LIMITED_API is not currently supported in the free-threaded build"
-#  endif
-
 #  if defined(_MSC_VER)
 #    include <intrin.h>             // __readgsqword()
 #  endif

Include/patchlevel.h

@@ -61,4 +61,32 @@
 #define PYTHON_ABI_VERSION 3
 #define PYTHON_ABI_STRING "3"
 
+
+/* Stable ABI for free-threaded builds (introduced in PEP 803)
+   is enabled by one of:
+     - Py_TARGET_ABI3T, or
+     - Py_LIMITED_API and Py_GIL_DISABLED.
+   "Output" macros to be used internally:
+     - Py_LIMITED_API (defines the subset of API we expose)
+     - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between
+       free-threaded & GIL)
+     (Don't use Py_TARGET_ABI3T directly: it's currently only used to set these
+      2 macros. It's also available for users' convenience.)
+ */
+#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \
+    && !defined(Py_TARGET_ABI3T)
+#  define Py_TARGET_ABI3T Py_LIMITED_API
+#endif
+#if defined(Py_TARGET_ABI3T)
+#  define _Py_OPAQUE_PYOBJECT
+#  if !defined(Py_LIMITED_API)
+#    define Py_LIMITED_API Py_TARGET_ABI3T
+#  elif Py_LIMITED_API > Py_TARGET_ABI3T
+     // if both are defined, use the *lower* version,
+     // i.e. maximum compatibility
+#    undef Py_LIMITED_API
+#    define Py_LIMITED_API Py_TARGET_ABI3T
+#  endif
+#endif
+
 #endif //_Py_PATCHLEVEL_H

Lib/test/test_cext/__init__.py

@@ -38,15 +38,15 @@ def test_build(self):
         self.check_build('_test_cext')
 
     def check_build(self, extension_name, std=None, limited=False,
-                    opaque_pyobject=False):
+                    abi3t=False):
         venv_dir = 'env'
         with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe:
             self._check_build(extension_name, python_exe,
                               std=std, limited=limited,
-                              opaque_pyobject=opaque_pyobject)
+                              abi3t=abi3t)
 
     def _check_build(self, extension_name, python_exe, std, limited,
-                     opaque_pyobject):
+                     abi3t):
         pkg_dir = 'pkg'
         os.mkdir(pkg_dir)
         shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP)))
@@ -60,8 +60,8 @@ def run_cmd(operation, cmd):
                 env['CPYTHON_TEST_STD'] = std
             if limited:
                 env['CPYTHON_TEST_LIMITED'] = '1'
-            if opaque_pyobject:
-                env['CPYTHON_TEST_OPAQUE_PYOBJECT'] = '1'
+            if abi3t:
+                env['CPYTHON_TEST_ABI3T'] = '1'
             env['CPYTHON_TEST_EXT_NAME'] = extension_name
             env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API))
             if support.verbose:
@@ -116,10 +116,9 @@ def test_build_limited_c11(self):
     def test_build_c11(self):
         self.check_build('_test_c11_cext', std='c11')
 
-    def test_build_opaque_pyobject(self):
-        # Test with _Py_OPAQUE_PYOBJECT
-        self.check_build('_test_limited_opaque_cext', limited=True,
-                         opaque_pyobject=True)
+    def test_build_abi3t(self):
+        # Test with Py_TARGET_ABI3T
+        self.check_build('_test_abi3t', abi3t=True)
 
     @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99")
     def test_build_c99(self):

Lib/test/test_cext/setup.py

@@ -59,7 +59,7 @@ def main():
     std = os.environ.get("CPYTHON_TEST_STD", "")
     module_name = os.environ["CPYTHON_TEST_EXT_NAME"]
     limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", ""))
-    opaque_pyobject = bool(os.environ.get("CPYTHON_TEST_OPAQUE_PYOBJECT", ""))
+    abi3t = bool(os.environ.get("CPYTHON_TEST_ABI3T", ""))
     internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0")))
 
     sources = [SOURCE]
@@ -91,14 +91,12 @@ def main():
         # CC env var overrides sysconfig CC variable in setuptools
         os.environ['CC'] = cmd
 
-    # Define Py_LIMITED_API macro
+    # Define opt-in macros
     if limited:
-        version = sys.hexversion
-        cflags.append(f'-DPy_LIMITED_API={version:#x}')
+        cflags.append(f'-DPy_LIMITED_API={sys.hexversion:#x}')
 
-    # Define _Py_OPAQUE_PYOBJECT macro
-    if opaque_pyobject:
-        cflags.append(f'-D_Py_OPAQUE_PYOBJECT')
+    if abi3t:
+        cflags.append(f'-DPy_TARGET_ABI3T={sys.hexversion:#x}')
 
     if internal:
         cflags.append('-DTEST_INTERNAL_C_API=1')

Lib/test/test_importlib/extension/test_finder.py

@@ -1,4 +1,4 @@
-from test.support import is_apple_mobile
+from test.support import is_apple_mobile, Py_GIL_DISABLED
 from test.test_importlib import abc, util
 
 machinery = util.import_importlib('importlib.machinery')
@@ -59,6 +59,20 @@ def test_module(self):
     def test_failure(self):
         self.assertIsNone(self.find_spec('asdfjkl;'))
 
+    def test_abi3_extension_suffixes(self):
+        suffixes = self.machinery.EXTENSION_SUFFIXES
+        if 'win32' in sys.platform:
+            # Either "_d.pyd" or ".pyd" must be in suffixes
+            self.assertTrue({"_d.pyd", ".pyd"}.intersection(suffixes))
+        elif 'cygwin' in sys.platform:
+            pass
+        else:
+            if Py_GIL_DISABLED:
+                self.assertNotIn(".abi3.so", suffixes)
+            else:
+                self.assertIn(".abi3.so", suffixes)
+            self.assertIn(".abi3t.so", suffixes)
+
 
 (Frozen_FinderTests,
  Source_FinderTests

Python/dynload_shlib.c

@@ -44,7 +44,10 @@ const char *_PyImport_DynLoadFiletab[] = {
 #ifdef ALT_SOABI
     "." ALT_SOABI ".so",
 #endif
+#ifndef Py_GIL_DISABLED
     ".abi" PYTHON_ABI_STRING ".so",
+#endif  /* Py_GIL_DISABLED */
+    ".abi" PYTHON_ABI_STRING "t.so",
     ".so",
 #endif  /* __CYGWIN__ */
     NULL,