From: Christian Hammond <christian@beanbaginc.com>
Date: Wed, 15 Dec 2021 23:12:36 -0800
Subject: Port ctypes and system libffi patches for arm64/macOS 10.15+ to Python 3.6.15

This ports the following ctypes and libffi pyenv patches for Python
2.7.18 to Python 3.6.15:

* `0004-Use-system-libffi-for-Mac-OS-10.15-and-up.patch`
* `0005-ctypes-use-the-correct-ABI-for-variadic-functions.patch`
* `0006-ctypes-probe-libffi-for-ffi_closure_alloc-and-ffi_pr.patch`

These patches enable use of system libffi (fixing a broken `ctypes`
module on arm64 targets) and enable calling variadic functions on arm64.
They've been combined from patches port from Homebrew to pyenv by Takumi
Sueda, updated to work on the Python 3.6.15 codebase.

The Homebrew patches are themselves backports of changes in Python 3.9
and 3.10. That patch can be found at:

https://github.com/Homebrew/formula-patches/blob/master/python/3.8.7.patch

diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst
index a7cc0c84df..f25f3c78d9 100644
--- a/Doc/library/ctypes.rst
+++ b/Doc/library/ctypes.rst
@@ -1551,6 +1551,13 @@ They are instances of a private class:
       value usable as argument (integer, string, ctypes instance).  This allows
       defining adapters that can adapt custom objects as function parameters.
 
+   .. attribute:: variadic
+
+      Assign a boolean to specify that the function takes a variable number of
+      arguments.   This does not matter on most platforms, but for Apple arm64
+      platforms variadic functions have a different calling convention than
+      normal functions.
+
    .. attribute:: errcheck
 
       Assign a Python function or another callable to this attribute. The
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index 56ab0ad732..013b1e9ce7 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -2443,11 +2443,14 @@ class CAPITest(unittest.TestCase):
     def test_from_format(self):
         support.import_module('ctypes')
         from ctypes import (
+            c_char_p,
             pythonapi, py_object, sizeof,
             c_int, c_long, c_longlong, c_ssize_t,
             c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p)
         name = "PyUnicode_FromFormat"
         _PyUnicode_FromFormat = getattr(pythonapi, name)
+        _PyUnicode_FromFormat.argtypes = (c_char_p,)
+        _PyUnicode_FromFormat.variadic = True
         _PyUnicode_FromFormat.restype = py_object
 
         def PyUnicode_FromFormat(format, *args):
diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index 24d64858ed..351f274ad3 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -3120,6 +3120,34 @@ PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored))
     }
 }
 
+static int
+PyCFuncPtr_set_variadic(PyCFuncPtrObject *self, PyObject *ob)
+{
+    StgDictObject *dict = PyObject_stgdict((PyObject *)self);
+    assert(dict);
+    int r = PyObject_IsTrue(ob);
+    if (r == 1) {
+        dict->flags |= FUNCFLAG_VARIADIC;
+        return 0;
+    } else if (r == 0) {
+        dict->flags &= ~FUNCFLAG_VARIADIC;
+        return 0;
+    } else {
+        return -1;
+    }
+}
+
+static PyObject *
+PyCFuncPtr_get_variadic(PyCFuncPtrObject *self)
+{
+    StgDictObject *dict = PyObject_stgdict((PyObject *)self);
+    assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */
+    if (dict->flags & FUNCFLAG_VARIADIC)
+        Py_RETURN_TRUE;
+    else
+        Py_RETURN_FALSE;
+}
+
 static int
 PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored))
 {
@@ -3166,6 +3194,8 @@ static PyGetSetDef PyCFuncPtr_getsets[] = {
     { "argtypes", (getter)PyCFuncPtr_get_argtypes,
       (setter)PyCFuncPtr_set_argtypes,
       "specify the argument types", NULL },
+    { "variadic", (getter)PyCFuncPtr_get_variadic, (setter)PyCFuncPtr_set_variadic,
+      "specify if function takes variable number of arguments", NULL },
     { NULL, NULL }
 };
 
@@ -5576,6 +5606,7 @@ PyInit__ctypes(void)
     PyModule_AddObject(m, "FUNCFLAG_USE_ERRNO", PyLong_FromLong(FUNCFLAG_USE_ERRNO));
     PyModule_AddObject(m, "FUNCFLAG_USE_LASTERROR", PyLong_FromLong(FUNCFLAG_USE_LASTERROR));
     PyModule_AddObject(m, "FUNCFLAG_PYTHONAPI", PyLong_FromLong(FUNCFLAG_PYTHONAPI));
+    PyModule_AddObject(m, "FUNCFLAG_VARIADIC", PyLong_FromLong(FUNCFLAG_VARIADIC));
     PyModule_AddStringConstant(m, "__version__", "1.1.0");
 
     PyModule_AddObject(m, "_memmove_addr", PyLong_FromVoidPtr(memmove));
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index 2bb289bce0..12951eadbf 100644
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -752,7 +754,8 @@ static int _call_function_pointer(int flags,
                                   ffi_type **atypes,
                                   ffi_type *restype,
                                   void *resmem,
-                                  int argcount)
+                                  int argcount,
+                                  int argtypecount)
 {
 #ifdef WITH_THREAD
     PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */
@@ -780,15 +783,39 @@ static int _call_function_pointer(int flags,
     if ((flags & FUNCFLAG_CDECL) == 0)
         cc = FFI_STDCALL;
 #endif
-    if (FFI_OK != ffi_prep_cif(&cif,
-                               cc,
-                               argcount,
-                               restype,
-                               atypes)) {
-        PyErr_SetString(PyExc_RuntimeError,
-                        "ffi_prep_cif failed");
-        return -1;
+
+#if HAVE_FFI_PREP_CIF_VAR
+    /* Everyone SHOULD set f.variadic=True on variadic function pointers, but
+     * lots of existing code will not.  If there's at least one arg and more
+     * args are passed than are defined in the prototype, then it must be a
+     * variadic function. */
+    if ((flags & FUNCFLAG_VARIADIC) ||
+        (argtypecount != 0 && argcount > argtypecount))
+    {
+        if (FFI_OK != ffi_prep_cif_var(&cif,
+                                       cc,
+                                       argtypecount,
+                                       argcount,
+                                       restype,
+                                       atypes)) {
+            PyErr_SetString(PyExc_RuntimeError,
+                            "ffi_prep_cif_var failed");
+            return -1;
+        }
+    } else {
+#endif
+        if (FFI_OK != ffi_prep_cif(&cif,
+                                   cc,
+                                   argcount,
+                                   restype,
+                                   atypes)) {
+            PyErr_SetString(PyExc_RuntimeError,
+                            "ffi_prep_cif failed");
+            return -1;
+        }
+#if HAVE_FFI_PREP_CIF_VAR
     }
+#endif
 
     if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) {
         error_object = _ctypes_get_errobj(&space);
@@ -1181,9 +1208,8 @@ PyObject *_ctypes_callproc(PPROC pProc,
 
     if (-1 == _call_function_pointer(flags, pProc, avalues, atypes,
                                      rtype, resbuf,
-                                     Py_SAFE_DOWNCAST(argcount,
-                                                      Py_ssize_t,
-                                                      int)))
+                                     Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int),
+                                     Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int)))
         goto cleanup;
 
 #ifdef WORDS_BIGENDIAN
diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h
index 5d3b966338..1299ec25aa 100644
--- a/Modules/_ctypes/ctypes.h
+++ b/Modules/_ctypes/ctypes.h
@@ -285,6 +285,7 @@ PyObject *_ctypes_callproc(PPROC pProc,
 #define FUNCFLAG_PYTHONAPI 0x4
 #define FUNCFLAG_USE_ERRNO 0x8
 #define FUNCFLAG_USE_LASTERROR 0x10
+#define FUNCFLAG_VARIADIC 0x20
 
 #define TYPEFLAG_ISPOINTER 0x100
 #define TYPEFLAG_HASPOINTER 0x200
diff --git a/setup.py b/setup.py
index 8e03c77997..3b124c4b7b 100644
--- a/setup.py
+++ b/setup.py
@@ -167,6 +167,13 @@ def macosx_sdk_specified():
     macosx_sdk_root()
     return MACOS_SDK_SPECIFIED

+def is_macosx_at_least(vers):
+    if host_platform == 'darwin':
+        dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
+        if dep_target:
+            return tuple(map(int, str(dep_target).split('.'))) >= vers
+    return False
+
 def is_macosx_sdk_path(path):
     """
     Returns True if 'path' can be located in an OSX SDK
@@ -175,6 +182,13 @@ def is_macosx_sdk_path(path):
                 or path.startswith('/System/')
                 or path.startswith('/Library/') )
 
+def grep_headers_for(function, headers):
+    for header in headers:
+        with open(header, 'r') as f:
+            if function in f.read():
+                return True
+    return False
+
 def find_file(filename, std_dirs, paths):
     """Searches for the directory where a given file is located,
     and returns a possibly-empty list of additional directories, or None
@@ -2115,7 +2129,11 @@ class PyBuildExt(build_ext):
         return True
 
     def detect_ctypes(self, inc_dirs, lib_dirs):
-        self.use_system_libffi = False
+        if not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and is_macosx_at_least((10,15)):
+            self.use_system_libffi = True
+        else:
+            self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS")
+
         include_dirs = []
         extra_compile_args = []
         extra_link_args = []
@@ -2162,20 +2180,29 @@ class PyBuildExt(build_ext):
                      libraries=math_libs)
         self.extensions.extend([ext, ext_test])
 
+        ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR")
+        ffi_lib = None
+
         if host_platform == 'darwin':
-            if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"):
-                return
-            # OS X 10.5 comes with libffi.dylib; the include files are
-            # in /usr/include/ffi
-            inc_dirs.append('/usr/include/ffi')
+             if not self.use_system_libffi:
+                 return
+             ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi")
+             if os.path.exists(ffi_in_sdk):
+                 ffi_inc = ffi_in_sdk
+                 ffi_lib = 'ffi'
+             else:
+                 # OS X 10.5 comes with libffi.dylib; the include files are
+                 # in /usr/include/ffi
+                 inc_dirs.append('/usr/include/ffi')
         elif '--without-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS"):
             return
 
-        ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")]
-        if not ffi_inc or ffi_inc[0] == '':
-            ffi_inc = find_file('ffi.h', [], inc_dirs)
-        if ffi_inc is not None:
-            ffi_h = ffi_inc[0] + '/ffi.h'
+        if not ffi_inc:
+            found = find_file('ffi.h', [], inc_dirs)
+            if found:
+                ffi_inc = found[0]
+        if ffi_inc:
+            ffi_h = ffi_inc + '/ffi.h'
             with open(ffi_h) as f:
                 for line in f:
                     line = line.strip()
@@ -2186,15 +2213,22 @@ class PyBuildExt(build_ext):
                     ffi_inc = None
                     print('Header file {} does not define LIBFFI_H or '
                           'ffi_wrapper_h'.format(ffi_h))
-        ffi_lib = None
-        if ffi_inc is not None:
+        if ffi_lib is None and ffi_inc is not None:
             for lib_name in ('ffi', 'ffi_pic'):
                 if (self.compiler.find_library_file(lib_dirs, lib_name)):
                     ffi_lib = lib_name
                     break
 
         if ffi_inc and ffi_lib:
-            ext.include_dirs.extend(ffi_inc)
+            ffi_headers = glob(os.path.join(ffi_inc, '*.h'))
+            if grep_headers_for('ffi_closure_alloc', ffi_headers):
+                try:
+                    sources.remove('_ctypes/malloc_closure.c')
+                except ValueError:
+                    pass
+            if grep_headers_for('ffi_prep_cif_var', ffi_headers):
+                ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1")
+            ext.include_dirs.append(ffi_inc)
             ext.libraries.append(ffi_lib)
             self.use_system_libffi = True

-- 
2.30.1 (Apple Git-130)

