github/pyenv
1
0
Fork 0
mirror of https://github.com/pyenv/pyenv.git synced 2024-11-21 20:47:00 -05:00
Code Issues Projects Releases Packages Wiki Activity

Fix Python 3.7.12 compilation on macOS arm64/M1. (#2190)

Browse source 
This change ports pyenv and Python patches to 3.7.12 to enable the
`ctypes `and `decimal` modules to compile.

While Python 3.7.12 itself compiles on arm64/M1, both of these modules
fail to compile, due to missing support for locating system libffi and
due to architecture gate-keeping. These issues have been fixed in newer
releases of Python, and in other pyenv patch bundles.

The following patches are provided:

1. `0001-Port-ctypes-and-system-libffi-patches-for-arm64-macO.patch` -
   Fixes system `ffi.h`/`libffi` path determination and usage and
   enables calling of variadic functions, fixing ctypes support
   (consolidated port of existing pyenv patches for 2.7.18 that iterate
   on this logic).

2. `0002-bpo-41100-fix-_decimal-for-arm64-Mac-OS-GH-21228.patch` -
   Adds arm64 to the list of allowable architectures for the
   `decimal` module (port of Python patch introduced in 3.8.10).
This commit is contained in:
Christian Hammond 2021-12-19 20:14:14 -08:00 committed by GitHub
parent 152457a428
commit 2d9e981d60
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 348 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

312
plugins/python-build/share/python-build/patches/3.7.12/Python-3.7.12/0001-Port-ctypes-and-system-libffi-patches-for-arm64-macO.patch Normal file
View file

@ -0,0 +1,312 @@
From: Christian Hammond <christian@beanbaginc.com>
Date: Wed, 15 Dec 2021 23:12:36 -0800
Subject: Port ctypes and system libffi patches for arm64/macOS 10.15+ to Python 3.7.12
This ports the following ctypes and libffi pyenv patches for Python
2.7.18 to Python 3.7.12:
* `0004-Use-system-libffi-for-Mac-OS-10.15-and-up.patch`
* `0005-ctypes-use-the-correct-ABI-for-variadic-functions.patch`
* `0006-ctypes-probe-libffi-for-ffi_closure_alloc-and-ffi_pr.patch`
These patches enable use of system libffi (fixing a broken `ctypes`
module on arm64 targets) and enable calling variadic functions on arm64.
They've been combined from patches port from Homebrew to pyenv by Takumi
Sueda, updated to work on the Python 3.7.12 codebase.
The Homebrew patches are themselves backports of changes in Python 3.9
and 3.10. That patch can be found at:
https://github.com/Homebrew/formula-patches/blob/master/python/3.8.7.patch
diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst
index 715d595b24..7743144978 100644
--- a/Doc/library/ctypes.rst
+++ b/Doc/library/ctypes.rst
@@ -1551,6 +1551,13 @@ They are instances of a private class:
value usable as argument (integer, string, ctypes instance). This allows
defining adapters that can adapt custom objects as function parameters.
+ .. attribute:: variadic
+
+ Assign a boolean to specify that the function takes a variable number of
+ arguments. This does not matter on most platforms, but for Apple arm64
+ platforms variadic functions have a different calling convention than
+ normal functions.
+
.. attribute:: errcheck
Assign a Python function or another callable to this attribute. The
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index 4ebd82d3e0..7b73c190b6 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -2458,11 +2458,14 @@ class CAPITest(unittest.TestCase):
def test_from_format(self):
support.import_module('ctypes')
from ctypes import (
+ c_char_p,
pythonapi, py_object, sizeof,
c_int, c_long, c_longlong, c_ssize_t,
c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p)
name = "PyUnicode_FromFormat"
_PyUnicode_FromFormat = getattr(pythonapi, name)
+ _PyUnicode_FromFormat.argtypes = (c_char_p,)
+ _PyUnicode_FromFormat.variadic = True
_PyUnicode_FromFormat.restype = py_object
def PyUnicode_FromFormat(format, *args):
diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index dd0c61fd8a..79137e1dc7 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -3174,6 +3174,34 @@ PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored))
}
}
+static int
+PyCFuncPtr_set_variadic(PyCFuncPtrObject *self, PyObject *ob)
+{
+ StgDictObject *dict = PyObject_stgdict((PyObject *)self);
+ assert(dict);
+ int r = PyObject_IsTrue(ob);
+ if (r == 1) {
+ dict->flags |= FUNCFLAG_VARIADIC;
+ return 0;
+ } else if (r == 0) {
+ dict->flags &= ~FUNCFLAG_VARIADIC;
+ return 0;
+ } else {
+ return -1;
+ }
+}
+
+static PyObject *
+PyCFuncPtr_get_variadic(PyCFuncPtrObject *self)
+{
+ StgDictObject *dict = PyObject_stgdict((PyObject *)self);
+ assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */
+ if (dict->flags & FUNCFLAG_VARIADIC)
+ Py_RETURN_TRUE;
+ else
+ Py_RETURN_FALSE;
+}
+
static int
PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored))
{
@@ -3219,6 +3247,8 @@ static PyGetSetDef PyCFuncPtr_getsets[] = {
{ "argtypes", (getter)PyCFuncPtr_get_argtypes,
(setter)PyCFuncPtr_set_argtypes,
"specify the argument types", NULL },
+ { "variadic", (getter)PyCFuncPtr_get_variadic, (setter)PyCFuncPtr_set_variadic,
+ "specify if function takes variable number of arguments", NULL },
{ NULL, NULL }
};
@@ -5632,6 +5662,7 @@ PyInit__ctypes(void)
PyModule_AddObject(m, "FUNCFLAG_USE_ERRNO", PyLong_FromLong(FUNCFLAG_USE_ERRNO));
PyModule_AddObject(m, "FUNCFLAG_USE_LASTERROR", PyLong_FromLong(FUNCFLAG_USE_LASTERROR));
PyModule_AddObject(m, "FUNCFLAG_PYTHONAPI", PyLong_FromLong(FUNCFLAG_PYTHONAPI));
+ PyModule_AddObject(m, "FUNCFLAG_VARIADIC", PyLong_FromLong(FUNCFLAG_VARIADIC));
PyModule_AddStringConstant(m, "__version__", "1.1.0");
PyModule_AddObject(m, "_memmove_addr", PyLong_FromVoidPtr(memmove));
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index 9cbf9801ad..e7fe11176b 100644
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -754,7 +756,8 @@ static int _call_function_pointer(int flags,
ffi_type **atypes,
ffi_type *restype,
void *resmem,
- int argcount)
+ int argcount,
+ int argtypecount)
{
PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */
PyObject *error_object = NULL;
@@ -780,15 +783,39 @@ static int _call_function_pointer(int flags,
if ((flags & FUNCFLAG_CDECL) == 0)
cc = FFI_STDCALL;
#endif
- if (FFI_OK != ffi_prep_cif(&cif,
- cc,
- argcount,
- restype,
- atypes)) {
- PyErr_SetString(PyExc_RuntimeError,
- "ffi_prep_cif failed");
- return -1;
+
+#if HAVE_FFI_PREP_CIF_VAR
+ /* Everyone SHOULD set f.variadic=True on variadic function pointers, but
+ * lots of existing code will not. If there's at least one arg and more
+ * args are passed than are defined in the prototype, then it must be a
+ * variadic function. */
+ if ((flags & FUNCFLAG_VARIADIC) ||
+ (argtypecount != 0 && argcount > argtypecount))
+ {
+ if (FFI_OK != ffi_prep_cif_var(&cif,
+ cc,
+ argtypecount,
+ argcount,
+ restype,
+ atypes)) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "ffi_prep_cif_var failed");
+ return -1;
+ }
+ } else {
+#endif
+ if (FFI_OK != ffi_prep_cif(&cif,
+ cc,
+ argcount,
+ restype,
+ atypes)) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "ffi_prep_cif failed");
+ return -1;
+ }
+#if HAVE_FFI_PREP_CIF_VAR
}
+#endif
if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) {
error_object = _ctypes_get_errobj(&space);
@@ -1187,9 +1214,8 @@ PyObject *_ctypes_callproc(PPROC pProc,
if (-1 == _call_function_pointer(flags, pProc, avalues, atypes,
rtype, resbuf,
- Py_SAFE_DOWNCAST(argcount,
- Py_ssize_t,
- int)))
+ Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int),
+ Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int)))
goto cleanup;
#ifdef WORDS_BIGENDIAN
diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h
index e58f85233c..e45975f6ad 100644
--- a/Modules/_ctypes/ctypes.h
+++ b/Modules/_ctypes/ctypes.h
@@ -285,6 +285,7 @@ PyObject *_ctypes_callproc(PPROC pProc,
#define FUNCFLAG_PYTHONAPI 0x4
#define FUNCFLAG_USE_ERRNO 0x8
#define FUNCFLAG_USE_LASTERROR 0x10
+#define FUNCFLAG_VARIADIC 0x20
#define TYPEFLAG_ISPOINTER 0x100
#define TYPEFLAG_HASPOINTER 0x200
diff --git a/setup.py b/setup.py
index bf90600eaa..48ff120e9a 100644
--- a/setup.py
+++ b/setup.py
@@ -142,6 +142,13 @@ def macosx_sdk_root():
return MACOS_SDK_ROOT
+def is_macosx_at_least(vers):
+ if host_platform == 'darwin':
+ dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
+ if dep_target:
+ return tuple(map(int, str(dep_target).split('.'))) >= vers
+ return False
+
def is_macosx_sdk_path(path):
"""
Returns True if 'path' can be located in an OSX SDK
@@ -150,6 +157,13 @@ def is_macosx_sdk_path(path):
or path.startswith('/System/')
or path.startswith('/Library/') )
+def grep_headers_for(function, headers):
+ for header in headers:
+ with open(header, 'r') as f:
+ if function in f.read():
+ return True
+ return False
+
def find_file(filename, std_dirs, paths):
"""Searches for the directory where a given file is located,
and returns a possibly-empty list of additional directories, or None
@@ -1972,7 +1986,11 @@ class PyBuildExt(build_ext):
return True
def detect_ctypes(self, inc_dirs, lib_dirs):
- self.use_system_libffi = False
+ if not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and is_macosx_at_least((10,15)):
+ self.use_system_libffi = True
+ else:
+ self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS")
+
include_dirs = []
extra_compile_args = []
extra_link_args = []
@@ -2018,30 +2036,49 @@ class PyBuildExt(build_ext):
libraries=['m'])
self.extensions.extend([ext, ext_test])
+ ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR")
+ ffi_lib = None
+
+ #inc_dirs = self.inc_dirs.copy()
+ inc_dirs = self.compiler.include_dirs[:]
if host_platform == 'darwin':
- if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"):
+ if not self.use_system_libffi:
return
- # OS X 10.5 comes with libffi.dylib; the include files are
- # in /usr/include/ffi
- inc_dirs.append('/usr/include/ffi')
-
- ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")]
- if not ffi_inc or ffi_inc[0] == '':
- ffi_inc = find_file('ffi.h', [], inc_dirs)
- if ffi_inc is not None:
- ffi_h = ffi_inc[0] + '/ffi.h'
+ ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi")
+ if os.path.exists(ffi_in_sdk):
+ ffi_inc = ffi_in_sdk
+ ffi_lib = 'ffi'
+ else:
+ # OS X 10.5 comes with libffi.dylib; the include files are
+ # in /usr/include/ffi
+ inc_dirs.append('/usr/include/ffi')
+
+ if not ffi_inc:
+ found = find_file('ffi.h', [], inc_dirs)
+ if found:
+ ffi_inc = found[0]
+ if ffi_inc:
+ ffi_h = ffi_inc + '/ffi.h'
if not os.path.exists(ffi_h):
ffi_inc = None
print('Header file {} does not exist'.format(ffi_h))
- ffi_lib = None
- if ffi_inc is not None:
- for lib_name in ('ffi', 'ffi_pic'):
+
+ if ffi_lib is None and ffi_inc:
+ for lib_name in ('ffi_convenience', 'ffi_pic', 'ffi'):
if (self.compiler.find_library_file(lib_dirs, lib_name)):
ffi_lib = lib_name
break
if ffi_inc and ffi_lib:
- ext.include_dirs.extend(ffi_inc)
+ ffi_headers = glob(os.path.join(ffi_inc, '*.h'))
+ if grep_headers_for('ffi_closure_alloc', ffi_headers):
+ try:
+ sources.remove('_ctypes/malloc_closure.c')
+ except ValueError:
+ pass
+ if grep_headers_for('ffi_prep_cif_var', ffi_headers):
+ ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1")
+ ext.include_dirs.append(ffi_inc)
ext.libraries.append(ffi_lib)
self.use_system_libffi = True
--
2.30.1 (Apple Git-130)

36
plugins/python-build/share/python-build/patches/3.7.12/Python-3.7.12/0002-bpo-41100-fix-_decimal-for-arm64-Mac-OS-GH-21228.patch Normal file
View file

@ -0,0 +1,36 @@
From 604d95e235d86465b8c17f02095edcaf18464d4c Mon Sep 17 00:00:00 2001
From: Lawrence D'Anna <64555057+lawrence-danna-apple@users.noreply.github.com>
Date: Tue, 30 Jun 2020 02:15:46 -0700
Subject: [PATCH] bpo-41100: fix _decimal for arm64 Mac OS (GH-21228)
Patch by Lawrence Danna.
---
.../Core and Builtins/2020-06-30-04-44-29.bpo-41100.PJwA6F.rst | 1 +
Modules/_decimal/libmpdec/mpdecimal.h | 3 +++
2 files changed, 4 insertions(+)
create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-06-30-04-44-29.bpo-41100.PJwA6F.rst
diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-30-04-44-29.bpo-41100.PJwA6F.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-30-04-44-29.bpo-41100.PJwA6F.rst
new file mode 100644
index 0000000000..d6176d69f0
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-30-04-44-29.bpo-41100.PJwA6F.rst
@@ -0,0 +1 @@
+add arm64 to the allowable Mac OS arches in mpdecimal.h
\ No newline at end of file
diff --git a/Modules/_decimal/libmpdec/mpdecimal.h b/Modules/_decimal/libmpdec/mpdecimal.h
index 108b76efa8..35ce429f60 100644
--- a/Modules/_decimal/libmpdec/mpdecimal.h
+++ b/Modules/_decimal/libmpdec/mpdecimal.h
@@ -135,6 +135,9 @@ const char *mpd_version(void);
#elif defined(__x86_64__)
#define CONFIG_64
#define ASM
+ #elif defined(__arm64__)
+ #define CONFIG_64
+ #define ANSI
#else
#error "unknown architecture for universal build."
#endif
--
2.30.1 (Apple Git-130)