From eb8afb00c2c139f83bda7630ead98523d1ac0248 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 4 Apr 2022 21:18:20 -0700 Subject: [PATCH] Update libffi commit (#2350) All libffi tests pass now. The only failing ctypes test is test_callback_too_many_args which doesn't segfault anymore, it only soft fails. Planning to submit a PR to cpython that fixes test_callback_too_many_args. See also: bugs.python.org/issue47208 https://github.com/emscripten-core/emscripten/pull/16658 --- conftest.py | 3 +- cpython/Makefile | 2 +- ...atch-in-keyboard-interrupt-handling.patch} | 4 +- ...ove-duplicate-symbols-from-cfield.c.patch} | 4 +- ...l-core-ctypes-tests-that-don-t-work.patch} | 4 +- ...rt_LoadDynamicModuleWithSpec-fpcast.patch} | 4 +- ...-void-return-type-handling-in-ctypes.patch | 52 +++++++++++++++++++ docs/project/changelog.md | 5 ++ src/tests/python_tests.yaml | 10 +--- 9 files changed, 69 insertions(+), 19 deletions(-) rename cpython/patches/{0007-Patch-in-keyboard-interrupt-handling.patch => 0006-Patch-in-keyboard-interrupt-handling.patch} (97%) rename cpython/patches/{0008-Remove-duplicate-symbols-from-cfield.c.patch => 0007-Remove-duplicate-symbols-from-cfield.c.patch} (92%) rename cpython/patches/{0009-xfail-core-ctypes-tests-that-don-t-work.patch => 0008-xfail-core-ctypes-tests-that-don-t-work.patch} (93%) rename cpython/patches/{0010-Fix-_PyImport_LoadDynamicModuleWithSpec-fpcast.patch => 0009-Fix-_PyImport_LoadDynamicModuleWithSpec-fpcast.patch} (89%) create mode 100644 cpython/patches/0010-bpo-47197-Fix-void-return-type-handling-in-ctypes.patch diff --git a/conftest.py b/conftest.py index 3077440f4..18c66bed8 100644 --- a/conftest.py +++ b/conftest.py @@ -387,7 +387,8 @@ class NodeWrapper(SeleniumWrapper): def init_node(self): os.chdir("build") self.p = pexpect.spawn( - f"node --expose-gc ../tools/node_test_driver.js {self.base_url}", timeout=60 + f"node --expose-gc --experimental-wasm-bigint ../tools/node_test_driver.js {self.base_url}", + timeout=60, ) self.p.setecho(False) self.p.delaybeforesend = None diff --git a/cpython/Makefile b/cpython/Makefile index ada5e92df..2cd888b30 100644 --- a/cpython/Makefile +++ b/cpython/Makefile @@ -27,7 +27,7 @@ BZIP2URL=https://sourceware.org/pub/bzip2/bzip2-1.0.2.tar.gz FFIBUILD=$(ROOT)/build/libffi LIBFFIREPO=https://github.com/hoodmane/libffi-emscripten -LIBFFI_COMMIT=d6666df2a2820e0548a66166021ddae04a11a2db +LIBFFI_COMMIT=008d7aafde297703eb2f259c969aebf301b01a6d all: $(INSTALL)/lib/$(LIB) $(INSTALL)/lib/libffi.a diff --git a/cpython/patches/0007-Patch-in-keyboard-interrupt-handling.patch b/cpython/patches/0006-Patch-in-keyboard-interrupt-handling.patch similarity index 97% rename from cpython/patches/0007-Patch-in-keyboard-interrupt-handling.patch rename to cpython/patches/0006-Patch-in-keyboard-interrupt-handling.patch index 46497da0a..5bb609631 100644 --- a/cpython/patches/0007-Patch-in-keyboard-interrupt-handling.patch +++ b/cpython/patches/0006-Patch-in-keyboard-interrupt-handling.patch @@ -1,7 +1,7 @@ -From d94b4b85f87cb2c0c83f605028acbe2de4ab028a Mon Sep 17 00:00:00 2001 +From 5269845b6390efedb10dbdaec4d5b4b73ca76445 Mon Sep 17 00:00:00 2001 From: Hood Date: Fri, 3 Sep 2021 18:08:26 -0700 -Subject: [PATCH 07/10] Patch in keyboard interrupt handling +Subject: [PATCH 06/10] Patch in keyboard interrupt handling This patch adds a callback called pyodide_callback with signature `int callback(void)` to the main loop in `ceval.c`. This function gets called once diff --git a/cpython/patches/0008-Remove-duplicate-symbols-from-cfield.c.patch b/cpython/patches/0007-Remove-duplicate-symbols-from-cfield.c.patch similarity index 92% rename from cpython/patches/0008-Remove-duplicate-symbols-from-cfield.c.patch rename to cpython/patches/0007-Remove-duplicate-symbols-from-cfield.c.patch index 807fabd3c..df95f0661 100644 --- a/cpython/patches/0008-Remove-duplicate-symbols-from-cfield.c.patch +++ b/cpython/patches/0007-Remove-duplicate-symbols-from-cfield.c.patch @@ -1,7 +1,7 @@ -From c1072b871866ac92abf18f159420dbf424902702 Mon Sep 17 00:00:00 2001 +From e053e51c4e4c84423c4389463b12defb5bca7b4d Mon Sep 17 00:00:00 2001 From: Hood Date: Tue, 22 Jun 2021 20:12:45 -0700 -Subject: [PATCH 08/10] Remove duplicate symbols from cfield.c +Subject: [PATCH 07/10] Remove duplicate symbols from cfield.c These symbols are already defined by libffi. --- diff --git a/cpython/patches/0009-xfail-core-ctypes-tests-that-don-t-work.patch b/cpython/patches/0008-xfail-core-ctypes-tests-that-don-t-work.patch similarity index 93% rename from cpython/patches/0009-xfail-core-ctypes-tests-that-don-t-work.patch rename to cpython/patches/0008-xfail-core-ctypes-tests-that-don-t-work.patch index 6965b66d4..f3f56edd4 100644 --- a/cpython/patches/0009-xfail-core-ctypes-tests-that-don-t-work.patch +++ b/cpython/patches/0008-xfail-core-ctypes-tests-that-don-t-work.patch @@ -1,7 +1,7 @@ -From 16abd462459b36c73db23b776f1eb5ab2c0c7813 Mon Sep 17 00:00:00 2001 +From fcd212640bb74988c31d3ba434bcac83cbcb087c Mon Sep 17 00:00:00 2001 From: Hood Date: Thu, 24 Jun 2021 14:55:10 -0700 -Subject: [PATCH 09/10] xfail core ctypes tests that don't work +Subject: [PATCH 08/10] xfail core ctypes tests that don't work PyCode_SetExtra doesn't work and ctypes doesn't seem to work with variadic functions including PyUnicode_FromFormat/ --- diff --git a/cpython/patches/0010-Fix-_PyImport_LoadDynamicModuleWithSpec-fpcast.patch b/cpython/patches/0009-Fix-_PyImport_LoadDynamicModuleWithSpec-fpcast.patch similarity index 89% rename from cpython/patches/0010-Fix-_PyImport_LoadDynamicModuleWithSpec-fpcast.patch rename to cpython/patches/0009-Fix-_PyImport_LoadDynamicModuleWithSpec-fpcast.patch index 95448a512..9c01bf153 100644 --- a/cpython/patches/0010-Fix-_PyImport_LoadDynamicModuleWithSpec-fpcast.patch +++ b/cpython/patches/0009-Fix-_PyImport_LoadDynamicModuleWithSpec-fpcast.patch @@ -1,7 +1,7 @@ -From 1775d9b346016fec4eb22f7b46d6aa8ece8aa761 Mon Sep 17 00:00:00 2001 +From 9a3c6ecd36a405e3d67059436cc54055cdcd465a Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 1 Mar 2022 17:06:53 -0800 -Subject: [PATCH 10/10] Fix _PyImport_LoadDynamicModuleWithSpec fpcast +Subject: [PATCH 09/10] Fix _PyImport_LoadDynamicModuleWithSpec fpcast --- Python/importdl.c | 11 ++++++++++- diff --git a/cpython/patches/0010-bpo-47197-Fix-void-return-type-handling-in-ctypes.patch b/cpython/patches/0010-bpo-47197-Fix-void-return-type-handling-in-ctypes.patch new file mode 100644 index 000000000..d3e3819e4 --- /dev/null +++ b/cpython/patches/0010-bpo-47197-Fix-void-return-type-handling-in-ctypes.patch @@ -0,0 +1,52 @@ +From 7b550d8ce20d48ebb1a4f240d6cd5e8a72a430b5 Mon Sep 17 00:00:00 2001 +From: Hood Chatham +Date: Fri, 1 Apr 2022 18:55:11 -0700 +Subject: [PATCH 10/10] bpo-47197: Fix void return type handling in ctypes + +_ctypes_get_ffi_type never returns ffi_type_void. If the +return type is specified as None, we need set the libffi +return type to void, but just taking the output from +_ctypes_get_ffi_type will make the return type be sint. + +This fixes two spots where ctypes accidentally converts +None return type to sint rather than void, causing crashes +on Emscripten targets. +--- + Modules/_ctypes/callbacks.c | 2 +- + Modules/_ctypes/callproc.c | 7 ++++++- + 2 files changed, 7 insertions(+), 2 deletions(-) + +diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c +index 5a4d1c543f..ef28b93551 100644 +--- a/Modules/_ctypes/callbacks.c ++++ b/Modules/_ctypes/callbacks.c +@@ -403,7 +403,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, + #endif + result = ffi_prep_cif(&p->cif, cc, + Py_SAFE_DOWNCAST(nArgs, Py_ssize_t, int), +- _ctypes_get_ffi_type(restype), ++ p->ffi_restype, + &p->atypes[0]); + if (result != FFI_OK) { + PyErr_Format(PyExc_RuntimeError, +diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c +index ddf289e3e8..d6513c0e1f 100644 +--- a/Modules/_ctypes/callproc.c ++++ b/Modules/_ctypes/callproc.c +@@ -1219,7 +1219,12 @@ PyObject *_ctypes_callproc(PPROC pProc, + } + } + +- rtype = _ctypes_get_ffi_type(restype); ++ if (restype == Py_None) { ++ rtype = &ffi_type_void; ++ } else { ++ rtype = _ctypes_get_ffi_type(restype); ++ } ++ + resbuf = alloca(max(rtype->size, sizeof(ffi_arg))); + + #ifdef _Py_MEMORY_SANITIZER +-- +2.25.1 + diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 47f693262..713061109 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -120,6 +120,11 @@ substitutions: possible to use an async Python function as a Javascript event handler. {pr}`2319` +- {{ Enhancement }} `libffi-emscripten` now passes all libffi tests. All + `ctypes` tests pass now except for `test_callback_too_many_args` (and we have + a plan to fix `test_callback_too_many_args` upstream). + {pr}`2350` + - {{ Enhancement }} Support ANSI escape codes in the Pyodide console. {pr}`2345` diff --git a/src/tests/python_tests.yaml b/src/tests/python_tests.yaml index 12e69b06a..aaf571e8b 100644 --- a/src/tests/python_tests.yaml +++ b/src/tests/python_tests.yaml @@ -190,16 +190,8 @@ - test_csv - test_ctypes: skip: + # See https://bugs.python.org/issue47208 - test_callback_too_many_args - - test_callback_large_struct - - CFunctions - - test_struct_return_2H - - FunctionTestCase - skip-node: - - test_longlong_callbacks - - test_longdouble - - test_longlong - - test_ulonglong - test_curses - test_dataclasses - test_datetime: