From b4f4bcffdcc556348e9bc9491aca1d28da9ad580 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sat, 19 Jun 2021 15:49:25 -0700 Subject: [PATCH] Update to Python 3.9.5 (#1637) Co-authored-by: Roman Yurchak --- .circleci/config.yml | 4 +- Dockerfile | 3 +- Makefile | 5 + Makefile.envs | 8 +- README.md | 2 +- cpython/Setup.local | 2 +- cpython/checksums | 2 +- .../patches/0001-Add-pyodide-callback.patch | 129 +++----- .../0001-disable-set_inheritable.patch | 10 +- ...0002-dont-test-undecodable-filenames.patch | 9 +- .../patches/0005-add-emscripten-host.patch | 35 ++- .../0006-fix-Py_Sigset_Converter.patch | 14 +- cpython/patches/0007-testing.patch | 50 +-- docs/development/building-from-sources.md | 2 +- docs/development/debugging.md | 2 +- docs/development/new-packages.md | 8 +- docs/index.rst | 2 +- docs/project/about.md | 2 +- docs/project/changelog.md | 4 + docs/usage/faq.md | 2 +- packages/Makefile | 3 + packages/lxml/meta.yaml | 18 +- packages/nltk/meta.yaml | 1 + packages/numpy/meta.yaml | 16 +- packages/pyyaml/meta.yaml | 8 +- packages/scikit-image/meta.yaml | 4 +- packages/scikit-image/test_skimage.py | 1 - packages/scipy/meta.yaml | 18 +- packages/statsmodels/meta.yaml | 6 +- packages/swiglpk/meta.yaml | 4 +- packages/test_common.py | 9 +- pyodide-build/pyodide_build/buildpkg.py | 2 +- pyodide-build/pyodide_build/pywasmcross.py | 3 + run_docker | 2 +- src/_testinternalcapi.py | 8 + src/core/jsproxy.c | 21 -- src/tests/make_test_list.py | 70 ++++- src/tests/python_tests.txt | 285 +++++++++--------- src/tests/test_console.py | 2 +- src/tests/test_pyodide.py | 10 +- src/tests/test_typeconversions.py | 6 + tools/dependency-check.sh | 8 +- 42 files changed, 447 insertions(+), 353 deletions(-) create mode 100644 src/_testinternalcapi.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 60d300338..0e18730cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ version: 2 defaults: &defaults working_directory: ~/repo docker: - - image: pyodide/pyodide-env:16 + - image: hoodmane/pyodide-env:17 environment: - EMSDK_NUM_CORES: 3 EMCC_CORES: 3 @@ -110,7 +110,7 @@ jobs: no_output_timeout: 1800 command: | ccache -z - make + make -C packages ccache -s - run: diff --git a/Dockerfile b/Dockerfile index bf64934b7..49e6d718f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM node:14.16.1-buster-slim AS node-image -FROM python:3.8.2-slim-buster +FROM python:3.9.5-slim-buster RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -18,6 +18,7 @@ RUN pip3 --no-cache-dir install \ hypothesis \ "mypy==0.812" \ pytest \ + pytest-asyncio \ pytest-cov \ pytest-httpserver \ pytest-instafail \ diff --git a/Makefile b/Makefile index 9aa4010c0..805f7fe85 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,8 @@ all: check \ build/webworker_dev.js echo -e "\nSUCCESS!" +$(CPYTHONLIB)/tzdata : + pip install tzdata --target=$(CPYTHONLIB) build/pyodide.asm.js: \ src/core/docstring.o \ @@ -38,7 +40,9 @@ build/pyodide.asm.js: \ src/core/python2js.o \ src/pystone.py \ src/_testcapi.py \ + src/_testinternalcapi.py \ src/webbrowser.py \ + $(CPYTHONLIB)/tzdata \ $(wildcard src/py/pyodide/*.py) \ $(CPYTHONLIB) date +"[%F %T] Building pyodide.asm.js..." @@ -48,6 +52,7 @@ build/pyodide.asm.js: \ --preload-file $(CPYTHONLIB)@/lib/python$(PYMINOR) \ --preload-file src/webbrowser.py@/lib/python$(PYMINOR)/webbrowser.py \ --preload-file src/_testcapi.py@/lib/python$(PYMINOR)/_testcapi.py \ + --preload-file src/_testinternalcapi.py@/lib/python$(PYMINOR)/_testinternalcapi.py \ --preload-file src/pystone.py@/lib/python$(PYMINOR)/pystone.py \ --preload-file src/py/pyodide@/lib/python$(PYMINOR)/site-packages/pyodide \ --preload-file src/py/_pyodide@/lib/python$(PYMINOR)/site-packages/_pyodide \ diff --git a/Makefile.envs b/Makefile.envs index def1b0b3f..0c74f27ae 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -8,10 +8,10 @@ export BASH_ENV := $(PYODIDE_ROOT)/pyodide_env.sh SHELL := /bin/bash export TOOLSDIR=$(PYODIDE_ROOT)/tools -export PYVERSION=3.8.2 +export PYVERSION=3.9.5 export PYMINOR=$(basename $(PYVERSION)) -export HOSTPYTHONROOT=$(shell python3.8 -c "import sys; print(sys.prefix)") -export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python3.8 +export HOSTPYTHONROOT=$(shell python3.9 -c "import sys; print(sys.prefix)") +export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python3.9 export TARGETPYTHONROOT=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION) export PYTHONINCLUDE=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION)/include/python$(PYMINOR) @@ -70,7 +70,7 @@ export MAIN_MODULE_CFLAGS= $(CFLAGS_BASE) \ -Werror=sometimes-uninitialized \ -Werror=int-conversion \ -Werror=incompatible-pointer-types \ - -I$(PYTHONINCLUDE) + -I$(PYTHONINCLUDE) .output_vars: diff --git a/README.md b/README.md index 02bfdd878..14d9babec 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Python with the scientific stack, compiled to WebAssembly. Pyodide may be used in any context where you want to run Python inside a web browser. -Pyodide brings the Python 3.8 runtime to the browser via WebAssembly, along with +Pyodide brings the Python 3.9 runtime to the browser via WebAssembly, along with the Python scientific stack including NumPy, Pandas, Matplotlib, SciPy, and scikit-learn. The [packages directory](packages) lists over 75 packages which are currently available. In addition it's possible to install pure Python wheels diff --git a/cpython/Setup.local b/cpython/Setup.local index e5b89d299..dffed2556 100644 --- a/cpython/Setup.local +++ b/cpython/Setup.local @@ -64,5 +64,5 @@ _codecs_tw cjkcodecs/_codecs_tw.c _lsprof _lsprof.c rotatingtree.c -_decimal _decimal/_decimal.c _decimal/libmpdec/basearith.c _decimal/libmpdec/constants.c _decimal/libmpdec/context.c _decimal/libmpdec/convolute.c _decimal/libmpdec/crt.c _decimal/libmpdec/difradix2.c _decimal/libmpdec/fnt.c _decimal/libmpdec/fourstep.c _decimal/libmpdec/io.c _decimal/libmpdec/memory.c _decimal/libmpdec/mpdecimal.c _decimal/libmpdec/numbertheory.c _decimal/libmpdec/sixstep.c _decimal/libmpdec/transpose.c -I$(srcdir)/Modules/_decimal/libmpdec +_decimal _decimal/_decimal.c _decimal/libmpdec/basearith.c _decimal/libmpdec/constants.c _decimal/libmpdec/context.c _decimal/libmpdec/convolute.c _decimal/libmpdec/crt.c _decimal/libmpdec/difradix2.c _decimal/libmpdec/fnt.c _decimal/libmpdec/fourstep.c _decimal/libmpdec/io.c _decimal/libmpdec/mpalloc.c _decimal/libmpdec/mpdecimal.c _decimal/libmpdec/numbertheory.c _decimal/libmpdec/sixstep.c _decimal/libmpdec/transpose.c -I$(srcdir)/Modules/_decimal/libmpdec mmap mmapmodule.c diff --git a/cpython/checksums b/cpython/checksums index b9838f099..f74ee6ca5 100644 --- a/cpython/checksums +++ b/cpython/checksums @@ -1 +1 @@ -f9f3768f757e34b342dbc06b41cbc844 downloads/Python-3.8.2.tgz +364158b3113cf8ac8db7868ce40ebc7b downloads/Python-3.9.5.tgz diff --git a/cpython/patches/0001-Add-pyodide-callback.patch b/cpython/patches/0001-Add-pyodide-callback.patch index 321733afb..5f33a35c5 100644 --- a/cpython/patches/0001-Add-pyodide-callback.patch +++ b/cpython/patches/0001-Add-pyodide-callback.patch @@ -1,14 +1,14 @@ -From ff5dbb071a679ee1ea158feac363388e878ce402 Mon Sep 17 00:00:00 2001 +From 55a3beea61b6e1d8c05fdfbb33d623dd03ffc8d6 Mon Sep 17 00:00:00 2001 From: Hood -Date: Sat, 16 Jan 2021 11:54:57 -0800 +Date: Sun, 13 Jun 2021 16:13:30 -0700 Subject: [PATCH] Add pyodide callback --- - Include/ceval.h | 1 + - Include/cpython/pystate.h | 3 ++ - Python/ceval.c | 60 +++++++++++++++++++++++---------------- + Include/cpython/ceval.h | 1 + + Include/cpython/pystate.h | 3 +++ + Python/ceval.c | 13 ++++++++++++- Python/pystate.c | 2 ++ - 4 files changed, 42 insertions(+), 24 deletions(-) + 4 files changed, 18 insertions(+), 1 deletion(-) This patch adds a callback called pyodide_callback with signature `int callback(void)` to the main loop in `ceval.c`. This function gets called once @@ -37,114 +37,72 @@ API `PyPyodide_SetPyodideCallback` to set `pyodide_callback`. This makes `PyPyodide_SetPyodideCallback` behave much like `PyEval_SetTrace`, except lighter weight. -diff --git a/Include/ceval.h b/Include/ceval.h -index 36fd014..5f5fbc4 100644 ---- a/Include/ceval.h -+++ b/Include/ceval.h -@@ -31,6 +31,7 @@ PyAPI_FUNC(PyObject *) PyEval_CallMethod(PyObject *obj, - #ifndef Py_LIMITED_API - PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); +diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h +index e1922a6..a4acd9b 100644 +--- a/Include/cpython/ceval.h ++++ b/Include/cpython/ceval.h +@@ -10,6 +10,7 @@ PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); + PyAPI_DATA(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); + PyAPI_FUNC(int) _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +PyAPI_FUNC(void) PyPyodide_SetPyodideCallback(PyPyodide_callback); - PyAPI_FUNC(void) _PyEval_SetCoroutineOriginTrackingDepth(int new_depth); PyAPI_FUNC(int) _PyEval_GetCoroutineOriginTrackingDepth(void); - PyAPI_FUNC(void) _PyEval_SetAsyncGenFirstiter(PyObject *); + PyAPI_FUNC(int) _PyEval_SetAsyncGenFirstiter(PyObject *); + PyAPI_FUNC(PyObject *) _PyEval_GetAsyncGenFirstiter(void); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h -index 94b0809..b215056 100644 +index f292da1..3577ac8 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -17,6 +17,7 @@ PyAPI_FUNC(PyObject *) _PyInterpreterState_GetMainModule(PyInterpreterState *); /* Py_tracefunc return -1 when raising an exception, or 0 for success. */ - typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *); + typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); +typedef int (*PyPyodide_callback)(void); /* The following values are used for 'what' for tracefunc functions * -@@ -73,6 +74,8 @@ struct _ts { - Py_tracefunc c_tracefunc; +@@ -75,6 +76,8 @@ struct _ts { PyObject *c_profileobj; PyObject *c_traceobj; -+ -+ PyPyodide_callback pyodide_callback; ++ PyPyodide_callback pyodide_callback; ++ /* The exception currently being raised */ PyObject *curexc_type; + PyObject *curexc_value; diff --git a/Python/ceval.c b/Python/ceval.c -index 3306fb9..86e7639 100644 +index 91e879e..f8d424d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c -@@ -1198,31 +1198,31 @@ main_loop: +@@ -1374,7 +1374,6 @@ main_loop: async I/O handler); see Py_AddPendingCall() and Py_MakePendingCalls() above. */ - if (_Py_atomic_load_relaxed(eval_breaker)) { -- opcode = _Py_OPCODE(*next_instr); -- if (opcode == SETUP_FINALLY || -- opcode == SETUP_WITH || -- opcode == BEFORE_ASYNC_WITH || -- opcode == YIELD_FROM) { -- /* Few cases where we skip running signal handlers and other -- pending calls: -- - If we're about to enter the 'with:'. It will prevent -- emitting a resource warning in the common idiom -- 'with open(path) as file:'. -- - If we're about to enter the 'async with:'. -- - If we're about to enter the 'try:' of a try/finally (not -- *very* useful, but might help in some cases and it's -- traditional) -- - If we're resuming a chain of nested 'yield from' or -- 'await' calls, then each frame is parked with YIELD_FROM -- as its next opcode. If the user hit control-C we want to -- wait until we've reached the innermost frame before -- running the signal handler and raising KeyboardInterrupt -- (see bpo-30039). -- */ -- goto fast_next_opcode; -- } -+ opcode = _Py_OPCODE(*next_instr); -+ if (opcode == SETUP_FINALLY || -+ opcode == SETUP_WITH || -+ opcode == BEFORE_ASYNC_WITH || -+ opcode == YIELD_FROM) { -+ /* Few cases where we skip running signal handlers and other -+ pending calls: -+ - If we're about to enter the 'with:'. It will prevent -+ emitting a resource warning in the common idiom -+ 'with open(path) as file:'. -+ - If we're about to enter the 'async with:'. -+ - If we're about to enter the 'try:' of a try/finally (not -+ *very* useful, but might help in some cases and it's -+ traditional) -+ - If we're resuming a chain of nested 'yield from' or -+ 'await' calls, then each frame is parked with YIELD_FROM -+ as its next opcode. If the user hit control-C we want to -+ wait until we've reached the innermost frame before -+ running the signal handler and raising KeyboardInterrupt -+ (see bpo-30039). -+ */ -+ goto fast_next_opcode; -+ } + opcode = _Py_OPCODE(*next_instr); + if (opcode == SETUP_FINALLY || + opcode == SETUP_WITH || +@@ -1399,11 +1398,18 @@ main_loop: + goto fast_next_opcode; + } + if (_Py_atomic_load_relaxed(eval_breaker)) { - if (_Py_atomic_load_relaxed(&ceval->signals_pending)) { - if (handle_signals(runtime) != 0) { - goto error; -@@ -1262,6 +1262,12 @@ main_loop: + if (eval_frame_handle_pending(tstate) != 0) { goto error; } } -+ + + if(tstate->pyodide_callback != NULL){ + if(tstate->pyodide_callback() != 0){ + goto error; + } -+ } - ++ } ++ fast_next_opcode: f->f_lasti = INSTR_OFFSET(); -@@ -4727,6 +4733,12 @@ PyEval_SetTrace(Py_tracefunc func, PyObject *arg) - || (tstate->c_profilefunc != NULL)); + +@@ -4816,6 +4822,11 @@ PyEval_SetTrace(Py_tracefunc func, PyObject *arg) + } } +void @@ -152,23 +110,22 @@ index 3306fb9..86e7639 100644 + PyThreadState *tstate = _PyThreadState_GET(); + tstate->pyodide_callback = pyodide_callback; +} -+ + void - _PyEval_SetCoroutineOriginTrackingDepth(int new_depth) - { + _PyEval_SetCoroutineOriginTrackingDepth(PyThreadState *tstate, int new_depth) diff --git a/Python/pystate.c b/Python/pystate.c -index aba673c..b809208 100644 +index 9beefa8..408b7ce 100644 --- a/Python/pystate.c +++ b/Python/pystate.c -@@ -592,6 +592,8 @@ new_threadstate(PyInterpreterState *interp, int init) - tstate->c_tracefunc = NULL; +@@ -602,6 +602,8 @@ new_threadstate(PyInterpreterState *interp, int init) tstate->c_profileobj = NULL; tstate->c_traceobj = NULL; -+ -+ tstate->pyodide_callback = NULL; ++ tstate->pyodide_callback = NULL; ++ tstate->trash_delete_nesting = 0; tstate->trash_delete_later = NULL; + tstate->on_delete = NULL; -- 2.17.1 diff --git a/cpython/patches/0001-disable-set_inheritable.patch b/cpython/patches/0001-disable-set_inheritable.patch index e15086b1d..042342780 100644 --- a/cpython/patches/0001-disable-set_inheritable.patch +++ b/cpython/patches/0001-disable-set_inheritable.patch @@ -3,12 +3,15 @@ From: Michael Droettboom Date: Sun, 5 Jul 2020 17:37:43 +0200 Subject: [PATCH] disable-set_inheritable +--- + Python/fileutils.c | 3 +++ + 1 file changed, 3 insertions(+) diff --git a/Python/fileutils.c b/Python/fileutils.c -index e79e732d1f..864dc78a57 100644 +index 769ab59..a784789 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c -@@ -1091,6 +1091,9 @@ _Py_get_inheritable(int fd) +@@ -1134,6 +1134,9 @@ _Py_get_inheritable(int fd) static int set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) { @@ -18,3 +21,6 @@ index e79e732d1f..864dc78a57 100644 #ifdef MS_WINDOWS HANDLE handle; DWORD flags; +-- +2.17.1 + diff --git a/cpython/patches/0002-dont-test-undecodable-filenames.patch b/cpython/patches/0002-dont-test-undecodable-filenames.patch index e5b3bdd70..1d216eaa5 100644 --- a/cpython/patches/0002-dont-test-undecodable-filenames.patch +++ b/cpython/patches/0002-dont-test-undecodable-filenames.patch @@ -3,12 +3,15 @@ From: Michael Droettboom Date: Sun, 5 Jul 2020 17:38:21 +0200 Subject: [PATCH] dont-test-undecodable-filenames +--- + Lib/test/support/__init__.py | 2 ++ + 1 file changed, 2 insertions(+) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py -index d6a7819cb0..17a13cf49d 100644 +index aee3737..d0be99f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py -@@ -952,6 +952,8 @@ if os.name == 'nt': +@@ -809,6 +809,8 @@ if os.name == 'nt': 'Unicode filename tests may not be effective' % (TESTFN_UNENCODABLE, TESTFN_ENCODING)) TESTFN_UNENCODABLE = None @@ -18,5 +21,5 @@ index d6a7819cb0..17a13cf49d 100644 elif sys.platform != 'darwin': try: -- -2.25.1 +2.17.1 diff --git a/cpython/patches/0005-add-emscripten-host.patch b/cpython/patches/0005-add-emscripten-host.patch index e0fffd1b5..8762b3342 100644 --- a/cpython/patches/0005-add-emscripten-host.patch +++ b/cpython/patches/0005-add-emscripten-host.patch @@ -1,6 +1,14 @@ ---- b/config.sub +--- + config.sub | 9 ++++++++- + configure | 6 ++++++ + configure.ac | 5 +++++ + 3 files changed, 19 insertions(+), 1 deletion(-) + +diff --git a/config.sub b/config.sub +index ba37cf9..0d22b33 100755 +--- a/config.sub +++ b/config.sub -@@ -118,7 +118,8 @@ +@@ -118,7 +118,8 @@ case $maybe_os in linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \ kopensolaris*-gnu* | cloudabi*-eabi* | \ @@ -10,7 +18,7 @@ os=-$maybe_os basic_machine=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` ;; -@@ -377,6 +378,7 @@ +@@ -377,6 +378,7 @@ case $basic_machine in | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ @@ -18,17 +26,17 @@ | avr-* | avr32-* \ | ba-* \ | be32-* | be64-* \ -@@ -527,6 +529,9 @@ +@@ -527,6 +529,9 @@ case $basic_machine in asmjs) basic_machine=asmjs-unknown ;; + wasm32) + basic_machine=wasm32-unknown -+ ;; ++ ;; aux) basic_machine=m68k-apple os=-aux -@@ -1522,6 +1527,8 @@ +@@ -1522,6 +1527,8 @@ case $os in ;; esac ;; @@ -37,9 +45,11 @@ -nacl*) ;; -ios) +diff --git a/configure b/configure +index 8dcdbf1..e1195d2 100755 --- a/configure +++ b/configure -@@ -3274,6 +3274,9 @@ then +@@ -3307,6 +3307,9 @@ then *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -49,7 +59,7 @@ *) # for now, limit cross builds to known configurations MACHDEP="unknown" -@@ -3324,6 +3327,9 @@ if test "$cross_compiling" = yes; then +@@ -3357,6 +3360,9 @@ if test "$cross_compiling" = yes; then *-*-vxworks*) _host_cpu=$host_cpu ;; @@ -59,9 +69,11 @@ *) # for now, limit cross builds to known configurations MACHDEP="unknown" +diff --git a/configure.ac b/configure.ac +index b1e4c6c..dbfae6d 100644 --- a/configure.ac +++ b/configure.ac -@@ -382,6 +382,9 @@ then +@@ -403,6 +403,9 @@ then *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -71,7 +83,7 @@ *) # for now, limit cross builds to known configurations MACHDEP="unknown" -@@ -430,6 +433,8 @@ if test "$cross_compiling" = yes; then +@@ -451,6 +454,8 @@ if test "$cross_compiling" = yes; then ;; *-*-vxworks*) _host_cpu=$host_cpu @@ -80,3 +92,6 @@ ;; *) # for now, limit cross builds to known configurations +-- +2.17.1 + diff --git a/cpython/patches/0006-fix-Py_Sigset_Converter.patch b/cpython/patches/0006-fix-Py_Sigset_Converter.patch index 11bd463dc..585dc8563 100644 --- a/cpython/patches/0006-fix-Py_Sigset_Converter.patch +++ b/cpython/patches/0006-fix-Py_Sigset_Converter.patch @@ -3,12 +3,16 @@ From: Roman Yurchak Date: Sun, 5 Jul 2020 21:17:10 +0200 Subject: [PATCH] fix Py_Sigset_Converter +--- + Modules/posixmodule.c | 7 +++++++ + Modules/posixmodule.h | 2 -- + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c -index 2f791d1df9..c775d7711d 100644 +index c984e2e..9d1bbee 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c -@@ -1337,6 +1337,13 @@ error: +@@ -1486,6 +1486,13 @@ error: Py_DECREF(iterator); return 0; } @@ -23,10 +27,10 @@ index 2f791d1df9..c775d7711d 100644 #ifdef MS_WINDOWS diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h -index 1e00562abc..229704a831 100644 +index 711ac68..5452ffb 100644 --- a/Modules/posixmodule.h +++ b/Modules/posixmodule.h -@@ -23,9 +23,7 @@ PyAPI_FUNC(int) _Py_Gid_Converter(PyObject *, void *); +@@ -23,9 +23,7 @@ PyAPI_FUNC(int) _Py_Gid_Converter(PyObject *, gid_t *); # define HAVE_SIGSET_T #endif @@ -37,5 +41,5 @@ index 1e00562abc..229704a831 100644 #ifdef __cplusplus -- -2.25.1 +2.17.1 diff --git a/cpython/patches/0007-testing.patch b/cpython/patches/0007-testing.patch index d241e527d..942a1748c 100644 --- a/cpython/patches/0007-testing.patch +++ b/cpython/patches/0007-testing.patch @@ -3,21 +3,21 @@ From: Michael Droettboom Date: Sun, 5 Jul 2020 19:56:47 +0200 Subject: [PATCH] testing +--- + Lib/platform.py | 2 +- + Lib/test/libregrtest/main.py | 4 +++- + Lib/test/support/script_helper.py | 5 +++++ + Lib/test/test_cgi.py | 1 + + Lib/test/test_fcntl.py | 2 ++ + Lib/test/test_gzip.py | 2 ++ + Lib/test/test_itertools.py | 1 + + 7 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Lib/platform.py b/Lib/platform.py -index 6fbb7b08c5..d46425659d 100755 +index 0bce438..8fdfe05 100755 --- a/Lib/platform.py +++ b/Lib/platform.py -@@ -604,7 +604,7 @@ def _syscmd_uname(option, default=''): - - """ Interface to the system's uname command. - """ -- if sys.platform in ('dos', 'win32', 'win16'): -+ if sys.platform in ('dos', 'win32', 'win16', 'emscripten'): - # XXX Others too ? - return default - -@@ -626,7 +626,7 @@ def _syscmd_file(target, default=''): +@@ -614,7 +614,7 @@ def _syscmd_file(target, default=''): default in case the command should fail. """ @@ -27,10 +27,10 @@ index 6fbb7b08c5..d46425659d 100755 return default diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py -index 76ad3359f2..b135966032 100644 +index e20bf9a..18f62f4 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py -@@ -635,7 +635,9 @@ class Regrtest: +@@ -639,7 +639,9 @@ class Regrtest: except SystemExit as exc: # bpo-38203: Python can hang at exit in Py_Finalize(), especially # on threading._shutdown() call: put a timeout @@ -42,7 +42,7 @@ index 76ad3359f2..b135966032 100644 sys.exit(exc.code) diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py -index 83519988e3..1410fe5c61 100644 +index 37e576d..d470e4a 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -9,6 +9,7 @@ import os.path @@ -52,7 +52,7 @@ index 83519988e3..1410fe5c61 100644 +import unittest from importlib.util import source_from_cache - from test.support import make_legacy_pyc, strip_python_stderr + from test.support import make_legacy_pyc @@ -34,6 +35,8 @@ def interpreter_requires_environment(): situation. PYTHONPATH or PYTHONUSERSITE are other common environment variables that might impact whether or not the interpreter can start. @@ -62,7 +62,7 @@ index 83519988e3..1410fe5c61 100644 global __cached_interp_requires_environment if __cached_interp_requires_environment is None: # If PYTHONHOME is set, assume that we need it -@@ -172,6 +175,8 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): +@@ -171,6 +174,8 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): kw is extra keyword args to pass to subprocess.Popen. Returns a Popen object. """ @@ -72,10 +72,10 @@ index 83519988e3..1410fe5c61 100644 if not interpreter_requires_environment(): cmd_line.append('-E') diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py -index ab8677199f..8eefd57776 100644 +index 4e1506a..8c02cdb 100644 --- a/Lib/test/test_cgi.py +++ b/Lib/test/test_cgi.py -@@ -188,6 +188,7 @@ Content-Length: 3 +@@ -221,6 +221,7 @@ Content-Length: 3 self.assertEqual(fs.getvalue(key), expect_val[0]) def test_log(self): @@ -84,7 +84,7 @@ index ab8677199f..8eefd57776 100644 cgi.logfp = StringIO() diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py -index c394f6945a..2ec020bd08 100644 +index 9ab68c6..3e74849 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -155,6 +155,7 @@ class TestFcntl(unittest.TestCase): @@ -104,10 +104,10 @@ index c394f6945a..2ec020bd08 100644 cmd = fcntl.LOCK_SH | fcntl.LOCK_NB fcntl.lockf(self.f, cmd) diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py -index 646828621a..b9663082b9 100644 +index 1af23c6..045b4c7 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py -@@ -731,6 +731,7 @@ class TestCommandLine(unittest.TestCase): +@@ -743,6 +743,7 @@ class TestCommandLine(unittest.TestCase): data = b'This is a simple test with gzip' def test_decompress_stdin_stdout(self): @@ -115,7 +115,7 @@ index 646828621a..b9663082b9 100644 with io.BytesIO() as bytes_io: with gzip.GzipFile(fileobj=bytes_io, mode='wb') as gzip_file: gzip_file.write(self.data) -@@ -767,6 +768,7 @@ class TestCommandLine(unittest.TestCase): +@@ -779,6 +780,7 @@ class TestCommandLine(unittest.TestCase): @create_and_remove_directory(TEMPDIR) def test_compress_stdin_outfile(self): @@ -124,10 +124,10 @@ index 646828621a..b9663082b9 100644 with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: out, err = proc.communicate(self.data) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py -index eaa6197bec..95ff99d15c 100644 +index 7101264..bf359a0 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py -@@ -1511,6 +1511,7 @@ class TestBasicOps(unittest.TestCase): +@@ -1513,6 +1513,7 @@ class TestBasicOps(unittest.TestCase): next(a) def test_tee_concurrent(self): @@ -136,5 +136,5 @@ index eaa6197bec..95ff99d15c 100644 finish = threading.Event() class I: -- -2.25.1 +2.17.1 diff --git a/docs/development/building-from-sources.md b/docs/development/building-from-sources.md index a9e7c6e3c..478f201ec 100644 --- a/docs/development/building-from-sources.md +++ b/docs/development/building-from-sources.md @@ -18,7 +18,7 @@ Additional build prerequisites are: - A working native compiler toolchain, enough to build [CPython](https://devguide.python.org/setup/#linux). -- A native Python 3.8 to run the build scripts. +- A native Python 3.9 to run the build scripts. - CMake - PyYAML - FreeType 2 development libraries to compile Matplotlib. diff --git a/docs/development/debugging.md b/docs/development/debugging.md index bfb5312a9..8357f1c6f 100644 --- a/docs/development/debugging.md +++ b/docs/development/debugging.md @@ -25,7 +25,7 @@ import socketserver from http.server import SimpleHTTPRequestHandler import pathlib -alternative_bases=["cpython/build/Python-3.8.2/","src/", "build/"] +alternative_bases=["cpython/build/Python-3.9.5/","src/", "build/"] def fixup_url(path): if pathlib.Path("." + path).exists(): return path diff --git a/docs/development/new-packages.md b/docs/development/new-packages.md index ecea5e87a..d71ab0831 100644 --- a/docs/development/new-packages.md +++ b/docs/development/new-packages.md @@ -228,7 +228,7 @@ packages the same way as CPython --- it looks for relevant files `.py` files in Suppose you have a Python library that consists of a single directory `/PATH/TO/LIB/` whose contents would go into -`/lib/python3.8/site-packages/PACKAGE_NAME/` under a normal Python +`/lib/python3.9/site-packages/PACKAGE_NAME/` under a normal Python installation. The simplest version of the corresponding Pyodide package contains two files @@ -237,7 +237,7 @@ The simplest version of the corresponding Pyodide package contains two files loading the package via `pyodide.loadPackage`, Pyodide will load and run `PACKAGE_NAME.js`. The script then fetches `PACKAGE_NAME.data` and extracts the contents to emscripten's virtual filesystem. Afterwards, since the files are -now in `/lib/python3.8/`, running `import PACKAGE_NAME` in Python will +now in `/lib/python3.9/`, running `import PACKAGE_NAME` in Python will successfully import the module as usual. To construct this bundle, we use the `file_packager.py` script from emscripten. @@ -246,7 +246,7 @@ We invoke it as follows: $ ./tools/file_packager.sh \ PACKAGE_NAME.data \ --js-output=PACKAGE_NAME.js \ - --preload /PATH/TO/LIB/@/lib/python3.8/site-packages/PACKAGE_NAME/ + --preload /PATH/TO/LIB/@/lib/python3.9/site-packages/PACKAGE_NAME/ ``` The arguments can be explained as follows: @@ -255,7 +255,7 @@ The arguments can be explained as follows: - `--preload` instructs the package to look for the file/directory before the separator `@` (namely `/PATH/TO/LIB/`) and place it at the path after the `@` in the virtual filesystem (namely - `/lib/python3.8/site-packages/PACKAGE_NAME/`). + `/lib/python3.9/site-packages/PACKAGE_NAME/`). `file_packager.sh` adds the following options: - `--lz4` to use LZ4 to compress the files diff --git a/docs/index.rst b/docs/index.rst index b3fa933c3..e0472d9cd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,7 +6,7 @@ Python with the scientific stack, compiled to WebAssembly. Pyodide may be used in any context where you want to run Python inside a web browser. -Pyodide brings the Python 3.8 runtime to the browser via WebAssembly, along +Pyodide brings the Python 3.9 runtime to the browser via WebAssembly, along with the Python scientific stack including NumPy, Pandas, Matplotlib, SciPy, and scikit-learn. Over 75 packages are currently available. In addition it's possible to install pure Python wheels from PyPi. diff --git a/docs/project/about.md b/docs/project/about.md index ea41422f8..195c58b0e 100644 --- a/docs/project/about.md +++ b/docs/project/about.md @@ -5,7 +5,7 @@ Python with the scientific stack, compiled to WebAssembly. Pyodide may be used in any context where you want to run Python inside a web browser. -Pyodide brings the Python 3.8 runtime to the browser via WebAssembly, along with +Pyodide brings the Python 3.9 runtime to the browser via WebAssembly, along with the Python scientific stack including NumPy, Pandas, Matplotlib, SciPy, and scikit-learn. The [packages directory](https://github.com/pyodide/pyodide/tree/main/packages) lists over diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 01472e62d..fee266898 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -4,6 +4,7 @@ substitutions: Enhancement : "Enhancement" Feature : "Feature" Fix : "Fix" + Update : "Update" --- @@ -37,6 +38,9 @@ substitutions: type. This is particularly important if the error is a `KeyboardInterrupt`. {pr}`1447` +- {{ Update }} Pyodide now runs Python 3.9.5. + {pr}`1637` + - {{ Enhancement }} It is now possible to import `Comlink` objects into Pyodide after using {any}`pyodide.registerComlink` {pr}`1642` diff --git a/docs/usage/faq.md b/docs/usage/faq.md index c772af04e..c09a13603 100644 --- a/docs/usage/faq.md +++ b/docs/usage/faq.md @@ -243,7 +243,7 @@ with redirect_stdin(StringIO("\n".join(["eval", "asyncio.ensure_future", "functo ``` it will print: ``` -Welcome to Python 3.8's help utility! +Welcome to Python 3.9's help utility! <...OMITTED LINES> Help on built-in function eval in module builtins: eval(source, globals=None, locals=None, /) diff --git a/packages/Makefile b/packages/Makefile index 39a423bc0..3f4b11eb0 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -2,6 +2,9 @@ export PYODIDE_ROOT=$(abspath ..) PYODIDE_LIBRARIES=$(abspath ./.artifacts) include ../Makefile.envs +export NUMPY_LIB=$(PYODIDE_ROOT)/packages/numpy/build/numpy-1.17.5/install/lib/python$(PYMINOR)/site-packages/numpy/core/lib/ + + ifeq ($(strip $(PYODIDE_PACKAGES)),) else ONLY_PACKAGES=--only "$(PYODIDE_PACKAGES)" diff --git a/packages/lxml/meta.yaml b/packages/lxml/meta.yaml index db0d5ce8e..ebb135457 100644 --- a/packages/lxml/meta.yaml +++ b/packages/lxml/meta.yaml @@ -6,17 +6,17 @@ source: url: https://files.pythonhosted.org/packages/c4/43/3f1e7d742e2a7925be180b6af5e0f67d38de2f37560365ac1a0b9a04c015/lxml-4.4.1.tar.gz build: cflags: | - -I../../../libxml/build/libxml2-2.9.10/include - -I../../../libxslt/build/libxslt-1.1.33/ - -L../../../zlib/build/zlib-1.2.11/include - -I../../../libiconv/build/libiconv-1.16/include + -I$(PYODIDE_ROOT)/packages/libxml/build/libxml2-2.9.10/include + -I$(PYODIDE_ROOT)/packages/libxslt/build/libxslt-1.1.33/ + -L$(PYODIDE_ROOT)/packages/zlib/build/zlib-1.2.11/include + -I$(PYODIDE_ROOT)/packages/libiconv/build/libiconv-1.16/include -Wno-implicit-function-declaration ldflags: | - -L../../../libxml/build/libxml2-2.9.10/.libs/ - -L../../../libxslt/build/libxslt-1.1.33/libxslt/.libs/ - -L../../../libxslt/build/libxslt-1.1.33/libexslt/.libs/ - -L../../../zlib/build/zlib-1.2.11/lib/ - -L../../../libiconv/build/libiconv-1.16/lib/.libs/ + -L$(PYODIDE_ROOT)/packages/libxml/build/libxml2-2.9.10/.libs/ + -L$(PYODIDE_ROOT)/packages/libxslt/build/libxslt-1.1.33/libxslt/.libs/ + -L$(PYODIDE_ROOT)/packages/libxslt/build/libxslt-1.1.33/libexslt/.libs/ + -L$(PYODIDE_ROOT)/packages/zlib/build/zlib-1.2.11/lib/ + -L$(PYODIDE_ROOT)/packages/libiconv/build/libiconv-1.16/lib/.libs/ requirements: run: - beautifulsoup4 diff --git a/packages/nltk/meta.yaml b/packages/nltk/meta.yaml index 2c414e90c..754ffdb12 100644 --- a/packages/nltk/meta.yaml +++ b/packages/nltk/meta.yaml @@ -10,3 +10,4 @@ requirements: test: imports: - nltk + - sqlite3 diff --git a/packages/numpy/meta.yaml b/packages/numpy/meta.yaml index 146b307a6..d3562c53f 100644 --- a/packages/numpy/meta.yaml +++ b/packages/numpy/meta.yaml @@ -24,12 +24,22 @@ source: build: skip_host: False - cflags: -include math.h -I../../config -Werror=implicit-function-declaration -Werror=mismatched-parameter-types -Werror=mismatched-return-types + # set linker and C flags to error on anything to do with function declarations being wrong. + # In webassembbly, any conflicts mean that a randomly selected 50% of calls to the function + # will fail. Better to fail at compile or link time. + cflags: | + -include math.h + -I $(PYODIDE_ROOT)/packages/numpy/config + -Werror=implicit-function-declaration + -Werror=mismatched-parameter-types + -Werror=mismatched-return-types + ldflags: | + -Wl,--fatal-warnings post: | # copy the correct numpy config into the build artifacts. Otherwise scipy will try to build with the config # from the build computer and bad things will happen - cp config/* $PKGDIR/../.artifacts/lib/python/numpy-1.17.5-py3.8-linux-x86_64.egg/numpy/core/include/numpy - cp -r $PKGDIR/../.artifacts/lib/python/numpy-1.17.5-py3.8-linux-x86_64.egg/numpy $PKGDIR/../.artifacts/lib/python/numpy + cp config/* $PKGDIR/../.artifacts/lib/python/numpy-1.17.5-py3.9-linux-x86_64.egg/numpy/core/include/numpy + cp -r $PKGDIR/../.artifacts/lib/python/numpy-1.17.5-py3.9-linux-x86_64.egg/numpy $PKGDIR/../.artifacts/lib/python test: imports: diff --git a/packages/pyyaml/meta.yaml b/packages/pyyaml/meta.yaml index 9048c810c..b31ae28f9 100644 --- a/packages/pyyaml/meta.yaml +++ b/packages/pyyaml/meta.yaml @@ -13,11 +13,11 @@ source: build: skip_host: True cflags: | - -I../../../../cpython/installs/python-3.8.2/include/python3.8 - -I../../../../emsdk/emsdk/fastcomp/emscripten/system/include/libc - -I../../../libyaml/build/include + -I$(PYTHONINCLUDE) + -I$(PYODIDE_ROOT)/emsdk/emsdk/fastcomp/emscripten/system/include/libc + -I$(PYODIDE_ROOT)/packages/libyaml/build/include ldflags: | - -L../../../libyaml/build/ + -L$(PYODIDE_ROOT)/packages/libyaml/build/ requirements: run: diff --git a/packages/scikit-image/meta.yaml b/packages/scikit-image/meta.yaml index 3ca425681..f5eb259d6 100644 --- a/packages/scikit-image/meta.yaml +++ b/packages/scikit-image/meta.yaml @@ -6,8 +6,8 @@ source: - patches/no-openmp.patch - patches/unwrap.patch - patches/preferred_plugins.patch - sha256: df111e654b47e5ea456c50553debe4c5ddd97258894c7ad3b7f2f9f10798e053 - url: https://files.pythonhosted.org/packages/e0/46/ca035f5d7d3414124a3a5ef22cd2e75c0c5149042a668375f1d44eb69f8f/scikit-image-0.15.0.tar.gz + sha256: 312dc7c49e3abdb90a42967fcf6129f0c9a45bd65f74b0cd3591390929b31a70 + url: https://github.com/scikit-image/scikit-image/archive/refs/tags/v0.15.0.tar.gz requirements: run: - distutils diff --git a/packages/scikit-image/test_skimage.py b/packages/scikit-image/test_skimage.py index b137d9990..cae7a6a47 100644 --- a/packages/scikit-image/test_skimage.py +++ b/packages/scikit-image/test_skimage.py @@ -30,7 +30,6 @@ def test_skimage(): from skimage.color import rgb2gray from skimage.filters import sobel from skimage.segmentation import felzenszwalb, slic, quickshift, watershed - from skimage.segmentation import mark_boundaries from skimage.util import img_as_float img = img_as_float(astronaut()[::2, ::2]) diff --git a/packages/scipy/meta.yaml b/packages/scipy/meta.yaml index 1051877be..0fafbd5b4 100644 --- a/packages/scipy/meta.yaml +++ b/packages/scipy/meta.yaml @@ -26,11 +26,19 @@ source: - patches/fix-typo-flapack-pyf.patch build: -# linker and C flags are set to error on anything to do with function declarations being wrong. By default these -# are just warnings, but in webassembbly, any conflicts mean that a randomly selected 50% of calls to the function -# will fail. This is understandably bad. Better to fail at compile or link time - cflags: -I ../../../numpy/config -I../../../CLAPACK/build/CLAPACK-3.2.1/INCLUDE -Werror=implicit-function-declaration -Werror=mismatched-parameter-types -DUNDERSCORE_G77 -Werror=mismatched-return-types - ldflags: -L$(PYODIDE_ROOT)/packages/numpy/build/numpy-1.17.5/install/lib/python3.8/site-packages/numpy/core/lib/ -Wl,--fatal-warnings + # set linker and C flags to error on anything to do with function declarations being wrong. + # In webassembbly, any conflicts mean that a randomly selected 50% of calls to the function + # will fail. Better to fail at compile or link time. + cflags: | + -I $(PYODIDE_ROOT)/packages/numpy/config + -I$(PYODIDE_ROOT)/packages/CLAPACK/build/CLAPACK-3.2.1/INCLUDE + -DUNDERSCORE_G77 + -Werror=implicit-function-declaration + -Werror=mismatched-parameter-types + -Werror=mismatched-return-types + ldflags: | + -L$(NUMPY_LIB) + -Wl,--fatal-warnings script: | # Some fixes were extracted from patches/make-int-return-values.patch as regex. find scipy -name "*.c*" | xargs sed -i 's/extern void F_FUNC/extern int F_FUNC/g' diff --git a/packages/statsmodels/meta.yaml b/packages/statsmodels/meta.yaml index dd6683823..4adfd5b7d 100644 --- a/packages/statsmodels/meta.yaml +++ b/packages/statsmodels/meta.yaml @@ -3,11 +3,11 @@ package: version: 0.9.0 source: - url: https://files.pythonhosted.org/packages/67/68/eb3ec6ab61f97216c257edddb853cc174cd76ea44b365cf4adaedcd44482/statsmodels-0.9.0.tar.gz - sha256: 6461f93a842c649922c2c9a9bc9d9c4834110b89de8c4af196a791ab8f42ba3b + url: https://github.com/statsmodels/statsmodels/archive/refs/tags/v0.9.0.tar.gz + sha256: 4ab219c776bf5b2187645c0ccb00cf8564c6c17541ae66ab96503f07368197d3 build: - ldflags: -L$(PYODIDE_ROOT)/packages/numpy/build/numpy-1.17.5/install/lib/python3.8/site-packages/numpy/core/lib/ + ldflags: -L$(NUMPY_LIB) requirements: run: diff --git a/packages/swiglpk/meta.yaml b/packages/swiglpk/meta.yaml index f149ff16a..95b3edc32 100644 --- a/packages/swiglpk/meta.yaml +++ b/packages/swiglpk/meta.yaml @@ -5,11 +5,11 @@ package: source: sha256: 0216db2930a6fe2c07ac7f0e28e76e9a2711a647836a3a4067113091c7ae221e url: https://files.pythonhosted.org/packages/0e/c9/9b0ec3e2ca942b6d067e84e52d72818fa9c3a1a1524671fcbec4841b54b2/swiglpk-4.65.1.tar.gz - extras: + extras: - [../glpk/build/glpk-4.65/src/glpk.h, ./glpk.h] build: - ldflags: -L../../../glpk/build/glpk-4.65/src/.libs/ + ldflags: -L$(PYODIDE_ROOT)/packages/glpk/build/glpk-4.65/src/.libs/ requirements: run: diff --git a/packages/test_common.py b/packages/test_common.py index c539ea0e2..1a6d58ee0 100644 --- a/packages/test_common.py +++ b/packages/test_common.py @@ -26,7 +26,10 @@ def registered_packages_meta(): } -UNSUPPORTED_PACKAGES: dict = {"chrome": ["scikit-image", "statsmodels"], "firefox": []} +UNSUPPORTED_PACKAGES: dict = { + "chrome": ["scikit-image", "statsmodels"], + "firefox": [], +} @pytest.mark.parametrize("name", registered_packages()) @@ -65,7 +68,7 @@ def test_import(name, selenium_standalone): baseline_pyc = selenium_standalone.run( """ len(list(glob.glob( - '/lib/python3.8/site-packages/**/*.pyc', + '/lib/python3.9/site-packages/**/*.pyc', recursive=True) )) """ @@ -78,7 +81,7 @@ def test_import(name, selenium_standalone): selenium_standalone.run( """ len(list(glob.glob( - '/lib/python3.8/site-packages/**/*.pyc', + '/lib/python3.9/site-packages/**/*.pyc', recursive=True) )) """ diff --git a/pyodide-build/pyodide_build/buildpkg.py b/pyodide-build/pyodide_build/buildpkg.py index 19b5d0140..820b2f03f 100755 --- a/pyodide-build/pyodide_build/buildpkg.py +++ b/pyodide-build/pyodide_build/buildpkg.py @@ -171,7 +171,7 @@ def compile(path: Path, srcpath: Path, pkg: Dict[str, Any], args): post = pkg.get("build", {}).get("post") if post is not None: - site_packages_dir = srcpath / "install" / "lib" / "python3.8" / "site-packages" + site_packages_dir = srcpath / "install" / "lib" / "python3.9" / "site-packages" pkgdir = path.parent.resolve() env = {"SITEPACKAGES": str(site_packages_dir), "PKGDIR": str(pkgdir)} subprocess.run(["bash", "-ce", post], env=env, check=True) diff --git a/pyodide-build/pyodide_build/pywasmcross.py b/pyodide-build/pyodide_build/pywasmcross.py index 86242ca0e..ac00b8845 100755 --- a/pyodide-build/pyodide_build/pywasmcross.py +++ b/pyodide-build/pyodide_build/pywasmcross.py @@ -315,6 +315,9 @@ def handle_command(line, args, dryrun=False): continue if arg == "-Wl,-Bsymbolic-functions": continue + # breaks emscripten see https://github.com/emscripten-core/emscripten/issues/14460 + if arg == "-Wl,--strip-all": + continue # threading is disabled for now if arg == "-pthread": continue diff --git a/run_docker b/run_docker index 80053ec64..0cd457ede 100755 --- a/run_docker +++ b/run_docker @@ -1,7 +1,7 @@ #!/usr/bin/env bash PYODIDE_IMAGE_REPO="pyodide" -PYODIDE_IMAGE_TAG="16" +PYODIDE_IMAGE_TAG="17" PYODIDE_PREBUILT_IMAGE_TAG="0.17.0" DEFAULT_PYODIDE_DOCKER_IMAGE="${PYODIDE_IMAGE_REPO}/pyodide-env:${PYODIDE_IMAGE_TAG}" DEFAULT_PYODIDE_SYSTEM_PORT="8000" diff --git a/src/_testinternalcapi.py b/src/_testinternalcapi.py new file mode 100644 index 000000000..9d18c956d --- /dev/null +++ b/src/_testinternalcapi.py @@ -0,0 +1,8 @@ +""" +A shim to skip tests involving the _testinternalcapi extension, since we don't build +it. +""" + +import unittest + +raise unittest.SkipTest("No _testinternalcapi") diff --git a/src/core/jsproxy.c b/src/core/jsproxy.c index 667d263f8..2949cae08 100644 --- a/src/core/jsproxy.c +++ b/src/core/jsproxy.c @@ -1740,27 +1740,6 @@ JsException_AsJs(PyObject* err) return hiwire_incref(js_error->js); } -// Copied from Python 3.9 -// TODO: remove once we update to Python 3.9 -static int -PyModule_AddType(PyObject* module, PyTypeObject* type) -{ - if (PyType_Ready(type) < 0) { - return -1; - } - - const char* name = _PyType_Name(type); - assert(name != NULL); - - Py_INCREF(type); - if (PyModule_AddObject(module, name, (PyObject*)type) < 0) { - Py_DECREF(type); - return -1; - } - - return 0; -} - int JsProxy_init(PyObject* core_module) { diff --git a/src/tests/make_test_list.py b/src/tests/make_test_list.py index 2466d16f3..6b8fbd68f 100644 --- a/src/tests/make_test_list.py +++ b/src/tests/make_test_list.py @@ -6,7 +6,67 @@ import os from pathlib import Path -TEST_DIR = Path(__file__).parents[1] / "cpython/build/3.8.2/host/lib/python3.8/test" +TEST_DIR = ( + Path(__file__).parents[2] / "cpython/installs/python-3.9.5/lib/python3.9/test/" +) + + +explanation = """\ +# This list is generated with test/make_test_list.py script, which needs +# to be re-run after each CPython update. +# +# Test modules with a failure reason after their name are either skipped +# or marked as a known failure in pytest. +# +# Following reason codes are skipped, as they lead to segfaults: +# - segfault-: segfault in the corresponding system call. +# +# While the below reason codes are marked as a known failure. By default, they +# are also skipped. To run them, provide --run-xfail argument to pytest, +# - platform-specific: This is testing something about a particular platform +# that isn't relevant here +# - async: relies on async +# - floating point: Failures caused by floating-point differences +# - threading: Failures due to lack of a threading implementation +# - subprocess: Failures caused by no subprocess module. Some of these are +# because the underlying functionality depends on subprocess, and others are +# just a side-effect of the way the test is written. The latter should +# probably be marked as "skip" or rearchitected so we don't have to skip the +# whole module. +# - networking: Fails because it tests low-level networking. +# - dbm: Failures due to no dbm module +# - strftime: Failures due to differences / shortcomings in WebAssembly's +# implementation of date/time formatting in strftime and strptime +# - permissions: Issues with the test writing to the virtual filesystem +# - locale: Fails due to limitations in the included locale implementation. +# - multiprocessing: Fails due to no multiprocessing implementation. +# - fs: Fails due to virtual filesystem issues. +# - nonsense: This functionality doesn't make sense in this context. Includes +# things like `pip`, `distutils` +# - crash: The Python interpreter just stopped without a traceback. Will require +# further investigation. This usually seems to be caused by calling into a +# system function that doesn't behave as one would expect. +# - crash-chrome: Same as crash but only affecting Chrome +# - crash-firefox: Same as crash but only affecting Firefox + +""" + + +def collect_old_error_flags(): + old_error_flags = {} + try: + with open(Path(__file__).parent / "python_tests.txt") as fp: + for line in fp: + line = line.strip() + if line.startswith("#") or not line: + continue + error_flags = line.split() + name = error_flags.pop(0) + if error_flags: + old_error_flags[name] = error_flags + except FileNotFoundError: + pass + return old_error_flags def collect_tests(base_dir): @@ -33,7 +93,13 @@ def collect_tests(base_dir): if __name__ == "__main__": + old_error_flags = collect_old_error_flags() tests = collect_tests(TEST_DIR) with open("python_tests.txt", "w") as fp: + fp.write(explanation) for test in tests: - fp.write(test + "\n") + error_flags = " ".join(old_error_flags.get(test, [])) + line = test + if error_flags: + line += " " + error_flags + fp.write(line + "\n") diff --git a/src/tests/python_tests.txt b/src/tests/python_tests.txt index 086d146ae..b088bc679 100644 --- a/src/tests/python_tests.txt +++ b/src/tests/python_tests.txt @@ -35,48 +35,51 @@ # - crash-chrome: Same as crash but only affecting Chrome # - crash-firefox: Same as crash but only affecting Firefox -test___all__ multiprocessing +test___all__ multiprocessing test___future__ -test__locale locale +test__locale locale test__opcode -test__osx_support platform-specific +test__osx_support platform-specific test__xxsubinterpreters test_abc test_abstract_numbers test_aifc -test_argparse crash-chrome +test_argparse crash-chrome test_array test_asdl_parser test_ast -test_asyncgen async -test_asynchat async -test_asyncio.test_base_events async -test_asyncio.test_buffered_proto async -test_asyncio.test_context async -test_asyncio.test_events async -test_asyncio.test_futures async -test_asyncio.test_locks async -test_asyncio.test_pep492 async -test_asyncio.test_proactor_events async +test_asyncgen async +test_asynchat async +test_asyncio.test_asyncio_waitfor async +test_asyncio.test_base_events async +test_asyncio.test_buffered_proto async +test_asyncio.test_context async +test_asyncio.test_events async +test_asyncio.test_futures async +test_asyncio.test_futures2 async +test_asyncio.test_locks async +test_asyncio.test_pep492 async +test_asyncio.test_proactor_events async test_asyncio.test_protocols -test_asyncio.test_queues async -test_asyncio.test_runners -test_asyncio.test_selector_events async segfault-socketcall -test_asyncio.test_sendfile async -test_asyncio.test_server async -test_asyncio.test_sock_lowlevel async +test_asyncio.test_queues async +test_asyncio.test_runners async +test_asyncio.test_selector_events async segfault-socketcall +test_asyncio.test_sendfile async +test_asyncio.test_server async +test_asyncio.test_sock_lowlevel async test_asyncio.test_sslproto -test_asyncio.test_streams async -test_asyncio.test_subprocess async -test_asyncio.test_tasks async +test_asyncio.test_streams async +test_asyncio.test_subprocess async +test_asyncio.test_tasks async +test_asyncio.test_threads async test_asyncio.test_transports -test_asyncio.test_unix_events async +test_asyncio.test_unix_events async test_asyncio.test_windows_events test_asyncio.test_windows_utils -test_asyncore bad ioctl syscall async +test_asyncore bad ioctl syscall async test_atexit test_audioop -test_audit subprocess +test_audit subprocess test_augassign test_base64 test_baseexception @@ -90,9 +93,9 @@ test_bisect test_bool test_buffer test_bufio -test_builtin floating point +test_builtin floating point test_bytes -test_bz2 threading +test_bz2 threading test_c_locale_coercion test_calendar test_call @@ -100,6 +103,7 @@ test_capi test_cgi test_cgitb test_charmapcodec +test_check_c_globals test_class test_clinic test_cmath @@ -125,18 +129,18 @@ test_codeop test_collections test_colorsys test_compare -test_compile crash -test_compileall multiprocessing +test_compile crash +test_compileall multiprocessing test_complex test_concurrent_futures test_configparser test_contains test_context test_contextlib -test_contextlib_async async +test_contextlib_async async test_copy -test_copyreg dbm -test_coroutines async +test_copyreg dbm +test_coroutines async test_cprofile test_crashers test_crypt @@ -146,10 +150,10 @@ test_curses test_dataclasses test_datetime test_dbm -test_dbm_dumb permissions +test_dbm_dumb permissions test_dbm_gnu test_dbm_ndbm -test_decimal floating point +test_decimal floating point test_decorators test_defaultdict test_deque @@ -162,22 +166,20 @@ test_dictcomps test_dictviews test_difflib test_dis -test_distutils nonsense -test_doctest subprocess +test_distutils nonsense +test_doctest subprocess test_doctest2 -test_docxmlrpc socket -test_dtrace platform -test_dummy_thread -test_dummy_threading +test_docxmlrpc socket +test_dtrace platform test_dynamic test_dynamicclassattribute -test_eintr subprocess +test_eintr subprocess test_email.test__encoded_words test_email.test__header_value_parser test_email.test_asian_codecs test_email.test_contentmanager test_email.test_defect_handling -test_email.test_email threading +test_email.test_email threading test_email.test_generator test_email.test_headerregistry test_email.test_inversion @@ -187,8 +189,8 @@ test_email.test_pickleable test_email.test_policy test_email.test_utils test_embed -test_ensurepip nonsense -test_enum threading +test_ensurepip nonsense +test_enum threading test_enumerate test_eof test_epoll @@ -200,23 +202,23 @@ test_extcall test_faulthandler test_fcntl test_file -test_file_eintr subprocess +test_file_eintr subprocess test_filecmp test_fileinput test_fileio test_finalization -test_float floating point +test_float floating point test_flufl test_fnmatch -test_fork1 threading +test_fork1 threading test_format test_fractions test_frame test_frozen test_fstring -test_ftplib socket +test_ftplib socket test_funcattrs -test_functools threading +test_functools threading test_future test_future3 test_future4 @@ -225,37 +227,39 @@ test_gc test_gdb test_generator_stop test_generators +test_genericalias multiprocessing test_genericclass -test_genericpath permissions +test_genericpath permissions test_genexps test_getargs2 test_getopt -test_getpass permissions +test_getpass permissions test_gettext -test_glob fs +test_glob fs test_global test_grammar +test_graphlib test_grp test_gzip test_hash -test_hashlib threading +test_hashlib threading test_heapq test_hmac test_html test_htmlparser test_http_cookiejar test_http_cookies -test_httplib socket segfault-socketcall -test_httpservers threading +test_httplib socket segfault-socketcall +test_httpservers threading test_idle -test_imaplib socket +test_imaplib socket test_imghdr -test_imp fs +test_imp fs test_importlib.builtin.test_finder test_importlib.builtin.test_loader test_importlib.extension.test_case_sensitivity -test_importlib.extension.test_finder fs -test_importlib.extension.test_loader fs +test_importlib.extension.test_finder fs +test_importlib.extension.test_loader fs test_importlib.extension.test_path_hook test_importlib.frozen.test_finder test_importlib.frozen.test_loader @@ -275,24 +279,27 @@ test_importlib.source.test_path_hook test_importlib.source.test_source_encoding test_importlib.test_abc test_importlib.test_api +test_importlib.test_files test_importlib.test_lazy -test_importlib.test_locks threading +test_importlib.test_locks threading test_importlib.test_main test_importlib.test_metadata_api test_importlib.test_namespace_pkgs test_importlib.test_open test_importlib.test_path +test_importlib.test_pkg_import test_importlib.test_read test_importlib.test_resource test_importlib.test_spec +test_importlib.test_threaded_import threading test_importlib.test_util test_importlib.test_windows test_importlib.test_zip test_index -test_inspect +test_inspect async test_int test_int_literal -test_io segfault-unknown +test_io segfault-unknown test_ioctl test_ipaddress test_isinstance @@ -314,32 +321,32 @@ test_json.test_recursion test_json.test_scanstring test_json.test_separators test_json.test_speedups -test_json.test_tool subprocess +test_json.test_tool subprocess test_json.test_unicode -test_keyword subprocess +test_keyword subprocess test_keywordonlyarg test_kqueue -test_largefile fs -test_lib2to3 nonsense +test_largefile fs +test_lib2to3 nonsense test_linecache test_list test_listcomps test_lltrace -test_locale locale -test_logging networking +test_locale locale +test_logging networking test_long test_longexp test_lzma -test_macpath platform-specific -test_mailbox fs +test_mailbox fs +test_mailcap not sure why it fails... test_marshal -test_math floating point +test_math floating point test_memoryio test_memoryview test_metaclass test_mimetypes test_minidom -test_mmap crash +test_mmap crash test_module test_modulefinder test_msilib @@ -352,36 +359,38 @@ test_named_expressions test_netrc test_nis test_nntplib -test_normalization -test_ntpath platform +test_ntpath platform test_numeric_tower test_opcodes -test_openpty platform-specific +test_openpty platform-specific test_operator test_optparse test_ordered_dict -test_os mmap +test_os mmap test_ossaudiodev test_osx_env test_parser -test_pathlib fs -test_pdb subprocess +test_pathlib fs +test_pdb subprocess test_peepholer -test_pickle dbm +test_peg_generator.test_c_parser +test_peg_generator.test_first_sets +test_peg_generator.test_pegen +test_peg_parser +test_pickle dbm test_picklebuffer -test_pickletools dbm -test_pipes platform-specific +test_pickletools dbm +test_pipes platform-specific test_pkg -test_pkgimport test_pkgutil -test_platform subprocess +test_platform subprocess test_plistlib -test_poll subprocess -test_popen subprocess -test_poplib bad ioctl socket +test_poll subprocess +test_popen subprocess +test_poplib bad ioctl socket test_positional_only_arg -test_posix segfault-fstatfs64 -test_posixpath crash +test_posix segfault-fstatfs64 +test_posixpath crash test_pow test_pprint test_print @@ -390,60 +399,60 @@ test_property test_pstats test_pty test_pulldom -test_pwd crash -test_py_compile +test_pwd crash +test_py_compile subprocess test_pyclbr -test_pydoc threading +test_pydoc threading test_pyexpat -test_queue threading -test_quopri subprocess +test_queue threading +test_quopri subprocess test_raise -test_random subprocess +test_random subprocess test_range -test_re locale +test_re locale test_readline -test_regrtest subprocess -test_repl subprocess +test_regrtest subprocess +test_repl subprocess test_reprlib test_resource test_richcmp -test_rlcompleter crash -test_robotparser socket -test_runpy +test_rlcompleter crash +test_robotparser socket +test_runpy subprocess test_sax -test_sched threading +test_sched threading test_scope test_script_helper test_secrets -test_select networking segfault-newselect -test_selectors networking segfault-newselect +test_select networking segfault-newselect +test_selectors networking segfault-newselect test_set test_setcomps test_shelve test_shlex -test_shutil crash +test_shutil crash test_signal -test_site subprocess +test_site subprocess test_slice test_smtpd -test_smtplib bad ioctl syscall 21537 +test_smtplib bad ioctl syscall 21537 test_smtpnet test_sndhdr test_socket test_socketserver test_sort -test_source_encoding subprocess, +test_source_encoding subprocess, test_spwd -test_sqlite threading +test_sqlite threading test_ssl test_startfile test_stat test_statistics -test_strftime strftime +test_strftime strftime test_string test_string_literals test_stringprep -test_strptime strftime +test_strptime strftime test_strtod test_struct test_structmembers @@ -453,27 +462,26 @@ test_subprocess test_sunau test_sundry test_super -test_support multiprocessing +test_support multiprocessing test_symbol test_symtable test_syntax -test_sys subprocess +test_sys subprocess test_sys_setprofile -test_sys_settrace async -test_sysconfig nonsense +test_sys_settrace async +test_sysconfig nonsense test_syslog test_tabnanny -test_tarfile crash +test_tarfile crash test_tcl -test_telnetlib bad ioctl syscall 21537 -test_tempfile fs +test_telnetlib bad ioctl syscall 21537 +test_tempfile fs test_textwrap -test_thread threading -test_threaded_import floating point -test_threadedtempfile threading -test_threading threading -test_threading_local threading -test_threadsignals threading +test_thread threading +test_threadedtempfile threading +test_threading threading +test_threading_local threading +test_threadsignals threading test_time test_timeit test_timeout @@ -490,9 +498,8 @@ test_tools.test_pdeps test_tools.test_pindent test_tools.test_reindent test_tools.test_sundry -test_tools.test_unparse test_trace -test_traceback subprocess +test_traceback subprocess test_tracemalloc test_ttk_guionly test_ttk_textonly @@ -509,13 +516,14 @@ test_unicode_file test_unicode_file_functions test_unicode_identifiers test_unicodedata -test_unittest os.kill +test_unittest os.kill test_univnewlines test_unpack test_unpack_ex -test_urllib segfault-socketcall -test_urllib2 subprocess -test_urllib2_localnet socket +test_unparse +test_urllib segfault-socketcall +test_urllib2 subprocess +test_urllib2_localnet socket test_urllib2net test_urllib_response test_urllibnet @@ -526,14 +534,14 @@ test_userstring test_utf8_mode test_utf8source test_uu -test_uuid subprocess -test_venv nonsense -test_wait3 threading -test_wait4 threading +test_uuid subprocess +test_venv nonsense +test_wait3 threading +test_wait4 threading test_wave -test_weakref threading +test_weakref threading test_weakset -test_webbrowser replaced +test_webbrowser replaced test_winconsoleio test_winreg test_winsound @@ -543,13 +551,14 @@ test_xdrlib test_xml_dom_minicompat test_xml_etree test_xml_etree_c -test_xmlrpc networking +test_xmlrpc networking test_xmlrpc_net test_xxtestfuzz test_yield_from test_zipapp -test_zipfile crash-chrome +test_zipfile crash-chrome test_zipfile64 test_zipimport test_zipimport_support test_zlib +test_zoneinfo.test_zoneinfo diff --git a/src/tests/test_console.py b/src/tests/test_console.py index c19d58b07..fc54bfedf 100644 --- a/src/tests/test_console.py +++ b/src/tests/test_console.py @@ -311,7 +311,7 @@ def test_console_html(console_html_fixture): `>>> 1+ [[;;;terminal-error] File "", line 1 1+ - ^ + ^ SyntaxError: invalid syntax]` ]); diff --git a/src/tests/test_pyodide.py b/src/tests/test_pyodide.py index d705ac578..1c0b0e412 100644 --- a/src/tests/test_pyodide.py +++ b/src/tests/test_pyodide.py @@ -487,7 +487,11 @@ def test_fatal_error(selenium_standalone): ) import re - strip_stack_trace = lambda x: re.sub("\n.*site-packages.*", "", x) + def strip_stack_trace(x): + x = re.sub("\n.*site-packages.*", "", x) + x = re.sub("/lib/python.*/", "", x) + return x + assert ( strip_stack_trace(selenium_standalone.logs) == dedent( @@ -502,8 +506,8 @@ def test_fatal_error(selenium_standalone): File "", line 6 in g File "", line 4 in f File "", line 9 in - File "/lib/python3.8/site-packages/pyodide/_base.py", line 242 in run - File "/lib/python3.8/site-packages/pyodide/_base.py", line 344 in eval_code + File "/lib/pythonxxx/site-packages/pyodide/_base.py", line 242 in run + File "/lib/pythonxxx/site-packages/pyodide/_base.py", line 344 in eval_code """ ) ).strip() diff --git a/src/tests/test_typeconversions.py b/src/tests/test_typeconversions.py index 101d1cfbd..41681fe30 100644 --- a/src/tests/test_typeconversions.py +++ b/src/tests/test_typeconversions.py @@ -116,6 +116,9 @@ def test_bigint_conversions(selenium_module_scope, n): def test_hyp_py2js2py(selenium_module_scope, obj): with selenium_context_manager(selenium_module_scope) as selenium: import pickle + from zoneinfo import ZoneInfo + + assume(not isinstance(obj, ZoneInfo)) # When we compare x == x, there are three possible outcomes: # 1. returns True @@ -180,6 +183,9 @@ def test_big_integer_py2js2py(selenium): def test_hyp_tojs_no_crash(selenium_module_scope, obj): with selenium_context_manager(selenium_module_scope) as selenium: import pickle + from zoneinfo import ZoneInfo + + assume(not isinstance(obj, ZoneInfo)) try: obj_bytes = list(pickle.dumps(obj)) diff --git a/tools/dependency-check.sh b/tools/dependency-check.sh index d545fe48e..a0a69ce2c 100755 --- a/tools/dependency-check.sh +++ b/tools/dependency-check.sh @@ -6,17 +6,17 @@ failure_exit() { } check_python_version() { - if ! command -v python3.8 &> /dev/null; then - echo >&2 "Must compile with python 3.8." + if ! command -v python3.9 &> /dev/null; then + echo >&2 "Must compile with python 3.9." exit 1 fi } check_python_headers() { local python_headers_present - python_headers_present="$(pkg-config --libs python-3.8)" + python_headers_present="$(pkg-config --libs python-3.9)" if [ ! "${python_headers_present}" ]; then - failure_exit "Python 3.8 headers" + failure_exit "Python 3.9 headers" fi }