From 5724bbc05c15e9a65c205a4d6549e9e9711bfce0 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 3 Mar 2022 19:13:58 -0800 Subject: [PATCH] MAINT use variables for Python version rather than hardcoding it (#2241) * As much as possible, use variable for Python version rather than hardcoding it This should make Python updates easier. --- Makefile.envs | 2 +- .../test_cpp_exceptions.py | 35 +++++++++++-------- packages/test_packages_common.py | 13 +++---- pyodide-build/pyodide_build/testing.py | 6 ++++ src/js/pyodide.ts | 4 ++- src/tests/make_test_list.py | 5 ++- src/tests/test_console.py | 6 ++-- src/tests/test_filesystem.py | 16 +++++---- src/tests/test_pyodide.py | 8 ++--- tools/dependency-check.sh | 8 ++--- 10 files changed, 62 insertions(+), 41 deletions(-) diff --git a/Makefile.envs b/Makefile.envs index 5022a4437..37e139f0f 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -30,7 +30,7 @@ export PYODIDE_BASE_URL?=./ # For packages that depend on numpy. # TODO: maybe move this somewhere else? -export NUMPY_LIB=$(PYODIDE_ROOT)/packages/numpy/build/numpy-1.21.4/build/temp.emscripten_wasm32-3.9 +export NUMPY_LIB=$(PYODIDE_ROOT)/packages/numpy/build/numpy-1.21.4/build/temp.emscripten_wasm32-$(PYMAJOR).$(PYMINOR) # This environment variable is used for packages to detect if they are built # for pyodide during build time diff --git a/packages/cpp-exceptions-test/test_cpp_exceptions.py b/packages/cpp-exceptions-test/test_cpp_exceptions.py index ea83a2e9b..5e5298a6c 100644 --- a/packages/cpp-exceptions-test/test_cpp_exceptions.py +++ b/packages/cpp-exceptions-test/test_cpp_exceptions.py @@ -1,11 +1,16 @@ +from pyodide_build.testing import PYVERSION + + def test_uncaught_cpp_exceptions(selenium): assert ( selenium.run_js( - """ + f""" await pyodide.loadPackage("cpp-exceptions-test"); const Tests = pyodide._api.tests; - const idx = pyodide._module.LDSO.loadedLibNames["/lib/python3.9/site-packages/cpp-exceptions-test-throw.so"] + const idx = pyodide._module.LDSO.loadedLibNames["/lib/{PYVERSION}/site-packages/cpp-exceptions-test-throw.so"] const throwlib = pyodide._module.LDSO.loadedLibs[idx].module; + """ + """\ function t(x){ try { throwlib.throw_exc(x); @@ -34,20 +39,22 @@ def test_uncaught_cpp_exceptions(selenium): def test_cpp_exception_catching(selenium): assert ( selenium.run_js( + f""" + await pyodide.loadPackage("cpp-exceptions-test"); + const Module = pyodide._module; + const idx = Module.LDSO.loadedLibNames["/lib/{PYVERSION}/site-packages/cpp-exceptions-test-catch.so"] + const catchlib = Module.LDSO.loadedLibs[idx].module; """ - await pyodide.loadPackage("cpp-exceptions-test"); - const Module = pyodide._module; - const idx = Module.LDSO.loadedLibNames["/lib/python3.9/site-packages/cpp-exceptions-test-catch.so"] - const catchlib = Module.LDSO.loadedLibs[idx].module; - function t(x){ - const ptr = catchlib.catch_exc(x); - const res = Module.UTF8ToString(ptr); - Module._free(ptr); - return res; - } + """\ + function t(x){ + const ptr = catchlib.catch_exc(x); + const res = Module.UTF8ToString(ptr); + Module._free(ptr); + return res; + } - return [t(1), t(2), t(3), t(5)]; - """ + return [t(1), t(2), t(3), t(5)]; + """ ) == [ "caught int 1000", diff --git a/packages/test_packages_common.py b/packages/test_packages_common.py index fce2e1b99..ee3cbb748 100644 --- a/packages/test_packages_common.py +++ b/packages/test_packages_common.py @@ -5,6 +5,7 @@ import pytest from conftest import ROOT_PATH, built_packages from pyodide_build.io import parse_package_config +from pyodide_build.testing import PYVERSION PKG_DIR = ROOT_PATH / "packages" @@ -72,9 +73,9 @@ def test_import(name, selenium_standalone): selenium_standalone.run("import glob, os") baseline_pyc = selenium_standalone.run( - """ + f""" len(list(glob.glob( - '/lib/python3.9/site-packages/**/*.pyc', + '/lib/{PYVERSION}/site-packages/**/*.pyc', recursive=True) )) """ @@ -85,9 +86,9 @@ def test_import(name, selenium_standalone): # files assert ( selenium_standalone.run( - """ + f""" len(list(glob.glob( - '/lib/python3.9/site-packages/**/*.pyc', + '/lib/{PYVERSION}/site-packages/**/*.pyc', recursive=True) )) """ @@ -97,9 +98,9 @@ def test_import(name, selenium_standalone): # Make sure no exe files were loaded! assert ( selenium_standalone.run( - """ + f""" len(list(glob.glob( - '/lib/python3.9/site-packages/**/*.exe', + '/lib/{PYVERSION}/site-packages/**/*.exe', recursive=True) )) """ diff --git a/pyodide-build/pyodide_build/testing.py b/pyodide-build/pyodide_build/testing.py index 6a947391a..dc3f0df6e 100644 --- a/pyodide-build/pyodide_build/testing.py +++ b/pyodide-build/pyodide_build/testing.py @@ -5,6 +5,12 @@ from typing import Callable, Collection, Optional import pytest +from .common import get_make_flag + +PYMAJOR = get_make_flag("PYMAJOR") +PYMINOR = get_make_flag("PYMINOR") +PYVERSION = f"python{PYMAJOR}.{PYMINOR}" + def _run_in_pyodide_get_source(f): lines, start_line = inspect.getsourcelines(f) diff --git a/src/js/pyodide.ts b/src/js/pyodide.ts index d55751f49..00f310bbd 100644 --- a/src/js/pyodide.ts +++ b/src/js/pyodide.ts @@ -79,8 +79,10 @@ function unpackPyodidePy(pyodide_py_tar: Uint8Array) { ); Module.FS.close(stream); const code_ptr = Module.stringToNewUTF8(` +from sys import version_info +pyversion = f"python{version_info.major}.{version_info.minor}" import shutil -shutil.unpack_archive("/pyodide_py.tar", "/lib/python3.9/site-packages/") +shutil.unpack_archive("/pyodide_py.tar", f"/lib/{pyversion}/site-packages/") del shutil import importlib importlib.invalidate_caches() diff --git a/src/tests/make_test_list.py b/src/tests/make_test_list.py index c6ad985d7..91c9901e3 100644 --- a/src/tests/make_test_list.py +++ b/src/tests/make_test_list.py @@ -4,9 +4,12 @@ Generate a list of test modules in the CPython distribution. import os from pathlib import Path +from sys import version_info TEST_DIR = ( - Path(__file__).parents[2] / "cpython/installs/python-3.9.5/lib/python3.9/test/" + Path(__file__).parents[2] / "cpython/installs" + f"/python-{version_info.major}.{version_info.minor}.{version_info.micro}" + f"/lib/python{version_info.major}.{version_info.minor}/test/" ) diff --git a/src/tests/test_console.py b/src/tests/test_console.py index a62d4a69e..5343d090d 100644 --- a/src/tests/test_console.py +++ b/src/tests/test_console.py @@ -8,7 +8,7 @@ from _pyodide import console from _pyodide.console import Console, _CommandCompiler, _Compile # noqa: E402 from conftest import selenium_common from pyodide import CodeRunner # noqa: E402 -from pyodide_build.testing import run_in_pyodide +from pyodide_build.testing import PYVERSION, run_in_pyodide def test_command_compiler(): @@ -417,7 +417,7 @@ def test_console_html(console_html_fixture): ).strip() ) == dedent( - """ + f""" >>> class Test: ... def __repr__(self): ... raise TypeError(\"hi\") @@ -425,7 +425,7 @@ def test_console_html(console_html_fixture): >>> Test() [[;;;terminal-error]Traceback (most recent call last): - File \"/lib/python3.9/site-packages/_pyodide/console.py\", line 465, in repr_shorten + File \"/lib/{PYVERSION}/site-packages/_pyodide/console.py\", line 465, in repr_shorten text = repr(value) File \"\", line 3, in __repr__ TypeError: hi] diff --git a/src/tests/test_filesystem.py b/src/tests/test_filesystem.py index 205dc5a1e..1746d7d83 100644 --- a/src/tests/test_filesystem.py +++ b/src/tests/test_filesystem.py @@ -4,6 +4,8 @@ for a basic nodejs-based test, see src/js/test/filesystem.test.js """ import pytest +from pyodide_build.testing import PYVERSION + @pytest.mark.skip_refcount_check @pytest.mark.skip_pyproxy_check @@ -17,16 +19,16 @@ def test_idbfs_persist_code(selenium_standalone): # create mount selenium.run_js( f""" - pyodide.FS.mkdir('/lib/python3.9/site-packages/test_idbfs'); - pyodide.FS.mount(pyodide.FS.filesystems.{fstype}, {{root : "."}}, "/lib/python3.9/site-packages/test_idbfs"); + pyodide.FS.mkdir('/lib/{PYVERSION}/site-packages/test_idbfs'); + pyodide.FS.mount(pyodide.FS.filesystems.{fstype}, {{root : "."}}, "/lib/{PYVERSION}/site-packages/test_idbfs"); """ ) # create file in mount selenium.run_js( - """ + f""" pyodide.runPython(` import pathlib - p = pathlib.Path('/lib/python3.9/site-packages/test_idbfs/__init__.py') + p = pathlib.Path('/lib/{PYVERSION}/site-packages/test_idbfs/__init__.py') p.write_text("def test(): return 7") from importlib import invalidate_caches invalidate_caches() @@ -69,8 +71,8 @@ def test_idbfs_persist_code(selenium_standalone): # re-mount selenium.run_js( f""" - pyodide.FS.mkdir('/lib/python3.9/site-packages/test_idbfs'); - pyodide.FS.mount(pyodide.FS.filesystems.{fstype}, {{root : "."}}, "/lib/python3.9/site-packages/test_idbfs"); + pyodide.FS.mkdir('/lib/{PYVERSION}/site-packages/test_idbfs'); + pyodide.FS.mount(pyodide.FS.filesystems.{fstype}, {{root : "."}}, "/lib/{PYVERSION}/site-packages/test_idbfs"); """ ) # sync FROM idbfs @@ -95,5 +97,5 @@ def test_idbfs_persist_code(selenium_standalone): ) # remove file selenium.run_js( - """pyodide.FS.unlink("/lib/python3.9/site-packages/test_idbfs/__init__.py")""" + f"""pyodide.FS.unlink("/lib/{PYVERSION}/site-packages/test_idbfs/__init__.py")""" ) diff --git a/src/tests/test_pyodide.py b/src/tests/test_pyodide.py index 13f1097bf..b19a88471 100644 --- a/src/tests/test_pyodide.py +++ b/src/tests/test_pyodide.py @@ -5,7 +5,7 @@ from typing import Any import pytest from pyodide import CodeRunner, eval_code, find_imports, should_quiet # noqa: E402 -from pyodide_build.testing import run_in_pyodide +from pyodide_build.testing import PYVERSION, run_in_pyodide def test_find_imports(): @@ -846,14 +846,14 @@ def test_js_stackframes(selenium): ["test.html", "d2"], ["test.html", "d1"], ["pyodide.js", "runPython"], - ["/lib/python3.9/site-packages/_pyodide/_base.py", "eval_code"], - ["/lib/python3.9/site-packages/_pyodide/_base.py", "run"], + [f"/lib/{PYVERSION}/site-packages/_pyodide/_base.py", "eval_code"], + [f"/lib/{PYVERSION}/site-packages/_pyodide/_base.py", "run"], ["", ""], ["", "c2"], ["", "c1"], ["test.html", "b"], ["pyodide.js", "pyimport"], - ["/lib/python3.9/importlib/__init__.py", "import_module"], + [f"/lib/{PYVERSION}/importlib/__init__.py", "import_module"], ] assert normalize_tb(res[: len(frames)]) == frames diff --git a/tools/dependency-check.sh b/tools/dependency-check.sh index a0a69ce2c..f8195b865 100755 --- a/tools/dependency-check.sh +++ b/tools/dependency-check.sh @@ -6,17 +6,17 @@ failure_exit() { } check_python_version() { - if ! command -v python3.9 &> /dev/null; then - echo >&2 "Must compile with python 3.9." + if ! command -v python$PYMAJOR.$PYMINOR &> /dev/null; then + echo >&2 "Must compile with python $PYMAJOR.$PYMINOR." exit 1 fi } check_python_headers() { local python_headers_present - python_headers_present="$(pkg-config --libs python-3.9)" + python_headers_present="$(pkg-config --libs python-$PYMAJOR.$PYMINOR)" if [ ! "${python_headers_present}" ]; then - failure_exit "Python 3.9 headers" + failure_exit "Python $PYMAJOR.$PYMINOR headers" fi }