diff --git a/.circleci/config.yml b/.circleci/config.yml index 38ea2ff81..ca839ae81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,7 @@ jobs: root: . paths: - ./build - - ./cpython/build/3.8.2/host + - ./packages/.artifacts - store_artifacts: path: /home/circleci/repo/build/ @@ -135,7 +135,7 @@ jobs: - run: name: benchmark command: | - python benchmark/benchmark.py cpython/build/3.8.2/host/bin/python3 build/benchmarks.json + python benchmark/benchmark.py /usr/local/bin/python3 build/benchmarks.json - store_artifacts: path: /home/circleci/repo/build/benchmarks.json diff --git a/.gitignore b/.gitignore index e8e27c995..c3fe699e5 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ packages/libxml/libxml* packages/libxslt/libxslt* packages/libiconv/libiconv* packages/zlib/zlib* +packages/.artifacts docs/python-api/ docs/micropip-api/ diff --git a/Makefile.envs b/Makefile.envs index fd66353e2..a098727b4 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -10,7 +10,7 @@ export BINARYEN_ROOT = $(PYODIDE_ROOT)/emsdk/emsdk/binaryen export PYVERSION=3.8.2 export PYMINOR=$(basename $(PYVERSION)) -export HOSTPYTHONROOT=$(PYODIDE_ROOT)/cpython/build/$(PYVERSION)/host +export HOSTPYTHONROOT=$(shell python -c "import sys; print(sys.prefix)") export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python3 export TARGETPYTHONROOT=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION) export PYTHONINCLUDE=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION)/include/python$(PYMINOR) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 3a5510390..c62c84376 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -23,10 +23,15 @@ def print_entry(name, res): def run_native(hostpython, code): + root = Path(__file__).resolve().parents[1] output = subprocess.check_output( [hostpython.resolve(), "-c", code], cwd=Path(__file__).resolve().parent, - env={"PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src")}, + env={ + "PYTHONPATH": str(root / "src") + + ":" + + str(root / "packages" / ".artifacts" / "lib" / "python") + }, ) return float(output.strip().split()[-1]) diff --git a/cpython/Makefile b/cpython/Makefile index 8fa194f7d..35acb4c7b 100644 --- a/cpython/Makefile +++ b/cpython/Makefile @@ -3,14 +3,7 @@ include ../Makefile.envs ROOT=$(abspath .) -HOSTINSTALL=$(ROOT)/build/$(PYVERSION)/host -HOSTBUILD=$(HOSTINSTALL)/Python-$(PYVERSION) -HOSTPYTHON=$(HOSTINSTALL)/bin/python3$(EXE) -HOSTPYTHON_CPPFLAGS="-I/usr/local/opt/openssl/include" -HOSTPYTHON_LDFLAGS="-L/usr/local/opt/openssl/lib" -HOSTPGEN=$(HOSTINSTALL)/bin/pgen$(EXE) - -BUILD=$(ROOT)/build/$(PYVERSION)/Python-$(PYVERSION) +BUILD=$(ROOT)/build/Python-$(PYVERSION) INSTALL=$(ROOT)/installs/python-$(PYVERSION) TARBALL=$(ROOT)/downloads/Python-$(PYVERSION).tgz URL=https://www.python.org/ftp/python/$(PYVERSION)/Python-$(PYVERSION).tgz @@ -39,15 +32,14 @@ $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB) cd $(BUILD); \ sed -i -e 's/libinstall:.*/libinstall:/' Makefile; \ touch $(BUILD)/$(LIB) ; \ - emmake make HOSTPYTHON=$(HOSTPYTHON) PYTHON_FOR_BUILD=$(HOSTPYTHON) CROSS_COMPILE=yes inclinstall libinstall $(LIB) -j $${PYODIDE_JOBS:-3} && \ + emmake make PYTHON_FOR_BUILD=$(HOSTPYTHON) CROSS_COMPILE=yes inclinstall libinstall $(LIB) -j $${PYODIDE_JOBS:-3} && \ cp $(LIB) $(INSTALL)/lib/ && \ - cp $(HOSTINSTALL)/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py $(INSTALL)/lib/python$(PYMINOR)/_sysconfigdata__emscripten_.py; \ + cp $(HOSTPYTHONROOT)/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py $(INSTALL)/lib/python$(PYMINOR)/_sysconfigdata__emscripten_.py; \ sed -i -e 's#'"$(PYODIDE_ROOT)"'##g' $(INSTALL)/lib/python$(PYMINOR)/_sysconfigdata__emscripten_.py; \ ) clean: - -rm -fr $(HOSTINSTALL) -rm -fr $(BUILD) -rm -fr $(INSTALL) @@ -73,17 +65,6 @@ $(BZIP2TARBALL): wget -q -O $@ $(BZIP2URL) -$(HOSTPYTHON) $(HOSTPGEN): $(TARBALL) - mkdir -p $(HOSTINSTALL) - [ -d $(HOSTBUILD) ] || tar -C $(HOSTINSTALL) -xf $(TARBALL) - ( \ - cd $(HOSTBUILD); \ - PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig" ./configure --prefix=$(HOSTINSTALL) || cat config.log && \ - make regen-grammar -j $${PYODIDE_JOBS:-3} && \ - make install -j $${PYODIDE_JOBS:-3} && \ - make distclean \ - ) - $(BUILD)/.patched: $(TARBALL) [ -d $(BUILD) ] || (mkdir -p $(dir $(BUILD)); tar -C $(dir $(BUILD)) -xf $(TARBALL)) @@ -135,13 +116,13 @@ $(BUILD)/Makefile: $(BUILD)/.patched $(ZLIBBUILD)/.patched $(SQLITEBUILD)/libsql ) -$(BUILD)/$(LIB): $(BUILD)/Makefile $(HOSTPYTHON) $(HOSTPGEN) Setup.local +$(BUILD)/$(LIB): $(BUILD)/Makefile Setup.local cp Setup.local $(BUILD)/Modules/ cat pyconfig.undefs.h >> $(BUILD)/pyconfig.h ( \ - cp build/$(PYVERSION)/host/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py build/$(PYVERSION)/Python-$(PYVERSION)/Lib/_sysconfigdata__emscripten_.py; \ + cp $(HOSTPYTHONROOT)/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py $(BUILD)/Lib/_sysconfigdata__emscripten_.py; \ cd $(BUILD); \ - emmake make HOSTPYTHON=$(HOSTPYTHON) HOSTPGEN=$(HOSTPGEN) CROSS_COMPILE=yes $(LIB) -j $${PYODIDE_JOBS:-3} \ + emmake make CROSS_COMPILE=yes $(LIB) -j $${PYODIDE_JOBS:-3} \ ) - sed -i -e 's/\-undefined dynamic_lookup//' build/$(PYVERSION)/Python-$(PYVERSION)/Lib/_sysconfigdata__emscripten_.py + sed -i -e 's/\-undefined dynamic_lookup//' $(BUILD)/Lib/_sysconfigdata__emscripten_.py touch $(BUILD)/$(LIB) diff --git a/cpython/Setup.local b/cpython/Setup.local index db025f2b4..336501974 100644 --- a/cpython/Setup.local +++ b/cpython/Setup.local @@ -24,7 +24,7 @@ select selectmodule.c _posixsubprocess _posixsubprocess.c binascii binascii.c -zlib zlibmodule.c -I../../zlib-1.2.11 +zlib zlibmodule.c -I../zlib-1.2.11 pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI -DXML_POOR_ENTROPY @@ -33,7 +33,7 @@ _sha256 sha256module.c _sha512 sha512module.c _sha3 _sha3/sha3module.c _md5 md5module.c -_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c ../../host/Python-3.8.2/Modules/_blake2/blake2s_impl.c +_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c _sqlite3 _sqlite/cache.c _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c -I$(SQLITEBUILD) -L$(SQLITEBUILD) -lsqlite3 _crypt _cryptmodule.c diff --git a/packages/Makefile b/packages/Makefile index e2cc102df..bb8b69a8e 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -1,9 +1,11 @@ PYODIDE_ROOT=$(abspath ..) +PYODIDE_LIBRARIES=$(abspath ./.artifacts) include ../Makefile.envs all: deps - ../bin/pyodide buildall . ../build \ - --package_abi=$(PYODIDE_PACKAGE_ABI) --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) --only $(PYODIDE_PACKAGES) + mkdir -p $(PYODIDE_LIBRARIES) + PYTHONPATH=$(PYODIDE_LIBRARIES)/lib/python ../bin/pyodide buildall . ../build \ + --package_abi=$(PYODIDE_PACKAGE_ABI) --target=$(TARGETPYTHONROOT) --only $(PYODIDE_PACKAGES) --install-dir $(PYODIDE_LIBRARIES) update-all: for pkg in $$(find . -maxdepth 1 -type d -exec basename {} \; | tail -n +2); do \ @@ -16,3 +18,4 @@ deps: clean: rm -rf ./*/build + rm -rf ./.artifacts diff --git a/pyodide_build/common.py b/pyodide_build/common.py index 005fbc23a..c353ae1d8 100644 --- a/pyodide_build/common.py +++ b/pyodide_build/common.py @@ -1,9 +1,9 @@ from pathlib import Path from typing import Optional, Set - +import sys ROOTDIR = Path(__file__).parents[1].resolve() / "tools" -HOSTPYTHON = ROOTDIR / ".." / "cpython" / "build" / "3.8.2" / "host" +HOSTPYTHON = sys.prefix TARGETPYTHON = ROOTDIR / ".." / "cpython" / "installs" / "python-3.8.2" DEFAULTCFLAGS = "" # fmt: off diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index d76ff2790..6f517f7fe 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -197,8 +197,8 @@ def handle_command(line, args, dryrun=False): -------- >>> from collections import namedtuple - >>> Args = namedtuple('args', ['cflags', 'ldflags']) - >>> args = Args(cflags='', ldflags='') + >>> Args = namedtuple('args', ['cflags', 'ldflags', 'host']) + >>> args = Args(cflags='', ldflags='', host='') >>> handle_command(['gcc', 'test.c'], args, dryrun=True) emcc test.c ['emcc', 'test.c'] @@ -239,11 +239,12 @@ def handle_command(line, args, dryrun=False): lapack_dir = None + host_dir = str(Path(args.host).resolve()) # Go through and adjust arguments for arg in line[1:]: if arg.startswith("-I"): if ( - str(Path(arg[2:]).resolve()).startswith(args.host) + str(Path(arg[2:]).resolve()).startswith(host_dir + "/include/python") and "site-packages" not in arg ): arg = arg.replace("-I" + args.host, "-I" + args.target) diff --git a/pyodide_build/tests/test_pywasmcross.py b/pyodide_build/tests/test_pywasmcross.py index bedeb99ec..64d4e4d14 100644 --- a/pyodide_build/tests/test_pywasmcross.py +++ b/pyodide_build/tests/test_pywasmcross.py @@ -31,8 +31,8 @@ f2c_wrap = _args_wrapper(f2c) def test_handle_command(): - Args = namedtuple("args", ["cflags", "ldflags"]) - args = Args(cflags="", ldflags="") + Args = namedtuple("args", ["cflags", "ldflags", "host"]) + args = Args(cflags="", ldflags="", host="") assert handle_command_wrap("gcc -print-multiarch", args) is None assert handle_command_wrap("gcc test.c", args) == "emcc test.c" assert ( @@ -41,7 +41,7 @@ def test_handle_command(): ) # check ldflags injection - args = Args(cflags="", ldflags="-lm") + args = Args(cflags="", ldflags="-lm", host="") assert ( handle_command_wrap("gcc -shared -c test.o -o test.so", args) == "emcc -lm -shared -c test.bc -o test.wasm" diff --git a/tools/dependency-check.sh b/tools/dependency-check.sh index 7abef5d95..9f4787e54 100755 --- a/tools/dependency-check.sh +++ b/tools/dependency-check.sh @@ -5,6 +5,12 @@ failure_exit() { exit 1 } +check_python_version() { + if ! command -v python3.8 &> /dev/null; then + echo >&2 "Must compile with python 3.8." + exit 1 + fi +} check_python_headers() { local python_headers_present python_headers_present="$(pkg-config --libs python-3.8)" @@ -44,6 +50,7 @@ check_pyyaml() { fi } +check_python_version check_pkgconfig #check_python_headers check_fortran_dependencies