diff --git a/CLAPACK/Makefile b/CLAPACK/Makefile index e6ae28f6d..701283623 100644 --- a/CLAPACK/Makefile +++ b/CLAPACK/Makefile @@ -19,7 +19,7 @@ $(SRC)/lapack_WA.bc: $(SRC)/Makefile # On target it gets rewritten by pywasmcross to the full patch of # blas_WA.bc, lapack_WA.bc which are linked statically in scipy # in each module that needs them. - emmake make -C CLAPACK-WA/ + emmake make -C CLAPACK-WA/ -j $${PYODIDE_JOBS:-3} $(SRC)/Makefile: diff --git a/Makefile b/Makefile index bd8a68895..be4a9ed06 100644 --- a/Makefile +++ b/Makefile @@ -77,10 +77,12 @@ all: check \ build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \ src/pyimport.bc src/pyproxy.bc src/python2js.bc src/python2js_buffer.bc \ src/runpython.bc src/hiwire.bc + date +"[%F %T] Building pyodide.asm.js..." [ -d build ] || mkdir build $(CXX) -s EXPORT_NAME="'pyodide'" -o build/pyodide.asm.html $(filter %.bc,$^) \ $(LDFLAGS) -s FORCE_FILESYSTEM=1 rm build/pyodide.asm.html + date +"[%F %T] done building pyodide.asm.js." env: @@ -223,23 +225,33 @@ $(PYODIDE_CXX): $(CPYTHONLIB): emsdk/emsdk/.complete $(PYODIDE_EMCC) $(PYODIDE_CXX) + date +"[%F %T] Building cpython..." make -C $(CPYTHONROOT) + date +"[%F %T] done building cpython..." $(LZ4LIB): + date +"[%F %T] Building lz4..." make -C lz4 + date +"[%F %T] done building lz4." $(SIX_LIBS): $(CPYTHONLIB) + date +"[%F %T] Building six..." make -C six + date +"[%F %T] done building six." $(JEDI_LIBS): $(CPYTHONLIB) + date +"[%F %T] Building jedi..." make -C jedi + date +"[%F %T] done building jedi." $(PARSO_LIBS): $(CPYTHONLIB) + date +"[%F %T] Building parso..." make -C parso + date +"[%F %T] done building parso." $(CLAPACK): $(CPYTHONLIB) @@ -249,16 +261,22 @@ ifdef PYODIDE_PACKAGES mkdir -p CLAPACK/CLAPACK-WA/ touch $(CLAPACK) else + date +"[%F %T] Building CLAPACK..." make -C CLAPACK + date +"[%F %T] done building CLAPACK." endif build/packages.json: $(CLAPACK) FORCE + date +"[%F %T] Building packages..." make -C packages + date +"[%F %T] done building packages..." emsdk/emsdk/.complete: + date +"[%F %T] Building emsdk..." make -C emsdk + date +"[%F %T] done building emsdk." FORCE: diff --git a/cpython/Makefile b/cpython/Makefile index 553a46fea..6e95ad452 100644 --- a/cpython/Makefile +++ b/cpython/Makefile @@ -39,7 +39,7 @@ $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB) cd $(BUILD); \ sed -i -e 's/libinstall:.*/libinstall:/' Makefile; \ touch $(BUILD)/$(LIB) ; \ - emmake make HOSTPYTHON=$(HOSTPYTHON) PYTHON_FOR_BUILD=$(HOSTPYTHON) CROSS_COMPILE=yes inclinstall libinstall $(LIB) && \ + emmake make HOSTPYTHON=$(HOSTPYTHON) PYTHON_FOR_BUILD=$(HOSTPYTHON) CROSS_COMPILE=yes inclinstall libinstall $(LIB) -j $${PYODIDE_JOBS:-3} && \ cp $(LIB) $(INSTALL)/lib/ && \ cp $(HOSTINSTALL)/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py $(INSTALL)/lib/python$(PYMINOR)/_sysconfigdata__emscripten_.py; \ sed -i -e 's#'"$(PYODIDE_ROOT)"'##g' $(INSTALL)/lib/python$(PYMINOR)/_sysconfigdata__emscripten_.py; \ @@ -79,8 +79,8 @@ $(HOSTPYTHON) $(HOSTPGEN): $(TARBALL) ( \ cd $(HOSTBUILD); \ PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig" ./configure --prefix=$(HOSTINSTALL) || cat config.log && \ - make regen-grammar && \ - make install && \ + make regen-grammar -j $${PYODIDE_JOBS:-3} && \ + make install -j $${PYODIDE_JOBS:-3} && \ cp Parser/pgen$(EXE) $(HOSTINSTALL)/bin/ && \ make distclean \ ) @@ -105,7 +105,7 @@ $(SQLITEBUILD)/libsqlite3.la: $(SQLITETARBALL) ( \ cd $(SQLITEBUILD); \ emconfigure ./configure; \ - emmake make; \ + emmake make -j $${PYODIDE_JOBS:-3}; \ ) @@ -114,7 +114,7 @@ $(BZIP2BUILD)/libbz2.a: $(BZIP2TARBALL) tar -C $(ROOT)/build/ -xf $(BZIP2TARBALL) ( \ cd $(BZIP2BUILD); \ - emmake make CC=emcc CFLAGS="-Wall -Winline -O2 -fomit-frame-pointer -D_FILE_OFFSET_BITS=64" AR=emar RANLIB=emranlib libbz2.a; \ + emmake make -j $${PYODIDE_JOBS:-3} CC=emcc CFLAGS="-Wall -Winline -O2 -fomit-frame-pointer -D_FILE_OFFSET_BITS=64" AR=emar RANLIB=emranlib libbz2.a; \ ) @@ -142,7 +142,7 @@ $(BUILD)/$(LIB): $(BUILD)/Makefile $(HOSTPYTHON) $(HOSTPGEN) Setup.local ( \ cp build/$(PYVERSION)/host/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py build/$(PYVERSION)/Python-$(PYVERSION)/Lib/_sysconfigdata__emscripten_.py; \ cd $(BUILD); \ - emmake make HOSTPYTHON=$(HOSTPYTHON) HOSTPGEN=$(HOSTPGEN) CROSS_COMPILE=yes $(LIB) \ + emmake make HOSTPYTHON=$(HOSTPYTHON) HOSTPGEN=$(HOSTPGEN) CROSS_COMPILE=yes $(LIB) -j $${PYODIDE_JOBS:-3} \ ) sed -i -e 's/\-undefined dynamic_lookup//' build/$(PYVERSION)/Python-$(PYVERSION)/Lib/_sysconfigdata__emscripten_.py touch $(BUILD)/$(LIB) diff --git a/docs/building_from_sources.md b/docs/building_from_sources.md index 4ea45b004..8523fb951 100644 --- a/docs/building_from_sources.md +++ b/docs/building_from_sources.md @@ -56,7 +56,7 @@ If running ``make`` deterministically stops at one point in each subsequent try, the maximum RAM usage available to the docker container might help [This is different from the physical RAM capacity inside the system]. Ideally, at least 3 GB of RAM should be available to the docker container to build `pyodide` smoothly. These settings can -be changed via Docker Preferences [See [here](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container)]. +be changed via Docker Preferences (See [here](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container)). You can edit the files in your source checkout on your host machine, and then repeatedly run `make` inside the Docker environment to test your changes. @@ -81,3 +81,8 @@ micropip and package is generally always included for any non empty value of If scipy is included in `PYODIDE_PACKAGES`, BLAS/LAPACK must be manually built first with `make -c CLAPACK`. + +## Environement variables + +Following environment variables additionally impact the build, + - `PYODIDE_JOBS`: the `-j` option passed to the `emmake make` command when applicable for parallel compilation. Default: 3. diff --git a/lz4/Makefile b/lz4/Makefile index e511e10d2..d2fe09da4 100644 --- a/lz4/Makefile +++ b/lz4/Makefile @@ -32,5 +32,5 @@ $(SRC)/Makefile: $(TARBALL) $(SRC)/lib/liblz4.a: $(SRC)/Makefile ( \ cd $(SRC) ; \ - emmake make ; \ + emmake make -j $${PYODIDE_JOBS:-3} ; \ ) diff --git a/packages/micropip/test_micropip.py b/packages/micropip/test_micropip.py index 339c982b5..71c93c416 100644 --- a/packages/micropip/test_micropip.py +++ b/packages/micropip/test_micropip.py @@ -68,4 +68,6 @@ def test_install_custom_url(selenium_standalone, web_server_secondary): base_url = f'http://{server_hostname}:{server_port}/test/data/' url = base_url + 'snowballstemmer-2.0.0-py2.py3-none-any.whl' selenium_standalone.run(f"micropip.install('{url}')") + # wait untill micropip is loaded + time.sleep(1) selenium_standalone.run("import snowballstemmer") diff --git a/pyodide_build/buildpkg.py b/pyodide_build/buildpkg.py index 599507892..cd7c77828 100755 --- a/pyodide_build/buildpkg.py +++ b/pyodide_build/buildpkg.py @@ -10,6 +10,7 @@ import os from pathlib import Path import shutil import subprocess +from datetime import datetime from . import common @@ -193,7 +194,12 @@ def needs_rebuild(pkg, path, buildpath): def build_package(path, args): pkg = common.parse_package(path) - packagedir = pkg['package']['name'] + '-' + pkg['package']['version'] + name = pkg['package']['name'] + t0 = datetime.now() + print("[{}] Building package {}...".format( + t0.strftime('%Y-%m-%d %H:%M:%S'), name) + ) + packagedir = name + '-' + pkg['package']['version'] dirpath = path.parent orig_path = Path.cwd() os.chdir(dirpath) @@ -211,6 +217,10 @@ def build_package(path, args): package_files(buildpath, srcpath, pkg, args) finally: os.chdir(orig_path) + t1 = datetime.now() + print("[{}] done building package {} in {:.1f} s.".format( + t1.strftime('%Y-%m-%d %H:%M:%S'), name, (t1 - t0).total_seconds() + )) def make_parser(parser):