BLD Build CLAPACK, cpython and lz4 in parallel (#701)

CLAPACK/Makefile

@@ -19,7 +19,7 @@ $(SRC)/lapack_WA.bc:  $(SRC)/Makefile
 	# On target it gets rewritten by pywasmcross to the full patch of
 	# blas_WA.bc, lapack_WA.bc which are linked statically in scipy
 	# in each module that needs them.
-	emmake make -C CLAPACK-WA/
+	emmake make -C CLAPACK-WA/ -j $${PYODIDE_JOBS:-3}
 
 
 $(SRC)/Makefile:

Makefile

@@ -77,10 +77,12 @@ all: check \
 build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \
 		src/pyimport.bc src/pyproxy.bc src/python2js.bc src/python2js_buffer.bc \
 		src/runpython.bc src/hiwire.bc
+	date +"[%F %T] Building pyodide.asm.js..."
 	[ -d build ] || mkdir build
 	$(CXX) -s EXPORT_NAME="'pyodide'" -o build/pyodide.asm.html $(filter %.bc,$^) \
 		$(LDFLAGS) -s FORCE_FILESYSTEM=1
 	rm build/pyodide.asm.html
+	date +"[%F %T] done building pyodide.asm.js."
 
 
 env:
@@ -223,23 +225,33 @@ $(PYODIDE_CXX):
 
 
 $(CPYTHONLIB): emsdk/emsdk/.complete $(PYODIDE_EMCC) $(PYODIDE_CXX)
+	date +"[%F %T] Building cpython..."
 	make -C $(CPYTHONROOT)
+	date +"[%F %T] done building cpython..."
 
 
 $(LZ4LIB):
+	date +"[%F %T] Building lz4..."
 	make -C lz4
+	date +"[%F %T] done building lz4."
 
 
 $(SIX_LIBS): $(CPYTHONLIB)
+	date +"[%F %T] Building six..."
 	make -C six
+	date +"[%F %T] done building six."
 
 
 $(JEDI_LIBS): $(CPYTHONLIB)
+	date +"[%F %T] Building jedi..."
 	make -C jedi
+	date +"[%F %T] done building jedi."
 
 
 $(PARSO_LIBS): $(CPYTHONLIB)
+	date +"[%F %T] Building parso..."
 	make -C parso
+	date +"[%F %T] done building parso."
 
 
 $(CLAPACK): $(CPYTHONLIB)
@@ -249,16 +261,22 @@ ifdef PYODIDE_PACKAGES
 	mkdir -p CLAPACK/CLAPACK-WA/
 	touch $(CLAPACK)
 else
+	date +"[%F %T] Building CLAPACK..."
 	make -C CLAPACK
+	date +"[%F %T] done building CLAPACK."
 endif
 
 
 
 build/packages.json: $(CLAPACK) FORCE
+	date +"[%F %T] Building packages..."
 	make -C packages
+	date +"[%F %T] done building packages..."
 
 emsdk/emsdk/.complete:
+	date +"[%F %T] Building emsdk..."
 	make -C emsdk
+	date +"[%F %T] done building emsdk."
 
 FORCE:
 

cpython/Makefile

@@ -39,7 +39,7 @@ $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB)
 		cd $(BUILD); \
 		sed -i -e 's/libinstall:.*/libinstall:/' Makefile; \
 		touch $(BUILD)/$(LIB) ; \
-		emmake make HOSTPYTHON=$(HOSTPYTHON) PYTHON_FOR_BUILD=$(HOSTPYTHON) CROSS_COMPILE=yes inclinstall libinstall $(LIB) && \
+		emmake make HOSTPYTHON=$(HOSTPYTHON) PYTHON_FOR_BUILD=$(HOSTPYTHON) CROSS_COMPILE=yes inclinstall libinstall $(LIB) -j $${PYODIDE_JOBS:-3} && \
 		cp $(LIB) $(INSTALL)/lib/ && \
 		cp $(HOSTINSTALL)/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py $(INSTALL)/lib/python$(PYMINOR)/_sysconfigdata__emscripten_.py; \
 		sed -i -e 's#'"$(PYODIDE_ROOT)"'##g' $(INSTALL)/lib/python$(PYMINOR)/_sysconfigdata__emscripten_.py; \
@@ -79,8 +79,8 @@ $(HOSTPYTHON) $(HOSTPGEN): $(TARBALL)
 	( \
 		cd $(HOSTBUILD); \
 		PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig" ./configure --prefix=$(HOSTINSTALL) || cat config.log && \
-	  make regen-grammar && \
+	  	make regen-grammar -j $${PYODIDE_JOBS:-3} && \
-		make install && \
+		make install -j $${PYODIDE_JOBS:-3} && \
 		cp Parser/pgen$(EXE) $(HOSTINSTALL)/bin/ && \
 		make distclean \
 	)
@@ -105,7 +105,7 @@ $(SQLITEBUILD)/libsqlite3.la: $(SQLITETARBALL)
 	( \
 		cd $(SQLITEBUILD); \
 		emconfigure ./configure; \
-		emmake make; \
+		emmake make -j $${PYODIDE_JOBS:-3}; \
 	)
 
 
@@ -114,7 +114,7 @@ $(BZIP2BUILD)/libbz2.a: $(BZIP2TARBALL)
 	tar -C $(ROOT)/build/ -xf $(BZIP2TARBALL)
 	( \
 		cd $(BZIP2BUILD); \
-		emmake make CC=emcc CFLAGS="-Wall -Winline -O2 -fomit-frame-pointer -D_FILE_OFFSET_BITS=64" AR=emar RANLIB=emranlib libbz2.a; \
+		emmake make -j $${PYODIDE_JOBS:-3} CC=emcc CFLAGS="-Wall -Winline -O2 -fomit-frame-pointer -D_FILE_OFFSET_BITS=64" AR=emar RANLIB=emranlib libbz2.a; \
 	)
 
 
@@ -142,7 +142,7 @@ $(BUILD)/$(LIB): $(BUILD)/Makefile $(HOSTPYTHON) $(HOSTPGEN) Setup.local
 	( \
 		cp build/$(PYVERSION)/host/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py build/$(PYVERSION)/Python-$(PYVERSION)/Lib/_sysconfigdata__emscripten_.py; \
 		cd $(BUILD); \
-		emmake make HOSTPYTHON=$(HOSTPYTHON) HOSTPGEN=$(HOSTPGEN) CROSS_COMPILE=yes $(LIB) \
+		emmake make HOSTPYTHON=$(HOSTPYTHON) HOSTPGEN=$(HOSTPGEN) CROSS_COMPILE=yes $(LIB) -j $${PYODIDE_JOBS:-3} \
 	)
 	sed -i -e 's/\-undefined dynamic_lookup//' build/$(PYVERSION)/Python-$(PYVERSION)/Lib/_sysconfigdata__emscripten_.py
 	touch $(BUILD)/$(LIB)

docs/building_from_sources.md

@@ -56,7 +56,7 @@ If running ``make`` deterministically stops at one point in each subsequent try,
 the maximum RAM usage available to the docker container might help [This is different
 from the physical RAM capacity inside the system]. Ideally, at least 3 GB of RAM
 should be available to the docker container to build `pyodide` smoothly. These settings can
-be changed via Docker Preferences [See [here](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container)].
+be changed via Docker Preferences (See [here](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container)).
 
 You can edit the files in your source checkout on your host machine, and then
 repeatedly run `make` inside the Docker environment to test your changes.
@@ -81,3 +81,8 @@ micropip and package is generally always included for any non empty value of
 
 If scipy is included in `PYODIDE_PACKAGES`, BLAS/LAPACK must be manually built
 first with `make -c CLAPACK`.
+
+## Environement variables
+
+Following environment variables additionally impact the build,
+ - `PYODIDE_JOBS`: the `-j` option passed to the `emmake make` command when applicable for parallel compilation. Default: 3.

lz4/Makefile

@@ -32,5 +32,5 @@ $(SRC)/Makefile: $(TARBALL)
 $(SRC)/lib/liblz4.a: $(SRC)/Makefile
 	( \
 		cd $(SRC) ; \
-		emmake make ; \
+		emmake make -j $${PYODIDE_JOBS:-3} ; \
 	)

packages/micropip/test_micropip.py

@@ -68,4 +68,6 @@ def test_install_custom_url(selenium_standalone, web_server_secondary):
     base_url = f'http://{server_hostname}:{server_port}/test/data/'
     url = base_url + 'snowballstemmer-2.0.0-py2.py3-none-any.whl'
     selenium_standalone.run(f"micropip.install('{url}')")
+    # wait untill micropip is loaded
+    time.sleep(1)
     selenium_standalone.run("import snowballstemmer")

pyodide_build/buildpkg.py

@@ -10,6 +10,7 @@ import os
 from pathlib import Path
 import shutil
 import subprocess
+from datetime import datetime
 
 
 from . import common
@@ -193,7 +194,12 @@ def needs_rebuild(pkg, path, buildpath):
 
 def build_package(path, args):
     pkg = common.parse_package(path)
-    packagedir = pkg['package']['name'] + '-' + pkg['package']['version']
+    name = pkg['package']['name']
+    t0 = datetime.now()
+    print("[{}] Building package {}...".format(
+        t0.strftime('%Y-%m-%d %H:%M:%S'), name)
+    )
+    packagedir = name + '-' + pkg['package']['version']
     dirpath = path.parent
     orig_path = Path.cwd()
     os.chdir(dirpath)
@@ -211,6 +217,10 @@ def build_package(path, args):
         package_files(buildpath, srcpath, pkg, args)
     finally:
         os.chdir(orig_path)
+        t1 = datetime.now()
+        print("[{}] done building package {} in {:.1f} s.".format(
+            t1.strftime('%Y-%m-%d %H:%M:%S'), name, (t1 - t0).total_seconds()
+        ))
 
 
 def make_parser(parser):