BLD Don't compile host python (#830)

.circleci/config.yml

@@ -54,7 +54,7 @@ jobs:
           root: .
           paths:
             - ./build
-            - ./cpython/build/3.8.2/host
+            - ./packages/.artifacts
 
       - store_artifacts:
           path: /home/circleci/repo/build/
@@ -135,7 +135,7 @@ jobs:
       - run:
           name: benchmark
           command: |
-            python benchmark/benchmark.py cpython/build/3.8.2/host/bin/python3 build/benchmarks.json
+            python benchmark/benchmark.py /usr/local/bin/python3 build/benchmarks.json
       - store_artifacts:
           path: /home/circleci/repo/build/benchmarks.json
 

.gitignore

@@ -35,6 +35,7 @@ packages/libxml/libxml*
 packages/libxslt/libxslt*
 packages/libiconv/libiconv*
 packages/zlib/zlib*
+packages/.artifacts
 
 docs/python-api/
 docs/micropip-api/

Makefile.envs

@@ -10,7 +10,7 @@ export BINARYEN_ROOT = $(PYODIDE_ROOT)/emsdk/emsdk/binaryen
 
 export PYVERSION=3.8.2
 export PYMINOR=$(basename $(PYVERSION))
-export HOSTPYTHONROOT=$(PYODIDE_ROOT)/cpython/build/$(PYVERSION)/host
+export HOSTPYTHONROOT=$(shell python -c "import sys; print(sys.prefix)")
 export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python3
 export TARGETPYTHONROOT=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION)
 export PYTHONINCLUDE=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION)/include/python$(PYMINOR)

benchmark/benchmark.py

@@ -23,10 +23,15 @@ def print_entry(name, res):
 
 
 def run_native(hostpython, code):
+    root = Path(__file__).resolve().parents[1]
     output = subprocess.check_output(
         [hostpython.resolve(), "-c", code],
         cwd=Path(__file__).resolve().parent,
-        env={"PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src")},
+        env={
+            "PYTHONPATH": str(root / "src")
+            + ":"
+            + str(root / "packages" / ".artifacts" / "lib" / "python")
+        },
     )
     return float(output.strip().split()[-1])
 

cpython/Makefile

@@ -3,14 +3,7 @@ include ../Makefile.envs
 
 ROOT=$(abspath .)
 
-HOSTINSTALL=$(ROOT)/build/$(PYVERSION)/host
-HOSTBUILD=$(HOSTINSTALL)/Python-$(PYVERSION)
-HOSTPYTHON=$(HOSTINSTALL)/bin/python3$(EXE)
-HOSTPYTHON_CPPFLAGS="-I/usr/local/opt/openssl/include"
-HOSTPYTHON_LDFLAGS="-L/usr/local/opt/openssl/lib"
-HOSTPGEN=$(HOSTINSTALL)/bin/pgen$(EXE)
-
-BUILD=$(ROOT)/build/$(PYVERSION)/Python-$(PYVERSION)
+BUILD=$(ROOT)/build/Python-$(PYVERSION)
 INSTALL=$(ROOT)/installs/python-$(PYVERSION)
 TARBALL=$(ROOT)/downloads/Python-$(PYVERSION).tgz
 URL=https://www.python.org/ftp/python/$(PYVERSION)/Python-$(PYVERSION).tgz
@@ -39,15 +32,14 @@ $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB)
 		cd $(BUILD); \
 		sed -i -e 's/libinstall:.*/libinstall:/' Makefile; \
 		touch $(BUILD)/$(LIB) ; \
-		emmake make HOSTPYTHON=$(HOSTPYTHON) PYTHON_FOR_BUILD=$(HOSTPYTHON) CROSS_COMPILE=yes inclinstall libinstall $(LIB) -j $${PYODIDE_JOBS:-3} && \
+		emmake make PYTHON_FOR_BUILD=$(HOSTPYTHON) CROSS_COMPILE=yes inclinstall libinstall $(LIB) -j $${PYODIDE_JOBS:-3} && \
 		cp $(LIB) $(INSTALL)/lib/ && \
-		cp $(HOSTINSTALL)/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py $(INSTALL)/lib/python$(PYMINOR)/_sysconfigdata__emscripten_.py; \
+		cp $(HOSTPYTHONROOT)/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py $(INSTALL)/lib/python$(PYMINOR)/_sysconfigdata__emscripten_.py; \
 		sed -i -e 's#'"$(PYODIDE_ROOT)"'##g' $(INSTALL)/lib/python$(PYMINOR)/_sysconfigdata__emscripten_.py; \
 	)
 
 
 clean:
-	-rm -fr $(HOSTINSTALL)
 	-rm -fr $(BUILD)
 	-rm -fr $(INSTALL)
 
@@ -73,17 +65,6 @@ $(BZIP2TARBALL):
 	wget -q -O $@ $(BZIP2URL)
 
 
-$(HOSTPYTHON) $(HOSTPGEN): $(TARBALL)
-	mkdir -p $(HOSTINSTALL)
-	[ -d $(HOSTBUILD) ] || tar -C $(HOSTINSTALL) -xf $(TARBALL)
-	( \
-		cd $(HOSTBUILD); \
-		PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig" ./configure --prefix=$(HOSTINSTALL) || cat config.log && \
-	  	make regen-grammar -j $${PYODIDE_JOBS:-3} && \
-		make install -j $${PYODIDE_JOBS:-3} && \
-		make distclean \
-	)
-
 
 $(BUILD)/.patched: $(TARBALL)
 	[ -d $(BUILD) ] || (mkdir -p $(dir $(BUILD)); tar -C $(dir $(BUILD)) -xf $(TARBALL))
@@ -135,13 +116,13 @@ $(BUILD)/Makefile: $(BUILD)/.patched $(ZLIBBUILD)/.patched $(SQLITEBUILD)/libsql
 	)
 
 
-$(BUILD)/$(LIB): $(BUILD)/Makefile $(HOSTPYTHON) $(HOSTPGEN) Setup.local
+$(BUILD)/$(LIB): $(BUILD)/Makefile Setup.local
 	cp Setup.local $(BUILD)/Modules/
 	cat pyconfig.undefs.h >> $(BUILD)/pyconfig.h
 	( \
-		cp build/$(PYVERSION)/host/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py build/$(PYVERSION)/Python-$(PYVERSION)/Lib/_sysconfigdata__emscripten_.py; \
+		cp $(HOSTPYTHONROOT)/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py $(BUILD)/Lib/_sysconfigdata__emscripten_.py; \
 		cd $(BUILD); \
-		emmake make HOSTPYTHON=$(HOSTPYTHON) HOSTPGEN=$(HOSTPGEN) CROSS_COMPILE=yes $(LIB) -j $${PYODIDE_JOBS:-3} \
+		emmake make CROSS_COMPILE=yes $(LIB) -j $${PYODIDE_JOBS:-3} \
 	)
-	sed -i -e 's/\-undefined dynamic_lookup//' build/$(PYVERSION)/Python-$(PYVERSION)/Lib/_sysconfigdata__emscripten_.py
+	sed -i -e 's/\-undefined dynamic_lookup//' $(BUILD)/Lib/_sysconfigdata__emscripten_.py
 	touch $(BUILD)/$(LIB)

cpython/Setup.local

@@ -24,7 +24,7 @@ select selectmodule.c
 _posixsubprocess _posixsubprocess.c
 binascii binascii.c
 
-zlib zlibmodule.c -I../../zlib-1.2.11
+zlib zlibmodule.c -I../zlib-1.2.11
 
 pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI -DXML_POOR_ENTROPY
 
@@ -33,7 +33,7 @@ _sha256 sha256module.c
 _sha512 sha512module.c
 _sha3 _sha3/sha3module.c
 _md5 md5module.c
-_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c ../../host/Python-3.8.2/Modules/_blake2/blake2s_impl.c
+_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
 
 _sqlite3 _sqlite/cache.c _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c -I$(SQLITEBUILD) -L$(SQLITEBUILD) -lsqlite3
 _crypt _cryptmodule.c

packages/Makefile

@@ -1,9 +1,11 @@
 PYODIDE_ROOT=$(abspath ..)
+PYODIDE_LIBRARIES=$(abspath ./.artifacts)
 include ../Makefile.envs
 
 all: deps
-	../bin/pyodide buildall . ../build \
-		--package_abi=$(PYODIDE_PACKAGE_ABI) --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) --only $(PYODIDE_PACKAGES)
+	mkdir -p $(PYODIDE_LIBRARIES)
+	PYTHONPATH=$(PYODIDE_LIBRARIES)/lib/python ../bin/pyodide buildall . ../build \
+		--package_abi=$(PYODIDE_PACKAGE_ABI) --target=$(TARGETPYTHONROOT) --only $(PYODIDE_PACKAGES) --install-dir $(PYODIDE_LIBRARIES)
 
 update-all:
 	for pkg in $$(find . -maxdepth 1 -type d -exec basename {} \; | tail -n +2); do \
@@ -16,3 +18,4 @@ deps:
 
 clean:
 	rm -rf ./*/build
+	rm -rf ./.artifacts

pyodide_build/common.py

@@ -1,9 +1,9 @@
 from pathlib import Path
 from typing import Optional, Set
-
+import sys
 
 ROOTDIR = Path(__file__).parents[1].resolve() / "tools"
-HOSTPYTHON = ROOTDIR / ".." / "cpython" / "build" / "3.8.2" / "host"
+HOSTPYTHON = sys.prefix
 TARGETPYTHON = ROOTDIR / ".." / "cpython" / "installs" / "python-3.8.2"
 DEFAULTCFLAGS = ""
 # fmt: off

pyodide_build/pywasmcross.py

@@ -197,8 +197,8 @@ def handle_command(line, args, dryrun=False):
     --------
 
     >>> from collections import namedtuple
-    >>> Args = namedtuple('args', ['cflags', 'ldflags'])
-    >>> args = Args(cflags='', ldflags='')
+    >>> Args = namedtuple('args', ['cflags', 'ldflags', 'host'])
+    >>> args = Args(cflags='', ldflags='', host='')
     >>> handle_command(['gcc', 'test.c'], args, dryrun=True)
     emcc test.c
     ['emcc', 'test.c']
@@ -239,11 +239,12 @@ def handle_command(line, args, dryrun=False):
 
     lapack_dir = None
 
+    host_dir = str(Path(args.host).resolve())
     # Go through and adjust arguments
     for arg in line[1:]:
         if arg.startswith("-I"):
             if (
-                str(Path(arg[2:]).resolve()).startswith(args.host)
+                str(Path(arg[2:]).resolve()).startswith(host_dir + "/include/python")
                 and "site-packages" not in arg
             ):
                 arg = arg.replace("-I" + args.host, "-I" + args.target)

pyodide_build/tests/test_pywasmcross.py

@@ -31,8 +31,8 @@ f2c_wrap = _args_wrapper(f2c)
 
 
 def test_handle_command():
-    Args = namedtuple("args", ["cflags", "ldflags"])
-    args = Args(cflags="", ldflags="")
+    Args = namedtuple("args", ["cflags", "ldflags", "host"])
+    args = Args(cflags="", ldflags="", host="")
     assert handle_command_wrap("gcc -print-multiarch", args) is None
     assert handle_command_wrap("gcc test.c", args) == "emcc test.c"
     assert (
@@ -41,7 +41,7 @@ def test_handle_command():
     )
 
     # check ldflags injection
-    args = Args(cflags="", ldflags="-lm")
+    args = Args(cflags="", ldflags="-lm", host="")
     assert (
         handle_command_wrap("gcc -shared -c test.o -o test.so", args)
         == "emcc -lm -shared -c test.bc -o test.wasm"

tools/dependency-check.sh

@@ -5,6 +5,12 @@ failure_exit() {
   exit 1
 }
 
+check_python_version() {
+  if ! command -v python3.8 &> /dev/null; then
+    echo >&2 "Must compile with python 3.8."
+    exit 1
+  fi
+}
 check_python_headers() {
   local python_headers_present
   python_headers_present="$(pkg-config --libs python-3.8)"
@@ -44,6 +50,7 @@ check_pyyaml() {
   fi
 }
 
+check_python_version
 check_pkgconfig
 #check_python_headers
 check_fortran_dependencies