More work on create_xbuildenv to allow building numpy (#2811)

This is more WIP on create_xbuildenv. I am including some of the WASM_LIB_DIR
but not all of it to keep size in check. The current cross build environment that we
would upload/download is 20mb.

To use this, we need an extra CLI entrypoint which I am working on.

Makefile.envs

@@ -23,7 +23,9 @@ export HOSTPYTHONROOT=$(shell python${PYMAJOR}.${PYMINOR} -c "import sys; print(
 export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python$(PYMAJOR).$(PYMINOR)
 
 export CPYTHONROOT=$(PYODIDE_ROOT)/cpython
-export CPYTHONLIB=$(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/python$(PYMAJOR).$(PYMINOR)
+export CPYTHONINSTALL=$(CPYTHONROOT)/installs/python-$(PYVERSION)
+export CPYTHONLIB=$(CPYTHONINSTALL)/lib/python$(PYMAJOR).$(PYMINOR)
+export SYSCONFIGDATA_DIR=$(CPYTHONINSTALL)/sysconfigdata/
 export CPYTHONBUILD=$(CPYTHONROOT)/build/Python-$(PYVERSION)/
 
 export TARGETINSTALLDIR=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION)
@@ -132,7 +134,7 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \
 
 export SIDE_MODULE_CXXFLAGS = $(CXXFLAGS_BASE)
 
-export SIDE_MODULE_CFLAGS= $(CFLAGS_BASE)
+export SIDE_MODULE_CFLAGS= $(CFLAGS_BASE) -I$(PYTHONINCLUDE)
 export MAIN_MODULE_CFLAGS= $(CFLAGS_BASE) \
 	-Wall \
 	-Wno-warn-absolute-paths \

cpython/Makefile

@@ -5,8 +5,8 @@ ROOT=$(abspath .)
 
 PYTHON_CFLAGS=$(CFLAGS_BASE)
 
-BUILD=$(ROOT)/build/Python-$(PYVERSION)
-INSTALL=$(ROOT)/installs/python-$(PYVERSION)
+BUILD=$(CPYTHONROOT)/build/Python-$(PYVERSION)
+INSTALL=$(CPYTHONINSTALL)
 TARBALL=$(ROOT)/downloads/Python-$(PYVERSION).tgz
 URL=https://www.python.org/ftp/python/$(PYVERSION)/Python-$(PYVERSION).tgz
 LIB=libpython$(PYMAJOR).$(PYMINOR).a
@@ -46,8 +46,8 @@ $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB) remove_modules.txt
 	$(eval PYBUILDDIR=`cat pybuilddir.txt`)
 	PYTHONPATH=$(PYBUILDDIR) python adjust_sysconfig.py
 	cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/
-	mkdir -p $(INSTALL)/sysconfigdata/
-	cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(INSTALL)/sysconfigdata/
+	mkdir -p $(SYSCONFIGDATA_DIR)
+	cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(SYSCONFIGDATA_DIR)
 	cd $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/ && rm -rf `cat $(ROOT)/remove_modules.txt`
 	rm -rf $(PYBUILDDIR)
 	rm pybuilddir.txt

pyodide-build/pyodide_build/common.py

@@ -223,7 +223,9 @@ def init_environment() -> None:
     if "sphinx" in sys.modules:
         os.environ["PYODIDE_ROOT"] = ""
 
-    if "PYODIDE_ROOT" not in os.environ:
+    if "PYODIDE_ROOT" in os.environ:
+        os.environ["PYODIDE_ROOT"] = str(Path(os.environ["PYODIDE_ROOT"]).resolve())
+    else:
         os.environ["PYODIDE_ROOT"] = str(search_pyodide_root(os.getcwd()))
 
     os.environ.update(get_make_environment_vars())

pyodide-build/pyodide_build/create_xbuildenv.py

@@ -19,23 +19,71 @@ def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
 def copy_xbuild_files(xbuildenv_path: Path) -> None:
     PYODIDE_ROOT = get_pyodide_root()
     site_packages = Path(get_make_flag("HOSTSITEPACKAGES"))
-    xbuild_site_packages = xbuildenv_path / "site-packages"
+    # Store package cross-build-files into site_packages_extras in the same tree
+    # structure as they would appear in the real package.
+    # In install_xbuildenv, we will use:
+    # pip install -t $HOSTSITEPACKAGES -r requirements.txt
+    # cp site-packages-extras $HOSTSITEPACKAGES
+    site_packages_extras = xbuildenv_path / "site-packages-extras"
     for pkg in (PYODIDE_ROOT / "packages").glob("**/meta.yaml"):
         config = parse_package_config(pkg, check=False)
         xbuild_files = config.get("build", {}).get("cross-build-files", [])
         for path in xbuild_files:
-            target = xbuild_site_packages / path
+            target = site_packages_extras / path
             target.parent.mkdir(parents=True, exist_ok=True)
             shutil.copy(site_packages / path, target)
 
 
+def get_relative_path(pyodide_root: Path, flag: str) -> Path:
+    return Path(get_make_flag(flag)).relative_to(pyodide_root)
+
+
+def copy_wasm_libs(xbuildenv_path: Path) -> None:
+    pyodide_root = get_pyodide_root()
+    pythoninclude = get_relative_path(pyodide_root, "PYTHONINCLUDE")
+    wasm_lib_dir = get_relative_path(pyodide_root, "WASM_LIBRARY_DIR")
+    sysconfig_dir = get_relative_path(pyodide_root, "SYSCONFIGDATA_DIR")
+    xbuildenv_root = xbuildenv_path / "pyodide-root"
+    xbuildenv_path.mkdir()
+    to_copy: list[Path] = [
+        pythoninclude,
+        sysconfig_dir,
+        Path("Makefile.envs"),
+        wasm_lib_dir / "CLAPACK",
+        wasm_lib_dir / "cmake",
+        Path("tools/pyo3_config.ini"),
+    ]
+    # Some ad-hoc stuff here to moderate size. We'd like to include all of
+    # wasm_lib_dir but there's 180mb of it. Better to leave out all the video
+    # codecs and stuff.
+    for pkg in ["ssl", "libcrypto", "zlib", "xml", "mpfr"]:
+        to_copy.extend(
+            x.relative_to(pyodide_root)
+            for x in (pyodide_root / wasm_lib_dir / "include").glob(f"**/*{pkg}*")
+            if "boost" not in str(x)
+        )
+        to_copy.extend(
+            x.relative_to(pyodide_root)
+            for x in (pyodide_root / wasm_lib_dir / "lib").glob(f"**/*{pkg}*")
+        )
+
+    for path in to_copy:
+        if (pyodide_root / path).is_dir():
+            shutil.copytree(
+                pyodide_root / path, xbuildenv_root / path, dirs_exist_ok=True
+            )
+        else:
+            (xbuildenv_root / path).parent.mkdir(exist_ok=True, parents=True)
+            shutil.copy(pyodide_root / path, xbuildenv_root / path)
+
+
 def main(args: argparse.Namespace) -> None:
     pyodide_root = get_pyodide_root()
     xbuildenv_path = pyodide_root / "xbuildenv"
     shutil.rmtree(xbuildenv_path, ignore_errors=True)
-    xbuildenv_path.mkdir()
-    shutil.copytree(get_make_flag("PYTHONINCLUDE"), xbuildenv_path / "python-include")
+
     copy_xbuild_files(xbuildenv_path)
+    copy_wasm_libs(xbuildenv_path)
     res = subprocess.run(
         ["pip", "freeze", "--path", get_make_flag("HOSTSITEPACKAGES")],
         stdout=subprocess.PIPE,

pyodide-build/pyodide_build/install_xbuildenv.py

@@ -15,16 +15,17 @@ def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
         "on numpy or scipy.\n"
         "Note: this is a private endpoint that should not be used outside of the Pyodide Makefile."
     )
+    parser.add_argument("xbuild_env", type=str, nargs=1)
     return parser
 
 
 def main(args: argparse.Namespace) -> None:
+    xbuildenv_path = Path(args.xbuild_env[0])
     pyodide_root = get_pyodide_root()
-    host_site_packages = Path(get_make_flag("HOSTSITEPACKAGES"))
-    xbuildenv_path = pyodide_root / "xbuildenv"
-    include_path = Path(get_make_flag("PYTHONINCLUDE"))
-    include_path.mkdir(exist_ok=True, parents=True)
-    shutil.copytree(xbuildenv_path / "python-include", include_path, dirs_exist_ok=True)
+    xbuildenv_root = xbuildenv_path / "pyodide-root"
+    host_site_packages = xbuildenv_root / Path(
+        get_make_flag("HOSTSITEPACKAGES")
+    ).relative_to(pyodide_root)
     host_site_packages.mkdir(exist_ok=True, parents=True)
     subprocess.run(
         [
@@ -36,6 +37,8 @@ def main(args: argparse.Namespace) -> None:
             xbuildenv_path / "requirements.txt",
         ]
     )
+    # Copy the site-packages-extras (coming from the cross-build-files meta.yaml
+    # key) over the site-packages directory with the newly installed packages.
     shutil.copytree(
-        xbuildenv_path / "site-packages", host_site_packages, dirs_exist_ok=True
+        xbuildenv_path / "site-packages-extras", host_site_packages, dirs_exist_ok=True
     )