diff --git a/Makefile.envs b/Makefile.envs index 447553458..67edb9eca 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -23,7 +23,9 @@ export HOSTPYTHONROOT=$(shell python${PYMAJOR}.${PYMINOR} -c "import sys; print( export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python$(PYMAJOR).$(PYMINOR) export CPYTHONROOT=$(PYODIDE_ROOT)/cpython -export CPYTHONLIB=$(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/python$(PYMAJOR).$(PYMINOR) +export CPYTHONINSTALL=$(CPYTHONROOT)/installs/python-$(PYVERSION) +export CPYTHONLIB=$(CPYTHONINSTALL)/lib/python$(PYMAJOR).$(PYMINOR) +export SYSCONFIGDATA_DIR=$(CPYTHONINSTALL)/sysconfigdata/ export CPYTHONBUILD=$(CPYTHONROOT)/build/Python-$(PYVERSION)/ export TARGETINSTALLDIR=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION) @@ -132,7 +134,7 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \ export SIDE_MODULE_CXXFLAGS = $(CXXFLAGS_BASE) -export SIDE_MODULE_CFLAGS= $(CFLAGS_BASE) +export SIDE_MODULE_CFLAGS= $(CFLAGS_BASE) -I$(PYTHONINCLUDE) export MAIN_MODULE_CFLAGS= $(CFLAGS_BASE) \ -Wall \ -Wno-warn-absolute-paths \ diff --git a/cpython/Makefile b/cpython/Makefile index 4184da524..37e772c7c 100644 --- a/cpython/Makefile +++ b/cpython/Makefile @@ -5,8 +5,8 @@ ROOT=$(abspath .) PYTHON_CFLAGS=$(CFLAGS_BASE) -BUILD=$(ROOT)/build/Python-$(PYVERSION) -INSTALL=$(ROOT)/installs/python-$(PYVERSION) +BUILD=$(CPYTHONROOT)/build/Python-$(PYVERSION) +INSTALL=$(CPYTHONINSTALL) TARBALL=$(ROOT)/downloads/Python-$(PYVERSION).tgz URL=https://www.python.org/ftp/python/$(PYVERSION)/Python-$(PYVERSION).tgz LIB=libpython$(PYMAJOR).$(PYMINOR).a @@ -46,8 +46,8 @@ $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB) remove_modules.txt $(eval PYBUILDDIR=`cat pybuilddir.txt`) PYTHONPATH=$(PYBUILDDIR) python adjust_sysconfig.py cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/ - mkdir -p $(INSTALL)/sysconfigdata/ - cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(INSTALL)/sysconfigdata/ + mkdir -p $(SYSCONFIGDATA_DIR) + cp $(PYBUILDDIR)/$(SYSCONFIG_NAME).py $(SYSCONFIGDATA_DIR) cd $(INSTALL)/lib/python$(PYMAJOR).$(PYMINOR)/ && rm -rf `cat $(ROOT)/remove_modules.txt` rm -rf $(PYBUILDDIR) rm pybuilddir.txt diff --git a/pyodide-build/pyodide_build/common.py b/pyodide-build/pyodide_build/common.py index e17ea34ae..5965a99dd 100644 --- a/pyodide-build/pyodide_build/common.py +++ b/pyodide-build/pyodide_build/common.py @@ -223,7 +223,9 @@ def init_environment() -> None: if "sphinx" in sys.modules: os.environ["PYODIDE_ROOT"] = "" - if "PYODIDE_ROOT" not in os.environ: + if "PYODIDE_ROOT" in os.environ: + os.environ["PYODIDE_ROOT"] = str(Path(os.environ["PYODIDE_ROOT"]).resolve()) + else: os.environ["PYODIDE_ROOT"] = str(search_pyodide_root(os.getcwd())) os.environ.update(get_make_environment_vars()) diff --git a/pyodide-build/pyodide_build/create_xbuildenv.py b/pyodide-build/pyodide_build/create_xbuildenv.py index 0cb6ea564..ca36d0877 100644 --- a/pyodide-build/pyodide_build/create_xbuildenv.py +++ b/pyodide-build/pyodide_build/create_xbuildenv.py @@ -19,23 +19,71 @@ def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: def copy_xbuild_files(xbuildenv_path: Path) -> None: PYODIDE_ROOT = get_pyodide_root() site_packages = Path(get_make_flag("HOSTSITEPACKAGES")) - xbuild_site_packages = xbuildenv_path / "site-packages" + # Store package cross-build-files into site_packages_extras in the same tree + # structure as they would appear in the real package. + # In install_xbuildenv, we will use: + # pip install -t $HOSTSITEPACKAGES -r requirements.txt + # cp site-packages-extras $HOSTSITEPACKAGES + site_packages_extras = xbuildenv_path / "site-packages-extras" for pkg in (PYODIDE_ROOT / "packages").glob("**/meta.yaml"): config = parse_package_config(pkg, check=False) xbuild_files = config.get("build", {}).get("cross-build-files", []) for path in xbuild_files: - target = xbuild_site_packages / path + target = site_packages_extras / path target.parent.mkdir(parents=True, exist_ok=True) shutil.copy(site_packages / path, target) +def get_relative_path(pyodide_root: Path, flag: str) -> Path: + return Path(get_make_flag(flag)).relative_to(pyodide_root) + + +def copy_wasm_libs(xbuildenv_path: Path) -> None: + pyodide_root = get_pyodide_root() + pythoninclude = get_relative_path(pyodide_root, "PYTHONINCLUDE") + wasm_lib_dir = get_relative_path(pyodide_root, "WASM_LIBRARY_DIR") + sysconfig_dir = get_relative_path(pyodide_root, "SYSCONFIGDATA_DIR") + xbuildenv_root = xbuildenv_path / "pyodide-root" + xbuildenv_path.mkdir() + to_copy: list[Path] = [ + pythoninclude, + sysconfig_dir, + Path("Makefile.envs"), + wasm_lib_dir / "CLAPACK", + wasm_lib_dir / "cmake", + Path("tools/pyo3_config.ini"), + ] + # Some ad-hoc stuff here to moderate size. We'd like to include all of + # wasm_lib_dir but there's 180mb of it. Better to leave out all the video + # codecs and stuff. + for pkg in ["ssl", "libcrypto", "zlib", "xml", "mpfr"]: + to_copy.extend( + x.relative_to(pyodide_root) + for x in (pyodide_root / wasm_lib_dir / "include").glob(f"**/*{pkg}*") + if "boost" not in str(x) + ) + to_copy.extend( + x.relative_to(pyodide_root) + for x in (pyodide_root / wasm_lib_dir / "lib").glob(f"**/*{pkg}*") + ) + + for path in to_copy: + if (pyodide_root / path).is_dir(): + shutil.copytree( + pyodide_root / path, xbuildenv_root / path, dirs_exist_ok=True + ) + else: + (xbuildenv_root / path).parent.mkdir(exist_ok=True, parents=True) + shutil.copy(pyodide_root / path, xbuildenv_root / path) + + def main(args: argparse.Namespace) -> None: pyodide_root = get_pyodide_root() xbuildenv_path = pyodide_root / "xbuildenv" shutil.rmtree(xbuildenv_path, ignore_errors=True) - xbuildenv_path.mkdir() - shutil.copytree(get_make_flag("PYTHONINCLUDE"), xbuildenv_path / "python-include") + copy_xbuild_files(xbuildenv_path) + copy_wasm_libs(xbuildenv_path) res = subprocess.run( ["pip", "freeze", "--path", get_make_flag("HOSTSITEPACKAGES")], stdout=subprocess.PIPE, diff --git a/pyodide-build/pyodide_build/install_xbuildenv.py b/pyodide-build/pyodide_build/install_xbuildenv.py index 11479c7ea..ffbb8eece 100644 --- a/pyodide-build/pyodide_build/install_xbuildenv.py +++ b/pyodide-build/pyodide_build/install_xbuildenv.py @@ -15,16 +15,17 @@ def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: "on numpy or scipy.\n" "Note: this is a private endpoint that should not be used outside of the Pyodide Makefile." ) + parser.add_argument("xbuild_env", type=str, nargs=1) return parser def main(args: argparse.Namespace) -> None: + xbuildenv_path = Path(args.xbuild_env[0]) pyodide_root = get_pyodide_root() - host_site_packages = Path(get_make_flag("HOSTSITEPACKAGES")) - xbuildenv_path = pyodide_root / "xbuildenv" - include_path = Path(get_make_flag("PYTHONINCLUDE")) - include_path.mkdir(exist_ok=True, parents=True) - shutil.copytree(xbuildenv_path / "python-include", include_path, dirs_exist_ok=True) + xbuildenv_root = xbuildenv_path / "pyodide-root" + host_site_packages = xbuildenv_root / Path( + get_make_flag("HOSTSITEPACKAGES") + ).relative_to(pyodide_root) host_site_packages.mkdir(exist_ok=True, parents=True) subprocess.run( [ @@ -36,6 +37,8 @@ def main(args: argparse.Namespace) -> None: xbuildenv_path / "requirements.txt", ] ) + # Copy the site-packages-extras (coming from the cross-build-files meta.yaml + # key) over the site-packages directory with the newly installed packages. shutil.copytree( - xbuildenv_path / "site-packages", host_site_packages, dirs_exist_ok=True + xbuildenv_path / "site-packages-extras", host_site_packages, dirs_exist_ok=True )