diff --git a/pyodide-build/pyodide_build/common.py b/pyodide-build/pyodide_build/common.py index 5965a99dd..776bfe362 100644 --- a/pyodide-build/pyodide_build/common.py +++ b/pyodide-build/pyodide_build/common.py @@ -229,11 +229,14 @@ def init_environment() -> None: os.environ["PYODIDE_ROOT"] = str(search_pyodide_root(os.getcwd())) os.environ.update(get_make_environment_vars()) - hostsitepackages = get_hostsitepackages() - pythonpath = [ - hostsitepackages, - ] - os.environ["PYTHONPATH"] = ":".join(pythonpath) + try: + hostsitepackages = get_hostsitepackages() + pythonpath = [ + hostsitepackages, + ] + os.environ["PYTHONPATH"] = ":".join(pythonpath) + except KeyError: + pass os.environ["BASH_ENV"] = "" get_unisolated_packages() @@ -251,13 +254,18 @@ def get_unisolated_packages() -> list[str]: if "UNISOLATED_PACKAGES" in os.environ: return json.loads(os.environ["UNISOLATED_PACKAGES"]) PYODIDE_ROOT = get_pyodide_root() - unisolated_packages = [] - for pkg in (PYODIDE_ROOT / "packages").glob("**/meta.yaml"): - config = parse_package_config(pkg, check=False) - if config.get("build", {}).get("cross-build-env", False): - unisolated_packages.append(config["package"]["name"]) - # TODO: remove setuptools_rust from this when they release the next version. - unisolated_packages.append("setuptools_rust") + unisolated_file = PYODIDE_ROOT / "unisolated.txt" + if unisolated_file.exists(): + # in xbuild env, read from file + unisolated_packages = unisolated_file.read_text().splitlines() + else: + unisolated_packages = [] + for pkg in (PYODIDE_ROOT / "packages").glob("**/meta.yaml"): + config = parse_package_config(pkg, check=False) + if config.get("build", {}).get("cross-build-env", False): + unisolated_packages.append(config["package"]["name"]) + # TODO: remove setuptools_rust from this when they release the next version. + unisolated_packages.append("setuptools_rust") os.environ["UNISOLATED_PACKAGES"] = json.dumps(unisolated_packages) return unisolated_packages diff --git a/pyodide-build/pyodide_build/create_xbuildenv.py b/pyodide-build/pyodide_build/create_xbuildenv.py index ca36d0877..54eb7afe3 100644 --- a/pyodide-build/pyodide_build/create_xbuildenv.py +++ b/pyodide-build/pyodide_build/create_xbuildenv.py @@ -3,7 +3,7 @@ import shutil import subprocess from pathlib import Path -from .common import get_make_flag, get_pyodide_root +from .common import get_make_flag, get_pyodide_root, get_unisolated_packages from .io import parse_package_config @@ -44,7 +44,7 @@ def copy_wasm_libs(xbuildenv_path: Path) -> None: wasm_lib_dir = get_relative_path(pyodide_root, "WASM_LIBRARY_DIR") sysconfig_dir = get_relative_path(pyodide_root, "SYSCONFIGDATA_DIR") xbuildenv_root = xbuildenv_path / "pyodide-root" - xbuildenv_path.mkdir() + xbuildenv_path.mkdir(exist_ok=True) to_copy: list[Path] = [ pythoninclude, sysconfig_dir, @@ -89,3 +89,6 @@ def main(args: argparse.Namespace) -> None: stdout=subprocess.PIPE, ) (xbuildenv_path / "requirements.txt").write_bytes(res.stdout) + (xbuildenv_path / "pyodide-root/unisolated.txt").write_text( + "\n".join(get_unisolated_packages()) + ) diff --git a/pyodide-build/pyodide_build/install_xbuildenv.py b/pyodide-build/pyodide_build/install_xbuildenv.py index ffbb8eece..aa216ae9f 100644 --- a/pyodide-build/pyodide_build/install_xbuildenv.py +++ b/pyodide-build/pyodide_build/install_xbuildenv.py @@ -15,12 +15,27 @@ def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: "on numpy or scipy.\n" "Note: this is a private endpoint that should not be used outside of the Pyodide Makefile." ) + parser.add_argument("--download", action="store_true", help="Download xbuild env") parser.add_argument("xbuild_env", type=str, nargs=1) return parser -def main(args: argparse.Namespace) -> None: - xbuildenv_path = Path(args.xbuild_env[0]) +def download_xbuild_env(version: str, xbuildenv_path: Path) -> None: + from shutil import rmtree, unpack_archive + from tempfile import NamedTemporaryFile + from urllib.request import urlretrieve + + rmtree(xbuildenv_path, ignore_errors=True) + with NamedTemporaryFile(suffix=".tar") as f: + urlretrieve( + f"http://pyodide-cache.s3-website-us-east-1.amazonaws.com/xbuildenv/{version}.tar", + f.name, + ) + unpack_archive(f.name, xbuildenv_path) + + +def install_xbuild_env(xbuildenv_path: Path) -> None: + xbuildenv_path = xbuildenv_path / "xbuildenv" pyodide_root = get_pyodide_root() xbuildenv_root = xbuildenv_path / "pyodide-root" host_site_packages = xbuildenv_root / Path( @@ -42,3 +57,11 @@ def main(args: argparse.Namespace) -> None: shutil.copytree( xbuildenv_path / "site-packages-extras", host_site_packages, dirs_exist_ok=True ) + + +def main(args: argparse.Namespace) -> None: + xbuildenv_path = Path(args.xbuild_env[0]) + version = "2" + if args.download: + download_xbuild_env(version, xbuildenv_path) + install_xbuild_env(xbuildenv_path) diff --git a/pyodide-build/pyodide_build/out_of_tree/__init__.py b/pyodide-build/pyodide_build/out_of_tree/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyodide-build/pyodide_build/out_of_tree/__main__.py b/pyodide-build/pyodide_build/out_of_tree/__main__.py new file mode 100644 index 000000000..325a7a56f --- /dev/null +++ b/pyodide-build/pyodide_build/out_of_tree/__main__.py @@ -0,0 +1,36 @@ +import argparse +import os +from pathlib import Path + +from . import build + + +def ensure_env_installed(env: Path) -> None: + if env.exists(): + return + from ..install_xbuildenv import download_xbuild_env, install_xbuild_env + + version = "2" + download_xbuild_env(version, env) + install_xbuild_env(env) + + +def main(): + main_parser = argparse.ArgumentParser(prog="pywasmbuild") + main_parser.description = "Tools for creating Python extension modules for the wasm32-unknown-emscripten platform" + subparsers = main_parser.add_subparsers(help="action") + for module in [build]: + modname = module.__name__.rpartition(".")[-1] + parser = module.make_parser(subparsers.add_parser(modname)) + parser.set_defaults(func=module.main) + + env = Path(".pyodide-xbuildenv") + os.environ["PYODIDE_ROOT"] = str(env / "xbuildenv/pyodide-root") + ensure_env_installed(env) + + args = main_parser.parse_args() + if hasattr(args, "func"): + # run the selected action + args.func(args) + else: + main_parser.print_help() diff --git a/pyodide-build/pyodide_build/out_of_tree/build.py b/pyodide-build/pyodide_build/out_of_tree/build.py new file mode 100644 index 000000000..85cc46722 --- /dev/null +++ b/pyodide-build/pyodide_build/out_of_tree/build.py @@ -0,0 +1,47 @@ +import argparse +import os + +from .. import common, pypabuild, pywasmcross + + +def run(exports, args): + cflags = common.get_make_flag("SIDE_MODULE_CFLAGS") + cflags += f" {os.environ.get('CFLAGS', '')}" + cxxflags = common.get_make_flag("SIDE_MODULE_CXXFLAGS") + cxxflags += f" {os.environ.get('CXXFLAGS', '')}" + ldflags = common.get_make_flag("SIDE_MODULE_LDFLAGS") + ldflags += f" {os.environ.get('LDFLAGS', '')}" + build_env_ctx = pywasmcross.get_build_env( + env=os.environ.copy(), + pkgname="", + cflags=cflags, + cxxflags=cxxflags, + ldflags=ldflags, + target_install_dir="", + exports=exports, + ) + + with build_env_ctx as env: + pypabuild.build(env, " ".join(args)) + + +def main(parser_args: argparse.Namespace) -> None: + run(parser_args.exports, parser_args.backend_args) + + +def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + parser.description = "Use pypa/build to build a Python package." + parser.add_argument( + "--exports", + choices=["pyinit", "requested", "whole_archive"], + default="requested", + help="Which symbols should be exported when linking .so files?", + ) + parser.add_argument( + "backend_args", + metavar="args", + type=str, + nargs=argparse.REMAINDER, + help="Arguments to pass on to the backend", + ) + return parser diff --git a/pyodide-build/setup.py b/pyodide-build/setup.py index 52a7cac56..6d6c0fd67 100644 --- a/pyodide-build/setup.py +++ b/pyodide-build/setup.py @@ -5,6 +5,7 @@ if __name__ == "__main__": setuptools.setup( entry_points={ "console_scripts": [ + "pywasm = pyodide_build.out_of_tree.__main__:main", "_pywasmcross = pyodide_build.pywasmcross:compiler_main", ] }