pyodide/pyodide-build/pyodide_build/create_xbuildenv.py

import argparse
import shutil
import subprocess
from pathlib import Path

from .common import (
    exit_with_stdio,
    get_make_flag,
    get_pyodide_root,
    get_unisolated_packages,
)
from .io import MetaConfig


def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
    parser.description = (
        "Create xbuild env.\n\n"
        "Note: this is a private endpoint that should not be used "
        "outside of the Pyodide Makefile."
    )
    return parser


def copy_xbuild_files(xbuildenv_path: Path) -> None:
    PYODIDE_ROOT = get_pyodide_root()
    site_packages = Path(get_make_flag("HOSTSITEPACKAGES"))
    # Store package cross-build-files into site_packages_extras in the same tree
    # structure as they would appear in the real package.
    # In install_xbuildenv, we will use:
    # pip install -t $HOSTSITEPACKAGES -r requirements.txt
    # cp site-packages-extras $HOSTSITEPACKAGES
    site_packages_extras = xbuildenv_path / "site-packages-extras"
    for pkg in (PYODIDE_ROOT / "packages").glob("**/meta.yaml"):
        config = MetaConfig.from_yaml(pkg)
        xbuild_files = config.build.cross_build_files
        for path in xbuild_files:
            target = site_packages_extras / path
            target.parent.mkdir(parents=True, exist_ok=True)
            shutil.copy(site_packages / path, target)


def get_relative_path(pyodide_root: Path, flag: str) -> Path:
    return Path(get_make_flag(flag)).relative_to(pyodide_root)


def copy_wasm_libs(xbuildenv_path: Path) -> None:
    pyodide_root = get_pyodide_root()
    pythoninclude = get_relative_path(pyodide_root, "PYTHONINCLUDE")
    wasm_lib_dir = get_relative_path(pyodide_root, "WASM_LIBRARY_DIR")
    sysconfig_dir = get_relative_path(pyodide_root, "SYSCONFIGDATA_DIR")
    xbuildenv_root = xbuildenv_path / "pyodide-root"
    xbuildenv_path.mkdir(exist_ok=True)
    to_copy: list[Path] = [
        pythoninclude,
        sysconfig_dir,
        Path("Makefile.envs"),
        wasm_lib_dir / "cmake",
        Path("tools/pyo3_config.ini"),
        Path("tools/python"),
        Path("tools/cmake"),
        Path("dist/repodata.json"),
        Path("dist/pyodide_py.tar"),
    ]
    to_copy.extend(
        x.relative_to(pyodide_root) for x in (pyodide_root / "dist").glob("pyodide.*")
    )
    # Some ad-hoc stuff here to moderate size. We'd like to include all of
    # wasm_lib_dir but there's 180mb of it. Better to leave out all the video
    # codecs and stuff.
    for pkg in ["ssl", "libcrypto", "zlib", "xml", "mpfr", "lapack", "blas", "f2c"]:
        to_copy.extend(
            x.relative_to(pyodide_root)
            for x in (pyodide_root / wasm_lib_dir / "include").glob(f"**/*{pkg}*")
            if "boost" not in str(x)
        )
        to_copy.extend(
            x.relative_to(pyodide_root)
            for x in (pyodide_root / wasm_lib_dir / "lib").glob(f"**/*{pkg}*")
        )

    for path in to_copy:
        if (pyodide_root / path).is_dir():
            shutil.copytree(
                pyodide_root / path, xbuildenv_root / path, dirs_exist_ok=True
            )
        else:
            (xbuildenv_root / path).parent.mkdir(exist_ok=True, parents=True)
            shutil.copy(pyodide_root / path, xbuildenv_root / path)


def main(args: argparse.Namespace) -> None:
    pyodide_root = get_pyodide_root()
    xbuildenv_path = pyodide_root / "xbuildenv"
    xbuildenv_root = xbuildenv_path / "pyodide-root"
    shutil.rmtree(xbuildenv_path, ignore_errors=True)

    copy_xbuild_files(xbuildenv_path)
    copy_wasm_libs(xbuildenv_path)

    (xbuildenv_root / "package.json").write_text("{}")
    res = subprocess.run(
        ["npm", "i", "node-fetch@2"],
        cwd=xbuildenv_root,
        capture_output=True,
        encoding="utf8",
    )
    if res.returncode != 0:
        print("Failed to install node-fetch:")
        exit_with_stdio(res)

    res = subprocess.run(
        ["pip", "freeze", "--path", get_make_flag("HOSTSITEPACKAGES")],
        capture_output=True,
        encoding="utf8",
    )
    if res.returncode != 0:
        print("Failed to run pip freeze:")
        exit_with_stdio(res)

    (xbuildenv_path / "requirements.txt").write_text(res.stdout)
    (xbuildenv_root / "unisolated.txt").write_text("\n".join(get_unisolated_packages()))