#!/usr/bin/env python3 """ Builds a Pyodide package. """ import argparse import cgi import hashlib import os from pathlib import Path import shutil import subprocess from urllib import request from datetime import datetime from . import common def check_checksum(path, pkg): """ Checks that a tarball matches the checksum in the package metadata. """ checksum_keys = {"md5", "sha256"}.intersection(pkg["source"]) if not checksum_keys: return elif len(checksum_keys) != 1: raise ValueError( "Only one checksum should be included in a package " "setup; found {}.".format(checksum_keys) ) checksum_algorithm = checksum_keys.pop() checksum = pkg["source"][checksum_algorithm] CHUNK_SIZE = 1 << 16 h = getattr(hashlib, checksum_algorithm)() with open(path, "rb") as fd: while True: chunk = fd.read(CHUNK_SIZE) h.update(chunk) if len(chunk) < CHUNK_SIZE: break if h.hexdigest() != checksum: raise ValueError("Invalid {} checksum".format(checksum_algorithm)) def download_and_extract(buildpath, packagedir, pkg, args): srcpath = buildpath / packagedir if "source" not in pkg: return srcpath if "url" in pkg["source"]: response = request.urlopen(pkg["source"]["url"]) _, parameters = cgi.parse_header( response.headers.get("Content-Disposition", "") ) if "filename" in parameters: tarballname = parameters["filename"] else: tarballname = Path(response.geturl()).name tarballpath = buildpath / tarballname if not tarballpath.is_file(): try: os.makedirs(os.path.dirname(tarballpath), exist_ok=True) with open(tarballpath, "wb") as f: f.write(response.read()) check_checksum(tarballpath, pkg) except Exception: tarballpath.unlink() raise if not srcpath.is_dir(): shutil.unpack_archive(str(tarballpath), str(buildpath)) for extension in [ ".tar.gz", ".tgz", ".tar", ".tar.bz2", ".tbz2", ".tar.xz", ".txz", ".zip", ]: if tarballname.endswith(extension): tarballname = tarballname[: -len(extension)] break return buildpath / tarballname elif "path" in pkg["source"]: srcdir = Path(pkg["source"]["path"]) if not srcdir.is_dir(): raise ValueError("'path' must point to a path") if not srcpath.is_dir(): shutil.copytree(srcdir, srcpath) return srcpath else: raise ValueError("Incorrect source provided") def patch(path, srcpath, pkg, args): if (srcpath / ".patched").is_file(): return # Apply all of the patches orig_dir = Path.cwd() pkgdir = path.parent.resolve() os.chdir(srcpath) try: for patch in pkg.get("source", {}).get("patches", []): subprocess.run( ["patch", "-p1", "--binary", "-i", pkgdir / patch], check=True ) finally: os.chdir(orig_dir) # Add any extra files for src, dst in pkg.get("source", {}).get("extras", []): shutil.copyfile(pkgdir / src, srcpath / dst) with open(srcpath / ".patched", "wb") as fd: fd.write(b"\n") def compile(path, srcpath, pkg, args): if (srcpath / ".built").is_file(): return orig_dir = Path.cwd() os.chdir(srcpath) env = dict(os.environ) if pkg.get("build", {}).get("skip_host", True): env["SKIP_HOST"] = "" try: subprocess.run( [ str(Path(args.host) / "bin" / "python3"), "-m", "pyodide_build", "pywasmcross", "--cflags", args.cflags + " " + pkg.get("build", {}).get("cflags", ""), "--ldflags", args.ldflags + " " + pkg.get("build", {}).get("ldflags", ""), "--host", args.host, "--target", args.target, ], env=env, check=True, ) finally: os.chdir(orig_dir) post = pkg.get("build", {}).get("post") if post is not None: site_packages_dir = srcpath / "install" / "lib" / "python3.8" / "site-packages" pkgdir = path.parent.resolve() env = {"SITEPACKAGES": site_packages_dir, "PKGDIR": pkgdir} subprocess.run(["bash", "-c", post], env=env, check=True) with open(srcpath / ".built", "wb") as fd: fd.write(b"\n") def package_files(buildpath, srcpath, pkg, args): if (buildpath / ".packaged").is_file(): return name = pkg["package"]["name"] install_prefix = (srcpath / "install").resolve() subprocess.run( [ "python", common.ROOTDIR / "file_packager.py", name + ".data", "--abi={0}".format(args.package_abi), "--lz4", "--preload", "{}@/".format(install_prefix), "--js-output={}".format(name + ".js"), "--export-name=pyodide._module", "--exclude", "*.wasm.pre", "--exclude", "*__pycache__*", "--use-preload-plugins", ], cwd=buildpath, check=True, ) subprocess.run( ["uglifyjs", buildpath / (name + ".js"), "-o", buildpath / (name + ".js")], check=True, ) with open(buildpath / ".packaged", "wb") as fd: fd.write(b"\n") def needs_rebuild(pkg, path, buildpath): """ Determines if a package needs a rebuild because its meta.yaml, patches, or sources are newer than the `.packaged` thunk. """ packaged_token = buildpath / ".packaged" if not packaged_token.is_file(): return True package_time = packaged_token.stat().st_mtime def source_files(): yield path yield from pkg.get("source", {}).get("patches", []) yield from (x[0] for x in pkg.get("source", {}).get("extras", [])) for source_file in source_files(): source_file = Path(source_file) if source_file.stat().st_mtime > package_time: return True def build_package(path, args): pkg = common.parse_package(path) name = pkg["package"]["name"] t0 = datetime.now() print("[{}] Building package {}...".format(t0.strftime("%Y-%m-%d %H:%M:%S"), name)) packagedir = name + "-" + pkg["package"]["version"] dirpath = path.parent orig_path = Path.cwd() os.chdir(dirpath) buildpath = dirpath / "build" try: if not needs_rebuild(pkg, path, buildpath): return if "source" in pkg: if buildpath.resolve().is_dir(): shutil.rmtree(buildpath) os.makedirs(buildpath) srcpath = download_and_extract(buildpath, packagedir, pkg, args) patch(path, srcpath, pkg, args) compile(path, srcpath, pkg, args) package_files(buildpath, srcpath, pkg, args) finally: os.chdir(orig_path) t1 = datetime.now() print( "[{}] done building package {} in {:.1f} s.".format( t1.strftime("%Y-%m-%d %H:%M:%S"), name, (t1 - t0).total_seconds() ) ) def make_parser(parser): parser.description = "Build a pyodide package." parser.add_argument( "package", type=str, nargs=1, help="Path to meta.yaml package description" ) parser.add_argument( "--package_abi", type=int, required=True, help="The ABI number for the package to be built", ) parser.add_argument( "--cflags", type=str, nargs="?", default=common.DEFAULTCFLAGS, help="Extra compiling flags", ) parser.add_argument( "--ldflags", type=str, nargs="?", default=common.DEFAULTLDFLAGS, help="Extra linking flags", ) parser.add_argument( "--host", type=str, nargs="?", default=common.HOSTPYTHON, help="The path to the host Python installation", ) parser.add_argument( "--target", type=str, nargs="?", default=common.TARGETPYTHON, help="The path to the target Python installation", ) return parser def main(args): path = Path(args.package[0]).resolve() build_package(path, args) if __name__ == "__main__": parser = make_parser(argparse.ArgumentParser()) args = parser.parse_args() main(args)