pyodide/pyodide_build/buildpkg.py

193 lines
5.5 KiB
Python
Raw Normal View History

2018-06-20 18:54:47 +00:00
#!/usr/bin/env python3
"""
Builds a Pyodide package.
"""
2018-06-20 18:54:47 +00:00
import argparse
import hashlib
import os
2018-08-03 16:48:22 +00:00
from pathlib import Path
2018-06-20 18:54:47 +00:00
import shutil
import subprocess
from . import common
2018-06-20 18:54:47 +00:00
def check_checksum(path, pkg):
"""
Checks that a tarball matches the checksum in the package metadata.
"""
2018-08-06 15:03:51 +00:00
checksum_keys = {'md5', 'sha256'}.intersection(pkg['source'])
if not checksum_keys:
return
2018-08-06 15:03:51 +00:00
elif len(checksum_keys) != 1:
raise ValueError('Only one checksum should be included in a package '
'setup; found {}.'.format(checksum_keys))
checksum_algorithm = checksum_keys.pop()
checksum = pkg['source'][checksum_algorithm]
2018-06-20 18:54:47 +00:00
CHUNK_SIZE = 1 << 16
2018-08-06 15:03:51 +00:00
h = getattr(hashlib, checksum_algorithm)()
2018-06-20 18:54:47 +00:00
with open(path, 'rb') as fd:
while True:
chunk = fd.read(CHUNK_SIZE)
h.update(chunk)
if len(chunk) < CHUNK_SIZE:
break
if h.hexdigest() != checksum:
2018-08-06 15:03:51 +00:00
raise ValueError("Invalid {} checksum".format(checksum_algorithm))
2018-06-20 18:54:47 +00:00
def download_and_extract(buildpath, packagedir, pkg, args):
2018-08-03 16:48:22 +00:00
tarballpath = buildpath / Path(pkg['source']['url']).name
if not tarballpath.is_file():
2018-06-20 19:05:13 +00:00
subprocess.run([
2018-08-03 16:48:22 +00:00
'wget', '-q', '-O', str(tarballpath), pkg['source']['url']
2018-06-20 19:05:13 +00:00
], check=True)
check_checksum(tarballpath, pkg)
2018-08-03 16:48:22 +00:00
srcpath = buildpath / packagedir
if not srcpath.is_dir():
shutil.unpack_archive(str(tarballpath), str(buildpath))
2018-06-20 18:54:47 +00:00
return srcpath
def patch(path, srcpath, pkg, args):
2018-08-03 16:48:22 +00:00
if (srcpath / '.patched').is_file():
2018-06-20 18:54:47 +00:00
return
# Apply all of the patches
2018-08-03 16:48:22 +00:00
orig_dir = Path.cwd()
pkgdir = path.parent.resolve()
2018-06-20 18:54:47 +00:00
os.chdir(srcpath)
try:
for patch in pkg['source'].get('patches', []):
subprocess.run([
2018-08-03 16:48:22 +00:00
'patch', '-p1', '--binary', '-i', pkgdir / patch
2018-06-20 19:05:13 +00:00
], check=True)
2018-06-20 18:54:47 +00:00
finally:
os.chdir(orig_dir)
# Add any extra files
2018-06-20 18:54:47 +00:00
for src, dst in pkg['source'].get('extras', []):
2018-08-03 16:48:22 +00:00
shutil.copyfile(pkgdir / src, srcpath / dst)
2018-06-20 18:54:47 +00:00
2018-08-03 16:48:22 +00:00
with open(srcpath / '.patched', 'wb') as fd:
2018-06-20 18:54:47 +00:00
fd.write(b'\n')
def compile(path, srcpath, pkg, args):
2018-08-03 16:48:22 +00:00
if (srcpath / '.built').is_file():
2018-06-20 18:54:47 +00:00
return
2018-08-03 16:48:22 +00:00
orig_dir = Path.cwd()
2018-06-20 18:54:47 +00:00
os.chdir(srcpath)
try:
subprocess.run([
2018-08-03 16:48:22 +00:00
str(Path(args.host) / 'bin' / 'python3'),
'-m', 'pyodide_build',
'pywasmcross',
2018-06-20 19:05:13 +00:00
'--cflags',
args.cflags + ' ' +
2018-06-20 19:05:13 +00:00
pkg.get('build', {}).get('cflags', ''),
'--ldflags',
args.ldflags + ' ' +
2018-06-20 19:05:13 +00:00
pkg.get('build', {}).get('ldflags', ''),
'--host', args.host,
'--target', args.target], check=True)
2018-06-20 18:54:47 +00:00
finally:
os.chdir(orig_dir)
post = pkg.get('build', {}).get('post')
if post is not None:
2018-07-18 13:26:18 +00:00
site_packages_dir = (
srcpath / 'install' / 'lib' / 'python3.7' / 'site-packages')
2018-08-03 16:48:22 +00:00
pkgdir = path.parent.resolve()
2018-06-20 18:54:47 +00:00
env = {
2018-07-18 13:26:18 +00:00
'SITEPACKAGES': site_packages_dir,
2018-06-20 18:54:47 +00:00
'PKGDIR': pkgdir
}
subprocess.run([
'bash', '-c', post], env=env, check=True)
2018-08-03 16:48:22 +00:00
with open(srcpath / '.built', 'wb') as fd:
2018-06-20 18:54:47 +00:00
fd.write(b'\n')
def package_files(buildpath, srcpath, pkg, args):
if (buildpath / '.packaged').is_file():
2018-06-20 18:54:47 +00:00
return
name = pkg['package']['name']
install_prefix = (srcpath / 'install').resolve()
2018-06-20 18:54:47 +00:00
subprocess.run([
2018-08-16 21:09:50 +00:00
'python',
common.TOOLSDIR / 'file_packager.py',
name + '.data',
'--lz4',
2018-06-20 18:54:47 +00:00
'--preload',
'{}@/'.format(install_prefix),
'--js-output={}'.format(name + '.js'),
'--export-name=pyodide._module',
2018-06-20 18:54:47 +00:00
'--exclude', '*.wasm.pre',
2018-06-23 12:28:34 +00:00
'--exclude', '__pycache__',
'--use-preload-plugins'],
cwd=buildpath, check=True)
2018-06-20 18:54:47 +00:00
subprocess.run([
'uglifyjs',
2018-08-03 16:48:22 +00:00
buildpath / (name + '.js'),
2018-06-20 18:54:47 +00:00
'-o',
2018-08-03 16:48:22 +00:00
buildpath / (name + '.js')], check=True)
2018-06-20 18:54:47 +00:00
2018-08-03 16:48:22 +00:00
with open(buildpath / '.packaged', 'wb') as fd:
2018-06-20 18:54:47 +00:00
fd.write(b'\n')
def build_package(path, args):
pkg = common.parse_package(path)
packagedir = pkg['package']['name'] + '-' + pkg['package']['version']
2018-08-03 16:48:22 +00:00
dirpath = path.parent
orig_path = Path.cwd()
2018-06-20 18:54:47 +00:00
os.chdir(dirpath)
try:
2018-08-03 16:48:22 +00:00
buildpath = dirpath / 'build'
if not buildpath.is_dir():
2018-06-20 18:54:47 +00:00
os.makedirs(buildpath)
srcpath = download_and_extract(buildpath, packagedir, pkg, args)
patch(path, srcpath, pkg, args)
compile(path, srcpath, pkg, args)
package_files(buildpath, srcpath, pkg, args)
finally:
os.chdir(orig_path)
def make_parser(parser):
parser.description = 'Build a pyodide package.'
parser.add_argument(
'package', type=str, nargs=1,
help="Path to meta.yaml package description")
2018-06-20 19:05:13 +00:00
parser.add_argument(
2018-06-22 18:49:52 +00:00
'--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS,
help='Extra compiling flags')
2018-06-20 19:05:13 +00:00
parser.add_argument(
2018-06-22 18:49:52 +00:00
'--ldflags', type=str, nargs='?', default=common.DEFAULTLDFLAGS,
help='Extra linking flags')
2018-06-20 19:05:13 +00:00
parser.add_argument(
2018-06-22 18:49:52 +00:00
'--host', type=str, nargs='?', default=common.HOSTPYTHON,
help='The path to the host Python installation')
2018-06-20 19:05:13 +00:00
parser.add_argument(
2018-06-22 18:49:52 +00:00
'--target', type=str, nargs='?', default=common.TARGETPYTHON,
help='The path to the target Python installation')
return parser
2018-06-20 18:54:47 +00:00
def main(args):
2018-08-03 16:48:22 +00:00
path = Path(args.package[0]).resolve()
2018-06-20 18:54:47 +00:00
build_package(path, args)
if __name__ == '__main__':
parser = make_parser(argparse.ArgumentParser())
args = parser.parse_args()
2018-06-20 18:54:47 +00:00
main(args)