2018-06-20 18:54:47 +00:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
2018-06-22 14:22:00 +00:00
|
|
|
"""
|
|
|
|
Builds a Pyodide package.
|
|
|
|
"""
|
|
|
|
|
2018-06-20 18:54:47 +00:00
|
|
|
import argparse
|
|
|
|
import hashlib
|
|
|
|
import os
|
2018-08-03 16:48:22 +00:00
|
|
|
from pathlib import Path
|
2018-06-20 18:54:47 +00:00
|
|
|
import shutil
|
|
|
|
import subprocess
|
|
|
|
|
|
|
|
|
2018-09-20 14:42:38 +00:00
|
|
|
from . import common
|
2018-06-20 18:54:47 +00:00
|
|
|
|
|
|
|
|
2018-06-22 14:22:00 +00:00
|
|
|
def check_checksum(path, pkg):
|
|
|
|
"""
|
|
|
|
Checks that a tarball matches the checksum in the package metadata.
|
|
|
|
"""
|
2018-08-06 15:03:51 +00:00
|
|
|
checksum_keys = {'md5', 'sha256'}.intersection(pkg['source'])
|
|
|
|
if not checksum_keys:
|
2018-06-22 14:22:00 +00:00
|
|
|
return
|
2018-08-06 15:03:51 +00:00
|
|
|
elif len(checksum_keys) != 1:
|
|
|
|
raise ValueError('Only one checksum should be included in a package '
|
|
|
|
'setup; found {}.'.format(checksum_keys))
|
|
|
|
checksum_algorithm = checksum_keys.pop()
|
|
|
|
checksum = pkg['source'][checksum_algorithm]
|
2018-06-20 18:54:47 +00:00
|
|
|
CHUNK_SIZE = 1 << 16
|
2018-08-06 15:03:51 +00:00
|
|
|
h = getattr(hashlib, checksum_algorithm)()
|
2018-06-20 18:54:47 +00:00
|
|
|
with open(path, 'rb') as fd:
|
|
|
|
while True:
|
|
|
|
chunk = fd.read(CHUNK_SIZE)
|
|
|
|
h.update(chunk)
|
|
|
|
if len(chunk) < CHUNK_SIZE:
|
|
|
|
break
|
|
|
|
if h.hexdigest() != checksum:
|
2018-08-06 15:03:51 +00:00
|
|
|
raise ValueError("Invalid {} checksum".format(checksum_algorithm))
|
2018-06-20 18:54:47 +00:00
|
|
|
|
|
|
|
|
|
|
|
def download_and_extract(buildpath, packagedir, pkg, args):
|
2018-08-03 16:48:22 +00:00
|
|
|
tarballpath = buildpath / Path(pkg['source']['url']).name
|
|
|
|
if not tarballpath.is_file():
|
2018-06-20 19:05:13 +00:00
|
|
|
subprocess.run([
|
2018-08-03 16:48:22 +00:00
|
|
|
'wget', '-q', '-O', str(tarballpath), pkg['source']['url']
|
2018-06-20 19:05:13 +00:00
|
|
|
], check=True)
|
2018-06-22 14:22:00 +00:00
|
|
|
check_checksum(tarballpath, pkg)
|
2018-08-03 16:48:22 +00:00
|
|
|
srcpath = buildpath / packagedir
|
|
|
|
if not srcpath.is_dir():
|
|
|
|
shutil.unpack_archive(str(tarballpath), str(buildpath))
|
2018-06-20 18:54:47 +00:00
|
|
|
return srcpath
|
|
|
|
|
|
|
|
|
|
|
|
def patch(path, srcpath, pkg, args):
|
2018-08-03 16:48:22 +00:00
|
|
|
if (srcpath / '.patched').is_file():
|
2018-06-20 18:54:47 +00:00
|
|
|
return
|
|
|
|
|
2018-06-22 14:22:00 +00:00
|
|
|
# Apply all of the patches
|
2018-08-03 16:48:22 +00:00
|
|
|
orig_dir = Path.cwd()
|
|
|
|
pkgdir = path.parent.resolve()
|
2018-06-20 18:54:47 +00:00
|
|
|
os.chdir(srcpath)
|
|
|
|
try:
|
|
|
|
for patch in pkg['source'].get('patches', []):
|
|
|
|
subprocess.run([
|
2018-08-03 16:48:22 +00:00
|
|
|
'patch', '-p1', '--binary', '-i', pkgdir / patch
|
2018-06-20 19:05:13 +00:00
|
|
|
], check=True)
|
2018-06-20 18:54:47 +00:00
|
|
|
finally:
|
|
|
|
os.chdir(orig_dir)
|
|
|
|
|
2018-06-22 14:22:00 +00:00
|
|
|
# Add any extra files
|
2018-06-20 18:54:47 +00:00
|
|
|
for src, dst in pkg['source'].get('extras', []):
|
2018-08-03 16:48:22 +00:00
|
|
|
shutil.copyfile(pkgdir / src, srcpath / dst)
|
2018-06-20 18:54:47 +00:00
|
|
|
|
2018-08-03 16:48:22 +00:00
|
|
|
with open(srcpath / '.patched', 'wb') as fd:
|
2018-06-20 18:54:47 +00:00
|
|
|
fd.write(b'\n')
|
|
|
|
|
|
|
|
|
|
|
|
def compile(path, srcpath, pkg, args):
|
2018-08-03 16:48:22 +00:00
|
|
|
if (srcpath / '.built').is_file():
|
2018-06-20 18:54:47 +00:00
|
|
|
return
|
|
|
|
|
2018-08-03 16:48:22 +00:00
|
|
|
orig_dir = Path.cwd()
|
2018-06-20 18:54:47 +00:00
|
|
|
os.chdir(srcpath)
|
|
|
|
try:
|
|
|
|
subprocess.run([
|
2018-08-03 16:48:22 +00:00
|
|
|
str(Path(args.host) / 'bin' / 'python3'),
|
2018-09-20 14:42:38 +00:00
|
|
|
'-m', 'pyodide_build',
|
|
|
|
'pywasmcross',
|
2018-06-20 19:05:13 +00:00
|
|
|
'--cflags',
|
2018-06-22 14:22:00 +00:00
|
|
|
args.cflags + ' ' +
|
2018-06-20 19:05:13 +00:00
|
|
|
pkg.get('build', {}).get('cflags', ''),
|
|
|
|
'--ldflags',
|
2018-06-22 14:22:00 +00:00
|
|
|
args.ldflags + ' ' +
|
2018-06-20 19:05:13 +00:00
|
|
|
pkg.get('build', {}).get('ldflags', ''),
|
2018-06-22 14:22:00 +00:00
|
|
|
'--host', args.host,
|
|
|
|
'--target', args.target], check=True)
|
2018-06-20 18:54:47 +00:00
|
|
|
finally:
|
|
|
|
os.chdir(orig_dir)
|
|
|
|
|
|
|
|
post = pkg.get('build', {}).get('post')
|
|
|
|
if post is not None:
|
2018-07-18 13:26:18 +00:00
|
|
|
site_packages_dir = (
|
|
|
|
srcpath / 'install' / 'lib' / 'python3.7' / 'site-packages')
|
2018-08-03 16:48:22 +00:00
|
|
|
pkgdir = path.parent.resolve()
|
2018-06-20 18:54:47 +00:00
|
|
|
env = {
|
2018-07-18 13:26:18 +00:00
|
|
|
'SITEPACKAGES': site_packages_dir,
|
2018-06-20 18:54:47 +00:00
|
|
|
'PKGDIR': pkgdir
|
|
|
|
}
|
|
|
|
subprocess.run([
|
|
|
|
'bash', '-c', post], env=env, check=True)
|
|
|
|
|
2018-08-03 16:48:22 +00:00
|
|
|
with open(srcpath / '.built', 'wb') as fd:
|
2018-06-20 18:54:47 +00:00
|
|
|
fd.write(b'\n')
|
|
|
|
|
|
|
|
|
|
|
|
def package_files(buildpath, srcpath, pkg, args):
|
2018-08-23 15:59:33 +00:00
|
|
|
if (buildpath / '.packaged').is_file():
|
2018-06-20 18:54:47 +00:00
|
|
|
return
|
|
|
|
|
|
|
|
name = pkg['package']['name']
|
2018-08-23 15:59:33 +00:00
|
|
|
install_prefix = (srcpath / 'install').resolve()
|
2018-06-20 18:54:47 +00:00
|
|
|
subprocess.run([
|
2018-08-16 21:09:50 +00:00
|
|
|
'python',
|
2018-09-20 14:42:38 +00:00
|
|
|
common.TOOLSDIR / 'file_packager.py',
|
2018-08-21 15:29:14 +00:00
|
|
|
name + '.data',
|
2018-09-17 19:30:25 +00:00
|
|
|
'--lz4',
|
2018-06-20 18:54:47 +00:00
|
|
|
'--preload',
|
2018-08-23 15:59:33 +00:00
|
|
|
'{}@/'.format(install_prefix),
|
2018-08-21 15:29:14 +00:00
|
|
|
'--js-output={}'.format(name + '.js'),
|
2018-09-19 14:42:49 +00:00
|
|
|
'--export-name=pyodide._module',
|
2018-06-20 18:54:47 +00:00
|
|
|
'--exclude', '*.wasm.pre',
|
2018-06-23 12:28:34 +00:00
|
|
|
'--exclude', '__pycache__',
|
2018-08-21 15:29:14 +00:00
|
|
|
'--use-preload-plugins'],
|
|
|
|
cwd=buildpath, check=True)
|
2018-06-20 18:54:47 +00:00
|
|
|
subprocess.run([
|
|
|
|
'uglifyjs',
|
2018-08-03 16:48:22 +00:00
|
|
|
buildpath / (name + '.js'),
|
2018-06-20 18:54:47 +00:00
|
|
|
'-o',
|
2018-08-03 16:48:22 +00:00
|
|
|
buildpath / (name + '.js')], check=True)
|
2018-06-20 18:54:47 +00:00
|
|
|
|
2018-08-03 16:48:22 +00:00
|
|
|
with open(buildpath / '.packaged', 'wb') as fd:
|
2018-06-20 18:54:47 +00:00
|
|
|
fd.write(b'\n')
|
|
|
|
|
|
|
|
|
|
|
|
def build_package(path, args):
|
|
|
|
pkg = common.parse_package(path)
|
|
|
|
packagedir = pkg['package']['name'] + '-' + pkg['package']['version']
|
2018-08-03 16:48:22 +00:00
|
|
|
dirpath = path.parent
|
|
|
|
orig_path = Path.cwd()
|
2018-06-20 18:54:47 +00:00
|
|
|
os.chdir(dirpath)
|
|
|
|
try:
|
2018-08-03 16:48:22 +00:00
|
|
|
buildpath = dirpath / 'build'
|
|
|
|
if not buildpath.is_dir():
|
2018-06-20 18:54:47 +00:00
|
|
|
os.makedirs(buildpath)
|
|
|
|
srcpath = download_and_extract(buildpath, packagedir, pkg, args)
|
|
|
|
patch(path, srcpath, pkg, args)
|
|
|
|
compile(path, srcpath, pkg, args)
|
|
|
|
package_files(buildpath, srcpath, pkg, args)
|
|
|
|
finally:
|
|
|
|
os.chdir(orig_path)
|
|
|
|
|
|
|
|
|
2018-09-20 14:42:38 +00:00
|
|
|
def make_parser(parser):
|
|
|
|
parser.description = 'Build a pyodide package.'
|
2018-06-22 14:22:00 +00:00
|
|
|
parser.add_argument(
|
|
|
|
'package', type=str, nargs=1,
|
|
|
|
help="Path to meta.yaml package description")
|
2018-06-20 19:05:13 +00:00
|
|
|
parser.add_argument(
|
2018-06-22 18:49:52 +00:00
|
|
|
'--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS,
|
2018-06-22 14:22:00 +00:00
|
|
|
help='Extra compiling flags')
|
2018-06-20 19:05:13 +00:00
|
|
|
parser.add_argument(
|
2018-06-22 18:49:52 +00:00
|
|
|
'--ldflags', type=str, nargs='?', default=common.DEFAULTLDFLAGS,
|
2018-06-22 14:22:00 +00:00
|
|
|
help='Extra linking flags')
|
2018-06-20 19:05:13 +00:00
|
|
|
parser.add_argument(
|
2018-06-22 18:49:52 +00:00
|
|
|
'--host', type=str, nargs='?', default=common.HOSTPYTHON,
|
2018-06-22 14:22:00 +00:00
|
|
|
help='The path to the host Python installation')
|
2018-06-20 19:05:13 +00:00
|
|
|
parser.add_argument(
|
2018-06-22 18:49:52 +00:00
|
|
|
'--target', type=str, nargs='?', default=common.TARGETPYTHON,
|
2018-06-22 14:22:00 +00:00
|
|
|
help='The path to the target Python installation')
|
2018-09-20 14:42:38 +00:00
|
|
|
return parser
|
2018-06-20 18:54:47 +00:00
|
|
|
|
|
|
|
|
|
|
|
def main(args):
|
2018-08-03 16:48:22 +00:00
|
|
|
path = Path(args.package[0]).resolve()
|
2018-06-20 18:54:47 +00:00
|
|
|
build_package(path, args)
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
2018-09-20 14:42:38 +00:00
|
|
|
parser = make_parser(argparse.ArgumentParser())
|
|
|
|
args = parser.parse_args()
|
2018-06-20 18:54:47 +00:00
|
|
|
main(args)
|