diff --git a/Makefile b/Makefile index 9c267f844..d879478ed 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,7 @@ clean: rm build/* rm src/*.bc make -C packages clean + make -C six clean echo "The Emsdk and CPython are not cleaned. cd into those directories to do so." diff --git a/packages/Makefile b/packages/Makefile index 1b0d9e122..c589cad6e 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -2,7 +2,7 @@ PYODIDE_ROOT=$(abspath ..) include ../Makefile.envs all: - ../tools/buildall . --output=../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) + ../tools/buildall . ../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) clean: rm -rf */build diff --git a/six/Makefile b/six/Makefile index 031b29b8d..89abf173a 100644 --- a/six/Makefile +++ b/six/Makefile @@ -14,7 +14,6 @@ URL=https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c all: $(BUILD)/__init__.py - clean: -rm -fr downloads -rm -fr $(SRC) diff --git a/six/README.md b/six/README.md new file mode 100644 index 000000000..b91b9dab0 --- /dev/null +++ b/six/README.md @@ -0,0 +1,2 @@ +Six is a special case package, since it's so commonly used, we want to include +it in the main Python package. diff --git a/tools/buildall b/tools/buildall index 789515318..828e32c35 100755 --- a/tools/buildall +++ b/tools/buildall @@ -1,5 +1,9 @@ #!/usr/bin/env python3 +""" +Build all of the packages in a given directory. +""" + import argparse import json import os @@ -10,9 +14,11 @@ import common import buildpkg -def build_package(pkgname, reqs, dependencies, packagesdir, args): +def build_package(pkgname, dependencies, packagesdir, args): + reqs = dependencies[pkgname] + # Make sure all of the package's requirements are built first for req in reqs: - build_package(req, dependencies[req], dependencies, packagesdir, args) + build_package(req, dependencies, packagesdir, args) if not os.path.isfile( os.path.join(packagesdir, pkgname, 'build', '.packaged')): print("BUILDING PACKAGE: " + pkgname) @@ -27,8 +33,8 @@ def build_package(pkgname, reqs, dependencies, packagesdir, args): def build_packages(packagesdir, args): - # We have to build the packages in order, so first we build a dependency - # tree + # We have to build the packages in the correct order (dependencies first), + # so first load in all of the package metadata and build a dependency map. dependencies = {} for pkgdir in os.listdir(packagesdir): pkgdir = os.path.join(packagesdir, pkgdir) @@ -39,8 +45,8 @@ def build_packages(packagesdir, args): reqs = pkg.get('requirements', {}).get('run', []) dependencies[name] = reqs - for pkgname, reqs in dependencies.items(): - build_package(pkgname, reqs, dependencies, packagesdir, args) + for pkgname in dependencies.keys(): + build_package(pkgname, dependencies, packagesdir, args) # This is done last so the main Makefile can use it as a completion token with open(os.path.join(args.output[0], 'packages.json'), 'w') as fd: @@ -48,17 +54,26 @@ def build_packages(packagesdir, args): def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('dir', type=str, nargs=1) - parser.add_argument('--cflags', type=str, nargs=1, default=['']) + parser = argparse.ArgumentParser( + "Build all of the packages in a given directory") parser.add_argument( - '--ldflags', type=str, nargs=1, default=[common.DEFAULT_LD]) + 'dir', type=str, nargs=1, + help='Input directory containing a tree of package definitions') parser.add_argument( - '--host', type=str, nargs=1, default=[common.HOSTPYTHON]) + 'output', type=str, nargs=1, + help='Output directory in which to put all built packages') parser.add_argument( - '--target', type=str, nargs=1, default=[common.TARGETPYTHON]) + '--cflags', type=str, nargs='?', default=[common.DEFAULTCFLAGS], + help='Extra compiling flags') parser.add_argument( - '--output', '-o', type=str, nargs=1) + '--ldflags', type=str, nargs='?', default=[common.DEFAULTLDFLAGS], + help='Extra linking flags') + parser.add_argument( + '--host', type=str, nargs='?', default=[common.HOSTPYTHON], + help='The path to the host Python installation') + parser.add_argument( + '--target', type=str, nargs='?', default=[common.TARGETPYTHON], + help='The path to the target Python installation') return parser.parse_args() diff --git a/tools/buildpkg.py b/tools/buildpkg.py index a826d15bd..477f22390 100755 --- a/tools/buildpkg.py +++ b/tools/buildpkg.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 +""" +Builds a Pyodide package. +""" + import argparse import hashlib import os @@ -13,7 +17,13 @@ import common ROOTDIR = os.path.abspath(os.path.dirname(__file__)) -def do_checksum(path, checksum): +def check_checksum(path, pkg): + """ + Checks that a tarball matches the checksum in the package metadata. + """ + if 'md5' not in pkg['source']: + return + checksum = pkg['source']['md5'] CHUNK_SIZE = 1 << 16 h = hashlib.md5() with open(path, 'rb') as fd: @@ -33,7 +43,7 @@ def download_and_extract(buildpath, packagedir, pkg, args): subprocess.run([ 'wget', '-q', '-O', tarballpath, pkg['source']['url'] ], check=True) - do_checksum(tarballpath, pkg['source']['md5']) + check_checksum(tarballpath, pkg) srcpath = os.path.join(buildpath, packagedir) if not os.path.isdir(srcpath): shutil.unpack_archive(tarballpath, buildpath) @@ -44,6 +54,7 @@ def patch(path, srcpath, pkg, args): if os.path.isfile(os.path.join(srcpath, '.patched')): return + # Apply all of the patches orig_dir = os.getcwd() pkgdir = os.path.abspath(os.path.dirname(path)) os.chdir(srcpath) @@ -55,6 +66,7 @@ def patch(path, srcpath, pkg, args): finally: os.chdir(orig_dir) + # Add any extra files for src, dst in pkg['source'].get('extras', []): shutil.copyfile(os.path.join(pkgdir, src), os.path.join(srcpath, dst)) @@ -63,8 +75,10 @@ def patch(path, srcpath, pkg, args): def get_libdir(srcpath, args): + # Get the name of the build/lib.XXX directory that distutils wrote its + # output to slug = subprocess.check_output([ - os.path.join(args.host[0], 'bin', 'python3'), + os.path.join(args.host, 'bin', 'python3'), '-c', 'import sysconfig, sys; ' 'print("{}-{}.{}".format(' @@ -87,15 +101,16 @@ def compile(path, srcpath, pkg, args): os.chdir(srcpath) try: subprocess.run([ + os.path.join(args.host, 'bin', 'python3'), os.path.join(ROOTDIR, 'pywasmcross'), '--cflags', - args.cflags[0] + ' ' + + args.cflags + ' ' + pkg.get('build', {}).get('cflags', ''), '--ldflags', - args.ldflags[0] + ' ' + + args.ldflags + ' ' + pkg.get('build', {}).get('ldflags', ''), - '--host', args.host[0], - '--target', args.target[0]], check=True) + '--host', args.host, + '--target', args.target], check=True) finally: os.chdir(orig_dir) @@ -159,16 +174,22 @@ def build_package(path, args): def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('package', type=str, nargs=1) + parser = argparse.ArgumentParser('Build a pyodide package.') parser.add_argument( - '--cflags', type=str, nargs=1, default=['']) + 'package', type=str, nargs=1, + help="Path to meta.yaml package description") parser.add_argument( - '--ldflags', type=str, nargs=1, default=[common.DEFAULT_LD]) + '--cflags', type=str, nargs='?', default=[common.DEFAULTCFLAGS], + help='Extra compiling flags') parser.add_argument( - '--host', type=str, nargs=1, default=[common.HOSTPYTHON]) + '--ldflags', type=str, nargs='?', default=[common.DEFAULTLDFLAGS], + help='Extra linking flags') parser.add_argument( - '--target', type=str, nargs=1, default=[common.TARGETPYTHON]) + '--host', type=str, nargs='?', default=[common.HOSTPYTHON], + help='The path to the host Python installation') + parser.add_argument( + '--target', type=str, nargs='?', default=[common.TARGETPYTHON], + help='The path to the target Python installation') return parser.parse_args() diff --git a/tools/common.py b/tools/common.py index 0b42dd2bf..301dbe60f 100644 --- a/tools/common.py +++ b/tools/common.py @@ -7,7 +7,8 @@ HOSTPYTHON = os.path.abspath( os.path.join(ROOTDIR, '..', 'cpython', 'build', '3.6.4', 'host')) TARGETPYTHON = os.path.abspath( os.path.join(ROOTDIR, '..', 'cpython', 'installs', 'python-3.6.4')) -DEFAULT_LD = ' '.join([ +DEFAULTCFLAGS = '' +DEFAULTLDFLAGS = ' '.join([ '-O3', '-s', "BINARYEN_METHOD='native-wasm'", '-Werror', diff --git a/tools/pywasmcross b/tools/pywasmcross index c28804e51..48fa4b478 100755 --- a/tools/pywasmcross +++ b/tools/pywasmcross @@ -1,5 +1,29 @@ #!/usr/bin/env python3 +"""Helper for cross-compiling distutils-based Python extensions. + +distutils has never had a proper cross-compilation story. This is a hack, which +miraculously works, to get around that. + +The gist is: + +- Compile the package natively, replacing calls to the compiler and linker with + wrappers that store the arguments in a log, and then delegate along to the + real native compiler and linker. + +- Remove all of the native build products. + +- Play back the log, replacing the native compiler with emscripten and + adjusting include paths and flags as necessary for cross-compiling to + emscripten. This overwrites the results from the original native compilation. + +While this results in more work than strictly necessary (it builds a native +version of the package, even though we then throw it away), it seems to be the +only reliable way to automatically build a package that interleaves +configuration with build. +""" + + import argparse import importlib.machinery import json @@ -9,26 +33,44 @@ import subprocess import sys +import common + + ROOTDIR = os.path.abspath(os.path.dirname(__file__)) symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc']) def collect_args(basename): + """ + This is called when this script is called through a symlink that looks like + a compiler or linker. + + It writes the arguments to the build.log, and then delegates to the real + native compiler or linker. + """ + # Remove the symlink compiler from the PATH, so we can delegate to the + # native compiler env = dict(os.environ) path = env['PATH'] while ROOTDIR + ':' in path: path = path.replace(ROOTDIR + ':', '') - env['PATH'] = path + with open('build.log', 'a') as fd: json.dump([basename] + sys.argv[1:], fd) fd.write('\n') - sys.exit(subprocess.run([basename] + sys.argv[1:], - env=env).returncode) + sys.exit( + subprocess.run( + [basename] + sys.argv[1:], + env=env).returncode) def make_symlinks(env): + """ + Makes sure all of the symlinks that make this script look like a compiler + exist. + """ exec_path = os.path.abspath(__file__) for symlink in symlinks: symlink_path = os.path.join(ROOTDIR, symlink) @@ -47,9 +89,9 @@ def capture_compile(args): env['PATH'] = ROOTDIR + ':' + os.environ['PATH'] result = subprocess.run( - [os.path.join(args.host[0], 'bin', 'python3'), - 'setup.py', - 'install'], env=env) + [os.path.join(args.host[0], 'bin', 'python3'), + 'setup.py', + 'install'], env=env) if result.returncode != 0: if os.path.exists('build.log'): os.remove('build.log') @@ -57,7 +99,8 @@ def capture_compile(args): def handle_command(line, args): - # This is a special case to skip the compilation tests in numpy + # This is a special case to skip the compilation tests in numpy that aren't + # actually part of the build for arg in line: if r'/file.c' in arg or '_configtest' in arg: return @@ -65,37 +108,35 @@ def handle_command(line, args): return if line[0] == 'ar': - line[0] = 'emar' + new_args = ['emar'] elif line[0] == 'c++': - line[0] = 'em++' + new_args = ['em++'] else: - line[0] = 'emcc' + new_args = ['emcc'] # distutils doesn't use the c++ compiler when compiling c++ - for arg in line: - if arg.endswith('.cpp'): - line[0] = 'em++' - break + if any(arg.endswith('.cpp') for arg in line): + new_args = ['em++'] shared = '-shared' in line - new_args = [line[0]] if shared: new_args.extend(args.ldflags[0].split()) - elif line[0] in ('emcc', 'em++'): + elif new_args[0] in ('emcc', 'em++'): new_args.extend(args.cflags[0].split()) - skip_next = False + # Go through and adjust arguments for arg in line[1:]: - if skip_next: - skip_next = False - continue if arg.startswith('-I'): + # Don't include any system directories + if arg[2:].startswith('/usr'): + continue if (os.path.abspath(arg[2:]).startswith(args.host[0]) and 'site-packages' not in arg): arg = arg.replace('-I' + args.host[0], '-I' + args.target[0]) - if arg[2:].startswith('/usr'): - continue + # Don't include any system directories if arg.startswith('-L/usr'): continue + # The native build is possibly multithreaded, but the emscripten one + # definitely isn't arg = re.sub(r'/python([0-9]\.[0-9]+)m', r'/python\1', arg) if arg.endswith('.o'): arg = arg[:-2] + '.bc' @@ -110,6 +151,7 @@ def handle_command(line, args): if result.returncode != 0: sys.exit(result.returncode) + # Emscripten .so files shouldn't have the native platform slug if shared: renamed = output[:-5] + '.so' for ext in importlib.machinery.EXTENSION_SUFFIXES: @@ -122,6 +164,8 @@ def handle_command(line, args): def replay_compile(args): + # If pure Python, there will be no build.log file, which is fine -- just do + # nothing if os.path.isfile('build.log'): with open('build.log', 'r') as fd: for line in fd: @@ -146,11 +190,21 @@ def build_wrap(args): def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--cflags', type=str, nargs=1, default=['']) - parser.add_argument('--ldflags', type=str, nargs=1, default=['']) - parser.add_argument('--host', type=str, nargs=1) - parser.add_argument('--target', type=str, nargs=1) + parser = argparse.ArgumentParser( + 'Cross compile a Python distutils package. ' + 'Run from the root directory of the package\'s source') + parser.add_argument( + '--cflags', type=str, nargs='?', default=[common.DEFAULTCFLAGS], + help='Extra compiling flags') + parser.add_argument( + '--ldflags', type=str, nargs='?', default=[common.DEFAULTLDFLAGS], + help='Extra linking flags') + parser.add_argument( + '--host', type=str, nargs='?', default=[common.HOSTPYTHON], + help='The path to the host Python installation') + parser.add_argument( + '--target', type=str, nargs='?', default=[common.TARGETPYTHON], + help='The path to the target Python installation') args = parser.parse_args() return args