mirror of https://github.com/pyodide/pyodide.git
Add documentation to new build tools.
Clean up argument parsing.
This commit is contained in:
Michael Droettboom
parent
d58f6b1f6e
commit
222c0248a1
1
Makefile
1
Makefile
|
|
@ -117,6 +117,7 @@ clean:
|
|||
rm build/*
|
||||
rm src/*.bc
|
||||
make -C packages clean
|
||||
make -C six clean
|
||||
echo "The Emsdk and CPython are not cleaned. cd into those directories to do so."
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ PYODIDE_ROOT=$(abspath ..)
|
|||
include ../Makefile.envs
|
||||
|
||||
all:
|
||||
../tools/buildall . --output=../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
|
||||
../tools/buildall . ../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
|
||||
|
||||
clean:
|
||||
rm -rf */build
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ URL=https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c
|
|||
all: $(BUILD)/__init__.py
|
||||
|
||||
|
||||
|
||||
clean:
|
||||
-rm -fr downloads
|
||||
-rm -fr $(SRC)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
Six is a special case package, since it's so commonly used, we want to include
|
||||
it in the main Python package.
|
||||
|
|
@ -1,5 +1,9 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Build all of the packages in a given directory.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
|
|
@ -10,9 +14,11 @@ import common
|
|||
import buildpkg
|
||||
|
||||
|
||||
def build_package(pkgname, reqs, dependencies, packagesdir, args):
|
||||
def build_package(pkgname, dependencies, packagesdir, args):
|
||||
reqs = dependencies[pkgname]
|
||||
# Make sure all of the package's requirements are built first
|
||||
for req in reqs:
|
||||
build_package(req, dependencies[req], dependencies, packagesdir, args)
|
||||
build_package(req, dependencies, packagesdir, args)
|
||||
if not os.path.isfile(
|
||||
os.path.join(packagesdir, pkgname, 'build', '.packaged')):
|
||||
print("BUILDING PACKAGE: " + pkgname)
|
||||
|
|
@ -27,8 +33,8 @@ def build_package(pkgname, reqs, dependencies, packagesdir, args):
|
|||
|
||||
|
||||
def build_packages(packagesdir, args):
|
||||
# We have to build the packages in order, so first we build a dependency
|
||||
# tree
|
||||
# We have to build the packages in the correct order (dependencies first),
|
||||
# so first load in all of the package metadata and build a dependency map.
|
||||
dependencies = {}
|
||||
for pkgdir in os.listdir(packagesdir):
|
||||
pkgdir = os.path.join(packagesdir, pkgdir)
|
||||
|
|
@ -39,8 +45,8 @@ def build_packages(packagesdir, args):
|
|||
reqs = pkg.get('requirements', {}).get('run', [])
|
||||
dependencies[name] = reqs
|
||||
|
||||
for pkgname, reqs in dependencies.items():
|
||||
build_package(pkgname, reqs, dependencies, packagesdir, args)
|
||||
for pkgname in dependencies.keys():
|
||||
build_package(pkgname, dependencies, packagesdir, args)
|
||||
|
||||
# This is done last so the main Makefile can use it as a completion token
|
||||
with open(os.path.join(args.output[0], 'packages.json'), 'w') as fd:
|
||||
|
|
@ -48,17 +54,26 @@ def build_packages(packagesdir, args):
|
|||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('dir', type=str, nargs=1)
|
||||
parser.add_argument('--cflags', type=str, nargs=1, default=[''])
|
||||
parser = argparse.ArgumentParser(
|
||||
"Build all of the packages in a given directory")
|
||||
parser.add_argument(
|
||||
'--ldflags', type=str, nargs=1, default=[common.DEFAULT_LD])
|
||||
'dir', type=str, nargs=1,
|
||||
help='Input directory containing a tree of package definitions')
|
||||
parser.add_argument(
|
||||
'--host', type=str, nargs=1, default=[common.HOSTPYTHON])
|
||||
'output', type=str, nargs=1,
|
||||
help='Output directory in which to put all built packages')
|
||||
parser.add_argument(
|
||||
'--target', type=str, nargs=1, default=[common.TARGETPYTHON])
|
||||
'--cflags', type=str, nargs='?', default=[common.DEFAULTCFLAGS],
|
||||
help='Extra compiling flags')
|
||||
parser.add_argument(
|
||||
'--output', '-o', type=str, nargs=1)
|
||||
'--ldflags', type=str, nargs='?', default=[common.DEFAULTLDFLAGS],
|
||||
help='Extra linking flags')
|
||||
parser.add_argument(
|
||||
'--host', type=str, nargs='?', default=[common.HOSTPYTHON],
|
||||
help='The path to the host Python installation')
|
||||
parser.add_argument(
|
||||
'--target', type=str, nargs='?', default=[common.TARGETPYTHON],
|
||||
help='The path to the target Python installation')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Builds a Pyodide package.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
|
|
@ -13,7 +17,13 @@ import common
|
|||
ROOTDIR = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
|
||||
def do_checksum(path, checksum):
|
||||
def check_checksum(path, pkg):
|
||||
"""
|
||||
Checks that a tarball matches the checksum in the package metadata.
|
||||
"""
|
||||
if 'md5' not in pkg['source']:
|
||||
return
|
||||
checksum = pkg['source']['md5']
|
||||
CHUNK_SIZE = 1 << 16
|
||||
h = hashlib.md5()
|
||||
with open(path, 'rb') as fd:
|
||||
|
|
@ -33,7 +43,7 @@ def download_and_extract(buildpath, packagedir, pkg, args):
|
|||
subprocess.run([
|
||||
'wget', '-q', '-O', tarballpath, pkg['source']['url']
|
||||
], check=True)
|
||||
do_checksum(tarballpath, pkg['source']['md5'])
|
||||
check_checksum(tarballpath, pkg)
|
||||
srcpath = os.path.join(buildpath, packagedir)
|
||||
if not os.path.isdir(srcpath):
|
||||
shutil.unpack_archive(tarballpath, buildpath)
|
||||
|
|
@ -44,6 +54,7 @@ def patch(path, srcpath, pkg, args):
|
|||
if os.path.isfile(os.path.join(srcpath, '.patched')):
|
||||
return
|
||||
|
||||
# Apply all of the patches
|
||||
orig_dir = os.getcwd()
|
||||
pkgdir = os.path.abspath(os.path.dirname(path))
|
||||
os.chdir(srcpath)
|
||||
|
|
@ -55,6 +66,7 @@ def patch(path, srcpath, pkg, args):
|
|||
finally:
|
||||
os.chdir(orig_dir)
|
||||
|
||||
# Add any extra files
|
||||
for src, dst in pkg['source'].get('extras', []):
|
||||
shutil.copyfile(os.path.join(pkgdir, src), os.path.join(srcpath, dst))
|
||||
|
||||
|
|
@ -63,8 +75,10 @@ def patch(path, srcpath, pkg, args):
|
|||
|
||||
|
||||
def get_libdir(srcpath, args):
|
||||
# Get the name of the build/lib.XXX directory that distutils wrote its
|
||||
# output to
|
||||
slug = subprocess.check_output([
|
||||
os.path.join(args.host[0], 'bin', 'python3'),
|
||||
os.path.join(args.host, 'bin', 'python3'),
|
||||
'-c',
|
||||
'import sysconfig, sys; '
|
||||
'print("{}-{}.{}".format('
|
||||
|
|
@ -87,15 +101,16 @@ def compile(path, srcpath, pkg, args):
|
|||
os.chdir(srcpath)
|
||||
try:
|
||||
subprocess.run([
|
||||
os.path.join(args.host, 'bin', 'python3'),
|
||||
os.path.join(ROOTDIR, 'pywasmcross'),
|
||||
'--cflags',
|
||||
args.cflags[0] + ' ' +
|
||||
args.cflags + ' ' +
|
||||
pkg.get('build', {}).get('cflags', ''),
|
||||
'--ldflags',
|
||||
args.ldflags[0] + ' ' +
|
||||
args.ldflags + ' ' +
|
||||
pkg.get('build', {}).get('ldflags', ''),
|
||||
'--host', args.host[0],
|
||||
'--target', args.target[0]], check=True)
|
||||
'--host', args.host,
|
||||
'--target', args.target], check=True)
|
||||
finally:
|
||||
os.chdir(orig_dir)
|
||||
|
||||
|
|
@ -159,16 +174,22 @@ def build_package(path, args):
|
|||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('package', type=str, nargs=1)
|
||||
parser = argparse.ArgumentParser('Build a pyodide package.')
|
||||
parser.add_argument(
|
||||
'--cflags', type=str, nargs=1, default=[''])
|
||||
'package', type=str, nargs=1,
|
||||
help="Path to meta.yaml package description")
|
||||
parser.add_argument(
|
||||
'--ldflags', type=str, nargs=1, default=[common.DEFAULT_LD])
|
||||
'--cflags', type=str, nargs='?', default=[common.DEFAULTCFLAGS],
|
||||
help='Extra compiling flags')
|
||||
parser.add_argument(
|
||||
'--host', type=str, nargs=1, default=[common.HOSTPYTHON])
|
||||
'--ldflags', type=str, nargs='?', default=[common.DEFAULTLDFLAGS],
|
||||
help='Extra linking flags')
|
||||
parser.add_argument(
|
||||
'--target', type=str, nargs=1, default=[common.TARGETPYTHON])
|
||||
'--host', type=str, nargs='?', default=[common.HOSTPYTHON],
|
||||
help='The path to the host Python installation')
|
||||
parser.add_argument(
|
||||
'--target', type=str, nargs='?', default=[common.TARGETPYTHON],
|
||||
help='The path to the target Python installation')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ HOSTPYTHON = os.path.abspath(
|
|||
os.path.join(ROOTDIR, '..', 'cpython', 'build', '3.6.4', 'host'))
|
||||
TARGETPYTHON = os.path.abspath(
|
||||
os.path.join(ROOTDIR, '..', 'cpython', 'installs', 'python-3.6.4'))
|
||||
DEFAULT_LD = ' '.join([
|
||||
DEFAULTCFLAGS = ''
|
||||
DEFAULTLDFLAGS = ' '.join([
|
||||
'-O3',
|
||||
'-s', "BINARYEN_METHOD='native-wasm'",
|
||||
'-Werror',
|
||||
|
|
|
|||
|
|
@ -1,5 +1,29 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""Helper for cross-compiling distutils-based Python extensions.
|
||||
|
||||
distutils has never had a proper cross-compilation story. This is a hack, which
|
||||
miraculously works, to get around that.
|
||||
|
||||
The gist is:
|
||||
|
||||
- Compile the package natively, replacing calls to the compiler and linker with
|
||||
wrappers that store the arguments in a log, and then delegate along to the
|
||||
real native compiler and linker.
|
||||
|
||||
- Remove all of the native build products.
|
||||
|
||||
- Play back the log, replacing the native compiler with emscripten and
|
||||
adjusting include paths and flags as necessary for cross-compiling to
|
||||
emscripten. This overwrites the results from the original native compilation.
|
||||
|
||||
While this results in more work than strictly necessary (it builds a native
|
||||
version of the package, even though we then throw it away), it seems to be the
|
||||
only reliable way to automatically build a package that interleaves
|
||||
configuration with build.
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import importlib.machinery
|
||||
import json
|
||||
|
|
@ -9,26 +33,44 @@ import subprocess
|
|||
import sys
|
||||
|
||||
|
||||
import common
|
||||
|
||||
|
||||
ROOTDIR = os.path.abspath(os.path.dirname(__file__))
|
||||
symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc'])
|
||||
|
||||
|
||||
def collect_args(basename):
|
||||
"""
|
||||
This is called when this script is called through a symlink that looks like
|
||||
a compiler or linker.
|
||||
|
||||
It writes the arguments to the build.log, and then delegates to the real
|
||||
native compiler or linker.
|
||||
"""
|
||||
# Remove the symlink compiler from the PATH, so we can delegate to the
|
||||
# native compiler
|
||||
env = dict(os.environ)
|
||||
path = env['PATH']
|
||||
while ROOTDIR + ':' in path:
|
||||
path = path.replace(ROOTDIR + ':', '')
|
||||
|
||||
env['PATH'] = path
|
||||
|
||||
with open('build.log', 'a') as fd:
|
||||
json.dump([basename] + sys.argv[1:], fd)
|
||||
fd.write('\n')
|
||||
|
||||
sys.exit(subprocess.run([basename] + sys.argv[1:],
|
||||
env=env).returncode)
|
||||
sys.exit(
|
||||
subprocess.run(
|
||||
[basename] + sys.argv[1:],
|
||||
env=env).returncode)
|
||||
|
||||
|
||||
def make_symlinks(env):
|
||||
"""
|
||||
Makes sure all of the symlinks that make this script look like a compiler
|
||||
exist.
|
||||
"""
|
||||
exec_path = os.path.abspath(__file__)
|
||||
for symlink in symlinks:
|
||||
symlink_path = os.path.join(ROOTDIR, symlink)
|
||||
|
|
@ -47,9 +89,9 @@ def capture_compile(args):
|
|||
env['PATH'] = ROOTDIR + ':' + os.environ['PATH']
|
||||
|
||||
result = subprocess.run(
|
||||
[os.path.join(args.host[0], 'bin', 'python3'),
|
||||
'setup.py',
|
||||
'install'], env=env)
|
||||
[os.path.join(args.host[0], 'bin', 'python3'),
|
||||
'setup.py',
|
||||
'install'], env=env)
|
||||
if result.returncode != 0:
|
||||
if os.path.exists('build.log'):
|
||||
os.remove('build.log')
|
||||
|
|
@ -57,7 +99,8 @@ def capture_compile(args):
|
|||
|
||||
|
||||
def handle_command(line, args):
|
||||
# This is a special case to skip the compilation tests in numpy
|
||||
# This is a special case to skip the compilation tests in numpy that aren't
|
||||
# actually part of the build
|
||||
for arg in line:
|
||||
if r'/file.c' in arg or '_configtest' in arg:
|
||||
return
|
||||
|
|
@ -65,37 +108,35 @@ def handle_command(line, args):
|
|||
return
|
||||
|
||||
if line[0] == 'ar':
|
||||
line[0] = 'emar'
|
||||
new_args = ['emar']
|
||||
elif line[0] == 'c++':
|
||||
line[0] = 'em++'
|
||||
new_args = ['em++']
|
||||
else:
|
||||
line[0] = 'emcc'
|
||||
new_args = ['emcc']
|
||||
# distutils doesn't use the c++ compiler when compiling c++ <sigh>
|
||||
for arg in line:
|
||||
if arg.endswith('.cpp'):
|
||||
line[0] = 'em++'
|
||||
break
|
||||
if any(arg.endswith('.cpp') for arg in line):
|
||||
new_args = ['em++']
|
||||
shared = '-shared' in line
|
||||
|
||||
new_args = [line[0]]
|
||||
if shared:
|
||||
new_args.extend(args.ldflags[0].split())
|
||||
elif line[0] in ('emcc', 'em++'):
|
||||
elif new_args[0] in ('emcc', 'em++'):
|
||||
new_args.extend(args.cflags[0].split())
|
||||
|
||||
skip_next = False
|
||||
# Go through and adjust arguments
|
||||
for arg in line[1:]:
|
||||
if skip_next:
|
||||
skip_next = False
|
||||
continue
|
||||
if arg.startswith('-I'):
|
||||
# Don't include any system directories
|
||||
if arg[2:].startswith('/usr'):
|
||||
continue
|
||||
if (os.path.abspath(arg[2:]).startswith(args.host[0]) and
|
||||
'site-packages' not in arg):
|
||||
arg = arg.replace('-I' + args.host[0], '-I' + args.target[0])
|
||||
if arg[2:].startswith('/usr'):
|
||||
continue
|
||||
# Don't include any system directories
|
||||
if arg.startswith('-L/usr'):
|
||||
continue
|
||||
# The native build is possibly multithreaded, but the emscripten one
|
||||
# definitely isn't
|
||||
arg = re.sub(r'/python([0-9]\.[0-9]+)m', r'/python\1', arg)
|
||||
if arg.endswith('.o'):
|
||||
arg = arg[:-2] + '.bc'
|
||||
|
|
@ -110,6 +151,7 @@ def handle_command(line, args):
|
|||
if result.returncode != 0:
|
||||
sys.exit(result.returncode)
|
||||
|
||||
# Emscripten .so files shouldn't have the native platform slug
|
||||
if shared:
|
||||
renamed = output[:-5] + '.so'
|
||||
for ext in importlib.machinery.EXTENSION_SUFFIXES:
|
||||
|
|
@ -122,6 +164,8 @@ def handle_command(line, args):
|
|||
|
||||
|
||||
def replay_compile(args):
|
||||
# If pure Python, there will be no build.log file, which is fine -- just do
|
||||
# nothing
|
||||
if os.path.isfile('build.log'):
|
||||
with open('build.log', 'r') as fd:
|
||||
for line in fd:
|
||||
|
|
@ -146,11 +190,21 @@ def build_wrap(args):
|
|||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--cflags', type=str, nargs=1, default=[''])
|
||||
parser.add_argument('--ldflags', type=str, nargs=1, default=[''])
|
||||
parser.add_argument('--host', type=str, nargs=1)
|
||||
parser.add_argument('--target', type=str, nargs=1)
|
||||
parser = argparse.ArgumentParser(
|
||||
'Cross compile a Python distutils package. '
|
||||
'Run from the root directory of the package\'s source')
|
||||
parser.add_argument(
|
||||
'--cflags', type=str, nargs='?', default=[common.DEFAULTCFLAGS],
|
||||
help='Extra compiling flags')
|
||||
parser.add_argument(
|
||||
'--ldflags', type=str, nargs='?', default=[common.DEFAULTLDFLAGS],
|
||||
help='Extra linking flags')
|
||||
parser.add_argument(
|
||||
'--host', type=str, nargs='?', default=[common.HOSTPYTHON],
|
||||
help='The path to the host Python installation')
|
||||
parser.add_argument(
|
||||
'--target', type=str, nargs='?', default=[common.TARGETPYTHON],
|
||||
help='The path to the target Python installation')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue