diff --git a/.circleci/config.yml b/.circleci/config.yml index a56908713..8f633e100 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,16 +34,6 @@ jobs: pip install pytest pytest-xdist pytest-instafail selenium PyYAML - # Get recent version of Firefox and geckodriver - wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-latest-ssl\&os\=linux64\&lang\=en-US - tar jxf firefox.tar.bz2 - wget https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-linux64.tar.gz - tar zxf geckodriver-v0.21.0-linux64.tar.gz -C firefox - - # Get recent version of chromedriver - wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip - unzip chromedriver_linux64.zip - mv chromedriver firefox - run: name: lint @@ -79,7 +69,6 @@ jobs: paths: - ./build - ./pyodide-env - - ./firefox - store_artifacts: path: /home/circleci/repo/build/ @@ -97,6 +86,12 @@ jobs: # causes Firefox to complain when loading it. Let's just add the new mime type. sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types" + # Get recent version of Firefox and geckodriver + wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-latest-ssl\&os\=linux64\&lang\=en-US + tar jxf firefox.tar.bz2 + wget https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-linux64.tar.gz + tar zxf geckodriver-v0.21.0-linux64.tar.gz -C firefox + source pyodide-env/bin/activate export PATH=$PWD/firefox:$PATH pytest test -v -k firefox @@ -114,8 +109,12 @@ jobs: # causes Firefox to complain when loading it. Let's just add the new mime type. sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types" + # Get recent version of chromedriver + wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip + unzip chromedriver_linux64.zip + mv chromedriver pyodide-env/bin/ + source pyodide-env/bin/activate - export PATH=$PWD/firefox:$PATH pytest test -v -k chrome deploy: diff --git a/.gitignore b/.gitignore index 6401e1112..200eb5c37 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ ccache /six/six-1.11.0 /lz4/lz4-1.8.3 +*.egg-info/ diff --git a/bin/pyodide b/bin/pyodide new file mode 100755 index 000000000..653c925c9 --- /dev/null +++ b/bin/pyodide @@ -0,0 +1,9 @@ +#!/bin/bash + +# get the absolute path to the root directory +ROOTDIR=$(python -c 'import pathlib, sys; \ + print(pathlib.Path(sys.argv[1]).resolve().parents[1])' \ + "${BASH_SOURCE[0]}") + +export PYTHONPATH="${PYTHONPATH}:${ROOTDIR}" +python -m pyodide_build "$@" diff --git a/packages/Makefile b/packages/Makefile index 22552d59e..bee738225 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -2,7 +2,8 @@ PYODIDE_ROOT=$(abspath ..) include ../Makefile.envs all: - ../tools/buildall . ../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) + ../bin/pyodide buildall . ../build \ + --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) clean: rm -rf ./*/build diff --git a/pyodide_build/__init__.py b/pyodide_build/__init__.py new file mode 100644 index 000000000..b794fd409 --- /dev/null +++ b/pyodide_build/__init__.py @@ -0,0 +1 @@ +__version__ = '0.1.0' diff --git a/pyodide_build/__main__.py b/pyodide_build/__main__.py new file mode 100644 index 000000000..ad5c7eb75 --- /dev/null +++ b/pyodide_build/__main__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import argparse + +from . import buildall +from . import buildpkg +from . import pywasmcross + + +def main(): + main_parser = argparse.ArgumentParser(prog='pyodide') + subparsers = main_parser.add_subparsers(help='action') + + for command_name, module in (("buildpkg", buildpkg), + ("buildall", buildall), + ("pywasmcross", pywasmcross)): + parser = module.make_parser(subparsers.add_parser(command_name)) + parser.set_defaults(func=module.main) + + args = main_parser.parse_args() + # run the selected action + args.func(args) + + +if __name__ == '__main__': + main() diff --git a/tools/buildall b/pyodide_build/buildall.py similarity index 92% rename from tools/buildall rename to pyodide_build/buildall.py index 1d50f3543..cc74b439b 100755 --- a/tools/buildall +++ b/pyodide_build/buildall.py @@ -10,8 +10,8 @@ from pathlib import Path import shutil -import common -import buildpkg +from . import common +from . import buildpkg def build_package(pkgname, dependencies, packagesdir, outputdir, args): @@ -54,9 +54,8 @@ def build_packages(packagesdir, outputdir, args): json.dump({'dependencies': dependencies}, fd) -def parse_args(): - parser = argparse.ArgumentParser( - "Build all of the packages in a given directory") +def make_parser(parser): + parser.description = "Build all of the packages in a given directory" parser.add_argument( 'dir', type=str, nargs=1, help='Input directory containing a tree of package definitions') @@ -75,7 +74,7 @@ def parse_args(): parser.add_argument( '--target', type=str, nargs='?', default=common.TARGETPYTHON, help='The path to the target Python installation') - return parser.parse_args() + return parser def main(args): @@ -85,5 +84,6 @@ def main(args): if __name__ == '__main__': - args = parse_args() + parser = make_parser(argparse.ArgumentParser()) + args = parser.parse_args() main(args) diff --git a/tools/buildpkg.py b/pyodide_build/buildpkg.py similarity index 94% rename from tools/buildpkg.py rename to pyodide_build/buildpkg.py index a4ee5c546..3aa0e720e 100755 --- a/tools/buildpkg.py +++ b/pyodide_build/buildpkg.py @@ -12,10 +12,7 @@ import shutil import subprocess -import common - - -ROOTDIR = Path(__file__).parent.resolve() +from . import common def check_checksum(path, pkg): @@ -88,7 +85,7 @@ def compile(path, srcpath, pkg, args): try: subprocess.run([ str(Path(args.host) / 'bin' / 'python3'), - str(ROOTDIR / 'pywasmcross'), + '-m', 'pyodide_build', 'pywasmcross', '--cflags', args.cflags + ' ' + pkg.get('build', {}).get('cflags', ''), @@ -124,7 +121,7 @@ def package_files(buildpath, srcpath, pkg, args): install_prefix = (srcpath / 'install').resolve() subprocess.run([ 'python', - Path(ROOTDIR) / 'file_packager.py', + common.ROOTDIR / 'file_packager.py', name + '.data', '--lz4', '--preload', @@ -163,8 +160,8 @@ def build_package(path, args): os.chdir(orig_path) -def parse_args(): - parser = argparse.ArgumentParser('Build a pyodide package.') +def make_parser(parser): + parser.description = 'Build a pyodide package.' parser.add_argument( 'package', type=str, nargs=1, help="Path to meta.yaml package description") @@ -180,7 +177,7 @@ def parse_args(): parser.add_argument( '--target', type=str, nargs='?', default=common.TARGETPYTHON, help='The path to the target Python installation') - return parser.parse_args() + return parser def main(args): @@ -189,5 +186,6 @@ def main(args): if __name__ == '__main__': - args = parse_args() + parser = make_parser(argparse.ArgumentParser()) + args = parser.parse_args() main(args) diff --git a/tools/common.py b/pyodide_build/common.py similarity index 92% rename from tools/common.py rename to pyodide_build/common.py index cae533e8b..52c77f4bf 100644 --- a/tools/common.py +++ b/pyodide_build/common.py @@ -1,7 +1,7 @@ from pathlib import Path -ROOTDIR = Path(__file__).parent.resolve() +ROOTDIR = Path(__file__).parents[1].resolve() / 'tools' HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.7.0' / 'host' TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.7.0' DEFAULTCFLAGS = '' diff --git a/tools/pywasmcross b/pyodide_build/pywasmcross.py similarity index 81% rename from tools/pywasmcross rename to pyodide_build/pywasmcross.py index a57dc9c5b..46d5ab5d6 100755 --- a/tools/pywasmcross +++ b/pyodide_build/pywasmcross.py @@ -34,10 +34,12 @@ import subprocess import sys -import common +# absolute import is necessary as this file will be symlinked +# under tools +from pyodide_build import common -ROOTDIR = Path(__file__).parent.resolve() +ROOTDIR = common.ROOTDIR symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc']) @@ -186,7 +188,7 @@ def clean_out_native_artifacts(): path.unlink() -def install_for_distribution(): +def install_for_distribution(args): subprocess.check_call( [Path(args.host) / 'bin' / 'python3', 'setup.py', @@ -202,33 +204,46 @@ def build_wrap(args): capture_compile(args) clean_out_native_artifacts() replay_compile(args) - install_for_distribution() + install_for_distribution(args) -def parse_args(): - parser = argparse.ArgumentParser( - 'Cross compile a Python distutils package. ' - 'Run from the root directory of the package\'s source') - parser.add_argument( - '--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS, - help='Extra compiling flags') - parser.add_argument( - '--ldflags', type=str, nargs='?', default=common.DEFAULTLDFLAGS, - help='Extra linking flags') - parser.add_argument( - '--host', type=str, nargs='?', default=common.HOSTPYTHON, - help='The path to the host Python installation') - parser.add_argument( - '--target', type=str, nargs='?', default=common.TARGETPYTHON, - help='The path to the target Python installation') - args = parser.parse_args() - return args +def make_parser(parser): + basename = Path(sys.argv[0]).name + if basename in symlinks: + # skip parsing of all arguments + parser._actions = [] + else: + parser.description = ( + 'Cross compile a Python distutils package. ' + 'Run from the root directory of the package\'s source') + parser.add_argument( + '--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS, + help='Extra compiling flags') + parser.add_argument( + '--ldflags', type=str, nargs='?', default=common.DEFAULTLDFLAGS, + help='Extra linking flags') + parser.add_argument( + '--host', type=str, nargs='?', default=common.HOSTPYTHON, + help='The path to the host Python installation') + parser.add_argument( + '--target', type=str, nargs='?', default=common.TARGETPYTHON, + help='The path to the target Python installation') + return parser + + +def main(args): + basename = Path(sys.argv[0]).name + if basename in symlinks: + collect_args(basename) + else: + build_wrap(args) if __name__ == '__main__': basename = Path(sys.argv[0]).name if basename in symlinks: - collect_args(basename) + main(None) else: - args = parse_args() - build_wrap(args) + parser = make_parser(argparse.ArgumentParser()) + args = parser.parse_args() + main(args) diff --git a/test/make_test_list.py b/test/make_test_list.py index cfa548c39..b2d2a5cb3 100644 --- a/test/make_test_list.py +++ b/test/make_test_list.py @@ -7,7 +7,7 @@ from pathlib import Path TEST_DIR = (Path(__file__).parent - / "cpython/build/3.6.4/host/lib/python3.7/test") + / "cpython/build/3.7.0/host/lib/python3.7/test") def collect_tests(base_dir): diff --git a/test/test_common.py b/test/test_common.py index 1d5d0fd69..5f7fe60b3 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -1,15 +1,11 @@ import pytest import os from pathlib import Path -import sys +from pyodide_build.common import parse_package BASE_DIR = Path(__file__).parent.parent PKG_DIR = BASE_DIR / 'packages' -# TODO: remove once we have a proper Python package for common functions -sys.path.append(str(BASE_DIR / 'tools')) -import common # noqa - def registered_packages(): """Returns a list of registred package names""" @@ -23,7 +19,7 @@ def registered_packages_meta(): for each registed package """ packages = registered_packages - return {name: common.parse_package(PKG_DIR / name / 'meta.yaml') + return {name: parse_package(PKG_DIR / name / 'meta.yaml') for name in packages} @@ -34,7 +30,7 @@ UNSUPPORTED_PACKAGES = {'chrome': ['pandas'], @pytest.mark.parametrize('name', registered_packages()) def test_import(name, selenium_standalone): # check that we can parse the meta.yaml - meta = common.parse_package(PKG_DIR / name / 'meta.yaml') + meta = parse_package(PKG_DIR / name / 'meta.yaml') if name in UNSUPPORTED_PACKAGES[selenium_standalone.browser]: pytest.xfail(