From 72293058b94808d417918393a6086bdd4803f896 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Thu, 20 Sep 2018 16:42:38 +0200 Subject: [PATCH 01/11] Converting tools scripts to a python package --- .gitignore | 1 + Makefile | 2 ++ packages/Makefile | 2 +- pyodide_build/__init__.py | 1 + tools/buildall => pyodide_build/buildall.py | 14 ++++---- {tools => pyodide_build}/buildpkg.py | 19 +++++----- {tools => pyodide_build}/common.py | 7 ++-- .../pywasmcross.py | 35 +++++++++++-------- setup.py | 16 +++++++++ 9 files changed, 62 insertions(+), 35 deletions(-) create mode 100644 pyodide_build/__init__.py rename tools/buildall => pyodide_build/buildall.py (92%) rename {tools => pyodide_build}/buildpkg.py (94%) rename {tools => pyodide_build}/common.py (73%) rename tools/pywasmcross => pyodide_build/pywasmcross.py (90%) create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 6401e1112..200eb5c37 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ ccache /six/six-1.11.0 /lz4/lz4-1.8.3 +*.egg-info/ diff --git a/Makefile b/Makefile index 91839a04b..a616e787b 100644 --- a/Makefile +++ b/Makefile @@ -204,6 +204,8 @@ ccache/em++: $(CPYTHONLIB): emsdk/emsdk/.complete ccache/emcc ccache/em++ make -C $(CPYTHONROOT) + python -m pip install -e . + "$(HOSTPYTHONROOT)/bin/python3" -m pip install -e . $(LZ4LIB): diff --git a/packages/Makefile b/packages/Makefile index 22552d59e..2e879dda3 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -2,7 +2,7 @@ PYODIDE_ROOT=$(abspath ..) include ../Makefile.envs all: - ../tools/buildall . ../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) + pyodide buildall . ../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) clean: rm -rf ./*/build diff --git a/pyodide_build/__init__.py b/pyodide_build/__init__.py new file mode 100644 index 000000000..b794fd409 --- /dev/null +++ b/pyodide_build/__init__.py @@ -0,0 +1 @@ +__version__ = '0.1.0' diff --git a/tools/buildall b/pyodide_build/buildall.py similarity index 92% rename from tools/buildall rename to pyodide_build/buildall.py index 1d50f3543..cc74b439b 100755 --- a/tools/buildall +++ b/pyodide_build/buildall.py @@ -10,8 +10,8 @@ from pathlib import Path import shutil -import common -import buildpkg +from . import common +from . import buildpkg def build_package(pkgname, dependencies, packagesdir, outputdir, args): @@ -54,9 +54,8 @@ def build_packages(packagesdir, outputdir, args): json.dump({'dependencies': dependencies}, fd) -def parse_args(): - parser = argparse.ArgumentParser( - "Build all of the packages in a given directory") +def make_parser(parser): + parser.description = "Build all of the packages in a given directory" parser.add_argument( 'dir', type=str, nargs=1, help='Input directory containing a tree of package definitions') @@ -75,7 +74,7 @@ def parse_args(): parser.add_argument( '--target', type=str, nargs='?', default=common.TARGETPYTHON, help='The path to the target Python installation') - return parser.parse_args() + return parser def main(args): @@ -85,5 +84,6 @@ def main(args): if __name__ == '__main__': - args = parse_args() + parser = make_parser(argparse.ArgumentParser()) + args = parser.parse_args() main(args) diff --git a/tools/buildpkg.py b/pyodide_build/buildpkg.py similarity index 94% rename from tools/buildpkg.py rename to pyodide_build/buildpkg.py index a4ee5c546..fd2de9f5b 100755 --- a/tools/buildpkg.py +++ b/pyodide_build/buildpkg.py @@ -12,10 +12,7 @@ import shutil import subprocess -import common - - -ROOTDIR = Path(__file__).parent.resolve() +from . import common def check_checksum(path, pkg): @@ -88,7 +85,8 @@ def compile(path, srcpath, pkg, args): try: subprocess.run([ str(Path(args.host) / 'bin' / 'python3'), - str(ROOTDIR / 'pywasmcross'), + '-m', 'pyodide_build', + 'pywasmcross', '--cflags', args.cflags + ' ' + pkg.get('build', {}).get('cflags', ''), @@ -124,7 +122,7 @@ def package_files(buildpath, srcpath, pkg, args): install_prefix = (srcpath / 'install').resolve() subprocess.run([ 'python', - Path(ROOTDIR) / 'file_packager.py', + common.TOOLSDIR / 'file_packager.py', name + '.data', '--lz4', '--preload', @@ -163,8 +161,8 @@ def build_package(path, args): os.chdir(orig_path) -def parse_args(): - parser = argparse.ArgumentParser('Build a pyodide package.') +def make_parser(parser): + parser.description = 'Build a pyodide package.' parser.add_argument( 'package', type=str, nargs=1, help="Path to meta.yaml package description") @@ -180,7 +178,7 @@ def parse_args(): parser.add_argument( '--target', type=str, nargs='?', default=common.TARGETPYTHON, help='The path to the target Python installation') - return parser.parse_args() + return parser def main(args): @@ -189,5 +187,6 @@ def main(args): if __name__ == '__main__': - args = parse_args() + parser = make_parser(argparse.ArgumentParser()) + args = parser.parse_args() main(args) diff --git a/tools/common.py b/pyodide_build/common.py similarity index 73% rename from tools/common.py rename to pyodide_build/common.py index cae533e8b..0afce09a5 100644 --- a/tools/common.py +++ b/pyodide_build/common.py @@ -1,9 +1,10 @@ from pathlib import Path -ROOTDIR = Path(__file__).parent.resolve() -HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.7.0' / 'host' -TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.7.0' +ROOTDIR = Path(__file__).parents[1].resolve() +TOOLSDIR = ROOTDIR / 'tools' +HOSTPYTHON = ROOTDIR / 'cpython' / 'build' / '3.7.0' / 'host' +TARGETPYTHON = ROOTDIR / 'cpython' / 'installs' / 'python-3.7.0' DEFAULTCFLAGS = '' DEFAULTLDFLAGS = ' '.join([ '-O3', diff --git a/tools/pywasmcross b/pyodide_build/pywasmcross.py similarity index 90% rename from tools/pywasmcross rename to pyodide_build/pywasmcross.py index a57dc9c5b..e2f5acd92 100755 --- a/tools/pywasmcross +++ b/pyodide_build/pywasmcross.py @@ -34,10 +34,12 @@ import subprocess import sys -import common +# absolute import is necessary as this file will be symlinked +# under tools +from pyodide_build import common -ROOTDIR = Path(__file__).parent.resolve() +TOOLSDIR = common.TOOLSDIR symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc']) @@ -53,8 +55,8 @@ def collect_args(basename): # native compiler env = dict(os.environ) path = env['PATH'] - while str(ROOTDIR) + ':' in path: - path = path.replace(str(ROOTDIR) + ':', '') + while str(TOOLSDIR) + ':' in path: + path = path.replace(str(TOOLSDIR) + ':', '') env['PATH'] = path with open('build.log', 'a') as fd: @@ -74,7 +76,7 @@ def make_symlinks(env): """ exec_path = Path(__file__).resolve() for symlink in symlinks: - symlink_path = ROOTDIR / symlink + symlink_path = TOOLSDIR / symlink if not symlink_path.exists(): symlink_path.symlink_to(exec_path) if symlink == 'c++': @@ -87,7 +89,7 @@ def make_symlinks(env): def capture_compile(args): env = dict(os.environ) make_symlinks(env) - env['PATH'] = str(ROOTDIR) + ':' + os.environ['PATH'] + env['PATH'] = str(TOOLSDIR) + ':' + os.environ['PATH'] result = subprocess.run( [Path(args.host) / 'bin' / 'python3', @@ -186,7 +188,7 @@ def clean_out_native_artifacts(): path.unlink() -def install_for_distribution(): +def install_for_distribution(args): subprocess.check_call( [Path(args.host) / 'bin' / 'python3', 'setup.py', @@ -202,11 +204,11 @@ def build_wrap(args): capture_compile(args) clean_out_native_artifacts() replay_compile(args) - install_for_distribution() + install_for_distribution(args) -def parse_args(): - parser = argparse.ArgumentParser( +def make_parser(parser): + parser.description = ( 'Cross compile a Python distutils package. ' 'Run from the root directory of the package\'s source') parser.add_argument( @@ -221,14 +223,19 @@ def parse_args(): parser.add_argument( '--target', type=str, nargs='?', default=common.TARGETPYTHON, help='The path to the target Python installation') - args = parser.parse_args() - return args + parser.add_argument('basename', type=str, nargs='?') + return parser -if __name__ == '__main__': +def main(args, unknown=None): basename = Path(sys.argv[0]).name if basename in symlinks: collect_args(basename) else: - args = parse_args() build_wrap(args) + + +if __name__ == '__main__': + parser = make_parser(argparse.ArgumentParser()) + args, unknown = parser.parse_known_args() + main(args, unknown) diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..89e67f2d8 --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, find_packages +from pyodide_build import __version__ + +with open('README.md', 'rt') as fh: + LONG_DESCRIPTION = fh.read() + +setup(name='pyodide_build', + version=__version__, + description='pyodide builder', + entry_points={ + 'console_scripts': [ + 'pyodide = pyodide_build.__main__:main' + ]}, + url="https://github.com/iodide-project/pyodide", + license='MPL', + packages=find_packages()) From 64ab6761713a508caa7e50821e956e954da39b55 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Thu, 20 Sep 2018 18:49:50 +0200 Subject: [PATCH 02/11] Add pyodide_build/__main__.py --- pyodide_build/__main__.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 pyodide_build/__main__.py diff --git a/pyodide_build/__main__.py b/pyodide_build/__main__.py new file mode 100644 index 000000000..375b15d6f --- /dev/null +++ b/pyodide_build/__main__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import argparse + +from . import buildall +from . import buildpkg +from . import pywasmcross + + +def main(): + main_parser = argparse.ArgumentParser() + subparsers = main_parser.add_subparsers(help='action') + + for command_name, module in (("buildpkg", buildpkg), + ("buildall", buildall), + ("pywasmcross", pywasmcross)): + parser = module.make_parser(subparsers.add_parser(command_name)) + parser.set_defaults(func=module.main) + + args = main_parser.parse_args() + # run the selected action + args.func(args) + + +if __name__ == '__main__': + main() From 3ee42aa779029fe5ef729747160dd7c3d157b125 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Thu, 20 Sep 2018 19:37:32 +0200 Subject: [PATCH 03/11] Use pyodide_build in tests --- .circleci/config.yml | 3 +++ pyodide_build/pywasmcross.py | 1 - test/make_test_list.py | 2 +- test/test_common.py | 10 +++------- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d803cbf2e..1c7464a1b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,6 +63,9 @@ jobs: ccache -z make ccache -s + # install all relevant pyodide_build files to + # the virtualenv that will be persisted + pip install . - save_cache: paths: diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index e2f5acd92..290b9ac14 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -223,7 +223,6 @@ def make_parser(parser): parser.add_argument( '--target', type=str, nargs='?', default=common.TARGETPYTHON, help='The path to the target Python installation') - parser.add_argument('basename', type=str, nargs='?') return parser diff --git a/test/make_test_list.py b/test/make_test_list.py index cfa548c39..b2d2a5cb3 100644 --- a/test/make_test_list.py +++ b/test/make_test_list.py @@ -7,7 +7,7 @@ from pathlib import Path TEST_DIR = (Path(__file__).parent - / "cpython/build/3.6.4/host/lib/python3.7/test") + / "cpython/build/3.7.0/host/lib/python3.7/test") def collect_tests(base_dir): diff --git a/test/test_common.py b/test/test_common.py index 1d5d0fd69..5f7fe60b3 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -1,15 +1,11 @@ import pytest import os from pathlib import Path -import sys +from pyodide_build.common import parse_package BASE_DIR = Path(__file__).parent.parent PKG_DIR = BASE_DIR / 'packages' -# TODO: remove once we have a proper Python package for common functions -sys.path.append(str(BASE_DIR / 'tools')) -import common # noqa - def registered_packages(): """Returns a list of registred package names""" @@ -23,7 +19,7 @@ def registered_packages_meta(): for each registed package """ packages = registered_packages - return {name: common.parse_package(PKG_DIR / name / 'meta.yaml') + return {name: parse_package(PKG_DIR / name / 'meta.yaml') for name in packages} @@ -34,7 +30,7 @@ UNSUPPORTED_PACKAGES = {'chrome': ['pandas'], @pytest.mark.parametrize('name', registered_packages()) def test_import(name, selenium_standalone): # check that we can parse the meta.yaml - meta = common.parse_package(PKG_DIR / name / 'meta.yaml') + meta = parse_package(PKG_DIR / name / 'meta.yaml') if name in UNSUPPORTED_PACKAGES[selenium_standalone.browser]: pytest.xfail( From da6bbb3b20389bd81ed6ff0029bbc8d5e6389112 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 21 Sep 2018 12:02:13 +0200 Subject: [PATCH 04/11] Set PYTHONPATH in Makefile.envs to avoiding installing pyodide_build --- .circleci/config.yml | 3 --- Makefile | 2 -- Makefile.envs | 1 + packages/Makefile | 3 ++- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c7464a1b..d803cbf2e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,9 +63,6 @@ jobs: ccache -z make ccache -s - # install all relevant pyodide_build files to - # the virtualenv that will be persisted - pip install . - save_cache: paths: diff --git a/Makefile b/Makefile index a616e787b..91839a04b 100644 --- a/Makefile +++ b/Makefile @@ -204,8 +204,6 @@ ccache/em++: $(CPYTHONLIB): emsdk/emsdk/.complete ccache/emcc ccache/em++ make -C $(CPYTHONROOT) - python -m pip install -e . - "$(HOSTPYTHONROOT)/bin/python3" -m pip install -e . $(LZ4LIB): diff --git a/Makefile.envs b/Makefile.envs index 0e93fe0fc..3cd1d5e52 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -1,4 +1,5 @@ export PATH := $(PYODIDE_ROOT)/ccache:$(PYODIDE_ROOT)/emsdk/emsdk:$(PYODIDE_ROOT)/emsdk/emsdk/clang/tag-e1.38.10/build_tag-e1.38.10_64/bin:$(PYODIDE_ROOT)/emsdk/emsdk/node/8.9.1_64bit/bin:$(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-1.38.10:$(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-1.38.10_64bit_binaryen/bin:$(PATH) +export PYTHONPATH := $(PYTHONPATH):$(PYODIDE_ROOT)/ export EMSDK = $(PYODIDE_ROOT)/emsdk/emsdk export EM_CONFIG = $(PYODIDE_ROOT)/emsdk/emsdk/.emscripten diff --git a/packages/Makefile b/packages/Makefile index 2e879dda3..569939cf7 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -2,7 +2,8 @@ PYODIDE_ROOT=$(abspath ..) include ../Makefile.envs all: - pyodide buildall . ../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) + python -m pyodide_build buildall . ../build \ + --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) clean: rm -rf ./*/build From ef3f4d2e94acf04b5a71612ef269fa7fb64b840a Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 21 Sep 2018 12:19:40 +0200 Subject: [PATCH 05/11] Only allow develop mode installation of pyodide_build --- setup.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 89e67f2d8..daf8a7180 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,19 @@ -from setuptools import setup, find_packages +from setuptools import setup +import sys from pyodide_build import __version__ +if 'install' in sys.argv or 'bdist_wheel' in sys.argv: + print("Error: pyodode_build is currently for fully standalone, " + "and can only be installed in develop mode. Use:\n" + " pip install -e . \n" + "to install it.") + sys.exit(1) + + with open('README.md', 'rt') as fh: LONG_DESCRIPTION = fh.read() + setup(name='pyodide_build', version=__version__, description='pyodide builder', @@ -13,4 +23,4 @@ setup(name='pyodide_build', ]}, url="https://github.com/iodide-project/pyodide", license='MPL', - packages=find_packages()) + packages=['pyodide_build']) From 8973c28642dea5c4c8848c8d7128d6f9ca07f3cd Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 21 Sep 2018 12:23:43 +0200 Subject: [PATCH 06/11] Install Crome/Firefox in corresponding test jobs --- .circleci/config.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d803cbf2e..2d3950775 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,16 +34,6 @@ jobs: pip install pytest pytest-xdist pytest-instafail selenium PyYAML - # Get recent version of Firefox and geckodriver - wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-latest-ssl\&os\=linux64\&lang\=en-US - tar jxf firefox.tar.bz2 - wget https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-linux64.tar.gz - tar zxf geckodriver-v0.21.0-linux64.tar.gz -C firefox - - # Get recent version of chromedriver - wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip - unzip chromedriver_linux64.zip - mv chromedriver firefox - run: name: lint @@ -75,7 +65,6 @@ jobs: paths: - ./build - ./pyodide-env - - ./firefox - store_artifacts: path: /home/circleci/repo/build/ @@ -93,6 +82,12 @@ jobs: # causes Firefox to complain when loading it. Let's just add the new mime type. sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types" + # Get recent version of Firefox and geckodriver + wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-latest-ssl\&os\=linux64\&lang\=en-US + tar jxf firefox.tar.bz2 + wget https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-linux64.tar.gz + tar zxf geckodriver-v0.21.0-linux64.tar.gz -C firefox + source pyodide-env/bin/activate export PATH=$PWD/firefox:$PATH pytest test -v -k firefox @@ -110,8 +105,12 @@ jobs: # causes Firefox to complain when loading it. Let's just add the new mime type. sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types" + # Get recent version of chromedriver + wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip + unzip chromedriver_linux64.zip + mv chromedriver pyodide-env/bin/ + source pyodide-env/bin/activate - export PATH=$PWD/firefox:$PATH pytest test -v -k chrome deploy: From 0c6b653fd01fc00a76b88ec8c70b8bf9266d1372 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 21 Sep 2018 13:30:28 +0200 Subject: [PATCH 07/11] Revert using ROOTDIR instead of TOOLSDIR --- pyodide_build/buildpkg.py | 2 +- pyodide_build/common.py | 7 +++---- pyodide_build/pywasmcross.py | 10 +++++----- setup.py | 4 ++-- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pyodide_build/buildpkg.py b/pyodide_build/buildpkg.py index fd2de9f5b..c1d569188 100755 --- a/pyodide_build/buildpkg.py +++ b/pyodide_build/buildpkg.py @@ -122,7 +122,7 @@ def package_files(buildpath, srcpath, pkg, args): install_prefix = (srcpath / 'install').resolve() subprocess.run([ 'python', - common.TOOLSDIR / 'file_packager.py', + common.ROOTDIR / 'file_packager.py', name + '.data', '--lz4', '--preload', diff --git a/pyodide_build/common.py b/pyodide_build/common.py index 0afce09a5..52c77f4bf 100644 --- a/pyodide_build/common.py +++ b/pyodide_build/common.py @@ -1,10 +1,9 @@ from pathlib import Path -ROOTDIR = Path(__file__).parents[1].resolve() -TOOLSDIR = ROOTDIR / 'tools' -HOSTPYTHON = ROOTDIR / 'cpython' / 'build' / '3.7.0' / 'host' -TARGETPYTHON = ROOTDIR / 'cpython' / 'installs' / 'python-3.7.0' +ROOTDIR = Path(__file__).parents[1].resolve() / 'tools' +HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.7.0' / 'host' +TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.7.0' DEFAULTCFLAGS = '' DEFAULTLDFLAGS = ' '.join([ '-O3', diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index 290b9ac14..e05b638a9 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -39,7 +39,7 @@ import sys from pyodide_build import common -TOOLSDIR = common.TOOLSDIR +ROOTDIR = common.ROOTDIR symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc']) @@ -55,8 +55,8 @@ def collect_args(basename): # native compiler env = dict(os.environ) path = env['PATH'] - while str(TOOLSDIR) + ':' in path: - path = path.replace(str(TOOLSDIR) + ':', '') + while str(ROOTDIR) + ':' in path: + path = path.replace(str(ROOTDIR) + ':', '') env['PATH'] = path with open('build.log', 'a') as fd: @@ -76,7 +76,7 @@ def make_symlinks(env): """ exec_path = Path(__file__).resolve() for symlink in symlinks: - symlink_path = TOOLSDIR / symlink + symlink_path = ROOTDIR / symlink if not symlink_path.exists(): symlink_path.symlink_to(exec_path) if symlink == 'c++': @@ -89,7 +89,7 @@ def make_symlinks(env): def capture_compile(args): env = dict(os.environ) make_symlinks(env) - env['PATH'] = str(TOOLSDIR) + ':' + os.environ['PATH'] + env['PATH'] = str(ROOTDIR) + ':' + os.environ['PATH'] result = subprocess.run( [Path(args.host) / 'bin' / 'python3', diff --git a/setup.py b/setup.py index daf8a7180..c16e331a6 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,8 @@ import sys from pyodide_build import __version__ if 'install' in sys.argv or 'bdist_wheel' in sys.argv: - print("Error: pyodode_build is currently for fully standalone, " - "and can only be installed in develop mode. Use:\n" + print("Error: pyodode_build is currently not fully standalone, " + "and can only be installed in development mode. Use:\n" " pip install -e . \n" "to install it.") sys.exit(1) From 3b429e61f3abfd5ac88e56ed60f9d730faa8ed50 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 21 Sep 2018 16:58:09 +0200 Subject: [PATCH 08/11] Address review comments --- Makefile.envs | 1 - bin/pyodide | 7 ++++++ packages/Makefile | 2 +- pyodide_build/__main__.py | 2 +- pyodide_build/buildpkg.py | 3 +-- pyodide_build/pywasmcross.py | 47 +++++++++++++++++++++--------------- setup.py | 26 -------------------- 7 files changed, 38 insertions(+), 50 deletions(-) create mode 100755 bin/pyodide delete mode 100644 setup.py diff --git a/Makefile.envs b/Makefile.envs index 3cd1d5e52..0e93fe0fc 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -1,5 +1,4 @@ export PATH := $(PYODIDE_ROOT)/ccache:$(PYODIDE_ROOT)/emsdk/emsdk:$(PYODIDE_ROOT)/emsdk/emsdk/clang/tag-e1.38.10/build_tag-e1.38.10_64/bin:$(PYODIDE_ROOT)/emsdk/emsdk/node/8.9.1_64bit/bin:$(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-1.38.10:$(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-1.38.10_64bit_binaryen/bin:$(PATH) -export PYTHONPATH := $(PYTHONPATH):$(PYODIDE_ROOT)/ export EMSDK = $(PYODIDE_ROOT)/emsdk/emsdk export EM_CONFIG = $(PYODIDE_ROOT)/emsdk/emsdk/.emscripten diff --git a/bin/pyodide b/bin/pyodide new file mode 100755 index 000000000..cac8846bf --- /dev/null +++ b/bin/pyodide @@ -0,0 +1,7 @@ +#!/bin/sh + +# get the absolute path to the root directory +ROOTDIR="$(cd ${BASH_SOURCE%/*}/../; pwd)"; + +export PYTHONPATH="${PYTHONPATH}:${ROOTDIR}" +python -m pyodide_build "$@" diff --git a/packages/Makefile b/packages/Makefile index 569939cf7..bee738225 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -2,7 +2,7 @@ PYODIDE_ROOT=$(abspath ..) include ../Makefile.envs all: - python -m pyodide_build buildall . ../build \ + ../bin/pyodide buildall . ../build \ --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) clean: diff --git a/pyodide_build/__main__.py b/pyodide_build/__main__.py index 375b15d6f..ad5c7eb75 100644 --- a/pyodide_build/__main__.py +++ b/pyodide_build/__main__.py @@ -7,7 +7,7 @@ from . import pywasmcross def main(): - main_parser = argparse.ArgumentParser() + main_parser = argparse.ArgumentParser(prog='pyodide') subparsers = main_parser.add_subparsers(help='action') for command_name, module in (("buildpkg", buildpkg), diff --git a/pyodide_build/buildpkg.py b/pyodide_build/buildpkg.py index c1d569188..3aa0e720e 100755 --- a/pyodide_build/buildpkg.py +++ b/pyodide_build/buildpkg.py @@ -85,8 +85,7 @@ def compile(path, srcpath, pkg, args): try: subprocess.run([ str(Path(args.host) / 'bin' / 'python3'), - '-m', 'pyodide_build', - 'pywasmcross', + '-m', 'pyodide_build', 'pywasmcross', '--cflags', args.cflags + ' ' + pkg.get('build', {}).get('cflags', ''), diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index e05b638a9..46d5ab5d6 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -208,25 +208,30 @@ def build_wrap(args): def make_parser(parser): - parser.description = ( - 'Cross compile a Python distutils package. ' - 'Run from the root directory of the package\'s source') - parser.add_argument( - '--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS, - help='Extra compiling flags') - parser.add_argument( - '--ldflags', type=str, nargs='?', default=common.DEFAULTLDFLAGS, - help='Extra linking flags') - parser.add_argument( - '--host', type=str, nargs='?', default=common.HOSTPYTHON, - help='The path to the host Python installation') - parser.add_argument( - '--target', type=str, nargs='?', default=common.TARGETPYTHON, - help='The path to the target Python installation') + basename = Path(sys.argv[0]).name + if basename in symlinks: + # skip parsing of all arguments + parser._actions = [] + else: + parser.description = ( + 'Cross compile a Python distutils package. ' + 'Run from the root directory of the package\'s source') + parser.add_argument( + '--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS, + help='Extra compiling flags') + parser.add_argument( + '--ldflags', type=str, nargs='?', default=common.DEFAULTLDFLAGS, + help='Extra linking flags') + parser.add_argument( + '--host', type=str, nargs='?', default=common.HOSTPYTHON, + help='The path to the host Python installation') + parser.add_argument( + '--target', type=str, nargs='?', default=common.TARGETPYTHON, + help='The path to the target Python installation') return parser -def main(args, unknown=None): +def main(args): basename = Path(sys.argv[0]).name if basename in symlinks: collect_args(basename) @@ -235,6 +240,10 @@ def main(args, unknown=None): if __name__ == '__main__': - parser = make_parser(argparse.ArgumentParser()) - args, unknown = parser.parse_known_args() - main(args, unknown) + basename = Path(sys.argv[0]).name + if basename in symlinks: + main(None) + else: + parser = make_parser(argparse.ArgumentParser()) + args = parser.parse_args() + main(args) diff --git a/setup.py b/setup.py deleted file mode 100644 index c16e331a6..000000000 --- a/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -from setuptools import setup -import sys -from pyodide_build import __version__ - -if 'install' in sys.argv or 'bdist_wheel' in sys.argv: - print("Error: pyodode_build is currently not fully standalone, " - "and can only be installed in development mode. Use:\n" - " pip install -e . \n" - "to install it.") - sys.exit(1) - - -with open('README.md', 'rt') as fh: - LONG_DESCRIPTION = fh.read() - - -setup(name='pyodide_build', - version=__version__, - description='pyodide builder', - entry_points={ - 'console_scripts': [ - 'pyodide = pyodide_build.__main__:main' - ]}, - url="https://github.com/iodide-project/pyodide", - license='MPL', - packages=['pyodide_build']) From 22e99f12d1f79b8f91455814adc586affd0df38b Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 24 Sep 2018 14:04:28 +0200 Subject: [PATCH 09/11] Debug failing build --- bin/pyodide | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/pyodide b/bin/pyodide index cac8846bf..5273734f8 100755 --- a/bin/pyodide +++ b/bin/pyodide @@ -1,4 +1,7 @@ #!/bin/sh +set -x + +ls -l /bin/sh # get the absolute path to the root directory ROOTDIR="$(cd ${BASH_SOURCE%/*}/../; pwd)"; From 71e495e20a004167e23834f06d2488d6456d2d12 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 24 Sep 2018 21:44:08 +0200 Subject: [PATCH 10/11] Use Path.resolve() to get the absolute path of bin/pyodide --- bin/pyodide | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/pyodide b/bin/pyodide index 5273734f8..adac9f553 100755 --- a/bin/pyodide +++ b/bin/pyodide @@ -1,10 +1,9 @@ #!/bin/sh set -x -ls -l /bin/sh - # get the absolute path to the root directory -ROOTDIR="$(cd ${BASH_SOURCE%/*}/../; pwd)"; +ROOTDIR=$(python -c 'import pathlib, sys; print(pathlib.Path(sys.argv[1]).resolve().parent)' \ + "${BASH_SOURCE[0]}") export PYTHONPATH="${PYTHONPATH}:${ROOTDIR}" python -m pyodide_build "$@" From bde0b96416016a958b1690492c02ecc4b4ab83c0 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Tue, 25 Sep 2018 11:30:22 +0200 Subject: [PATCH 11/11] Use bash instead of dash shell on ubuntu --- bin/pyodide | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/pyodide b/bin/pyodide index adac9f553..653c925c9 100755 --- a/bin/pyodide +++ b/bin/pyodide @@ -1,8 +1,8 @@ -#!/bin/sh -set -x +#!/bin/bash # get the absolute path to the root directory -ROOTDIR=$(python -c 'import pathlib, sys; print(pathlib.Path(sys.argv[1]).resolve().parent)' \ +ROOTDIR=$(python -c 'import pathlib, sys; \ + print(pathlib.Path(sys.argv[1]).resolve().parents[1])' \ "${BASH_SOURCE[0]}") export PYTHONPATH="${PYTHONPATH}:${ROOTDIR}"