diff --git a/docs/new_packages.md b/docs/new_packages.md index 2b8ccb70f..a959b9081 100644 --- a/docs/new_packages.md +++ b/docs/new_packages.md @@ -90,6 +90,17 @@ source tree (the expanded tarball). ### `build` +#### `build/skip_host` + +Skip building C extensions for the host environment. Default: `True`. + +Setting this to `False` will result in ~2x slower builds for packages that +include C extensions. It should only be needed when a package is a build +time dependency for other packages. For instance, numpy is imported during +installation of matplotlib, importing numpy also imports included C extensions, +therefore it is built both for host and target. + + #### `build/cflags` Extra arguments to pass to the compiler when building for WebAssembly. diff --git a/packages/numpy/meta.yaml b/packages/numpy/meta.yaml index 2effe8f43..201821f0b 100644 --- a/packages/numpy/meta.yaml +++ b/packages/numpy/meta.yaml @@ -18,6 +18,7 @@ source: - patches/fix-install-with-skip-build.patch build: + skip_host: False cflags: -include math.h -I../../config diff --git a/pyodide_build/buildpkg.py b/pyodide_build/buildpkg.py index 3aa0e720e..d92e0ff97 100755 --- a/pyodide_build/buildpkg.py +++ b/pyodide_build/buildpkg.py @@ -82,6 +82,10 @@ def compile(path, srcpath, pkg, args): orig_dir = Path.cwd() os.chdir(srcpath) + env = dict(os.environ) + if pkg.get('build', {}).get('skip_host', True): + env['SKIP_HOST'] = '' + try: subprocess.run([ str(Path(args.host) / 'bin' / 'python3'), @@ -93,7 +97,7 @@ def compile(path, srcpath, pkg, args): args.ldflags + ' ' + pkg.get('build', {}).get('ldflags', ''), '--host', args.host, - '--target', args.target], check=True) + '--target', args.target], env=env, check=True) finally: os.chdir(orig_dir) diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index 23db1479a..e2d02a371 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -60,10 +60,37 @@ def collect_args(basename): path = path.replace(str(ROOTDIR) + ':', '') env['PATH'] = path + skip_host = 'SKIP_HOST' in os.environ + + # Skip compilations of C/Fortran extensions for the target environement. + # We still need to generate the output files for distutils to continue + # the build. + # TODO: This may need slight tuning for new projects. In particular, + # currently ar is not skipped, so a known failure would happen when + # we create some object files (that are empty as gcc is skipped), on + # which we run the actual ar command. + skip = False + if (basename in ['gcc', 'cc', 'c++', 'gfortran', 'ld'] + and '-o' in sys.argv[1:] + # do not skip numpy as it is needed as build time + # dependency by other packages (e.g. matplotlib) + and skip_host): + out_idx = sys.argv.index('-o') + if (out_idx + 1) < len(sys.argv): + # get the index of the output file path + out_idx += 1 + with open(sys.argv[out_idx], 'wb') as fh: + fh.write(b'') + skip = True + with open('build.log', 'a') as fd: + # TODO: store skip status in the build.log json.dump([basename] + sys.argv[1:], fd) fd.write('\n') + if skip: + sys.exit(0) + sys.exit( subprocess.run( [basename] + sys.argv[1:], diff --git a/test/test_common.py b/test/test_common.py index c5ea3c966..f5d04f376 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -27,6 +27,18 @@ UNSUPPORTED_PACKAGES = {'chrome': ['pandas'], 'firefox': []} +@pytest.mark.parametrize('name', registered_packages()) +def test_parse_package(name): + # check that we can parse the meta.yaml + meta = parse_package(PKG_DIR / name / 'meta.yaml') + + skip_host = meta.get('build', {}).get('skip_host', True) + if name == 'numpy': + assert skip_host is False + elif name == 'pandas': + assert skip_host is True + + @pytest.mark.parametrize('name', registered_packages()) def test_import(name, selenium_standalone): # check that we can parse the meta.yaml