diff --git a/.circleci/config.yml b/.circleci/config.yml index b54a6ec09..2f7ad8e79 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,6 +50,43 @@ jobs: - store_artifacts: path: /home/circleci/repo/build/ + build-test-minimal: + <<: *defaults + steps: + - checkout + + - run: + name: lint + command: | + make lint + + # this cache is generated by the main build job, we never store it here + - restore_cache: + keys: + - v1-emsdk-{{ checksum "emsdk/Makefile" }}-v14- + + - run: + name: build + no_output_timeout: 1200 + command: | + ccache -z + PYODIDE_PACKAGES='micropip' make + ccache -s + + - run: + name: check-size + command: ls -lh build/ + + - run: + name: test + command: | + export PYODIDE_PACKAGES='micropip' + # only check common tests, all other are checked in the main test jobs + pytest test/test_common.py -v -k firefox + + - store_artifacts: + path: /home/circleci/repo/build/ + test-firefox: <<: *defaults steps: @@ -140,6 +177,7 @@ workflows: filters: tags: only: /.*/ + - build-test-minimal - test-chrome: requires: - build diff --git a/Makefile b/Makefile index 2aa184062..d7f2ea04c 100644 --- a/Makefile +++ b/Makefile @@ -243,7 +243,15 @@ $(PARSO_LIBS): $(CPYTHONLIB) $(CLAPACK): $(CPYTHONLIB) +ifdef PYODIDE_PACKAGES + echo "Skipping BLAS/LAPACK build due to PYODIDE_PACKAGES being defined." + echo "Build it manually with make -C CLAPACK if needed." + mkdir -p CLAPACK/CLAPACK-WA/ + touch $(CLAPACK) +else make -C CLAPACK +endif + build/packages.json: $(CLAPACK) FORCE diff --git a/docs/building_from_sources.md b/docs/building_from_sources.md new file mode 100644 index 000000000..078fad156 --- /dev/null +++ b/docs/building_from_sources.md @@ -0,0 +1,26 @@ +# Building from sources + +To install Pyodide from sources follow the steps in the +[readme](./rootdir.html#building-from-source). + + +## Partial builds + +To build a subset of available packages in pyodide, set the environment +variable `PYODIDE_PACKAGES` to a comma separated list of packages. For +instance, + +``` +PYODIDE_PACKAGES="toolz,attrs" make +``` + +Note that this environment variable must contain both the packages and their +dependencies. The package names must match the folder names in `packages/` +exactly; in particular they are case sensitive. + +To build a minimal version of pyodide, set `PYODIDE_PACKAGES="micropip"`. The +micropip and package is generally always included for any non empty value of +`PYODIDE_PACKAGES`. + +If scipy is included in `PYODIDE_PACKAGES`, BLAS/LAPACK must be manually built +first with `make -c CLAPACK`. diff --git a/docs/index.rst b/docs/index.rst index 7eb677fab..de0615405 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -57,6 +57,7 @@ information about the project's organization. :maxdepth: 1 :caption: Development + building_from_sources.md new_packages.md type_conversions.md diff --git a/packages/Makefile b/packages/Makefile index d7fd22a43..1432ea82f 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -3,7 +3,7 @@ include ../Makefile.envs all: deps ../bin/pyodide buildall . ../build \ - --package_abi=$(PYODIDE_PACKAGE_ABI) --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) + --package_abi=$(PYODIDE_PACKAGE_ABI) --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) --only $(PYODIDE_PACKAGES) deps: # Install build dependencies $(HOSTPYTHON) -m pip install Cython Tempita diff --git a/pyodide_build/buildall.py b/pyodide_build/buildall.py index b64a1b161..162374693 100755 --- a/pyodide_build/buildall.py +++ b/pyodide_build/buildall.py @@ -9,7 +9,6 @@ import json from pathlib import Path import shutil - from . import common from . import buildpkg @@ -33,7 +32,27 @@ def build_packages(packagesdir, outputdir, args): # so first load in all of the package metadata and build a dependency map. dependencies = {} import_name_to_package_name = {} + included_packages = common._parse_package_subset(args.only) + if included_packages is not None: + # check that the specified packages exist + for name in included_packages: + if not (packagesdir / name).exists(): + raise ValueError( + f'package name {name} does not exist. ' + f'The value of PYODIDE_PACKAGES is likely incorrect.' + ) + for pkgdir in packagesdir.iterdir(): + if ( + included_packages is not None + and pkgdir.name not in included_packages + ): + print( + f"Warning: skiping build of {pkgdir.name} due " + f"to specified PYODIDE_PACKAGES" + ) + continue + pkgpath = pkgdir / 'meta.yaml' if pkgdir.is_dir() and pkgpath.is_file(): pkg = common.parse_package(pkgpath) @@ -60,7 +79,10 @@ def build_packages(packagesdir, outputdir, args): def make_parser(parser): - parser.description = "Build all of the packages in a given directory" + parser.description = ( + "Build all of the packages in a given directory\n\n" + "Unless the --only option is provided" + ) parser.add_argument( 'dir', type=str, nargs=1, help='Input directory containing a tree of package definitions') @@ -82,6 +104,10 @@ def make_parser(parser): parser.add_argument( '--target', type=str, nargs='?', default=common.TARGETPYTHON, help='The path to the target Python installation') + parser.add_argument( + '--only', type=str, nargs='?', default=None, + help=('Only build the specified packages, provided as a comma ' + 'separated list')) return parser diff --git a/pyodide_build/common.py b/pyodide_build/common.py index f681b1206..a82c10d30 100644 --- a/pyodide_build/common.py +++ b/pyodide_build/common.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Optional, Set ROOTDIR = Path(__file__).parents[1].resolve() / 'tools' @@ -24,3 +25,19 @@ def parse_package(package): # TODO: Validate against a schema with open(package) as fd: return yaml.load(fd) + + +def _parse_package_subset(query: Optional[str]) -> Optional[Set[str]]: + """Parse the list of packages specified with PYODIDE_PACKAGES env var. + + Also add the list of mandatory packages: ['micropip', 'distlib'] + + Returns: + a set of package names to build or None. + """ + if query is None: + return None + packages = query.split(',') + packages = [el.strip() for el in packages] + packages = ['micropip', 'distlib'] + packages + return set(packages) diff --git a/test/pyodide_build/test_common.py b/test/pyodide_build/test_common.py new file mode 100644 index 000000000..5fd05c1fb --- /dev/null +++ b/test/pyodide_build/test_common.py @@ -0,0 +1,19 @@ +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).parents[2])) + +from pyodide_build.common import _parse_package_subset # noqa + + +def test_parse_package_subset(): + assert _parse_package_subset(None) is None + # micropip is always included + assert _parse_package_subset("numpy,pandas") == { + 'micropip', 'distlib', 'numpy', 'pandas' + } + + # duplicates are removed + assert _parse_package_subset("numpy,numpy") == { + 'micropip', 'distlib', 'numpy' + } diff --git a/test/test_common.py b/test/test_common.py index 980051da1..a8e6aab9b 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -1,7 +1,7 @@ import pytest import os from pathlib import Path -from pyodide_build.common import parse_package +from pyodide_build.common import parse_package, _parse_package_subset BASE_DIR = Path(__file__).parent.parent PKG_DIR = BASE_DIR / 'packages' @@ -49,6 +49,11 @@ def test_import(name, selenium_standalone): '{} fails to load and is not supported on {}.' .format(name, selenium_standalone.browser)) + built_packages = _parse_package_subset(os.environ.get('PYODIDE_PACKAGES')) + # only a subset of packages were built + if built_packages is not None and name not in built_packages: + pytest.skip(f'{name} was skipped due to PYODIDE_PACKAGES') + selenium_standalone.run("import glob, os") baseline_pyc = selenium_standalone.run(