Merge pull request #645 from rth/partial-build

Support partial builds

.circleci/config.yml

@@ -50,6 +50,43 @@ jobs:
       - store_artifacts:
           path: /home/circleci/repo/build/
 
+  build-test-minimal:
+    <<: *defaults
+    steps:
+      - checkout
+
+      - run:
+          name: lint
+          command: |
+            make lint
+
+      # this cache is generated by the main build job, we never store it here
+      - restore_cache:
+          keys:
+          - v1-emsdk-{{ checksum "emsdk/Makefile" }}-v14-
+
+      - run:
+          name: build
+          no_output_timeout: 1200
+          command: |
+            ccache -z
+            PYODIDE_PACKAGES='micropip' make
+            ccache -s
+
+      - run:
+          name: check-size
+          command: ls -lh build/
+
+      - run:
+          name: test
+          command: |
+            export PYODIDE_PACKAGES='micropip'
+            # only check common tests, all other are checked in the main test jobs
+            pytest test/test_common.py -v -k firefox
+
+      - store_artifacts:
+          path: /home/circleci/repo/build/
+
   test-firefox:
     <<: *defaults
     steps:
@@ -140,6 +177,7 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - build-test-minimal
       - test-chrome:
           requires:
             - build

Makefile

@@ -243,7 +243,15 @@ $(PARSO_LIBS): $(CPYTHONLIB)
 
 
 $(CLAPACK): $(CPYTHONLIB)
+ifdef PYODIDE_PACKAGES
+	echo "Skipping BLAS/LAPACK build due to PYODIDE_PACKAGES being defined."
+	echo "Build it manually with make -C CLAPACK if needed."
+	mkdir -p CLAPACK/CLAPACK-WA/
+	touch $(CLAPACK)
+else
 	make -C CLAPACK
+endif
+
 
 
 build/packages.json: $(CLAPACK) FORCE

docs/building_from_sources.md

@@ -0,0 +1,26 @@
+# Building from sources
+
+To install Pyodide from sources follow the steps in the
+[readme](./rootdir.html#building-from-source).
+
+
+## Partial builds
+
+To build a subset of available packages in pyodide, set the environment
+variable `PYODIDE_PACKAGES` to a comma separated list of packages. For
+instance,
+
+```
+PYODIDE_PACKAGES="toolz,attrs" make
+```
+
+Note that this environment variable must contain both the packages and their
+dependencies. The package names must match the folder names in `packages/`
+exactly; in particular they are case sensitive.
+
+To build a minimal version of pyodide, set `PYODIDE_PACKAGES="micropip"`. The
+micropip and package is generally always included for any non empty value of
+`PYODIDE_PACKAGES`.
+
+If scipy is included in `PYODIDE_PACKAGES`, BLAS/LAPACK must be manually built
+first with `make -c CLAPACK`.

docs/index.rst

@@ -57,6 +57,7 @@ information about the project's organization.
    :maxdepth: 1
    :caption: Development
 
+   building_from_sources.md
    new_packages.md
    type_conversions.md
 

packages/Makefile

@@ -3,7 +3,7 @@ include ../Makefile.envs
 
 all: deps
 	../bin/pyodide buildall . ../build \
-		--package_abi=$(PYODIDE_PACKAGE_ABI) --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
+		--package_abi=$(PYODIDE_PACKAGE_ABI) --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT) --only $(PYODIDE_PACKAGES)
 deps:
 	# Install build dependencies
 	$(HOSTPYTHON) -m pip install Cython Tempita

pyodide_build/buildall.py

@@ -9,7 +9,6 @@ import json
 from pathlib import Path
 import shutil
 
-
 from . import common
 from . import buildpkg
 
@@ -33,7 +32,27 @@ def build_packages(packagesdir, outputdir, args):
     # so first load in all of the package metadata and build a dependency map.
     dependencies = {}
     import_name_to_package_name = {}
+    included_packages = common._parse_package_subset(args.only)
+    if included_packages is not None:
+        # check that the specified packages exist
+        for name in included_packages:
+            if not (packagesdir / name).exists():
+                raise ValueError(
+                    f'package name {name} does not exist. '
+                    f'The value of PYODIDE_PACKAGES is likely incorrect.'
+                )
+
     for pkgdir in packagesdir.iterdir():
+        if (
+            included_packages is not None
+            and pkgdir.name not in included_packages
+        ):
+            print(
+                f"Warning: skiping build of {pkgdir.name} due "
+                f"to specified PYODIDE_PACKAGES"
+            )
+            continue
+
         pkgpath = pkgdir / 'meta.yaml'
         if pkgdir.is_dir() and pkgpath.is_file():
             pkg = common.parse_package(pkgpath)
@@ -60,7 +79,10 @@ def build_packages(packagesdir, outputdir, args):
 
 
 def make_parser(parser):
-    parser.description = "Build all of the packages in a given directory"
+    parser.description = (
+        "Build all of the packages in a given directory\n\n"
+        "Unless the --only option is provided"
+    )
     parser.add_argument(
         'dir', type=str, nargs=1,
         help='Input directory containing a tree of package definitions')
@@ -82,6 +104,10 @@ def make_parser(parser):
     parser.add_argument(
         '--target', type=str, nargs='?', default=common.TARGETPYTHON,
         help='The path to the target Python installation')
+    parser.add_argument(
+        '--only', type=str, nargs='?', default=None,
+        help=('Only build the specified packages, provided as a comma '
+              'separated list'))
     return parser
 
 

pyodide_build/common.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Optional, Set
 
 
 ROOTDIR = Path(__file__).parents[1].resolve() / 'tools'
@@ -24,3 +25,19 @@ def parse_package(package):
     # TODO: Validate against a schema
     with open(package) as fd:
         return yaml.load(fd)
+
+
+def _parse_package_subset(query: Optional[str]) -> Optional[Set[str]]:
+    """Parse the list of packages specified with PYODIDE_PACKAGES env var.
+
+    Also add the list of mandatory packages: ['micropip', 'distlib']
+
+    Returns:
+      a set of package names to build or None.
+    """
+    if query is None:
+        return None
+    packages = query.split(',')
+    packages = [el.strip() for el in packages]
+    packages = ['micropip', 'distlib'] + packages
+    return set(packages)

test/pyodide_build/test_common.py

@@ -0,0 +1,19 @@
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).parents[2]))
+
+from pyodide_build.common import _parse_package_subset  # noqa
+
+
+def test_parse_package_subset():
+    assert _parse_package_subset(None) is None
+    # micropip is always included
+    assert _parse_package_subset("numpy,pandas") == {
+        'micropip', 'distlib', 'numpy', 'pandas'
+    }
+
+    # duplicates are removed
+    assert _parse_package_subset("numpy,numpy") == {
+        'micropip', 'distlib', 'numpy'
+    }

test/test_common.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from pathlib import Path
-from pyodide_build.common import parse_package
+from pyodide_build.common import parse_package, _parse_package_subset
 
 BASE_DIR = Path(__file__).parent.parent
 PKG_DIR = BASE_DIR / 'packages'
@@ -49,6 +49,11 @@ def test_import(name, selenium_standalone):
                 '{} fails to load and is not supported on {}.'
                 .format(name, selenium_standalone.browser))
 
+    built_packages = _parse_package_subset(os.environ.get('PYODIDE_PACKAGES'))
+    # only a subset of packages were built
+    if built_packages is not None and name not in built_packages:
+        pytest.skip(f'{name} was skipped due to PYODIDE_PACKAGES')
+
     selenium_standalone.run("import glob, os")
 
     baseline_pyc = selenium_standalone.run(