diff --git a/.dockerignore b/.dockerignore index 0ab130acb..5e9654a08 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,4 @@ emsdk/emsdk/ cpython/installs/ cpython/build/ -packages/ +packages/*/build/ diff --git a/.github/workflows/prebuilt_docker_image.yaml b/.github/workflows/prebuilt_docker_image.yaml new file mode 100644 index 000000000..5cdd99756 --- /dev/null +++ b/.github/workflows/prebuilt_docker_image.yaml @@ -0,0 +1,39 @@ +on: + workflow_dispatch: + inputs: + version: + description: 'Version of the docker image to build. + The same version of the pyodide-env image must already exist' + required: true + +jobs: + build_image: + runs-on: ubuntu-latest +# This is the recommended use of github's docker build-push action +# See https://github.com/marketplace/actions/build-and-push-docker-images#git-context + steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + file: ./Dockerfile-prebuilt + push: true + build-args: | + VERSION=${{ github.event.inputs.version }} + tags: "iodide/pyodide:${{ github.event.inputs.version }}" + - + name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/Dockerfile-prebuilt b/Dockerfile-prebuilt new file mode 100644 index 000000000..26bbc685d --- /dev/null +++ b/Dockerfile-prebuilt @@ -0,0 +1,14 @@ +ARG VERSION + +FROM iodide/pyodide-env:${VERSION} + +USER root + +ENV EMSDK_NUM_CORES=4 EMCC_CORES=4 PYODIDE_JOBS=4 + +COPY . pyodide + +# the rm at the end deletes the build results such that the resulting image can still be used for building pyodide +# from source (including partial and customized builds). Due to the previous run of make, builds +# executed with this image will be much faster than the ones executed with pyodide-env +RUN cd pyodide && make && rm -r pyodide/build diff --git a/docs/building_from_sources.md b/docs/building_from_sources.md index 3e5dee37a..2eff381a4 100644 --- a/docs/building_from_sources.md +++ b/docs/building_from_sources.md @@ -45,16 +45,21 @@ make ## Using Docker We provide a Debian-based Docker image on Docker Hub with the dependencies -already installed to make it easier to build Pyodide. Note that building from -the Docker image is *very* slow on Mac, building on the host machine is -preferred if at all possible. +already installed to make it easier to build Pyodide. On top of that we provide a +pre-built image which can be used for fast custom and partial builds of pyodide. +Note that building from the non pre-built the Docker image is *very* slow on Mac, +building on the host machine is preferred if at all possible. 1. Install Docker -2. From a git checkout of Pyodide, run `./run_docker` +2. From a git checkout of Pyodide, run `./run_docker` or `./run_docker --pre-built` 3. Run `make` to build. +Note: You can control the resources allocated to the build by setting the env vars +`EMSDK_NUM_CORE`, `EMCC_CORES` and `PYODIDE_JOBS` (the default for each is 4). + + If running ``make`` deterministically stops at one point in each subsequent try, increasing the maximum RAM usage available to the docker container might help [This is different from the physical RAM capacity inside the system]. Ideally, at least 3 GB of RAM diff --git a/run_docker b/run_docker index 2d40ff2e7..31f26a202 100755 --- a/run_docker +++ b/run_docker @@ -1,13 +1,77 @@ -#!/bin/sh +#!/usr/bin/env bash + +set -eo pipefail + + +function usage() { + cat > /dev/stdout < System port to which to forward. + This is ignored if the env var PYODIDE_SYSTEM_PORT is set + + Prerequisites: + Docker has to be set up on your system +EOF +} + +function error() { + usage + exit 255 +} + + +PYODIDE_IMAGE_TAG="0.16.1" +DEFAULT_PYODIDE_DOCKER_IMAGE="iodide/pyodide-env:${PYODIDE_IMAGE_TAG}" +DEFAULT_PYODIDE_SYSTEM_PORT="8000" + +while [[ $# -gt 0 ]] +do + key="$1" + case $key in + -h|--help) + usage + exit 0 + ;; + --pre-built) + if [[ -n ${PYODIDE_DOCKER_IMAGE} ]]; then + echo "WARNING: will use the env var PYODIDE_DOCKER_IMAGE=${PYODIDE_DOCKER_IMAGE}, + the flag --pre-built has no effect" + fi + DEFAULT_PYODIDE_DOCKER_IMAGE="iodide/pyodide:${PYODIDE_IMAGE_TAG}" + shift + ;; + -p|--port) + if [ "$#" -lt 2 ]; then + >&2 echo "port cannot be empty" + error + fi + if [[ -n ${PYODIDE_SYSTEM_PORT} ]]; then + echo "WARNING: will use the env var PYODIDE_SYSTEM_PORT=${PYODIDE_SYSTEM_PORT} instead of the provided port" + fi + DEFAULT_PYODIDE_SYSTEM_PORT=$2 + shift 2 + ;; + *) + >&2 echo "Unknown option $1" + error + ;; + esac +done PYODIDE_DOCKER_PORT=${PYODIDE_DOCKER_PORT:-"8000"} -PYODIDE_SYSTEM_PORT=${PYODIDE_SYSTEM_PORT:-"8000"} -PYODIDE_DOCKER_IMAGE=${PYODIDE_DOCKER_IMAGE:-"iodide/pyodide-env:0.16.1"} +PYODIDE_SYSTEM_PORT=${PYODIDE_SYSTEM_PORT:-${DEFAULT_PYODIDE_SYSTEM_PORT}} +PYODIDE_DOCKER_IMAGE=${PYODIDE_DOCKER_IMAGE:-${DEFAULT_PYODIDE_DOCKER_IMAGE}} exec docker run \ - -p $PYODIDE_SYSTEM_PORT:$PYODIDE_DOCKER_PORT \ + -p "$PYODIDE_SYSTEM_PORT":"$PYODIDE_DOCKER_PORT" \ -it --rm \ -v $PWD:/src \ --user root -e NB_UID=$UID -e NB_GID=$GID \ - ${PYODIDE_DOCKER_IMAGE} \ + "${PYODIDE_DOCKER_IMAGE}" \ /bin/bash diff --git a/setup.cfg b/setup.cfg index eb7648e31..9ecf68d45 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ addopts = -r sxX [bumpversion] -current_version = 0.15.0 +current_version = 0.16.1 commit = True tag = True tag_name = {new_version} diff --git a/test/pyodide_build/test_run_docker.py b/test/pyodide_build/test_run_docker.py new file mode 100644 index 000000000..fcebf7674 --- /dev/null +++ b/test/pyodide_build/test_run_docker.py @@ -0,0 +1,24 @@ +from pathlib import Path +import subprocess + +BASE_DIR = Path(__file__).parents[2] + + +def test_run_docker_script(): + res = subprocess.run( + ["bash", str(BASE_DIR / "run_docker"), "--help"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + assert "Usage:\n run_docker" in res.stdout.decode("utf-8") + + res = subprocess.run( + ["bash", str(BASE_DIR / "run_docker"), "--invalid-param"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert res.returncode > 0 + assert "Unknown option --invalid-param" in res.stderr.decode("utf-8")