From cd1b0c2c240eb4e36c04179d4ed3971f74b15921 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 15 Nov 2021 17:22:24 +0100 Subject: [PATCH] DOC Improve build instructions [skip ci] (#1960) Co-authored-by: Hood Chatham Co-authored-by: Jan Max Meyer --- docs/conf.py | 1 + docs/development/building-from-sources.md | 121 ++++++++++++++-------- docs/requirements-doc.txt | 1 + 3 files changed, 81 insertions(+), 42 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index dd195edd7..d88a11035 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,6 +50,7 @@ extensions = [ "myst_parser", "sphinx_js", "autodocsumm", + "sphinx_panels", "sphinx_pyodide", "sphinx_argparse_cli", # "versionwarning.extension", diff --git a/docs/development/building-from-sources.md b/docs/development/building-from-sources.md index fba588eb3..e6bb58d26 100644 --- a/docs/development/building-from-sources.md +++ b/docs/development/building-from-sources.md @@ -2,51 +2,116 @@ # Building from sources -Building is easiest on Linux and relatively straightforward on Mac. For Windows, -we currently recommend using the Docker image (described below) to build -Pyodide. Another option for building on Windows is to use +```{warning} +If you are building the latest development version of Pyodide from the `main` +branch, please make sure to follow the build instructions from the dev +version of the documentation at +[pyodide.org/en/latest/](https://pyodide.org/en/latest/development/building-from-sources.html) +``` + +Building on any operating system is easiest using the Pyodide Docker image. This approach works +with any native operating system as long as Docker is installed. You can also build on your +native Linux OS if the correct build prerequisites are installed. Building on MacOS is +possible, but there are known issues as of version 0.18 that you will need to work around. +It is not possible to build on Windows, but you can use [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to create a Linux build environment. -## Build using `make` +## Build instructions + +### Using Docker + +We provide a Debian-based Docker image +([`pyodide/pyodide-env`](https://hub.docker.com/r/pyodide/pyodide-env)) on +Docker Hub with the dependencies already installed to make it easier to build +Pyodide. On top of that we provide +a pre-built image +([`pyodide/pyodide`](https://hub.docker.com/r/pyodide/pyodide)) which can be +used for fast custom and partial builds. Note that building from the non +pre-built Docker image is _very_ slow on Mac, building on the host machine +is preferred if at all possible. + +1. Install Docker + +1. From a git checkout of Pyodide, run `./run_docker` or `./run_docker --pre-built` + +1. Run `make` to build. + +```{note} +You can control the resources allocated to the build by setting the env +vars `EMSDK_NUM_CORE`, `EMCC_CORES` and `PYODIDE_JOBS` (the default for each is +4). +``` + +If running `make` deterministically stops at some point, +increasing the maximum RAM usage available to the docker container might help. +(The RAM available to the container is different from the physical RAM capacity of the machine.) +Ideally, +at least 3 GB of RAM should be available to the docker container to build +Pyodide smoothly. These settings can be changed via Docker preferences (see +[here](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container)). + +You can edit the files in the shared `pyodide` source folder on your host +machine (outside of Docker), and then repeatedly run `make` inside the Docker +environment to test your changes. + +## Using `make` Make sure the prerequisites for [emsdk](https://github.com/emscripten-core/emsdk) are installed. Pyodide will build a custom, patched version of emsdk, so there is no need to build it yourself prior. +You would need Python 3.9.5 to run the build scripts. To make sure that the +correct Python is used during build it is recommended to use a [Python virtual +environment](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment), + +```{tabbed} Linux + Additional build prerequisites are: - A working native compiler toolchain, enough to build [CPython](https://devguide.python.org/setup/#linux). -- A native Python 3.9 to run the build scripts. - CMake -- PyYAML - FreeType 2 development libraries to compile Matplotlib. -- Cython to compile SciPy -- SWIG to compile NLopt - gfortran (GNU Fortran 95 compiler) - [f2c](http://www.netlib.org/f2c/) - [ccache](https://ccache.samba.org) (optional) _highly_ recommended for much faster rebuilds. +- (optional) SWIG to compile NLopt -You can install the python dependencies from the requirement file at the root of pyodide folder: -`pip install -r requirements.txt` +``` -On Mac, you will also need: +```{tabbed} MacOS + +To build on MacOS, you need: - [Homebrew](https://brew.sh/) for installing dependencies - System libraries in the root directory ( `sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /` should do it, see https://github.com/pyenv/pyenv/issues/1219#issuecomment-428305417) -- coreutils for md5sum and other essential Unix utilities (`brew install coreutils`) +- coreutils for md5sum and other essential Unix utilities (`brew install coreutils`). - cmake (`brew install cmake`) -- Cython to compile SciPy (`brew install cython`) -- SWIG to compile NLopt (`brew install swig`) - pkg-config (`brew install pkg-config`) - openssl (`brew install openssl`) - gfortran (`brew cask install gfortran`) - f2c: Install wget (`brew install wget`), and then run the buildf2c script from the root directory (`sudo ./tools/buildf2c`) +- It is also recommended installing the GNU patch (`brew install gpatch`), and + GNU sed (`brew install gnu-sed`) and [re-defining them temporarily as `patch` and + `sed`](https://formulae.brew.sh/formula/gnu-sed). +- (optional) SWIG to compile NLopt (`brew install swig`) + +``` + +```{note} +If you encounter issues with the requirements, it is useful to check the exact +list in the +[Dockerfile](https://github.com/pyodide/pyodide/blob/main/Dockerfile) which is +tested in the CI. +``` + +You can install the Python dependencies from the requirement file at the root of Pyodide folder: +`pip install -r requirements.txt` After installing the build prerequisites, run from the command line: @@ -54,34 +119,6 @@ After installing the build prerequisites, run from the command line: make ``` -## Using Docker - -We provide a Debian-based Docker image on Docker Hub with the dependencies -already installed to make it easier to build Pyodide. On top of that we provide -a pre-built image which can be used for fast custom and partial builds of -Pyodide. Note that building from the non pre-built the Docker image is _very_ -slow on Mac, building on the host machine is preferred if at all possible. - -1. Install Docker - -2. From a git checkout of Pyodide, run `./run_docker` or `./run_docker --pre-built` - -3. Run `make` to build. - -Note: You can control the resources allocated to the build by setting the env -vars `EMSDK_NUM_CORE`, `EMCC_CORES` and `PYODIDE_JOBS` (the default for each is -4). - -If running `make` deterministically stops at one point in each subsequent try, -increasing the maximum RAM usage available to the docker container might help -[This is different from the physical RAM capacity inside the system]. Ideally, -at least 3 GB of RAM should be available to the docker container to build -Pyodide smoothly. These settings can be changed via Docker Preferences (See -[here](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container)). - -You can edit the files in your source checkout on your host machine, and then -repeatedly run `make` inside the Docker environment to test your changes. - (partial-builds)= ## Partial builds diff --git a/docs/requirements-doc.txt b/docs/requirements-doc.txt index 75326bb99..92ada9f31 100644 --- a/docs/requirements-doc.txt +++ b/docs/requirements-doc.txt @@ -9,3 +9,4 @@ sphinxcontrib-napoleon sphinx-issues sphinx-js==3.1 sphinx-version-warning~=1.1.2 +sphinx-panels