diff --git a/Makefile b/Makefile index 9079eb34a..804990ce8 100644 --- a/Makefile +++ b/Makefile @@ -190,7 +190,6 @@ minimal : PYODIDE_PACKAGES="micropip" make debug : - EXTRA_CFLAGS="-D DEBUG_F" \ - EXTRA_LDFLAGS="-s ASSERTIONS=2" \ + EXTRA_CFLAGS+="-D DEBUG_F" \ PYODIDE_PACKAGES+="micropip,pyparsing,pytz,packaging,kiwisolver" \ make diff --git a/docs/development/building-from-sources.md b/docs/development/building-from-sources.md index 69b383877..7cf1b8b3a 100644 --- a/docs/development/building-from-sources.md +++ b/docs/development/building-from-sources.md @@ -1,19 +1,23 @@ (building_from_sources)= # Building from sources -Building is easiest on Linux and relatively straightforward on Mac. For -Windows, we currently recommend using the Docker image (described below) to -build Pyodide. Another option for building on Windows is to use [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to create a Linux build environment. +Building is easiest on Linux and relatively straightforward on Mac. For Windows, +we currently recommend using the Docker image (described below) to build +Pyodide. Another option for building on Windows is to use +[WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to create a +Linux build environment. ## Build using `make` -Make sure the prerequisites for [emsdk](https://github.com/emscripten-core/emsdk) are -installed. Pyodide will build a custom, patched version of emsdk, so there is no -need to build it yourself prior. +Make sure the prerequisites for +[emsdk](https://github.com/emscripten-core/emsdk) are installed. Pyodide will +build a custom, patched version of emsdk, so there is no need to build it +yourself prior. Additional build prerequisites are: -- A working native compiler toolchain, enough to build [CPython](https://devguide.python.org/setup/#linux). +- A working native compiler toolchain, enough to build + [CPython](https://devguide.python.org/setup/#linux). - A native Python 3.8 to run the build scripts. - CMake - PyYAML @@ -27,7 +31,9 @@ Additional build prerequisites are: On Mac, you will also need: - [Homebrew](https://brew.sh/) for installing dependencies -- System libraries in the root directory (`sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /` should do it, see https://github.com/pyenv/pyenv/issues/1219#issuecomment-428305417) +- System libraries in the root directory ( + `sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /` + should do it, see https://github.com/pyenv/pyenv/issues/1219#issuecomment-428305417) - coreutils for md5sum and other essential Unix utilities (`brew install coreutils`) - cmake (`brew install cmake`) - Cython to compile SciPy (`brew install cython`) @@ -35,7 +41,8 @@ On Mac, you will also need: - pkg-config (`brew install pkg-config`) - openssl (`brew install openssl`) - gfortran (`brew cask install gfortran`) -- f2c: Install wget (`brew install wget`), and then run the buildf2c script from the root directory (`sudo ./tools/buildf2c`) +- f2c: Install wget (`brew install wget`), and then run the buildf2c script from + the root directory (`sudo ./tools/buildf2c`) After installing the build prerequisites, run from the command line: @@ -47,10 +54,10 @@ make ## Using Docker We provide a Debian-based Docker image on Docker Hub with the dependencies -already installed to make it easier to build Pyodide. On top of that we provide a -pre-built image which can be used for fast custom and partial builds of pyodide. -Note that building from the non pre-built the Docker image is *very* slow on Mac, -building on the host machine is preferred if at all possible. +already installed to make it easier to build Pyodide. On top of that we provide +a pre-built image which can be used for fast custom and partial builds of +pyodide. Note that building from the non pre-built the Docker image is *very* +slow on Mac, building on the host machine is preferred if at all possible. 1. Install Docker @@ -58,15 +65,17 @@ building on the host machine is preferred if at all possible. 3. Run `make` to build. -Note: You can control the resources allocated to the build by setting the env vars -`EMSDK_NUM_CORE`, `EMCC_CORES` and `PYODIDE_JOBS` (the default for each is 4). +Note: You can control the resources allocated to the build by setting the env +vars `EMSDK_NUM_CORE`, `EMCC_CORES` and `PYODIDE_JOBS` (the default for each is +4). -If running ``make`` deterministically stops at one point in each subsequent try, increasing -the maximum RAM usage available to the docker container might help [This is different -from the physical RAM capacity inside the system]. Ideally, at least 3 GB of RAM -should be available to the docker container to build Pyodide smoothly. These settings can -be changed via Docker Preferences (See [here](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container)). +If running ``make`` deterministically stops at one point in each subsequent try, +increasing the maximum RAM usage available to the docker container might help +[This is different from the physical RAM capacity inside the system]. Ideally, +at least 3 GB of RAM should be available to the docker container to build +Pyodide smoothly. These settings can be changed via Docker Preferences (See +[here](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container)). You can edit the files in your source checkout on your host machine, and then repeatedly run `make` inside the Docker environment to test your changes. @@ -74,27 +83,42 @@ repeatedly run `make` inside the Docker environment to test your changes. (partial-builds)= ## Partial builds -To build a subset of available packages in Pyodide, set the environment -variable `PYODIDE_PACKAGES` to a comma separated list of packages. For -instance, +To build a subset of available packages in Pyodide, set the environment variable +`PYODIDE_PACKAGES` to a comma separated list of packages. For instance, ``` PYODIDE_PACKAGES="toolz,attrs" make ``` -Dependencies of the listed packages will be built automatically as well. -The package names must match the folder names in `packages/` exactly; in -particular they are case sensitive. +Dependencies of the listed packages will be built automatically as well. The +package names must match the folder names in `packages/` exactly; in particular +they are case sensitive. To build a minimal version of Pyodide, set `PYODIDE_PACKAGES="micropip"`. The packages micropip and distutils are always automatically included (but an empty -`PYODIDE_PACKAGES` is interpreted as unset). +`PYODIDE_PACKAGES` is interpreted as unset). As a shorthand for this, one can +say `make minimal`. ## Environment variables Following environment variables additionally impact the build, - - `PYODIDE_JOBS`: the `-j` option passed to the `emmake make` command when applicable for parallel compilation. Default: 3. - - `PYODIDE_BASE_URL`: Base URL where Pyodide packages are deployed. It must - end with a trailing `/`. Default: `./` to load Pyodide packages from the - same base URL path as where `pyodide.js` is located. Example: + - `PYODIDE_JOBS`: the `-j` option passed to the `emmake make` command when + applicable for parallel compilation. Default: 3. + - `PYODIDE_BASE_URL`: Base URL where Pyodide packages are deployed. It must end + with a trailing `/`. Default: `./` to load Pyodide packages from the same + base URL path as where `pyodide.js` is located. Example: `https://cdn.jsdelivr.net/pyodide/dev/full/` + - `EXTRA_CFLAGS` : Add extra compilation flags. + - `EXTRA_LDFLAGS` : Add extra linker flags. + +Setting `EXTRA_CFLAGS="-D DEBUG_F"` provides detailed diagnostic information +whenever error branches are taken inside of the Pyodide core code. These error +messages are frequently helpful even when the problem is a fatal configuration +problem and Pyodide cannot even be initialized. These error branches occur also +in correctly working code, but they are relatively uncommon so in practice the +amount of noise generated isn't too large. The shorthand `make debug` +automatically sets this flag. + +In certain cases, setting `EXTRA_LDFLAGS="-s ASSERTIONS=1` or `ASSERTIONS=2` can +also be helpful, but this slows down the linking and the runtime speed of +Pyodide a lot and generates a large amount of noise in the console. diff --git a/src/core/error_handling.c b/src/core/error_handling.c index ff736bf22..b242577fa 100644 --- a/src/core/error_handling.c +++ b/src/core/error_handling.c @@ -21,7 +21,7 @@ EM_JS_NUM(errcode, log_error, (char* msg), { // Right now this is dead code (probably), please don't remove it. // Intended for debugging purposes. -EM_JS_NUM(errcode, log_error_obj, (JsRef obj), { +EM_JS_NUM(errcode, console_error_obj, (JsRef obj), { console.error(Module.hiwire.get_value(obj)); }); @@ -114,14 +114,7 @@ wrap_exception(bool attach_python_error) success = true; finally: // Log an appropriate warning. - if (success) { - EM_ASM( - { - let msg = Module.hiwire.get_value($0).message; - console.warn("Python exception:\n" + msg + "\n"); - }, - jserror); - } else { + if (!success) { PySys_WriteStderr("Error occurred while formatting traceback:\n"); PyErr_Print(); if (type != NULL) { @@ -146,11 +139,19 @@ finally: return jserror; } +EM_JS_NUM(errcode, log_python_error, (JsRef jserror), { + let msg = Module.hiwire.get_value(jserror).message; + console.warn("Python exception:\n" + msg + "\n"); + return 0; +}); + void _Py_NO_RETURN pythonexc2js() { JsRef jserror = wrap_exception(false); - if (jserror == NULL) { + if (jserror != NULL) { + log_python_error(jserror); + } else { jserror = new_error("Error occurred while formatting traceback", Js_undefined); } diff --git a/src/core/error_handling.h b/src/core/error_handling.h index 7b02505b1..ca8ffddbe 100644 --- a/src/core/error_handling.h +++ b/src/core/error_handling.h @@ -22,6 +22,11 @@ extern PyObject* conversion_error; JsRef wrap_exception(bool attach_python_error); +/** + * Argument should be output of wrap_exception. + */ +errcode log_python_error(JsRef); + /** * Convert the active Python exception into a Javascript Error object and print * it to the console. @@ -29,13 +34,14 @@ wrap_exception(bool attach_python_error); void pythonexc2js(); +// Used by LOG_EM_JS_ERROR (behind DEBUG_F flag) errcode -log_error(char* msg); +console_error(char* msg); // Right now this is dead code (probably), please don't remove it. // Intended for debugging purposes. errcode -log_error_obj(JsRef obj); +console_error_obj(JsRef obj); /** * EM_JS Wrappers @@ -138,7 +144,7 @@ log_error_obj(JsRef obj); __LINE__, \ __func__, \ __FILE__); \ - log_error(msg); \ + console_error(msg); \ free(msg); \ goto finally; \ } while (0) diff --git a/src/core/hiwire.c b/src/core/hiwire.c index 83f53e4ee..13bc1f4c2 100644 --- a/src/core/hiwire.c +++ b/src/core/hiwire.c @@ -46,6 +46,11 @@ EM_JS_NUM(int, hiwire_init, (), { _hiwire.objects.set(Module.hiwire.TRUE, true); _hiwire.objects.set(Module.hiwire.FALSE, false); +#ifdef DEBUG_F + Module.hiwire._hiwire = _hiwire; + let many_objects_warning_threshold = 200; +#endif + Module.hiwire.new_value = function(jsval) { // Should we guard against duplicating standard values? @@ -60,6 +65,14 @@ EM_JS_NUM(int, hiwire_init, (), { let idval = _hiwire.counter[0]; _hiwire.objects.set(idval, jsval); _hiwire.counter[0] += 2; +#ifdef DEBUG_F + if (_hiwire.objects.size > many_objects_warning_threshold) { + console.warn( + "A fairly large number of hiwire objects are present, this could " + + "be a sign of a memory leak."); + many_objects_warning_threshold += 100; + } +#endif return idval; }; @@ -68,13 +81,27 @@ EM_JS_NUM(int, hiwire_init, (), { Module.hiwire.get_value = function(idval) { if (!idval) { + // clang-format off // This might have happened because the error indicator is set. Let's // check. if (_PyErr_Occurred()) { // This will lead to a more helpful error message. - _pythonexc2js(); + let exc = _wrap_exception(); + let e = Module.hiwire.pop_value(exc); + console.error( + `Internal error: Argument '${idval}' to hiwire.get_value is falsy. ` + + "This was probably because the Python error indicator was set when get_value was called. " + + "The Python error that caused this was:", + e + ); + throw e; + } else { + throw new Error( + `Internal error: Argument '${idval}' to hiwire.get_value is falsy` + + ' (but error indicator is not set).' + ); } - throw new Error("Argument to hiwire.get_value is undefined"); + // clang-format on } if (!_hiwire.objects.has(idval)) { // clang-format off