Default to -sSIDE_MODULE=2 in packages (#2712)

Drops the size of the dist directory from 271mb to 226 mb which is a 17% size reduction.

Makefile.envs

@@ -84,9 +84,6 @@ export LDFLAGS_BASE=\
 	$(DBGFLAGS) \
 	$(DBG_LDFLAGS) \
 	-s MODULARIZE=1 \
-	-s LINKABLE=1 \
-	-s EXPORT_ALL=1 \
-	-s WASM=1 \
 	-std=c++14 \
 	-s LZ4=1 \
 	-L $(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/ \
@@ -106,6 +103,7 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \
 	-s FORCE_FILESYSTEM=1 \
 	-s TOTAL_MEMORY=20971520 \
 	-s ALLOW_MEMORY_GROWTH=1 \
+	-s EXPORT_ALL=1 \
 	-s POLYFILL \
 	\
 	-lpython$(PYMAJOR).$(PYMINOR) \

docs/development/meta-yaml.md

@@ -103,6 +103,19 @@ Extra arguments to pass to the linker when building for WebAssembly.
 
 (This key is not in the Conda spec).
 
+### `build/exports`
+
+Which symbols should be exported from the shared object files. Possible values
+are:
+
+- `pyinit`: The default. Only export Python module initialization symbols of
+  the form `PyInit_some_module`.
+- `explicit`: Export the functions that are marked as exported in the object
+  files. Switch to this if `pyinit` doesn't work. Useful for packages that use
+  `ctypes` or `dlsym` to access symbols.
+- `whole_archive`: Uses `-Wl,--whole-archive` to force inclusion of all
+  symbols. Use this when neither `pyinit` nor `explicit` work.
+
 ### `build/backend-flags`
 
 Extra flags to pass to the build backend (e.g., `setuptools`, `flit`, etc).

packages/astropy/meta.yaml

@@ -8,6 +8,7 @@ build:
   # The test module is imported from the top level `__init__.py`
   # so it cannot be unvendored
   unvendor-tests: false
+  exports: "requested" # Astropy uses dlsym so we need to export more than just PyInit_astropy
 requirements:
   run:
     - distutils

packages/galpy/meta.yaml

@@ -15,6 +15,7 @@ build:
     $(LIBGSL_INCLUDE_PATH)
   ldflags: |
     $(LIBGSL_LIBRARY_PATH)
+  exports: "requested"
 requirements:
   run:
     - numpy

packages/xgboost/meta.yaml

@@ -16,6 +16,9 @@ build:
   script: |
     # export VERBOSE=1
     export CMAKE_TOOLCHAIN_FILE=$PYODIDE_ROOT/packages/xgboost/cmake/Toolchain.cmake
+  # xgboost uses dlsym so we need to export more than just PyInit_xgboost.
+  # We can't currently use "explicit" because there are too many exports.
+  exports: "whole_archive"
 requirements:
   run:
     - numpy

pyodide-build/pyodide_build/buildpkg.py

@@ -449,6 +449,7 @@ def compile(
             ldflags=build_metadata["ldflags"],
             target_install_dir=target_install_dir,
             replace_libs=replace_libs,
+            exports=build_metadata.get("exports", "pyinit"),
         )
 
 

pyodide-build/pyodide_build/io.py

@@ -20,6 +20,7 @@ PACKAGE_CONFIG_SPEC: dict[str, dict[str, Any]] = {
         "extras": list,  # List[Tuple[str, str]],
     },
     "build": {
+        "exports": str | list,  # list[str]
         "backend-flags": str,
         "cflags": str,
         "cxxflags": str,
@@ -126,6 +127,13 @@ def _check_config_build(config: dict[str, Any]) -> Iterator[str]:
     build_metadata = config["build"]
     library = build_metadata.get("library", False)
     sharedlibrary = build_metadata.get("sharedlibrary", False)
+    exports = build_metadata.get("exports", "pyinit")
+    if isinstance(exports, str) and exports not in [
+        "pyinit",
+        "requested",
+        "whole_archive",
+    ]:
+        yield f"build/exports must be 'pyinit', 'explicit', 'all', or a list of strings not {build_metadata['exports']}"
     if not library and not sharedlibrary:
         return
     if library and sharedlibrary:

pyodide-build/pyodide_build/pywasmcross.py

@@ -25,7 +25,7 @@ import re
 import subprocess
 from collections import namedtuple
 from pathlib import Path, PurePosixPath
-from typing import Any, MutableMapping, NoReturn
+from typing import Any, Iterator, MutableMapping, NoReturn
 
 from pyodide_build import common
 from pyodide_build._f2c_fixes import fix_f2c_input, fix_f2c_output, scipy_fixes
@@ -48,6 +48,7 @@ ReplayArgs = namedtuple(
         "replace_libs",
         "builddir",
         "pythoninclude",
+        "exports",
     ],
 )
 
@@ -85,6 +86,7 @@ def compile(
     ldflags: str,
     target_install_dir: str,
     replace_libs: str,
+    exports: str | list[str],
 ) -> None:
     kwargs = dict(
         pkgname=pkgname,
@@ -99,6 +101,7 @@ def compile(
     args = environment_substitute_args(kwargs, env)
     backend_flags = args.pop("backend_flags")
     args["builddir"] = str(Path(".").absolute())
+    args["exports"] = exports
 
     env = dict(env)
     SYMLINKDIR = symlink_dir()
@@ -368,6 +371,43 @@ def replay_genargs_handle_argument(arg: str) -> str | None:
     return arg
 
 
+def calculate_exports(line: list[str], export_all: bool) -> Iterator[str]:
+    """
+    Collect up all the object files and archive files being linked and list out
+    symbols in them that are marked as public. If ``export_all`` is ``True``,
+    then return all public symbols. If not, return only the public symbols that
+    begin with `PyInit`.
+    """
+    objects = [arg for arg in line if arg.endswith(".a") or arg.endswith(".o")]
+    args = ["emnm", "-j", "--export-symbols"] + objects
+    result = subprocess.run(
+        args, encoding="utf8", capture_output=True, env={"PATH": os.environ["PATH"]}
+    )
+    if result.returncode:
+        print(f"Command '{' '.join(args)}' failed. Output to stderr was:")
+        print(result.stderr)
+        sys.exit(result.returncode)
+
+    condition = (lambda x: True) if export_all else (lambda x: x.startswith("PyInit"))
+    return (x for x in result.stdout.splitlines() if condition(x))
+
+
+def get_export_flags(line, exports):
+    """
+    If "whole_archive" was requested, no action is needed. Otherwise, add
+    `-sSIDE_MODULE=2` and the appropriate export list.
+    """
+    if exports == "whole_archive":
+        return
+    yield "-sSIDE_MODULE=2"
+    if isinstance(exports, str):
+        export_list = calculate_exports(line, exports == "requested")
+    else:
+        export_list = exports
+    prefixed_exports = ["_" + x for x in export_list]
+    yield f"-sEXPORTED_FUNCTIONS={prefixed_exports!r}"
+
+
 def handle_command_generate_args(
     line: list[str], args: ReplayArgs, is_link_command: bool
 ) -> list[str]:
@@ -436,6 +476,8 @@ def handle_command_generate_args(
 
     if is_link_command:
         new_args.extend(args.ldflags.split())
+        new_args.extend(get_export_flags(line, args.exports))
+
     if "-c" in line:
         if new_args[0] == "emcc":
             new_args.extend(args.cflags.split())
@@ -537,8 +579,6 @@ def handle_command(
         new_args = _new_args
 
     returncode = subprocess.run(new_args).returncode
-    if returncode != 0:
-        sys.exit(returncode)
 
     sys.exit(returncode)
 

pyodide-build/pyodide_build/tests/test_pywasmcross.py

@@ -18,6 +18,7 @@ class BuildArgs:
     replace_libs: str = ""
     target_install_dir: str = ""
     pythoninclude: str = "python/include"
+    exports: str = "whole_archive"
 
 
 def _args_wrapper(func):
@@ -134,7 +135,6 @@ def test_handle_command_ldflags():
 def test_handle_command_optflags(in_ext, out_ext, executable, flag_name):
     # Make sure that when multiple optflags are present those in cflags,
     # cxxflags, or ldflags has priority
-
     args = BuildArgs(**{flag_name: "-Oz"})
     assert (
         generate_args(f"gcc -O3 -c test.{in_ext} -o test.{out_ext}", args, True)