diff --git a/Makefile.envs b/Makefile.envs index d5ff44a6d..46ec62275 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -84,9 +84,6 @@ export LDFLAGS_BASE=\ $(DBGFLAGS) \ $(DBG_LDFLAGS) \ -s MODULARIZE=1 \ - -s LINKABLE=1 \ - -s EXPORT_ALL=1 \ - -s WASM=1 \ -std=c++14 \ -s LZ4=1 \ -L $(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/ \ @@ -106,6 +103,7 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \ -s FORCE_FILESYSTEM=1 \ -s TOTAL_MEMORY=20971520 \ -s ALLOW_MEMORY_GROWTH=1 \ + -s EXPORT_ALL=1 \ -s POLYFILL \ \ -lpython$(PYMAJOR).$(PYMINOR) \ diff --git a/docs/development/meta-yaml.md b/docs/development/meta-yaml.md index a0ab26b8f..fae4dc2ed 100644 --- a/docs/development/meta-yaml.md +++ b/docs/development/meta-yaml.md @@ -103,6 +103,19 @@ Extra arguments to pass to the linker when building for WebAssembly. (This key is not in the Conda spec). +### `build/exports` + +Which symbols should be exported from the shared object files. Possible values +are: + +- `pyinit`: The default. Only export Python module initialization symbols of + the form `PyInit_some_module`. +- `explicit`: Export the functions that are marked as exported in the object + files. Switch to this if `pyinit` doesn't work. Useful for packages that use + `ctypes` or `dlsym` to access symbols. +- `whole_archive`: Uses `-Wl,--whole-archive` to force inclusion of all + symbols. Use this when neither `pyinit` nor `explicit` work. + ### `build/backend-flags` Extra flags to pass to the build backend (e.g., `setuptools`, `flit`, etc). diff --git a/packages/astropy/meta.yaml b/packages/astropy/meta.yaml index d5fa8c192..94a773494 100644 --- a/packages/astropy/meta.yaml +++ b/packages/astropy/meta.yaml @@ -8,6 +8,7 @@ build: # The test module is imported from the top level `__init__.py` # so it cannot be unvendored unvendor-tests: false + exports: "requested" # Astropy uses dlsym so we need to export more than just PyInit_astropy requirements: run: - distutils diff --git a/packages/galpy/meta.yaml b/packages/galpy/meta.yaml index 9b53d9f2c..ef6f5dada 100644 --- a/packages/galpy/meta.yaml +++ b/packages/galpy/meta.yaml @@ -15,6 +15,7 @@ build: $(LIBGSL_INCLUDE_PATH) ldflags: | $(LIBGSL_LIBRARY_PATH) + exports: "requested" requirements: run: - numpy diff --git a/packages/xgboost/meta.yaml b/packages/xgboost/meta.yaml index 339c88135..29618fd78 100644 --- a/packages/xgboost/meta.yaml +++ b/packages/xgboost/meta.yaml @@ -16,6 +16,9 @@ build: script: | # export VERBOSE=1 export CMAKE_TOOLCHAIN_FILE=$PYODIDE_ROOT/packages/xgboost/cmake/Toolchain.cmake + # xgboost uses dlsym so we need to export more than just PyInit_xgboost. + # We can't currently use "explicit" because there are too many exports. + exports: "whole_archive" requirements: run: - numpy diff --git a/pyodide-build/pyodide_build/buildpkg.py b/pyodide-build/pyodide_build/buildpkg.py index bcdaa17b5..bdb38c59e 100755 --- a/pyodide-build/pyodide_build/buildpkg.py +++ b/pyodide-build/pyodide_build/buildpkg.py @@ -449,6 +449,7 @@ def compile( ldflags=build_metadata["ldflags"], target_install_dir=target_install_dir, replace_libs=replace_libs, + exports=build_metadata.get("exports", "pyinit"), ) diff --git a/pyodide-build/pyodide_build/io.py b/pyodide-build/pyodide_build/io.py index 66dc2c75d..26b783faa 100644 --- a/pyodide-build/pyodide_build/io.py +++ b/pyodide-build/pyodide_build/io.py @@ -20,6 +20,7 @@ PACKAGE_CONFIG_SPEC: dict[str, dict[str, Any]] = { "extras": list, # List[Tuple[str, str]], }, "build": { + "exports": str | list, # list[str] "backend-flags": str, "cflags": str, "cxxflags": str, @@ -126,6 +127,13 @@ def _check_config_build(config: dict[str, Any]) -> Iterator[str]: build_metadata = config["build"] library = build_metadata.get("library", False) sharedlibrary = build_metadata.get("sharedlibrary", False) + exports = build_metadata.get("exports", "pyinit") + if isinstance(exports, str) and exports not in [ + "pyinit", + "requested", + "whole_archive", + ]: + yield f"build/exports must be 'pyinit', 'explicit', 'all', or a list of strings not {build_metadata['exports']}" if not library and not sharedlibrary: return if library and sharedlibrary: diff --git a/pyodide-build/pyodide_build/pywasmcross.py b/pyodide-build/pyodide_build/pywasmcross.py index b66869b21..0332716ce 100755 --- a/pyodide-build/pyodide_build/pywasmcross.py +++ b/pyodide-build/pyodide_build/pywasmcross.py @@ -25,7 +25,7 @@ import re import subprocess from collections import namedtuple from pathlib import Path, PurePosixPath -from typing import Any, MutableMapping, NoReturn +from typing import Any, Iterator, MutableMapping, NoReturn from pyodide_build import common from pyodide_build._f2c_fixes import fix_f2c_input, fix_f2c_output, scipy_fixes @@ -48,6 +48,7 @@ ReplayArgs = namedtuple( "replace_libs", "builddir", "pythoninclude", + "exports", ], ) @@ -85,6 +86,7 @@ def compile( ldflags: str, target_install_dir: str, replace_libs: str, + exports: str | list[str], ) -> None: kwargs = dict( pkgname=pkgname, @@ -99,6 +101,7 @@ def compile( args = environment_substitute_args(kwargs, env) backend_flags = args.pop("backend_flags") args["builddir"] = str(Path(".").absolute()) + args["exports"] = exports env = dict(env) SYMLINKDIR = symlink_dir() @@ -368,6 +371,43 @@ def replay_genargs_handle_argument(arg: str) -> str | None: return arg +def calculate_exports(line: list[str], export_all: bool) -> Iterator[str]: + """ + Collect up all the object files and archive files being linked and list out + symbols in them that are marked as public. If ``export_all`` is ``True``, + then return all public symbols. If not, return only the public symbols that + begin with `PyInit`. + """ + objects = [arg for arg in line if arg.endswith(".a") or arg.endswith(".o")] + args = ["emnm", "-j", "--export-symbols"] + objects + result = subprocess.run( + args, encoding="utf8", capture_output=True, env={"PATH": os.environ["PATH"]} + ) + if result.returncode: + print(f"Command '{' '.join(args)}' failed. Output to stderr was:") + print(result.stderr) + sys.exit(result.returncode) + + condition = (lambda x: True) if export_all else (lambda x: x.startswith("PyInit")) + return (x for x in result.stdout.splitlines() if condition(x)) + + +def get_export_flags(line, exports): + """ + If "whole_archive" was requested, no action is needed. Otherwise, add + `-sSIDE_MODULE=2` and the appropriate export list. + """ + if exports == "whole_archive": + return + yield "-sSIDE_MODULE=2" + if isinstance(exports, str): + export_list = calculate_exports(line, exports == "requested") + else: + export_list = exports + prefixed_exports = ["_" + x for x in export_list] + yield f"-sEXPORTED_FUNCTIONS={prefixed_exports!r}" + + def handle_command_generate_args( line: list[str], args: ReplayArgs, is_link_command: bool ) -> list[str]: @@ -436,6 +476,8 @@ def handle_command_generate_args( if is_link_command: new_args.extend(args.ldflags.split()) + new_args.extend(get_export_flags(line, args.exports)) + if "-c" in line: if new_args[0] == "emcc": new_args.extend(args.cflags.split()) @@ -537,8 +579,6 @@ def handle_command( new_args = _new_args returncode = subprocess.run(new_args).returncode - if returncode != 0: - sys.exit(returncode) sys.exit(returncode) diff --git a/pyodide-build/pyodide_build/tests/test_pywasmcross.py b/pyodide-build/pyodide_build/tests/test_pywasmcross.py index 561570c72..bfa93e5cc 100644 --- a/pyodide-build/pyodide_build/tests/test_pywasmcross.py +++ b/pyodide-build/pyodide_build/tests/test_pywasmcross.py @@ -18,6 +18,7 @@ class BuildArgs: replace_libs: str = "" target_install_dir: str = "" pythoninclude: str = "python/include" + exports: str = "whole_archive" def _args_wrapper(func): @@ -134,7 +135,6 @@ def test_handle_command_ldflags(): def test_handle_command_optflags(in_ext, out_ext, executable, flag_name): # Make sure that when multiple optflags are present those in cflags, # cxxflags, or ldflags has priority - args = BuildArgs(**{flag_name: "-Oz"}) assert ( generate_args(f"gcc -O3 -c test.{in_ext} -o test.{out_ext}", args, True)