diff --git a/pyodide-build/pyodide_build/_f2c_fixes.py b/pyodide-build/pyodide_build/_f2c_fixes.py index 9012f71ae..1eb5e4b76 100644 --- a/pyodide-build/pyodide_build/_f2c_fixes.py +++ b/pyodide-build/pyodide_build/_f2c_fixes.py @@ -1,4 +1,5 @@ import re +import subprocess from collections.abc import Iterable, Iterator from pathlib import Path from textwrap import dedent # for doctests @@ -513,3 +514,87 @@ def scipy_fixes(args: list[str]) -> None: for arg in args: if arg.endswith(".c"): scipy_fix_cfile(arg) + + +def replay_f2c(args: list[str], dryrun: bool = False) -> list[str] | None: + """Apply f2c to compilation arguments + + Parameters + ---------- + args + input compiler arguments + dryrun + if False run f2c on detected fortran files + + Returns + ------- + new_args + output compiler arguments + + + Examples + -------- + + >>> replay_f2c(['gfortran', 'test.f'], dryrun=True) + ['gcc', 'test.c'] + """ + + new_args = ["gcc"] + found_source = False + for arg in args[1:]: + if arg.endswith(".f") or arg.endswith(".F"): + filepath = Path(arg).resolve() + if not dryrun: + fix_f2c_input(arg) + if arg.endswith(".F"): + # .F files apparently expect to be run through the C + # preprocessor (they have #ifdef's in them) + # Use gfortran frontend, as gcc frontend might not be + # present on osx + # The file-system might be not case-sensitive, + # so take care to handle this by renaming. + # For preprocessing and further operation the + # expected file-name and extension needs to be preserved. + subprocess.check_call( + [ + "gfortran", + "-E", + "-C", + "-P", + filepath, + "-o", + filepath.with_suffix(".f77"), + ] + ) + filepath = filepath.with_suffix(".f77") + # -R flag is important, it means that Fortran functions that + # return real e.g. sdot will be transformed into C functions + # that return float. For historic reasons, by default f2c + # transform them into functions that return a double. Using -R + # allows to match what OpenBLAS has done when they f2ced their + # Fortran files, see + # https://github.com/xianyi/OpenBLAS/pull/3539#issuecomment-1493897254 + # for more details + with ( + open(filepath) as input_pipe, + open(filepath.with_suffix(".c"), "w") as output_pipe, + ): + subprocess.check_call( + ["f2c", "-R"], + stdin=input_pipe, + stdout=output_pipe, + cwd=filepath.parent, + ) + fix_f2c_output(arg[:-2] + ".c") + new_args.append(arg[:-2] + ".c") + found_source = True + else: + new_args.append(arg) + + new_args_str = " ".join(args) + if ".so" in new_args_str and "libgfortran.so" not in new_args_str: + found_source = True + + if not found_source: + return None + return new_args diff --git a/pyodide-build/pyodide_build/build_env.py b/pyodide-build/pyodide_build/build_env.py index 017d6b53b..15539ba12 100644 --- a/pyodide-build/pyodide_build/build_env.py +++ b/pyodide-build/pyodide_build/build_env.py @@ -1,5 +1,6 @@ # This file contains functions for managing the Pyodide build environment. +import dataclasses import functools import os import re @@ -68,6 +69,21 @@ BUILD_VARS: set[str] = { } +@dataclasses.dataclass(eq=False, order=False, kw_only=True) +class BuildArgs: + """ + Common arguments for building a package. + """ + + pkgname: str = "" + cflags: str = "" + cxxflags: str = "" + ldflags: str = "" + target_install_dir: str = "" # The path to the target Python installation + host_install_dir: str = "" # Directory for installing built host packages. + builddir: str = "" # The path to run pypa/build + + def init_environment(*, quiet: bool = False) -> None: """ Initialize Pyodide build environment. diff --git a/pyodide-build/pyodide_build/buildall.py b/pyodide-build/pyodide_build/buildall.py index 8917e697b..efb4c7f74 100755 --- a/pyodide-build/pyodide_build/buildall.py +++ b/pyodide-build/pyodide_build/buildall.py @@ -30,6 +30,7 @@ from rich.spinner import Spinner from rich.table import Table from . import build_env, recipe +from .build_env import BuildArgs from .buildpkg import needs_rebuild from .common import ( extract_wheel_metadata_file, @@ -39,7 +40,6 @@ from .common import ( ) from .io import MetaConfig, _BuildSpecTypes from .logger import console_stdout, logger -from .pywasmcross import BuildArgs class BuildError(Exception): @@ -915,9 +915,5 @@ def set_default_build_args(build_args: BuildArgs) -> BuildArgs: args.target_install_dir = build_env.get_build_flag("TARGETINSTALLDIR") # type: ignore[unreachable] if args.host_install_dir is None: args.host_install_dir = build_env.get_build_flag("HOSTINSTALLDIR") # type: ignore[unreachable] - if args.compression_level is None: - args.compression_level = int( # type: ignore[unreachable] - build_env.get_build_flag("PYODIDE_ZIP_COMPRESSION_LEVEL") - ) return args diff --git a/pyodide-build/pyodide_build/buildpkg.py b/pyodide-build/pyodide_build/buildpkg.py index 541409b37..89c6526fe 100755 --- a/pyodide-build/pyodide_build/buildpkg.py +++ b/pyodide-build/pyodide_build/buildpkg.py @@ -21,6 +21,7 @@ from . import common, pypabuild from .bash_runner import BashRunnerWithSharedEnvironment, get_bash_runner from .build_env import ( RUST_BUILD_PRELUDE, + BuildArgs, get_build_environment_vars, get_build_flag, pyodide_tags, @@ -37,7 +38,6 @@ from .common import ( ) from .io import MetaConfig, _SourceSpec from .logger import logger -from .pywasmcross import BuildArgs def _make_whlfile( diff --git a/pyodide-build/pyodide_build/cli/build_recipes.py b/pyodide-build/pyodide_build/cli/build_recipes.py index d61edf149..9aed5977a 100644 --- a/pyodide-build/pyodide_build/cli/build_recipes.py +++ b/pyodide-build/pyodide_build/cli/build_recipes.py @@ -5,10 +5,9 @@ from pathlib import Path import typer from .. import build_env, buildall, buildpkg -from ..build_env import init_environment +from ..build_env import BuildArgs, init_environment from ..common import get_num_cores from ..logger import logger -from ..pywasmcross import BuildArgs @dataclasses.dataclass(eq=False, order=False, kw_only=True) @@ -195,6 +194,7 @@ def build_recipes( ), compression_level: int = typer.Option( 6, + envvar="PYODIDE_ZIP_COMPRESSION_LEVEL", help="Level of zip compression to apply when installing. 0 means no compression.", ), ) -> None: diff --git a/pyodide-build/pyodide_build/pywasmcross.py b/pyodide-build/pyodide_build/pywasmcross.py index 556c4c1d5..f359f818d 100755 --- a/pyodide-build/pyodide_build/pywasmcross.py +++ b/pyodide-build/pyodide_build/pywasmcross.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 -"""Helper for cross-compiling distutils-based Python extensions. +"""Helper for cross-compiling Python binary extensions. -distutils has never had a proper cross-compilation story. This is a hack, which +Python has never had a proper cross-compilation story. This is a hack, which miraculously works, to get around that. - The gist is we compile the package replacing calls to the compiler and linker with wrappers that adjusting include paths and flags as necessary for cross-compiling and then pass the command long to emscripten. @@ -57,128 +56,40 @@ if IS_COMPILER_INVOCATION: __name__ = PYWASMCROSS_ARGS.pop("orig__name__") -import dataclasses -import re -import shutil import subprocess from collections.abc import Iterable, Iterator -from typing import Literal +from typing import Literal, NamedTuple -@dataclasses.dataclass(eq=False, order=False, kw_only=True) -class BuildArgs: +class CrossCompileArgs(NamedTuple): """ - Common arguments for building a package. + Arguments for cross-compiling a package. """ - pkgname: str = "" cflags: str = "" cxxflags: str = "" ldflags: str = "" + + # The name of the package being compiled + # This is used to apply package-specific fixes, such as scipy + pkgname: str = "" target_install_dir: str = "" # The path to the target Python installation - host_install_dir: str = "" # Directory for installing built host packages. - builddir: str = "" # The path to run pypa/build - pythoninclude: str = "" + pythoninclude: str = "" # path to the cross-compiled Python include directory exports: Literal["whole_archive", "requested", "pyinit"] | list[str] = "pyinit" - compression_level: int = 6 -def replay_f2c(args: list[str], dryrun: bool = False) -> list[str] | None: - """Apply f2c to compilation arguments - - Parameters - ---------- - args - input compiler arguments - dryrun - if False run f2c on detected fortran files - - Returns - ------- - new_args - output compiler arguments - - - Examples - -------- - - >>> replay_f2c(['gfortran', 'test.f'], dryrun=True) - ['gcc', 'test.c'] +def is_link_cmd(line: list[str]) -> bool: """ - - from pyodide_build._f2c_fixes import fix_f2c_input, fix_f2c_output - - new_args = ["gcc"] - found_source = False - for arg in args[1:]: - if arg.endswith(".f") or arg.endswith(".F"): - filepath = Path(arg).resolve() - if not dryrun: - fix_f2c_input(arg) - if arg.endswith(".F"): - # .F files apparently expect to be run through the C - # preprocessor (they have #ifdef's in them) - # Use gfortran frontend, as gcc frontend might not be - # present on osx - # The file-system might be not case-sensitive, - # so take care to handle this by renaming. - # For preprocessing and further operation the - # expected file-name and extension needs to be preserved. - subprocess.check_call( - [ - "gfortran", - "-E", - "-C", - "-P", - filepath, - "-o", - filepath.with_suffix(".f77"), - ] - ) - filepath = filepath.with_suffix(".f77") - # -R flag is important, it means that Fortran functions that - # return real e.g. sdot will be transformed into C functions - # that return float. For historic reasons, by default f2c - # transform them into functions that return a double. Using -R - # allows to match what OpenBLAS has done when they f2ced their - # Fortran files, see - # https://github.com/xianyi/OpenBLAS/pull/3539#issuecomment-1493897254 - # for more details - with ( - open(filepath) as input_pipe, - open(filepath.with_suffix(".c"), "w") as output_pipe, - ): - subprocess.check_call( - ["f2c", "-R"], - stdin=input_pipe, - stdout=output_pipe, - cwd=filepath.parent, - ) - fix_f2c_output(arg[:-2] + ".c") - new_args.append(arg[:-2] + ".c") - found_source = True - else: - new_args.append(arg) - - new_args_str = " ".join(args) - if ".so" in new_args_str and "libgfortran.so" not in new_args_str: - found_source = True - - if not found_source: - return None - return new_args - - -def get_library_output(line: list[str]) -> str | None: - """ - Check if the command is a linker invocation. If so, return the name of the - output file. + Check if the command is a linker invocation. """ + import re + SHAREDLIB_REGEX = re.compile(r"\.so(.\d+)*$") for arg in line: if not arg.startswith("-") and SHAREDLIB_REGEX.search(arg): - return arg - return None + return True + + return False def replay_genargs_handle_dashl(arg: str, used_libs: set[str]) -> str | None: @@ -425,6 +336,8 @@ def _calculate_object_exports_readobj_parse(output: str) -> list[str]: def calculate_object_exports_readobj(objects: list[str]) -> list[str] | None: + import shutil + readobj_path = shutil.which("llvm-readobj") if not readobj_path: which_emcc = shutil.which("emcc") @@ -518,7 +431,7 @@ def get_export_flags( def handle_command_generate_args( # noqa: C901 - line: list[str], build_args: BuildArgs, is_link_command: bool + line: list[str], build_args: CrossCompileArgs ) -> list[str]: """ A helper command for `handle_command` that generates the new arguments for @@ -534,42 +447,35 @@ def handle_command_generate_args( # noqa: C901 build_args The arguments that pywasmcross was invoked with - is_link_command Is this a linker invocation? - Returns ------- An updated argument list suitable for use with emscripten. - Examples -------- >>> from collections import namedtuple >>> Args = namedtuple('args', ['cflags', 'cxxflags', 'ldflags', 'target_install_dir']) >>> args = Args(cflags='', cxxflags='', ldflags='', target_install_dir='') - >>> handle_command_generate_args(['gcc', 'test.c'], args, False) + >>> handle_command_generate_args(['gcc', 'test.c'], args) ['emcc', 'test.c', '-Werror=implicit-function-declaration', '-Werror=mismatched-parameter-types', '-Werror=return-type'] """ if "-print-multiarch" in line: return ["echo", "wasm32-emscripten"] - for arg in line: - if arg.startswith("-print-file-name"): - return line if len(line) == 2 and line[1] == "-v": return ["emcc", "-v"] cmd = line[0] - if cmd == "ar": - line[0] = "emar" - return line - elif cmd == "c++" or cmd == "g++": + if cmd == "c++" or cmd == "g++": new_args = ["em++"] elif cmd in ("cc", "gcc", "ld", "lld"): new_args = ["emcc"] # distutils doesn't use the c++ compiler when compiling c++ if any(arg.endswith((".cpp", ".cc")) for arg in line): new_args = ["em++"] - + elif cmd == "ar": + line[0] = "emar" + return line elif cmd == "cmake": # If it is a build/install command, or running a script, we don't do anything. if "--build" in line or "--install" in line or "-P" in line: @@ -644,7 +550,7 @@ def handle_command_generate_args( # noqa: C901 # set linker and C flags to error on anything to do with function declarations being wrong. # Better to fail at compile or link time. - if is_link_command: + if is_link_cmd(line): new_args.append("-Wl,--fatal-warnings") new_args.extend(build_args.ldflags.split()) new_args.extend(get_export_flags(line, build_args.exports)) @@ -663,7 +569,7 @@ def handle_command_generate_args( # noqa: C901 def handle_command( line: list[str], - build_args: BuildArgs, + build_args: CrossCompileArgs, ) -> int: """Handle a compilation command. Exit with an appropriate exit code when done. @@ -674,10 +580,10 @@ def handle_command( build_args : BuildArgs a container with additional compilation options """ - # some libraries have different names on wasm e.g. png16 = png - is_link_cmd = get_library_output(line) is not None if line[0] == "gfortran": + from pyodide_build._f2c_fixes import replay_f2c + tmp = replay_f2c(line) if tmp is None: # No source file, it's a query for information about the compiler. Pretend we're @@ -686,7 +592,7 @@ def handle_command( line = tmp - new_args = handle_command_generate_args(line, build_args, is_link_cmd) + new_args = handle_command_generate_args(line, build_args) if build_args.pkgname == "scipy": from pyodide_build._f2c_fixes import scipy_fixes @@ -698,7 +604,15 @@ def handle_command( def compiler_main(): - build_args = BuildArgs(**PYWASMCROSS_ARGS) + build_args = CrossCompileArgs( + pkgname=PYWASMCROSS_ARGS["pkgname"], + cflags=PYWASMCROSS_ARGS["cflags"], + cxxflags=PYWASMCROSS_ARGS["cxxflags"], + ldflags=PYWASMCROSS_ARGS["ldflags"], + target_install_dir=PYWASMCROSS_ARGS["target_install_dir"], + pythoninclude=PYWASMCROSS_ARGS["pythoninclude"], + exports=PYWASMCROSS_ARGS["exports"], + ) basename = Path(sys.argv[0]).name args = list(sys.argv) args[0] = basename diff --git a/pyodide-build/pyodide_build/tests/test_buildall.py b/pyodide-build/pyodide_build/tests/test_buildall.py index fcceab079..5e0dc0b70 100644 --- a/pyodide-build/pyodide_build/tests/test_buildall.py +++ b/pyodide-build/pyodide_build/tests/test_buildall.py @@ -7,7 +7,7 @@ import pytest from pyodide_lock.spec import PackageSpec from pyodide_build import buildall -from pyodide_build.pywasmcross import BuildArgs +from pyodide_build.build_env import BuildArgs RECIPE_DIR = Path(__file__).parent / "_test_recipes" BUILD_DIR = RECIPE_DIR diff --git a/pyodide-build/pyodide_build/tests/test_buildpkg.py b/pyodide-build/pyodide_build/tests/test_buildpkg.py index 0e64b3740..a225c2eb5 100644 --- a/pyodide-build/pyodide_build/tests/test_buildpkg.py +++ b/pyodide-build/pyodide_build/tests/test_buildpkg.py @@ -7,9 +7,9 @@ import pydantic import pytest from pyodide_build import build_env, buildpkg, common +from pyodide_build.build_env import BuildArgs from pyodide_build.buildpkg import RecipeBuilder from pyodide_build.io import _SourceSpec -from pyodide_build.pywasmcross import BuildArgs RECIPE_DIR = Path(__file__).parent / "_test_recipes" WHEEL_DIR = Path(__file__).parent / "_test_wheels" diff --git a/pyodide-build/pyodide_build/tests/test_f2c_fixes.py b/pyodide-build/pyodide_build/tests/test_f2c_fixes.py new file mode 100644 index 000000000..d53b71aed --- /dev/null +++ b/pyodide-build/pyodide_build/tests/test_f2c_fixes.py @@ -0,0 +1,34 @@ +from pyodide_build._f2c_fixes import ( + replay_f2c, +) + + +def _args_wrapper(func): + """Convert function to take as input / return a string instead of a + list of arguments + + Also sets dryrun=True + """ + + def _inner(line, *pargs): + args = line.split() + res = func(args, *pargs, dryrun=True) + if hasattr(res, "__len__"): + return " ".join(res) + else: + return res + + return _inner + + +f2c_wrap = _args_wrapper(replay_f2c) + + +def test_f2c(): + assert f2c_wrap("gfortran test.f") == "gcc test.c" + assert f2c_wrap("gcc test.c") is None + assert f2c_wrap("gfortran --version") is None + assert ( + f2c_wrap("gfortran --shared -c test.o -o test.so") + == "gcc --shared -c test.o -o test.so" + ) diff --git a/pyodide-build/pyodide_build/tests/test_pywasmcross.py b/pyodide-build/pyodide_build/tests/test_pywasmcross.py index 4a3b1d02b..23a5f774e 100644 --- a/pyodide-build/pyodide_build/tests/test_pywasmcross.py +++ b/pyodide-build/pyodide_build/tests/test_pywasmcross.py @@ -3,54 +3,31 @@ import subprocess import pytest from pyodide_build.pywasmcross import ( - BuildArgs, + CrossCompileArgs, calculate_exports, filter_objects, get_cmake_compiler_flags, - get_library_output, handle_command_generate_args, - replay_f2c, + is_link_cmd, replay_genargs_handle_dashI, ) @pytest.fixture(scope="function") def build_args(): - yield BuildArgs( + yield CrossCompileArgs( cflags="", cxxflags="", ldflags="", target_install_dir="", - host_install_dir="", pythoninclude="python/include", exports="whole_archive", ) -def _args_wrapper(func): - """Convert function to take as input / return a string instead of a - list of arguments - - Also sets dryrun=True - """ - - def _inner(line, *pargs): - args = line.split() - res = func(args, *pargs, dryrun=True) - if hasattr(res, "__len__"): - return " ".join(res) - else: - return res - - return _inner - - -f2c_wrap = _args_wrapper(replay_f2c) - - -def generate_args(line: str, args: BuildArgs, is_link_cmd: bool = False) -> str: +def generate_args(line: str, args: CrossCompileArgs, is_link_cmd: bool = False) -> str: splitline = line.split() - res = handle_command_generate_args(splitline, args, is_link_cmd) + res = handle_command_generate_args(splitline, args) if res[0] in ("emcc", "em++"): for arg in [ @@ -76,7 +53,7 @@ def generate_args(line: str, args: BuildArgs, is_link_cmd: bool = False) -> str: def test_handle_command(build_args): args = build_args - assert handle_command_generate_args(["gcc", "-print-multiarch"], args, True) == [ + assert handle_command_generate_args(["gcc", "-print-multiarch"], args) == [ "echo", "wasm32-emscripten", ] @@ -101,7 +78,7 @@ def test_handle_command(build_args): ) # check cxxflags injection and cpp detection - args = BuildArgs( + args = CrossCompileArgs( cflags="-I./lib2", cxxflags="-std=c++11", ldflags="-lm", @@ -113,7 +90,7 @@ def test_handle_command(build_args): ) # check ldflags injection - args = BuildArgs( + args = CrossCompileArgs( cflags="", cxxflags="", ldflags="-lm", @@ -127,8 +104,8 @@ def test_handle_command(build_args): # Test that repeated libraries are removed assert ( - generate_args("gcc test.o -lbob -ljim -ljim -lbob -o test.so", args) - == "emcc test.o -lbob -ljim -o test.so" + generate_args("gcc test.o -lbob -ljim -ljim -lbob -o test.so", args, True) + == "emcc test.o -lbob -ljim -o test.so -lm" ) @@ -170,25 +147,15 @@ def test_replay_genargs_handle_dashI(monkeypatch): ) -def test_f2c(): - assert f2c_wrap("gfortran test.f") == "gcc test.c" - assert f2c_wrap("gcc test.c") is None - assert f2c_wrap("gfortran --version") is None - assert ( - f2c_wrap("gfortran --shared -c test.o -o test.so") - == "gcc --shared -c test.o -o test.so" - ) - - def test_conda_unsupported_args(build_args): # Check that compile arguments that are not supported by emcc and are sometimes # used in conda are removed. args = build_args - assert generate_args("gcc -c test.o -B /compiler_compat -o test.so", args) == ( - "emcc -c test.o -o test.so" - ) + assert generate_args( + "gcc -c test.o -B /compiler_compat -o test.so", args, True + ) == ("emcc -c test.o -o test.so") - assert generate_args("gcc -c test.o -Wl,--sysroot=/ -o test.so", args) == ( + assert generate_args("gcc -c test.o -Wl,--sysroot=/ -o test.so", args, True) == ( "emcc -c test.o -o test.so" ) @@ -285,16 +252,13 @@ def test_get_cmake_compiler_flags(): def test_handle_command_cmake(build_args): args = build_args - assert "--fresh" in handle_command_generate_args(["cmake", "./"], args, False) + assert "--fresh" in handle_command_generate_args(["cmake", "./"], args) build_cmd = ["cmake", "--build", "." "--target", "target"] - assert handle_command_generate_args(build_cmd, args, False) == build_cmd + assert handle_command_generate_args(build_cmd, args) == build_cmd -def test_get_library_output(): - assert get_library_output(["test.so"]) == "test.so" - assert get_library_output(["test.so.1.2.3"]) == "test.so.1.2.3" - assert ( - get_library_output(["test", "test.a", "test.o", "test.c", "test.cpp", "test.h"]) - is None - ) +def test_is_link_cmd(): + assert is_link_cmd(["test.so"]) + assert is_link_cmd(["test.so.1.2.3"]) + assert not is_link_cmd(["test", "test.a", "test.o", "test.c", "test.cpp", "test.h"])