Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

MAINT Minor pywasmcross optimizations (#4458) 

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Gyeongjae Choi 2024-02-09 14:06:31 +09:00 committed by GitHub
parent 84c77ff472
commit 4a1c0ba55a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 200 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
pyodide-build/pyodide_build/_f2c_fixes.py
View File

@ -1,4 +1,5 @@
import re
import subprocess
from collections.abc import Iterable, Iterator
from pathlib import Path
from textwrap import dedent # for doctests
@ -513,3 +514,87 @@ def scipy_fixes(args: list[str]) -> None:
for arg in args:
if arg.endswith(".c"):
scipy_fix_cfile(arg)
def replay_f2c(args: list[str], dryrun: bool = False) -> list[str] | None:
"""Apply f2c to compilation arguments
Parameters
----------
args
input compiler arguments
dryrun
if False run f2c on detected fortran files
Returns
-------
new_args
output compiler arguments
Examples
--------
>>> replay_f2c(['gfortran', 'test.f'], dryrun=True)
['gcc', 'test.c']
"""
new_args = ["gcc"]
found_source = False
for arg in args[1:]:
if arg.endswith(".f") or arg.endswith(".F"):
filepath = Path(arg).resolve()
if not dryrun:
fix_f2c_input(arg)
if arg.endswith(".F"):
# .F files apparently expect to be run through the C
# preprocessor (they have #ifdef's in them)
# Use gfortran frontend, as gcc frontend might not be
# present on osx
# The file-system might be not case-sensitive,
# so take care to handle this by renaming.
# For preprocessing and further operation the
# expected file-name and extension needs to be preserved.
subprocess.check_call(
[
"gfortran",
"-E",
"-C",
"-P",
filepath,
"-o",
filepath.with_suffix(".f77"),
]
)
filepath = filepath.with_suffix(".f77")
# -R flag is important, it means that Fortran functions that
# return real e.g. sdot will be transformed into C functions
# that return float. For historic reasons, by default f2c
# transform them into functions that return a double. Using -R
# allows to match what OpenBLAS has done when they f2ced their
# Fortran files, see
# https://github.com/xianyi/OpenBLAS/pull/3539#issuecomment-1493897254
# for more details
with (
open(filepath) as input_pipe,
open(filepath.with_suffix(".c"), "w") as output_pipe,
):
subprocess.check_call(
["f2c", "-R"],
stdin=input_pipe,
stdout=output_pipe,
cwd=filepath.parent,
)
fix_f2c_output(arg[:-2] + ".c")
new_args.append(arg[:-2] + ".c")
found_source = True
else:
new_args.append(arg)
new_args_str = " ".join(args)
if ".so" in new_args_str and "libgfortran.so" not in new_args_str:
found_source = True
if not found_source:
return None
return new_args

16
pyodide-build/pyodide_build/build_env.py
View File

@ -1,5 +1,6 @@
# This file contains functions for managing the Pyodide build environment.
import dataclasses
import functools
import os
import re
@ -68,6 +69,21 @@ BUILD_VARS: set[str] = {
}
@dataclasses.dataclass(eq=False, order=False, kw_only=True)
class BuildArgs:
"""
Common arguments for building a package.
"""
pkgname: str = ""
cflags: str = ""
cxxflags: str = ""
ldflags: str = ""
target_install_dir: str = "" # The path to the target Python installation
host_install_dir: str = "" # Directory for installing built host packages.
builddir: str = "" # The path to run pypa/build
def init_environment(*, quiet: bool = False) -> None:
"""
Initialize Pyodide build environment.

6
pyodide-build/pyodide_build/buildall.py
View File

@ -30,6 +30,7 @@ from rich.spinner import Spinner
from rich.table import Table
from . import build_env, recipe
from .build_env import BuildArgs
from .buildpkg import needs_rebuild
from .common import (
extract_wheel_metadata_file,
@ -39,7 +40,6 @@ from .common import (
)
from .io import MetaConfig, _BuildSpecTypes
from .logger import console_stdout, logger
from .pywasmcross import BuildArgs
class BuildError(Exception):
@ -915,9 +915,5 @@ def set_default_build_args(build_args: BuildArgs) -> BuildArgs:
args.target_install_dir = build_env.get_build_flag("TARGETINSTALLDIR") # type: ignore[unreachable]
if args.host_install_dir is None:
args.host_install_dir = build_env.get_build_flag("HOSTINSTALLDIR") # type: ignore[unreachable]
if args.compression_level is None:
args.compression_level = int( # type: ignore[unreachable]
build_env.get_build_flag("PYODIDE_ZIP_COMPRESSION_LEVEL")
)
return args

2
pyodide-build/pyodide_build/buildpkg.py
View File

@ -21,6 +21,7 @@ from . import common, pypabuild
from .bash_runner import BashRunnerWithSharedEnvironment, get_bash_runner
from .build_env import (
RUST_BUILD_PRELUDE,
BuildArgs,
get_build_environment_vars,
get_build_flag,
pyodide_tags,
@ -37,7 +38,6 @@ from .common import (
)
from .io import MetaConfig, _SourceSpec
from .logger import logger
from .pywasmcross import BuildArgs
def _make_whlfile(

4
pyodide-build/pyodide_build/cli/build_recipes.py
View File

@ -5,10 +5,9 @@ from pathlib import Path
import typer
from .. import build_env, buildall, buildpkg
from ..build_env import init_environment
from ..build_env import BuildArgs, init_environment
from ..common import get_num_cores
from ..logger import logger
from ..pywasmcross import BuildArgs
@dataclasses.dataclass(eq=False, order=False, kw_only=True)
@ -195,6 +194,7 @@ def build_recipes(
),
compression_level: int = typer.Option(
6,
envvar="PYODIDE_ZIP_COMPRESSION_LEVEL",
help="Level of zip compression to apply when installing. 0 means no compression.",
),
) -> None:

164
pyodide-build/pyodide_build/pywasmcross.py
View File

@ -1,9 +1,8 @@
#!/usr/bin/env python3
"""Helper for cross-compiling distutils-based Python extensions.
"""Helper for cross-compiling Python binary extensions.
distutils has never had a proper cross-compilation story. This is a hack, which
Python has never had a proper cross-compilation story. This is a hack, which
miraculously works, to get around that.
The gist is we compile the package replacing calls to the compiler and linker
with wrappers that adjusting include paths and flags as necessary for
cross-compiling and then pass the command long to emscripten.
@ -57,128 +56,40 @@ if IS_COMPILER_INVOCATION:
__name__ = PYWASMCROSS_ARGS.pop("orig__name__")
import dataclasses
import re
import shutil
import subprocess
from collections.abc import Iterable, Iterator
from typing import Literal
from typing import Literal, NamedTuple
@dataclasses.dataclass(eq=False, order=False, kw_only=True)
class BuildArgs:
class CrossCompileArgs(NamedTuple):
"""
Common arguments for building a package.
Arguments for cross-compiling a package.
"""
pkgname: str = ""
cflags: str = ""
cxxflags: str = ""
ldflags: str = ""
# The name of the package being compiled
# This is used to apply package-specific fixes, such as scipy
pkgname: str = ""
target_install_dir: str = "" # The path to the target Python installation
host_install_dir: str = "" # Directory for installing built host packages.
builddir: str = "" # The path to run pypa/build
pythoninclude: str = ""
pythoninclude: str = "" # path to the cross-compiled Python include directory
exports: Literal["whole_archive", "requested", "pyinit"] | list[str] = "pyinit"
compression_level: int = 6
def replay_f2c(args: list[str], dryrun: bool = False) -> list[str] | None:
"""Apply f2c to compilation arguments
Parameters
----------
args
input compiler arguments
dryrun
if False run f2c on detected fortran files
Returns
-------
new_args
output compiler arguments
Examples
--------
>>> replay_f2c(['gfortran', 'test.f'], dryrun=True)
['gcc', 'test.c']
def is_link_cmd(line: list[str]) -> bool:
"""
from pyodide_build._f2c_fixes import fix_f2c_input, fix_f2c_output
new_args = ["gcc"]
found_source = False
for arg in args[1:]:
if arg.endswith(".f") or arg.endswith(".F"):
filepath = Path(arg).resolve()
if not dryrun:
fix_f2c_input(arg)
if arg.endswith(".F"):
# .F files apparently expect to be run through the C
# preprocessor (they have #ifdef's in them)
# Use gfortran frontend, as gcc frontend might not be
# present on osx
# The file-system might be not case-sensitive,
# so take care to handle this by renaming.
# For preprocessing and further operation the
# expected file-name and extension needs to be preserved.
subprocess.check_call(
[
"gfortran",
"-E",
"-C",
"-P",
filepath,
"-o",
filepath.with_suffix(".f77"),
]
)
filepath = filepath.with_suffix(".f77")
# -R flag is important, it means that Fortran functions that
# return real e.g. sdot will be transformed into C functions
# that return float. For historic reasons, by default f2c
# transform them into functions that return a double. Using -R
# allows to match what OpenBLAS has done when they f2ced their
# Fortran files, see
# https://github.com/xianyi/OpenBLAS/pull/3539#issuecomment-1493897254
# for more details
with (
open(filepath) as input_pipe,
open(filepath.with_suffix(".c"), "w") as output_pipe,
):
subprocess.check_call(
["f2c", "-R"],
stdin=input_pipe,
stdout=output_pipe,
cwd=filepath.parent,
)
fix_f2c_output(arg[:-2] + ".c")
new_args.append(arg[:-2] + ".c")
found_source = True
else:
new_args.append(arg)
new_args_str = " ".join(args)
if ".so" in new_args_str and "libgfortran.so" not in new_args_str:
found_source = True
if not found_source:
return None
return new_args
def get_library_output(line: list[str]) -> str | None:
"""
Check if the command is a linker invocation. If so, return the name of the
output file.
Check if the command is a linker invocation.
"""
import re
SHAREDLIB_REGEX = re.compile(r"\.so(.\d+)*$")
for arg in line:
if not arg.startswith("-") and SHAREDLIB_REGEX.search(arg):
return arg
return None
return True
return False
def replay_genargs_handle_dashl(arg: str, used_libs: set[str]) -> str | None:
@ -425,6 +336,8 @@ def _calculate_object_exports_readobj_parse(output: str) -> list[str]:
def calculate_object_exports_readobj(objects: list[str]) -> list[str] | None:
import shutil
readobj_path = shutil.which("llvm-readobj")
if not readobj_path:
which_emcc = shutil.which("emcc")
@ -518,7 +431,7 @@ def get_export_flags(
def handle_command_generate_args( # noqa: C901
line: list[str], build_args: BuildArgs, is_link_command: bool
line: list[str], build_args: CrossCompileArgs
) -> list[str]:
"""
A helper command for `handle_command` that generates the new arguments for
@ -534,42 +447,35 @@ def handle_command_generate_args( # noqa: C901
build_args The arguments that pywasmcross was invoked with
is_link_command Is this a linker invocation?
Returns
-------
An updated argument list suitable for use with emscripten.
Examples
--------
>>> from collections import namedtuple
>>> Args = namedtuple('args', ['cflags', 'cxxflags', 'ldflags', 'target_install_dir'])
>>> args = Args(cflags='', cxxflags='', ldflags='', target_install_dir='')
>>> handle_command_generate_args(['gcc', 'test.c'], args, False)
>>> handle_command_generate_args(['gcc', 'test.c'], args)
['emcc', 'test.c', '-Werror=implicit-function-declaration', '-Werror=mismatched-parameter-types', '-Werror=return-type']
"""
if "-print-multiarch" in line:
return ["echo", "wasm32-emscripten"]
for arg in line:
if arg.startswith("-print-file-name"):
return line
if len(line) == 2 and line[1] == "-v":
return ["emcc", "-v"]
cmd = line[0]
if cmd == "ar":
line[0] = "emar"
return line
elif cmd == "c++" or cmd == "g++":
if cmd == "c++" or cmd == "g++":
new_args = ["em++"]
elif cmd in ("cc", "gcc", "ld", "lld"):
new_args = ["emcc"]
# distutils doesn't use the c++ compiler when compiling c++ <sigh>
if any(arg.endswith((".cpp", ".cc")) for arg in line):
new_args = ["em++"]
elif cmd == "ar":
line[0] = "emar"
return line
elif cmd == "cmake":
# If it is a build/install command, or running a script, we don't do anything.
if "--build" in line or "--install" in line or "-P" in line:
@ -644,7 +550,7 @@ def handle_command_generate_args( # noqa: C901
# set linker and C flags to error on anything to do with function declarations being wrong.
# Better to fail at compile or link time.
if is_link_command:
if is_link_cmd(line):
new_args.append("-Wl,--fatal-warnings")
new_args.extend(build_args.ldflags.split())
new_args.extend(get_export_flags(line, build_args.exports))
@ -663,7 +569,7 @@ def handle_command_generate_args( # noqa: C901
def handle_command(
line: list[str],
build_args: BuildArgs,
build_args: CrossCompileArgs,
) -> int:
"""Handle a compilation command. Exit with an appropriate exit code when done.
@ -674,10 +580,10 @@ def handle_command(
build_args : BuildArgs
a container with additional compilation options
"""
# some libraries have different names on wasm e.g. png16 = png
is_link_cmd = get_library_output(line) is not None
if line[0] == "gfortran":
from pyodide_build._f2c_fixes import replay_f2c
tmp = replay_f2c(line)
if tmp is None:
# No source file, it's a query for information about the compiler. Pretend we're
@ -686,7 +592,7 @@ def handle_command(
line = tmp
new_args = handle_command_generate_args(line, build_args, is_link_cmd)
new_args = handle_command_generate_args(line, build_args)
if build_args.pkgname == "scipy":
from pyodide_build._f2c_fixes import scipy_fixes
@ -698,7 +604,15 @@ def handle_command(
def compiler_main():
build_args = BuildArgs(**PYWASMCROSS_ARGS)
build_args = CrossCompileArgs(
pkgname=PYWASMCROSS_ARGS["pkgname"],
cflags=PYWASMCROSS_ARGS["cflags"],
cxxflags=PYWASMCROSS_ARGS["cxxflags"],
ldflags=PYWASMCROSS_ARGS["ldflags"],
target_install_dir=PYWASMCROSS_ARGS["target_install_dir"],
pythoninclude=PYWASMCROSS_ARGS["pythoninclude"],
exports=PYWASMCROSS_ARGS["exports"],
)
basename = Path(sys.argv[0]).name
args = list(sys.argv)
args[0] = basename

2
pyodide-build/pyodide_build/tests/test_buildall.py
View File

@ -7,7 +7,7 @@ import pytest
from pyodide_lock.spec import PackageSpec
from pyodide_build import buildall
from pyodide_build.pywasmcross import BuildArgs
from pyodide_build.build_env import BuildArgs
RECIPE_DIR = Path(__file__).parent / "_test_recipes"
BUILD_DIR = RECIPE_DIR

2
pyodide-build/pyodide_build/tests/test_buildpkg.py
View File

@ -7,9 +7,9 @@ import pydantic
import pytest
from pyodide_build import build_env, buildpkg, common
from pyodide_build.build_env import BuildArgs
from pyodide_build.buildpkg import RecipeBuilder
from pyodide_build.io import _SourceSpec
from pyodide_build.pywasmcross import BuildArgs
RECIPE_DIR = Path(__file__).parent / "_test_recipes"
WHEEL_DIR = Path(__file__).parent / "_test_wheels"

34
pyodide-build/pyodide_build/tests/test_f2c_fixes.py Normal file
View File

@ -0,0 +1,34 @@
from pyodide_build._f2c_fixes import (
replay_f2c,
)
def _args_wrapper(func):
"""Convert function to take as input / return a string instead of a
list of arguments
Also sets dryrun=True
"""
def _inner(line, *pargs):
args = line.split()
res = func(args, *pargs, dryrun=True)
if hasattr(res, "__len__"):
return " ".join(res)
else:
return res
return _inner
f2c_wrap = _args_wrapper(replay_f2c)
def test_f2c():
assert f2c_wrap("gfortran test.f") == "gcc test.c"
assert f2c_wrap("gcc test.c") is None
assert f2c_wrap("gfortran --version") is None
assert (
f2c_wrap("gfortran --shared -c test.o -o test.so")
== "gcc --shared -c test.o -o test.so"
)

76
pyodide-build/pyodide_build/tests/test_pywasmcross.py
View File

@ -3,54 +3,31 @@ import subprocess
import pytest
from pyodide_build.pywasmcross import (
BuildArgs,
CrossCompileArgs,
calculate_exports,
filter_objects,
get_cmake_compiler_flags,
get_library_output,
handle_command_generate_args,
replay_f2c,
is_link_cmd,
replay_genargs_handle_dashI,
)
@pytest.fixture(scope="function")
def build_args():
yield BuildArgs(
yield CrossCompileArgs(
cflags="",
cxxflags="",
ldflags="",
target_install_dir="",
host_install_dir="",
pythoninclude="python/include",
exports="whole_archive",
)
def _args_wrapper(func):
"""Convert function to take as input / return a string instead of a
list of arguments
Also sets dryrun=True
"""
def _inner(line, *pargs):
args = line.split()
res = func(args, *pargs, dryrun=True)
if hasattr(res, "__len__"):
return " ".join(res)
else:
return res
return _inner
f2c_wrap = _args_wrapper(replay_f2c)
def generate_args(line: str, args: BuildArgs, is_link_cmd: bool = False) -> str:
def generate_args(line: str, args: CrossCompileArgs, is_link_cmd: bool = False) -> str:
splitline = line.split()
res = handle_command_generate_args(splitline, args, is_link_cmd)
res = handle_command_generate_args(splitline, args)
if res[0] in ("emcc", "em++"):
for arg in [
@ -76,7 +53,7 @@ def generate_args(line: str, args: BuildArgs, is_link_cmd: bool = False) -> str:
def test_handle_command(build_args):
args = build_args
assert handle_command_generate_args(["gcc", "-print-multiarch"], args, True) == [
assert handle_command_generate_args(["gcc", "-print-multiarch"], args) == [
"echo",
"wasm32-emscripten",
]
@ -101,7 +78,7 @@ def test_handle_command(build_args):
)
# check cxxflags injection and cpp detection
args = BuildArgs(
args = CrossCompileArgs(
cflags="-I./lib2",
cxxflags="-std=c++11",
ldflags="-lm",
@ -113,7 +90,7 @@ def test_handle_command(build_args):
)
# check ldflags injection
args = BuildArgs(
args = CrossCompileArgs(
cflags="",
cxxflags="",
ldflags="-lm",
@ -127,8 +104,8 @@ def test_handle_command(build_args):
# Test that repeated libraries are removed
assert (
generate_args("gcc test.o -lbob -ljim -ljim -lbob -o test.so", args)
== "emcc test.o -lbob -ljim -o test.so"
generate_args("gcc test.o -lbob -ljim -ljim -lbob -o test.so", args, True)
== "emcc test.o -lbob -ljim -o test.so -lm"
)
@ -170,25 +147,15 @@ def test_replay_genargs_handle_dashI(monkeypatch):
)
def test_f2c():
assert f2c_wrap("gfortran test.f") == "gcc test.c"
assert f2c_wrap("gcc test.c") is None
assert f2c_wrap("gfortran --version") is None
assert (
f2c_wrap("gfortran --shared -c test.o -o test.so")
== "gcc --shared -c test.o -o test.so"
)
def test_conda_unsupported_args(build_args):
# Check that compile arguments that are not supported by emcc and are sometimes
# used in conda are removed.
args = build_args
assert generate_args("gcc -c test.o -B /compiler_compat -o test.so", args) == (
"emcc -c test.o -o test.so"
)
assert generate_args(
"gcc -c test.o -B /compiler_compat -o test.so", args, True
) == ("emcc -c test.o -o test.so")
assert generate_args("gcc -c test.o -Wl,--sysroot=/ -o test.so", args) == (
assert generate_args("gcc -c test.o -Wl,--sysroot=/ -o test.so", args, True) == (
"emcc -c test.o -o test.so"
)
@ -285,16 +252,13 @@ def test_get_cmake_compiler_flags():
def test_handle_command_cmake(build_args):
args = build_args
assert "--fresh" in handle_command_generate_args(["cmake", "./"], args, False)
assert "--fresh" in handle_command_generate_args(["cmake", "./"], args)
build_cmd = ["cmake", "--build", "." "--target", "target"]
assert handle_command_generate_args(build_cmd, args, False) == build_cmd
assert handle_command_generate_args(build_cmd, args) == build_cmd
def test_get_library_output():
assert get_library_output(["test.so"]) == "test.so"
assert get_library_output(["test.so.1.2.3"]) == "test.so.1.2.3"
assert (
get_library_output(["test", "test.a", "test.o", "test.c", "test.cpp", "test.h"])
is None
)
def test_is_link_cmd():
assert is_link_cmd(["test.so"])
assert is_link_cmd(["test.so.1.2.3"])
assert not is_link_cmd(["test", "test.a", "test.o", "test.c", "test.cpp", "test.h"])