From 6a5c37799e9e30a068185693f3c9fd7dbb054e71 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 18 Jul 2018 09:32:16 -0400 Subject: [PATCH 1/4] Add support for transforming Fortran files with f2c --- pyodide_build/pywasmcross.py | 38 ++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index 3fda9a726..dd2bf8ec0 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -41,6 +41,7 @@ from pyodide_build import common ROOTDIR = common.ROOTDIR symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc']) +symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc', 'gfortran']) def collect_args(basename): @@ -105,6 +106,24 @@ def capture_compile(args): sys.exit(result.returncode) +def f2c(args): + new_args = [] + found_source = False + for arg in args: + if arg.endswith('.f'): + filename = os.path.abspath(arg) + subprocess.check_call( + ['f2c', os.path.basename(filename)], + cwd=os.path.dirname(filename)) + new_args.append(arg[:-2] + '.c') + found_source = True + else: + new_args.append(arg) + if not found_source: + return None + return new_args + + def handle_command(line, args, dryrun=False): """Handle a compilation command @@ -137,8 +156,16 @@ def handle_command(line, args, dryrun=False): return if arg == '-print-multiarch': return + if arg.startswith('/tmp'): + return - if line[0] == 'ar': + if line[0] == 'gfortran': + result = f2c(line) + if result is None: + return + line = result + new_args = ['emcc'] + elif line[0] == 'ar': new_args = ['emar'] elif line[0] == 'c++': new_args = ['em++'] @@ -153,6 +180,8 @@ def handle_command(line, args, dryrun=False): new_args.extend(args.ldflags.split()) elif new_args[0] in ('emcc', 'em++'): new_args.extend(args.cflags.split()) + if new_args[0] == 'em++': + new_args.append('-std=c++98') # Go through and adjust arguments for arg in line[1:]: @@ -171,11 +200,16 @@ def handle_command(line, args, dryrun=False): arg = re.sub(r'/python([0-9]\.[0-9]+)m', r'/python\1', arg) if arg.endswith('.o'): arg = arg[:-2] + '.bc' - if shared and arg.endswith('.so'): + output = arg + elif shared and arg.endswith('.so'): arg = arg[:-3] + '.wasm' output = arg new_args.append(arg) + if os.path.isfile(output): + print('SKIPPING: ' + ' '.join(new_args)) + return + print(' '.join(new_args)) if not dryrun: From 2bdba761e38d743157b91acfd09c5f0b331860f1 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 26 Oct 2018 17:22:10 +0200 Subject: [PATCH 2/4] More fixes for f2c --- pyodide_build/pywasmcross.py | 73 +++++++++++++++++++++----- test/pyodide_build/test_pywasmcross.py | 11 +++- 2 files changed, 71 insertions(+), 13 deletions(-) diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index dd2bf8ec0..c0524d3a4 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -106,20 +106,48 @@ def capture_compile(args): sys.exit(result.returncode) -def f2c(args): +def f2c(args, dryrun=False): + """Apply f2c to compilation arguments + + Parameters + ---------- + args : iterable + input compiler arguments + dryrun : bool, default=True + if False run f2c on detected fortran files + + Returns + ------- + new_args : list + output compiler arguments + + + Examples + -------- + + >>> f2c(['gfortran', 'test.f'], dryrun=True) + ['gfortran', 'test.c'] + """ new_args = [] found_source = False for arg in args: if arg.endswith('.f'): filename = os.path.abspath(arg) - subprocess.check_call( - ['f2c', os.path.basename(filename)], - cwd=os.path.dirname(filename)) + if not dryrun: + subprocess.check_call( + ['f2c', os.path.basename(filename)], + cwd=os.path.dirname(filename)) new_args.append(arg[:-2] + '.c') found_source = True else: new_args.append(arg) + + new_args_str = ' '.join(args) + if ".so" in new_args_str and "libgfortran.so" not in new_args_str: + found_source = True + if not found_source: + print(f'f2c: source not found, skipping: {new_args_str}') return None return new_args @@ -147,6 +175,7 @@ def handle_command(line, args, dryrun=False): emcc test.c ['emcc', 'test.c'] """ + # This is a special case to skip the compilation tests in numpy that aren't # actually part of the build for arg in line: @@ -180,8 +209,8 @@ def handle_command(line, args, dryrun=False): new_args.extend(args.ldflags.split()) elif new_args[0] in ('emcc', 'em++'): new_args.extend(args.cflags.split()) - if new_args[0] == 'em++': - new_args.append('-std=c++98') + + lapack_dir = None # Go through and adjust arguments for arg in line[1:]: @@ -204,11 +233,23 @@ def handle_command(line, args, dryrun=False): elif shared and arg.endswith('.so'): arg = arg[:-3] + '.wasm' output = arg + # Fix for scipy to link to the correct BLAS/LAPACK files + if arg.startswith('-L') and 'CLAPACK-WA' in arg: + lapack_dir = arg.replace('-L', '') + for lib_name in ['F2CLIBS/libf2c.bc', + 'blas_WA.bc', + 'lapack_WA.bc']: + arg = os.path.join(lapack_dir, f"{lib_name}") + new_args.append(arg) + continue + new_args.append(arg) - if os.path.isfile(output): - print('SKIPPING: ' + ' '.join(new_args)) - return + # This can only be used for incremental rebuilds -- it generates + # an error during clean build of numpy + # if os.path.isfile(output): + # print('SKIPPING: ' + ' '.join(new_args)) + # return print(' '.join(new_args)) @@ -251,13 +292,21 @@ def clean_out_native_artifacts(): def install_for_distribution(args): - subprocess.check_call( - [Path(args.host) / 'bin' / 'python3', + commands = [ + Path(args.host) / 'bin' / 'python3', 'setup.py', 'install', '--skip-build', '--prefix=install', - '--old-and-unmanageable']) + '--old-and-unmanageable' + ] + try: + subprocess.check_call(commands) + except Exception: + # XXX: temporary hack to remove --old-and-unmanageable with distutils + # see https://github.com/iodide-project/pyodide/issues/220 + # for a better solution. + subprocess.check_call(commands[:-1]) def build_wrap(args): diff --git a/test/pyodide_build/test_pywasmcross.py b/test/pyodide_build/test_pywasmcross.py index 6669695f0..157e943fe 100644 --- a/test/pyodide_build/test_pywasmcross.py +++ b/test/pyodide_build/test_pywasmcross.py @@ -5,6 +5,7 @@ import sys sys.path.append(str(Path(__file__).parents[2])) from pyodide_build.pywasmcross import handle_command # noqa: E402 +from pyodide_build.pywasmcross import f2c # noqa: E402 def _args_wrapper(func): @@ -24,7 +25,7 @@ def _args_wrapper(func): handle_command_wrap = _args_wrapper(handle_command) -# TODO: add f2c here +f2c_wrap = _args_wrapper(f2c) def test_handle_command(): @@ -42,3 +43,11 @@ def test_handle_command(): # compilation checks in numpy assert handle_command_wrap('gcc /usr/file.c', args) is None + + +def test_f2c(): + assert f2c_wrap('gfortran test.f') == 'gfortran test.c' + assert f2c_wrap('gcc test.c') is None + assert f2c_wrap('gfortran --version') is None + assert f2c_wrap('gfortran --shared -c test.o -o test.so') == \ + 'gfortran --shared -c test.o -o test.so' From 8ec53afdc7a234920258dfa8c9910233c9936fba Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 26 Oct 2018 18:02:26 +0200 Subject: [PATCH 3/4] Remove --old-and-unmanageable with distutils --- pyodide_build/pywasmcross.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index c0524d3a4..90fc84ab0 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -175,7 +175,6 @@ def handle_command(line, args, dryrun=False): emcc test.c ['emcc', 'test.c'] """ - # This is a special case to skip the compilation tests in numpy that aren't # actually part of the build for arg in line: @@ -185,8 +184,6 @@ def handle_command(line, args, dryrun=False): return if arg == '-print-multiarch': return - if arg.startswith('/tmp'): - return if line[0] == 'gfortran': result = f2c(line) @@ -210,8 +207,6 @@ def handle_command(line, args, dryrun=False): elif new_args[0] in ('emcc', 'em++'): new_args.extend(args.cflags.split()) - lapack_dir = None - # Go through and adjust arguments for arg in line[1:]: if arg.startswith('-I'): @@ -233,15 +228,6 @@ def handle_command(line, args, dryrun=False): elif shared and arg.endswith('.so'): arg = arg[:-3] + '.wasm' output = arg - # Fix for scipy to link to the correct BLAS/LAPACK files - if arg.startswith('-L') and 'CLAPACK-WA' in arg: - lapack_dir = arg.replace('-L', '') - for lib_name in ['F2CLIBS/libf2c.bc', - 'blas_WA.bc', - 'lapack_WA.bc']: - arg = os.path.join(lapack_dir, f"{lib_name}") - new_args.append(arg) - continue new_args.append(arg) @@ -303,9 +289,10 @@ def install_for_distribution(args): try: subprocess.check_call(commands) except Exception: - # XXX: temporary hack to remove --old-and-unmanageable with distutils - # see https://github.com/iodide-project/pyodide/issues/220 - # for a better solution. + print(f'Warning: {" ".join(commands)} failed with distutils, possibly ' + f'due to the use if distutils that does not support the ' + f'--old-and-unmanageable argument. Re-trying the install ' + f'without this argument.') subprocess.check_call(commands[:-1]) From 0429185745b14783f1441c53be878b9b9c5ba353 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 26 Oct 2018 19:04:32 +0200 Subject: [PATCH 4/4] Fix typo --- pyodide_build/pywasmcross.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index 90fc84ab0..23db1479a 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -290,7 +290,7 @@ def install_for_distribution(args): subprocess.check_call(commands) except Exception: print(f'Warning: {" ".join(commands)} failed with distutils, possibly ' - f'due to the use if distutils that does not support the ' + f'due to the use of distutils that does not support the ' f'--old-and-unmanageable argument. Re-trying the install ' f'without this argument.') subprocess.check_call(commands[:-1])