From 3e1d6348a7e320062c6c21199ab2c93fc6ed7b12 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Thu, 18 Oct 2018 17:19:20 +0200 Subject: [PATCH] Statically link BLAS/LAPACK --- .circleci/config.yml | 2 +- Makefile | 13 +- packages/scipy/meta.yaml | 3 +- .../patches/disable_modules_with_blas.patch | 282 ------------------ packages/scipy/patches/fix-blas.patch | 93 ++++++ packages/scipy/patches/fix_mmap.patch | 27 ++ pyodide_build/pywasmcross.py | 10 + test/packages/test_scipy.py | 12 +- 8 files changed, 147 insertions(+), 295 deletions(-) delete mode 100644 packages/scipy/patches/disable_modules_with_blas.patch create mode 100644 packages/scipy/patches/fix-blas.patch create mode 100644 packages/scipy/patches/fix_mmap.patch diff --git a/.circleci/config.yml b/.circleci/config.yml index c8f3dc69e..67029c6cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ jobs: sudo apt-get install gfortran f2c # Download BLAS/LAPACK - git clone https://github.com/adrianbg/CLAPACK-WA.git packages/scipy/CLAPACK-WA + git clone https://github.com/rth/CLAPACK-WA.git packages/scipy/CLAPACK-WA - restore_cache: keys: diff --git a/Makefile b/Makefile index f88030b47..7f2989dd4 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ OPTFLAGS=-O3 CFLAGS=$(OPTFLAGS) -g -I$(PYTHONINCLUDE) -Wno-warn-absolute-paths CXXFLAGS=$(CFLAGS) -std=c++14 + # __ZNKSt3__220__vector_base_commonILb1EE20__throw_length_errorEv is in # EXPORTED_FUNCTIONS to keep the C++ standard library in the core, even though # there isn't any C++ there, for the sake of loading dynamic modules written in @@ -134,6 +135,7 @@ clean: rm -fr src/*.bc make -C packages clean make -C six clean + make -C packages/scipy/CLAPACK-WA cleanall echo "The Emsdk and CPython are not cleaned. cd into those directories to do so." @@ -208,8 +210,14 @@ $(LZ4LIB): $(SIX_LIBS): $(CPYTHONLIB) make -C six -$(LAPACK): $(CPYTHONLIB) - emmake make -C packages/scipy/CLAPACK-WA +clapack: $(CPYTHONLIB) + # We build BLAS/LAPACK only for target. + # On host we include -LCLAPACK-WA path which has no effect on host. + # On target it gets rewritten by pywasmcross to the full patch of + # blas_WA.bc, lapack_WA.bc which are linked statically in scipy + # in each module that needs them. + make -C $(LAPACK_DIR)F2CLIBS/libf2c/ arith.h + emmake make -C $(LAPACK_DIR) $(CLAPACK): $(CPYTHONLIB) make -C CLAPACK @@ -218,6 +226,5 @@ $(CLAPACK): $(CPYTHONLIB) build/packages.json: $(CPYTHONLIB) $(CLAPACK) make -C packages - emsdk/emsdk/.complete: make -C emsdk diff --git a/packages/scipy/meta.yaml b/packages/scipy/meta.yaml index 7e7193955..2a3bf8fc4 100644 --- a/packages/scipy/meta.yaml +++ b/packages/scipy/meta.yaml @@ -15,13 +15,12 @@ source: # these patches can be found as commits in # https://github.com/rth/scipy/tree/0.17.1-pyodide # on top of the v0.17.1 tag - - patches/disable_modules_with_blas.patch + - patches/fix-blas.patch - patches/fix-build-gcc5-a80460.patch - patches/force_malloc.patch - patches/disable_scipy_stats_mvn.patch - patches/skip-fortran-fails-to-link.patch - patches/dummy_threading.patch - - patches/skip-blas-imports.patch - patches/skip_ellip_harm_2_pyx_ctypes.patch build: diff --git a/packages/scipy/patches/disable_modules_with_blas.patch b/packages/scipy/patches/disable_modules_with_blas.patch deleted file mode 100644 index 4de10e9c4..000000000 --- a/packages/scipy/patches/disable_modules_with_blas.patch +++ /dev/null @@ -1,282 +0,0 @@ -commit 771537885db80fb664d5671af5048e445b6fa821 -Author: Roman Yurchak -Date: Mon Oct 8 10:53:11 2018 +0200 - - Removes files that require LAPACK - -diff --git a/scipy/linalg/setup.py b/scipy/linalg/setup.py -index 2c9b9ba22..035b4bb80 100755 ---- a/scipy/linalg/setup.py -+++ b/scipy/linalg/setup.py -@@ -16,8 +16,8 @@ def configuration(parent_package='', top_path=None): - - lapack_opt = get_info('lapack_opt') - -- if not lapack_opt: -- raise NotFoundError('no lapack/blas resources found') -+ # if not lapack_opt: -+ # raise NotFoundError('no lapack/blas resources found') - - atlas_version = ([v[3:-3] for k, v in lapack_opt.get('define_macros', []) - if k == 'ATLAS_INFO']+[None])[0] -@@ -29,45 +29,45 @@ def configuration(parent_package='', top_path=None): - sources += get_g77_abi_wrappers(lapack_opt) - sources += get_sgemv_fix(lapack_opt) - -- config.add_extension('_fblas', -- sources=sources, -- depends=['fblas_l?.pyf.src'], -- extra_info=lapack_opt -- ) -+ #config.add_extension('_fblas', -+ # sources=sources, -+ # depends=['fblas_l?.pyf.src'], -+ # extra_info=lapack_opt -+ # ) - - # flapack: -- sources = ['flapack.pyf.src'] -- sources += get_g77_abi_wrappers(lapack_opt) -- dep_pfx = join('src', 'lapack_deprecations') -- deprecated_lapack_routines = [join(dep_pfx, c + 'gegv.f') for c in 'cdsz'] -- sources += deprecated_lapack_routines -- -- config.add_extension('_flapack', -- sources=sources, -- depends=['flapack_user.pyf.src'], -- extra_info=lapack_opt -- ) -- -- if atlas_version is not None: -- # cblas: -- config.add_extension('_cblas', -- sources=['cblas.pyf.src'], -- depends=['cblas.pyf.src', 'cblas_l1.pyf.src'], -- extra_info=lapack_opt -- ) -- -- # clapack: -- config.add_extension('_clapack', -- sources=['clapack.pyf.src'], -- depends=['clapack.pyf.src'], -- extra_info=lapack_opt -- ) -- -- # _flinalg: -- config.add_extension('_flinalg', -- sources=[join('src', 'det.f'), join('src', 'lu.f')], -- extra_info=lapack_opt -- ) -+ #sources = ['flapack.pyf.src'] -+ #sources += get_g77_abi_wrappers(lapack_opt) -+ #dep_pfx = join('src', 'lapack_deprecations') -+ #deprecated_lapack_routines = [join(dep_pfx, c + 'gegv.f') for c in 'cdsz'] -+ #sources += deprecated_lapack_routines -+ -+ #config.add_extension('_flapack', -+ # sources=sources, -+ # depends=['flapack_user.pyf.src'], -+ # extra_info=lapack_opt -+ # ) -+ -+ #if atlas_version is not None: -+ # # cblas: -+ # config.add_extension('_cblas', -+ # sources=['cblas.pyf.src'], -+ # depends=['cblas.pyf.src', 'cblas_l1.pyf.src'], -+ # extra_info=lapack_opt -+ # ) -+ -+ # # clapack: -+ # config.add_extension('_clapack', -+ # sources=['clapack.pyf.src'], -+ # depends=['clapack.pyf.src'], -+ # extra_info=lapack_opt -+ # ) -+ -+ ## _flinalg: -+ #config.add_extension('_flinalg', -+ # sources=[join('src', 'det.f'), join('src', 'lu.f')], -+ # extra_info=lapack_opt -+ # ) - - # _interpolative: - routines_to_split = [ -@@ -116,15 +116,15 @@ def configuration(parent_package='', top_path=None): - dirname = os.path.split(os.path.abspath(__file__))[0] - fnames = split_fortran_files(join(dirname, 'src', 'id_dist', 'src'), - routines_to_split) -- fnames = [join('src', 'id_dist', 'src', f) for f in fnames] -- config.add_extension('_interpolative', fnames + ["interpolative.pyf"], -- extra_info=lapack_opt -- ) -+ #fnames = [join('src', 'id_dist', 'src', f) for f in fnames] -+ #config.add_extension('_interpolative', fnames + ["interpolative.pyf"], -+ # extra_info=lapack_opt -+ # ) - - # _calc_lwork: -- config.add_extension('_calc_lwork', -- [join('src', 'calc_lwork.f')], -- extra_info=lapack_opt) -+ #config.add_extension('_calc_lwork', -+ # [join('src', 'calc_lwork.f')], -+ # extra_info=lapack_opt) - - # _solve_toeplitz: - config.add_extension('_solve_toeplitz', -@@ -137,27 +137,27 @@ def configuration(parent_package='', top_path=None): - config.add_data_files('cython_blas.pxd') - config.add_data_files('cython_lapack.pxd') - -- sources = ['_blas_subroutine_wrappers.f', '_lapack_subroutine_wrappers.f'] -- sources += get_g77_abi_wrappers(lapack_opt) -- sources += get_sgemv_fix(lapack_opt) -- includes = numpy_info().get_include_dirs() + [get_python_inc()] -- config.add_library('fwrappers', sources=sources, include_dirs=includes) -- -- config.add_extension('cython_blas', -- sources=['cython_blas.c'], -- depends=['cython_blas.pyx', 'cython_blas.pxd', -- 'fortran_defs.h', '_blas_subroutines.h'], -- include_dirs=['.'], -- libraries=['fwrappers'], -- extra_info=lapack_opt) -- -- config.add_extension('cython_lapack', -- sources=['cython_lapack.c'], -- depends=['cython_lapack.pyx', 'cython_lapack.pxd', -- 'fortran_defs.h', '_lapack_subroutines.h'], -- include_dirs=['.'], -- libraries=['fwrappers'], -- extra_info=lapack_opt) -+ #sources = ['_blas_subroutine_wrappers.f', '_lapack_subroutine_wrappers.f'] -+ #sources += get_g77_abi_wrappers(lapack_opt) -+ #sources += get_sgemv_fix(lapack_opt) -+ #includes = numpy_info().get_include_dirs() + [get_python_inc()] -+ #config.add_library('fwrappers', sources=sources, include_dirs=includes) -+ -+ #config.add_extension('cython_blas', -+ # sources=['cython_blas.c'], -+ # depends=['cython_blas.pyx', 'cython_blas.pxd', -+ # 'fortran_defs.h', '_blas_subroutines.h'], -+ # include_dirs=['.'], -+ # libraries=['fwrappers'], -+ # extra_info=lapack_opt) -+ -+ #config.add_extension('cython_lapack', -+ # sources=['cython_lapack.c'], -+ # depends=['cython_lapack.pyx', 'cython_lapack.pxd', -+ # 'fortran_defs.h', '_lapack_subroutines.h'], -+ # include_dirs=['.'], -+ # libraries=['fwrappers'], -+ # extra_info=lapack_opt) - - config.add_extension('_decomp_update', - sources=['_decomp_update.c']) -diff --git a/scipy/sparse/linalg/eigen/arpack/setup.py b/scipy/sparse/linalg/eigen/arpack/setup.py -index a8175a9d5..c63b76dda 100755 ---- a/scipy/sparse/linalg/eigen/arpack/setup.py -+++ b/scipy/sparse/linalg/eigen/arpack/setup.py -@@ -13,28 +13,28 @@ def configuration(parent_package='',top_path=None): - - lapack_opt = get_info('lapack_opt') - -- if not lapack_opt: -- raise NotFoundError('no lapack/blas resources found') -+ #if not lapack_opt: -+ # raise NotFoundError('no lapack/blas resources found') - - config = Configuration('arpack', parent_package, top_path) - -- arpack_sources = [join('ARPACK','SRC', '*.f')] -- arpack_sources.extend([join('ARPACK','UTIL', '*.f')]) -- arpack_sources.extend([join('ARPACK','LAPACK', '*.f')]) -+ #arpack_sources = [join('ARPACK','SRC', '*.f')] -+ #arpack_sources.extend([join('ARPACK','UTIL', '*.f')]) -+ #arpack_sources.extend([join('ARPACK','LAPACK', '*.f')]) - -- arpack_sources += get_g77_abi_wrappers(lapack_opt) -+ #arpack_sources += get_g77_abi_wrappers(lapack_opt) - -- config.add_library('arpack_scipy', sources=arpack_sources, -- include_dirs=[join('ARPACK', 'SRC')]) -+ #config.add_library('arpack_scipy', sources=arpack_sources, -+ # include_dirs=[join('ARPACK', 'SRC')]) - -- ext_sources = ['arpack.pyf.src'] -- ext_sources += get_sgemv_fix(lapack_opt) -- config.add_extension('_arpack', -- sources=ext_sources, -- libraries=['arpack_scipy'], -- extra_info=lapack_opt, -- depends=arpack_sources, -- ) -+ #ext_sources = ['arpack.pyf.src'] -+ #ext_sources += get_sgemv_fix(lapack_opt) -+ #config.add_extension('_arpack', -+ # sources=ext_sources, -+ # libraries=['arpack_scipy'], -+ # extra_info=lapack_opt, -+ # depends=arpack_sources, -+ # ) - - config.add_data_dir('tests') - return config -diff --git a/scipy/sparse/linalg/isolve/setup.py b/scipy/sparse/linalg/isolve/setup.py -index becb9237a..17288207e 100755 ---- a/scipy/sparse/linalg/isolve/setup.py -+++ b/scipy/sparse/linalg/isolve/setup.py -@@ -13,29 +13,29 @@ def configuration(parent_package='',top_path=None): - - lapack_opt = get_info('lapack_opt') - -- if not lapack_opt: -- raise NotFoundError('no lapack/blas resources found') -- -- # iterative methods -- methods = ['BiCGREVCOM.f.src', -- 'BiCGSTABREVCOM.f.src', -- 'CGREVCOM.f.src', -- 'CGSREVCOM.f.src', --# 'ChebyREVCOM.f.src', -- 'GMRESREVCOM.f.src', --# 'JacobiREVCOM.f.src', -- 'QMRREVCOM.f.src', --# 'SORREVCOM.f.src' -- ] -- -- Util = ['STOPTEST2.f.src','getbreak.f.src'] -- sources = Util + methods + ['_iterative.pyf.src'] -- sources = [join('iterative', x) for x in sources] -- sources += get_g77_abi_wrappers(lapack_opt) -- -- config.add_extension('_iterative', -- sources=sources, -- extra_info=lapack_opt) -+ #if not lapack_opt: -+ # raise NotFoundError('no lapack/blas resources found') -+ -+ ## iterative methods -+ #methods = ['BiCGREVCOM.f.src', -+ # 'BiCGSTABREVCOM.f.src', -+ # 'CGREVCOM.f.src', -+ # 'CGSREVCOM.f.src', -+# # 'ChebyREVCOM.f.src', -+ # 'GMRESREVCOM.f.src', -+# # 'JacobiREVCOM.f.src', -+ # 'QMRREVCOM.f.src', -+# # 'SORREVCOM.f.src' -+ # ] -+ -+ #Util = ['STOPTEST2.f.src','getbreak.f.src'] -+ #sources = Util + methods + ['_iterative.pyf.src'] -+ #sources = [join('iterative', x) for x in sources] -+ #sources += get_g77_abi_wrappers(lapack_opt) -+ -+ #config.add_extension('_iterative', -+ # sources=sources, -+ # extra_info=lapack_opt) - - config.add_data_dir('tests') - diff --git a/packages/scipy/patches/fix-blas.patch b/packages/scipy/patches/fix-blas.patch new file mode 100644 index 000000000..7f46ad7c3 --- /dev/null +++ b/packages/scipy/patches/fix-blas.patch @@ -0,0 +1,93 @@ +commit 926f2e3cf302f2e04717675bb8e6cb77c1c9ee3d +Author: Roman Yurchak +Date: Mon Oct 8 10:53:11 2018 +0200 + + Partial fixes for BLAS/LAPACK + +diff --git a/scipy/linalg/setup.py b/scipy/linalg/setup.py +index 2c9b9ba22..f52f775d6 100755 +--- a/scipy/linalg/setup.py ++++ b/scipy/linalg/setup.py +@@ -14,7 +14,14 @@ def configuration(parent_package='', top_path=None): + + config = Configuration('linalg', parent_package, top_path) + +- lapack_opt = get_info('lapack_opt') ++ # lapack_opt = get_info('lapack_opt') ++ lapack_opt = { # libraries will be auto-generated by pywasmcross ++ 'libraries': [], ++ 'include_dirs': [], ++ 'library_dirs': ['../../CLAPACK-WA/'], ++ 'language': 'c', ++ 'define_macros': [('NO_ATLAS_INFO', 1), ++ ('HAVE_CBLAS', None)]} + + if not lapack_opt: + raise NotFoundError('no lapack/blas resources found') +@@ -117,9 +124,22 @@ def configuration(parent_package='', top_path=None): + fnames = split_fortran_files(join(dirname, 'src', 'id_dist', 'src'), + routines_to_split) + fnames = [join('src', 'id_dist', 'src', f) for f in fnames] +- config.add_extension('_interpolative', fnames + ["interpolative.pyf"], +- extra_info=lapack_opt +- ) ++ # TODO: The following fails with, ++ # scipy/linalg/src/id_dist/src/idd_sfft.c:114:22: error: conflicting types for 'idd_sffti1__' ++ # /* Subroutine */ int idd_sffti1__(integer *ind, integer *n, doublereal *wsave) ++ # ^ ++ # scipy/linalg/src/id_dist/src/idd_sfft.c:72:33: note: previous declaration is here ++ # extern /* Subroutine */ int idd_sffti1__(integer *, integer *, ++ # ^ ++ # scipy/linalg/src/id_dist/src/idd_sfft.c:371:22: error: conflicting types for 'idd_sfft1__' ++ # /* Subroutine */ int idd_sfft1__(integer *ind, integer *n, doublereal *v, ++ # ^ ++ # scipy/linalg/src/id_dist/src/idd_sfft.c:311:33: note: previous declaration is here ++ # extern /* Subroutine */ int idd_sfft1__(integer *, integer *, doublereal * ++ # ++ #config.add_extension('_interpolative', fnames + ["interpolative.pyf"], ++ # extra_info=lapack_opt ++ # ) + + # _calc_lwork: + config.add_extension('_calc_lwork', +diff --git a/scipy/sparse/linalg/eigen/arpack/setup.py b/scipy/sparse/linalg/eigen/arpack/setup.py +index a8175a9d5..f7af2f0d9 100755 +--- a/scipy/sparse/linalg/eigen/arpack/setup.py ++++ b/scipy/sparse/linalg/eigen/arpack/setup.py +@@ -11,7 +11,15 @@ def configuration(parent_package='',top_path=None): + + config = Configuration('arpack',parent_package,top_path) + +- lapack_opt = get_info('lapack_opt') ++ # lapack_opt = get_info('lapack_opt') ++ ++ lapack_opt = { # libraries will be auto-generated by pywasmcross ++ 'libraries': [], ++ 'include_dirs': [], ++ 'library_dirs': ['../../CLAPACK-WA/'], ++ 'language': 'c', ++ 'define_macros': [('NO_ATLAS_INFO', 1), ++ ('HAVE_CBLAS', None)]} + + if not lapack_opt: + raise NotFoundError('no lapack/blas resources found') +diff --git a/scipy/sparse/linalg/isolve/setup.py b/scipy/sparse/linalg/isolve/setup.py +index becb9237a..971e85b6b 100755 +--- a/scipy/sparse/linalg/isolve/setup.py ++++ b/scipy/sparse/linalg/isolve/setup.py +@@ -11,7 +11,14 @@ def configuration(parent_package='',top_path=None): + + config = Configuration('isolve',parent_package,top_path) + +- lapack_opt = get_info('lapack_opt') ++ # lapack_opt = get_info('lapack_opt') ++ lapack_opt = { # libraries will be auto-generated by pywasmcross ++ 'libraries': [], ++ 'include_dirs': [], ++ 'library_dirs': ['../../CLAPACK-WA/'], ++ 'language': 'c', ++ 'define_macros': [('NO_ATLAS_INFO', 1), ++ ('HAVE_CBLAS', None)]} + + if not lapack_opt: + raise NotFoundError('no lapack/blas resources found') diff --git a/packages/scipy/patches/fix_mmap.patch b/packages/scipy/patches/fix_mmap.patch new file mode 100644 index 000000000..daa9ad865 --- /dev/null +++ b/packages/scipy/patches/fix_mmap.patch @@ -0,0 +1,27 @@ +commit 6d4c53df9fecd236cb569b5f1997908af1daea6b +Author: Roman Yurchak +Date: Thu Oct 25 15:28:05 2018 +0200 + + Fix mmap imports + +diff --git a/scipy/io/netcdf.py b/scipy/io/netcdf.py +index 841e97830..f06759015 100644 +--- a/scipy/io/netcdf.py ++++ b/scipy/io/netcdf.py +@@ -39,7 +39,6 @@ __all__ = ['netcdf_file'] + import warnings + import weakref + from operator import mul +-import mmap as mm + + import numpy as np + from numpy.compat import asbytes, asstr +@@ -219,6 +218,8 @@ class netcdf_file(object): + def __init__(self, filename, mode='r', mmap=None, version=1, + maskandscale=False): + """Initialize netcdf_file from fileobj (str or file-like).""" ++ import mmap as mm ++ + if mode not in 'rwa': + raise ValueError("Mode must be either 'r', 'w' or 'a'.") + diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index 6f869f7b8..d17ab1023 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -236,6 +236,8 @@ def handle_command(line, args, dryrun=False): elif new_args[0] in ('emcc', 'em++'): new_args.extend(args.cflags.split()) + lapack_dir = None + # Go through and adjust arguments for arg in line[1:]: if arg.startswith('-I'): @@ -257,6 +259,14 @@ def handle_command(line, args, dryrun=False): elif shared and arg.endswith('.so'): arg = arg[:-3] + '.wasm' output = arg + # Fix for scipy to link to the correct BLAS/LAPACK files + if arg.startswith('-L') and 'CLAPACK-WA' in arg: + lapack_dir = arg.replace('-L', '') + for lib_name in ['F2CLIBS/libf2c.bc', + 'blas_WA.bc', 'lapack_WA.bc']: + arg = os.path.join(lapack_dir, f"{lib_name}") + new_args.append(arg) + continue new_args.append(arg) diff --git a/test/packages/test_scipy.py b/test/packages/test_scipy.py index c3f53f176..ad1e602e9 100644 --- a/test/packages/test_scipy.py +++ b/test/packages/test_scipy.py @@ -23,17 +23,15 @@ def test_scipy_import(selenium_standalone, request): """) # supported modules - for module in ['constants', 'fftpack', 'odr', 'sparse', - 'linalg', # heavily patched - 'misc', 'ndimage', 'special' + for module in ['cluster', 'constants', 'fftpack', 'odr', 'sparse', + 'interpolate', + 'linalg', + 'misc', 'ndimage', 'spatial', 'special' ]: selenium.run(f"import scipy.{module}") # not yet built modules - for module in ['cluster', # needs sparse - 'spatial', # needs sparse - 'integrate', # needs special - 'interpolate', # needs linalg + for module in ['integrate', # needs special 'signal', # needs special 'stats', # need special 'optimize', # needs _odepack