diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d1b29019..2bacc8d61 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,6 +19,8 @@ jobs: - run: name: dependencies command: | + sudo apt-get install gfortran f2c + # Download BLAS/LAPACK git clone https://github.com/adrianbg/CLAPACK-WA.git packages/scipy/CLAPACK-WA diff --git a/emsdk/patches/num_params.patch b/emsdk/patches/num_params.patch index 977bad2fd..7b435c9b1 100644 --- a/emsdk/patches/num_params.patch +++ b/emsdk/patches/num_params.patch @@ -7,7 +7,7 @@ index 013e9403..d95fc282 100644 // to match when dynamically linking, and also dynamic linking is why we // can't just detect this automatically in the module we see.) -static const int NUM_PARAMS = 15; -+static const int NUM_PARAMS = 37; ++static const int NUM_PARAMS = 45; // Converts a value to the ABI type of i64. static Expression* toABI(Expression* value, Module* module) { diff --git a/packages/scipy/meta.yaml b/packages/scipy/meta.yaml index 6edc659f0..844c86a79 100644 --- a/packages/scipy/meta.yaml +++ b/packages/scipy/meta.yaml @@ -20,10 +20,11 @@ source: - patches/fix-build-gcc5-a80460.patch - patches/force_malloc.patch - patches/disable_scipy_stats_mvn.patch - - patches/skip-scipy-special-_ufuncs.patch + - patches/skip-fortran-fails-to-link.patch + - patches/disable_FuncCastEmulation_num_params.patch build: - cflags: -I/home/rth/src/pyodide/packages/scipy/CLAPACK-WA/INCLUDE -Wno-implicit-function-declaration + cflags: -I../../CLAPACK-WA/INCLUDE -Wno-implicit-function-declaration cxxflags: requirements: diff --git a/packages/scipy/patches/disable_FuncCastEmulation_num_params.patch b/packages/scipy/patches/disable_FuncCastEmulation_num_params.patch new file mode 100644 index 000000000..cf4b36935 --- /dev/null +++ b/packages/scipy/patches/disable_FuncCastEmulation_num_params.patch @@ -0,0 +1,77 @@ +commit dbfb06104aa7f3f36737f394b66a76a29018d5e7 +Author: Roman Yurchak +Date: Fri Oct 12 10:36:48 2018 +0200 + + Disable F2C function that need larger FuncCastEmulation::NUM_PARAMS + + scipy/interpolate/_fitpack seems to require NUM_PARAMS + 43 then 45 then 47 (each requiring to rebuild emsdk), so disabling this + for now. + scipy/ord/_odrpack requires NUM_PARAMS 57 + +diff --git a/scipy/interpolate/setup.py b/scipy/interpolate/setup.py +index 2c8279403..9934376e8 100755 +--- a/scipy/interpolate/setup.py ++++ b/scipy/interpolate/setup.py +@@ -22,18 +22,18 @@ def configuration(parent_package='',top_path=None): + sources=['_ppoly.c'], + **lapack_opt) + +- config.add_extension('_fitpack', +- sources=['src/_fitpackmodule.c'], +- libraries=['fitpack'], +- depends=(['src/__fitpack.h','src/multipack.h'] +- + fitpack_src) +- ) +- +- config.add_extension('dfitpack', +- sources=['src/fitpack.pyf'], +- libraries=['fitpack'], +- depends=fitpack_src, +- ) ++ #config.add_extension('_fitpack', ++ # sources=['src/_fitpackmodule.c'], ++ # libraries=['fitpack'], ++ # depends=(['src/__fitpack.h','src/multipack.h'] ++ # + fitpack_src) ++ # ) ++ ++ #config.add_extension('dfitpack', ++ # sources=['src/fitpack.pyf'], ++ # libraries=['fitpack'], ++ # depends=fitpack_src, ++ # ) + + #config.add_extension('_interpolate', + # sources=['src/_interpolate.cpp'], +diff --git a/scipy/odr/setup.py b/scipy/odr/setup.py +index 9974dfac6..d7560896f 100644 +--- a/scipy/odr/setup.py ++++ b/scipy/odr/setup.py +@@ -22,18 +22,18 @@ def configuration(parent_package='', top_path=None): + libodr_files.append('d_lpkbls.f') + + odrpack_src = [join('odrpack', x) for x in libodr_files] +- config.add_library('odrpack', sources=odrpack_src) ++ #config.add_library('odrpack', sources=odrpack_src) + + sources = ['__odrpack.c'] + libraries = ['odrpack'] + blas_info.pop('libraries', []) + include_dirs = ['.'] + blas_info.pop('include_dirs', []) +- config.add_extension('__odrpack', +- sources=sources, +- libraries=libraries, +- include_dirs=include_dirs, +- depends=(['odrpack.h'] + odrpack_src), +- **blas_info +- ) ++ #config.add_extension('__odrpack', ++ # sources=sources, ++ # libraries=libraries, ++ # include_dirs=include_dirs, ++ # depends=(['odrpack.h'] + odrpack_src), ++ # **blas_info ++ #) + + config.add_data_dir('tests') + return config diff --git a/packages/scipy/patches/force_malloc.patch b/packages/scipy/patches/force_malloc.patch index 9cc5dea1a..8f4cd5ed3 100644 --- a/packages/scipy/patches/force_malloc.patch +++ b/packages/scipy/patches/force_malloc.patch @@ -1,3 +1,68 @@ +commit d755dab2147732e9213af29f55ab38c38b7ba8d2 +Author: Roman Yurchak +Date: Mon Oct 8 12:13:55 2018 +0200 + + Force malloc + + This aims to avoid the "invalid call target: $_malloc" issues + + - fixes scipy/optimize/zeros.c + - skips scipy/integrate/dop.pyf + - skips scipy/optimize/cobyla + +diff --git a/scipy/integrate/setup.py b/scipy/integrate/setup.py +index 2be454c70..0545dc759 100755 +--- a/scipy/integrate/setup.py ++++ b/scipy/integrate/setup.py +@@ -67,10 +67,10 @@ def configuration(parent_package='',top_path=None): + # **lapack_opt) + + # dop +- config.add_extension('_dop', +- sources=['dop.pyf'], +- libraries=['dop'], +- depends=dop_src) ++ #config.add_extension('_dop', ++ # sources=['dop.pyf'], ++ # libraries=['dop'], ++ # depends=dop_src) + + config.add_extension('_test_multivariate', + sources=quadpack_test_src) +diff --git a/scipy/optimize/__init__.py b/scipy/optimize/__init__.py +index 17ba78371..5deca7498 100644 +--- a/scipy/optimize/__init__.py ++++ b/scipy/optimize/__init__.py +@@ -236,7 +236,7 @@ from .minpack import * + from .zeros import * + from .lbfgsb import fmin_l_bfgs_b, LbfgsInvHessProduct + from .tnc import fmin_tnc +-from .cobyla import fmin_cobyla ++#from .cobyla import fmin_cobyla + from .nonlin import * + from .slsqp import fmin_slsqp + from .nnls import nnls +diff --git a/scipy/optimize/setup.py b/scipy/optimize/setup.py +index 7d2b987cb..1861e7563 100755 +--- a/scipy/optimize/setup.py ++++ b/scipy/optimize/setup.py +@@ -50,11 +50,11 @@ def configuration(parent_package='',top_path=None): + depends=[join('tnc','tnc.h')], + **numpy_nodepr_api) + +- config.add_extension('_cobyla', +- sources=[join('cobyla',x) for x in ['cobyla.pyf', +- 'cobyla2.f', +- 'trstlp.f']], +- **numpy_nodepr_api) ++ #config.add_extension('_cobyla', ++ # sources=[join('cobyla',x) for x in ['cobyla.pyf', ++ # 'cobyla2.f', ++ # 'trstlp.f']], ++ # **numpy_nodepr_api) + + sources = ['minpack2.pyf', 'dcsrch.f', 'dcstep.f'] + config.add_extension('minpack2', diff --git a/scipy/optimize/zeros.c b/scipy/optimize/zeros.c index e557f37cb..d450b3ed4 100644 --- a/scipy/optimize/zeros.c diff --git a/packages/scipy/patches/skip-fortran-fails-to-link.patch b/packages/scipy/patches/skip-fortran-fails-to-link.patch new file mode 100644 index 000000000..b9d978070 --- /dev/null +++ b/packages/scipy/patches/skip-fortran-fails-to-link.patch @@ -0,0 +1,83 @@ +commit 0822e53ae255433469257625e8f292abd13ae562 +Author: Roman Yurchak +Date: Thu Oct 11 17:10:36 2018 +0200 + + Skip fortran code that fails to link + + scipy.integrate.odepack + +diff --git a/scipy/integrate/setup.py b/scipy/integrate/setup.py +index 9d607af8d..2be454c70 100755 +--- a/scipy/integrate/setup.py ++++ b/scipy/integrate/setup.py +@@ -27,7 +27,7 @@ def configuration(parent_package='',top_path=None): + config.add_library('mach', sources=mach_src, + config_fc={'noopt':(__file__,1)}) + config.add_library('quadpack', sources=quadpack_src) +- config.add_library('odepack', sources=odepack_src) ++ #config.add_library('odepack', sources=odepack_src) + config.add_library('dop', sources=dop_src) + + # Extensions +@@ -44,27 +44,27 @@ def configuration(parent_package='',top_path=None): + + odepack_opts = lapack_opt.copy() + odepack_opts.update(numpy_nodepr_api) +- config.add_extension('_odepack', +- sources=['_odepackmodule.c'], +- libraries=odepack_libs, +- depends=(odepack_src + mach_src), +- **odepack_opts) ++ #config.add_extension('_odepack', ++ # sources=['_odepackmodule.c'], ++ # libraries=odepack_libs, ++ # depends=(odepack_src + mach_src), ++ # **odepack_opts) + + # vode +- config.add_extension('vode', +- sources=['vode.pyf'], +- libraries=odepack_libs, +- depends=(odepack_src +- + mach_src), +- **lapack_opt) ++ #config.add_extension('vode', ++ # sources=['vode.pyf'], ++ # libraries=odepack_libs, ++ # depends=(odepack_src ++ # + mach_src), ++ # **lapack_opt) + + # lsoda +- config.add_extension('lsoda', +- sources=['lsoda.pyf'], +- libraries=odepack_libs, +- depends=(odepack_src +- + mach_src), +- **lapack_opt) ++ #config.add_extension('lsoda', ++ # sources=['lsoda.pyf'], ++ # libraries=odepack_libs, ++ # depends=(odepack_src ++ # + mach_src), ++ # **lapack_opt) + + # dop + config.add_extension('_dop', +@@ -76,11 +76,11 @@ def configuration(parent_package='',top_path=None): + sources=quadpack_test_src) + + # Fortran+f2py extension module for testing odeint. +- config.add_extension('_test_odeint_banded', +- sources=odeint_banded_test_src, +- libraries=odepack_libs, +- depends=(odepack_src + mach_src), +- **lapack_opt) ++ #config.add_extension('_test_odeint_banded', ++ # sources=odeint_banded_test_src, ++ # libraries=odepack_libs, ++ # depends=(odepack_src + mach_src), ++ # **lapack_opt) + + config.add_data_dir('tests') + return config diff --git a/packages/scipy/patches/skip-scipy-special-_ufuncs.patch b/packages/scipy/patches/skip-scipy-special-_ufuncs.patch deleted file mode 100644 index 211b4209b..000000000 --- a/packages/scipy/patches/skip-scipy-special-_ufuncs.patch +++ /dev/null @@ -1,114 +0,0 @@ -commit 066faced67d1d0a8a4a89588194fcbd2faa0a525 -Author: Roman Yurchak -Date: Wed Oct 10 17:48:17 2018 +0200 - - Skip scipy.special._ufuncs - -diff --git a/scipy/special/__init__.py b/scipy/special/__init__.py -index f013b818e..ab4956825 100644 ---- a/scipy/special/__init__.py -+++ b/scipy/special/__init__.py -@@ -624,7 +624,11 @@ Convenience Functions - - from __future__ import division, print_function, absolute_import - --from ._ufuncs import * -+try: -+ from ._ufuncs import * -+except ImportError: -+ # TODO -+ pass - - from .basic import * - from . import specfun -diff --git a/scipy/special/_ellip_harm.py b/scipy/special/_ellip_harm.py -index 6972c2ad1..0558ee6d3 100644 ---- a/scipy/special/_ellip_harm.py -+++ b/scipy/special/_ellip_harm.py -@@ -3,7 +3,11 @@ from __future__ import division, print_function, absolute_import - import threading - import numpy as np - --from ._ufuncs import _ellip_harm -+try: -+ from ._ufuncs import _ellip_harm -+except ImportError: -+ # TODO -+ pass - from ._ellip_harm_2 import _ellipsoid, _ellipsoid_norm - - -diff --git a/scipy/special/basic.py b/scipy/special/basic.py -index bb251e745..f1c788c3d 100644 ---- a/scipy/special/basic.py -+++ b/scipy/special/basic.py -@@ -11,9 +11,13 @@ from scipy._lib.six import xrange - from numpy import (pi, asarray, floor, isscalar, iscomplex, real, imag, sqrt, - where, mgrid, sin, place, issubdtype, extract, - less, inexact, nan, zeros, atleast_1d, sinc) --from ._ufuncs import (ellipkm1, mathieu_a, mathieu_b, iv, jv, gamma, psi, zeta, -- hankel1, hankel2, yv, kv, gammaln, ndtri, errprint, poch, -- binom) -+try: -+ from ._ufuncs import (ellipkm1, mathieu_a, mathieu_b, iv, jv, gamma, psi, zeta, -+ hankel1, hankel2, yv, kv, gammaln, ndtri, errprint, poch, -+ binom) -+except ImportError: -+ # TODO -+ pass - from . import specfun - from . import orthogonal - -diff --git a/scipy/special/lambertw.py b/scipy/special/lambertw.py -index 4bdac30f9..8a890ce51 100644 ---- a/scipy/special/lambertw.py -+++ b/scipy/special/lambertw.py -@@ -1,6 +1,10 @@ - from __future__ import division, print_function, absolute_import - --from ._ufuncs import _lambertw -+try: -+ from ._ufuncs import _lambertw -+except ImportError: -+ # TODO -+ pass - - - def lambertw(z, k=0, tol=1e-8): -diff --git a/scipy/special/orthogonal.py b/scipy/special/orthogonal.py -index c2b8dead3..b33ab96ac 100755 ---- a/scipy/special/orthogonal.py -+++ b/scipy/special/orthogonal.py -@@ -102,8 +102,12 @@ from scipy import linalg - from scipy.special import airy - - # Local imports. --from . import _ufuncs as cephes --_gam = cephes.gamma -+try: -+ from . import _ufuncs as cephes -+ _gam = cephes.gamma -+except ImportError: -+ # TODO -+ pass - from . import specfun - - __all__ = ['legendre', 'chebyt', 'chebyu', 'chebyc', 'chebys', -@@ -1657,8 +1661,12 @@ def sh_legendre(n, monic=False): - # ----------------------------------------------------------------------------- - # Vectorized functions for evaluation - # ----------------------------------------------------------------------------- --from ._ufuncs import (binom, eval_jacobi, eval_sh_jacobi, eval_gegenbauer, -- eval_chebyt, eval_chebyu, eval_chebys, eval_chebyc, -- eval_sh_chebyt, eval_sh_chebyu, eval_legendre, -- eval_sh_legendre, eval_genlaguerre, eval_laguerre, -- eval_hermite, eval_hermitenorm) -+try: -+ from ._ufuncs import (binom, eval_jacobi, eval_sh_jacobi, eval_gegenbauer, -+ eval_chebyt, eval_chebyu, eval_chebys, eval_chebyc, -+ eval_sh_chebyt, eval_sh_chebyu, eval_legendre, -+ eval_sh_legendre, eval_genlaguerre, eval_laguerre, -+ eval_hermite, eval_hermitenorm) -+except ImportError: -+ # TODO -+ pass diff --git a/test/conftest.py b/test/conftest.py index b4a973148..99ef23453 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -16,7 +16,8 @@ import shutil TEST_PATH = pathlib.Path(__file__).parents[0].resolve() BUILD_PATH = TEST_PATH / '..' / 'build' -sys.path.append(TEST_PATH / '..') +sys.path.append(str(TEST_PATH / '..')) +print(sys.path) from pyodide_build._fixes import _selenium_is_connectable # noqa: E402 import selenium.webdriver.common.utils # noqa: E402 diff --git a/test/test_scipy.py b/test/test_scipy.py new file mode 100644 index 000000000..06844a85c --- /dev/null +++ b/test/test_scipy.py @@ -0,0 +1,83 @@ +from pathlib import Path +import subprocess +import sys +from textwrap import dedent + +import pytest + +sys.path.append(str(Path(__file__).parents[1])) + +from pyodide_build.common import HOSTPYTHON # noqa: E402 + + +def test_scipy_import(selenium_standalone): + from selenium.common.exceptions import JavascriptException + selenium = selenium_standalone + selenium.load_package("scipy") + selenium.run(""" + import scipy + """) + + # supported modules + for module in ['constants', 'fftpack']: + selenium.run(f"import scipy.{module}") + + # not yet built modules + for module in ['cluster', # needs sparse + 'spatial', # needs sparse + 'sparse', + 'integrate', # needs special + 'interpolate', # needs linalg + 'linalg', + 'misc', # needs special + 'odrpack', + 'signal', # needs special + 'ndimage', # needs special + 'stats', # need special + 'optimize', # needs minpack2 + 'special']: + print(module) + with pytest.raises(JavascriptException) as err: + selenium.run(f"import scipy.{module}") + assert ('ModuleNotFoundError' in str(err.value) + or 'ImportError' in str(err.value)) + + print(selenium.logs) + + +def test_built_so(selenium_standalone): + selenium = selenium_standalone + selenium.load_package("scipy") + + cmd = dedent(r""" + import scipy as sp + import os + + base_dir = os.path.dirname(sp.__file__) + + out = [] + for (dirpath, dirnames, filenames) in os.walk(base_dir): + for path in filenames: + if path.endswith('.so'): + rel_path = os.path.relpath(dirpath, base_dir) + out.append(os.path.join(rel_path, path)) + print("\n".join(out)) + out + """) + + out = subprocess.check_output( + [HOSTPYTHON / 'bin' / 'python3', '-c', cmd]) + modules_host = out.decode('utf-8').split('\n') + + def _get_modules_name(modules): + return set([path.split('.')[0] for path in modules if path]) + + modules_host = _get_modules_name(modules_host) + + modules_target = selenium.run(cmd) + modules_target = _get_modules_name(modules_target) + + print(f'Included modules: {len(modules_target)}') + print(f' {modules_target} ') + print(f'\nMissing modules: {len(modules_host.difference(modules_target))}') + print(f' {modules_host.difference(modules_target)}')