Run scipy tests as part of the Github Action CI (#4935)

Triggered only on push and on PR commits if [scipy] is in the commit message.

.github/workflows/main.yml

@@ -52,8 +52,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-14]
+        include:
+          - os: ubuntu-latest
+            pyodide_packages: "tag:core,numpy${{ needs.test-scipy-trigger.outputs.test-scipy == 'true' && ',scipy' || '' }}"
+          - os: macos-14
+            pyodide_packages: "tag:core,numpy"
     runs-on: ${{ matrix.os }}
+    needs: [test-scipy-trigger]
     env:
       EMSDK_NUM_CORES: 3
       EMCC_CORES: 3
@@ -124,13 +129,13 @@ jobs:
           make -C cpython
           ccache -s
 
-      - name: build Pyodide core + numpy
+      - name: build Pyodide with packages ${{ matrix.pyodide_packages }}
         shell: bash -l {0}
         run: |
           # This is necessary to use the ccache from emsdk
           source pyodide_env.sh
           ccache -z
-          PYODIDE_PACKAGES="tag:core,numpy" make
+          PYODIDE_PACKAGES=${{ matrix.pyodide_packages }} make
           ccache -s
 
       - name: check-size
@@ -225,6 +230,54 @@ jobs:
           paths: "test-results/*.xml"
         if: always()
 
+  test-scipy-trigger:
+    name: test-scipy-trigger
+    runs-on: ubuntu-latest
+    outputs:
+      test-scipy: ${{ steps.check-build-trigger.outputs.trigger }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - id: check-build-trigger
+        name: Check build trigger
+        run: bash tools/check_build_trigger.sh
+
+  test-scipy:
+    runs-on: ${{ matrix.os }}
+    needs: [test-scipy-trigger, build-core]
+    if: needs.test-scipy-trigger.outputs.test-scipy
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download build artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: core-build-${{ runner.os }}
+          path: ./dist/
+
+      - name: run scipy tests inside node
+        shell: bash -l {0}
+        run: |
+          npm install pyodide
+          cp -f dist/* node_modules/pyodide
+
+          # This uses conftest.py on the current working directory to
+          # skip/xfail tests in scipy
+          cd packages/scipy
+          # XXX for some unknown reason adding a conftest.py in the repo throws off the
+          # other tests trying to import from conftest they find the scipy one ...
+          mv scipy-conftest.py conftest.py
+
+          node scipy-pytest.js --pyargs scipy -m 'not slow' -ra -v
+
   test-bun:
     runs-on: ${{ matrix.os }}
 

packages/scipy/scipy-conftest.py

@@ -0,0 +1,263 @@
+import re
+
+import pytest
+
+xfail = pytest.mark.xfail
+skip = pytest.mark.skip
+
+fp_exception_msg = (
+    "no floating point exceptions, "
+    "see https://github.com/numpy/numpy/pull/21895#issuecomment-1311525881"
+)
+process_msg = "no process support"
+thread_msg = "no thread support"
+todo_signature_mismatch_msg = "TODO signature mismatch"
+todo_memory_corruption_msgt = "TODO memory corruption"
+todo_genuine_difference_msg = "TODO genuine difference to be investigated"
+
+tests_to_mark = [
+    # scipy/_lib/tests
+    (
+        "test__threadsafety.py::test_parallel_threads",
+        xfail,
+        thread_msg,
+    ),
+    ("test__threadsafety.py::test_parallel_threads", xfail, thread_msg),
+    ("test__util.py::test_pool", xfail, process_msg),
+    ("test__util.py::test_mapwrapper_parallel", xfail, process_msg),
+    ("test_ccallback.py::test_threadsafety", xfail, thread_msg),
+    ("test_import_cycles.py::test_modules_importable", xfail, process_msg),
+    ("test_import_cycles.py::test_public_modules_importable", xfail, process_msg),
+    # scipy/datasets/tests
+    ("test_data.py::TestDatasets", xfail, "TODO datasets not working right now"),
+    # scipy/fft/tests
+    (
+        r"test_basic.py::TestFFT1D.test_dtypes\[float32-numpy\]",
+        xfail,
+        "TODO small floating point difference on the CI but not locally",
+    ),
+    ("test_basic.py::TestFFTThreadSafe", xfail, thread_msg),
+    ("test_basic.py::test_multiprocess", xfail, process_msg),
+    ("test_fft_function.py::test_fft_function", xfail, process_msg),
+    ("test_multithreading.py::test_threaded_same", xfail, thread_msg),
+    (
+        "test_multithreading.py::test_mixed_threads_processes",
+        xfail,
+        thread_msg,
+    ),
+    # scipy/integrate tests
+    ("test__quad_vec.py::test_quad_vec_pool", xfail, process_msg),
+    (
+        "test_quadpack.py.+TestCtypesQuad.test_ctypes.*",
+        xfail,
+        "Test relying on finding libm.so shared library",
+    ),
+    (
+        "test_quadrature.py.+TestQMCQuad.test_basic",
+        xfail,
+        todo_genuine_difference_msg,
+    ),
+    (
+        "test_quadrature.py.+TestQMCQuad.test_sign",
+        xfail,
+        todo_genuine_difference_msg,
+    ),
+    # scipy/interpolate
+    (
+        "test_fitpack.+test_kink",
+        xfail,
+        "TODO error not raised, maybe due to no floating point exception?",
+    ),
+    # scipy/io
+    (
+        "test_mmio.py::.+fast_matrix_market",
+        xfail,
+        thread_msg,
+    ),
+    (
+        "test_mmio.py::TestMMIOCoordinate.test_precision",
+        xfail,
+        thread_msg,
+    ),
+    (
+        "test_paths.py::TestPaths.test_mmio_(read|write)",
+        xfail,
+        thread_msg,
+    ),
+    # scipy/linalg tests
+    ("test_blas.+test_complex_dotu", skip, todo_signature_mismatch_msg),
+    ("test_cython_blas.+complex", skip, todo_signature_mismatch_msg),
+    ("test_lapack.py.+larfg_larf", skip, todo_signature_mismatch_msg),
+    # scipy/ndimage/tests
+    ("test_filters.py::TestThreading", xfail, thread_msg),
+    # scipy/optimize/tests
+    (
+        "test__differential_evolution.py::"
+        "TestDifferentialEvolutionSolver.test_immediate_updating",
+        xfail,
+        process_msg,
+    ),
+    (
+        "test__differential_evolution.py::TestDifferentialEvolutionSolver.test_parallel",
+        xfail,
+        process_msg,
+    ),
+    (
+        "test__shgo.py.+test_19_parallelization",
+        xfail,
+        process_msg,
+    ),
+    (
+        "test__shgo.py.+",
+        xfail,
+        "Test failing on 32bit (skipped on win32)",
+    ),
+    (
+        "test_linprog.py::TestLinprogSimplexNoPresolve.test_bounds_infeasible_2",
+        xfail,
+        "TODO no warnings emitted maybe due to no floating point exception?",
+    ),
+    ("test_minpack.py::TestFSolve.test_concurrent.+", xfail, process_msg),
+    ("test_minpack.py::TestLeastSq.test_concurrent+", xfail, process_msg),
+    ("test_optimize.py::test_cobyla_threadsafe", xfail, thread_msg),
+    ("test_optimize.py::TestBrute.test_workers", xfail, process_msg),
+    # scipy/signal/tests
+    (
+        "test_signaltools.py::TestMedFilt.test_medfilt2d_parallel",
+        xfail,
+        thread_msg,
+    ),
+    # scipy/sparse/tests
+    ("test_arpack.py::test_parallel_threads", xfail, thread_msg),
+    ("test_array_api.py::test_sparse_dense_divide", xfail, fp_exception_msg),
+    # TODO remove when scipy 1.13 is packaged in Pyodide
+    (
+        "test_base.py.+(COO|DIA|BSR).+multiple_ellipsis_slicing",
+        xfail,
+        "DeprecationWarning for scipy 1.13 not raised not important",
+    ),
+    ("test_linsolve.py::TestSplu.test_threads_parallel", xfail, thread_msg),
+    ("test_propack", skip, todo_signature_mismatch_msg),
+    ("test_sparsetools.py::test_threads", xfail, thread_msg),
+    # scipy/sparse/csgraph/tests
+    ("test_shortest_path.py::test_gh_17782_segfault", xfail, thread_msg),
+    # scipy/spatial/tests
+    (
+        "test_kdtree.py::test_query_ball_point_multithreading",
+        xfail,
+        thread_msg,
+    ),
+    ("test_kdtree.py::test_ckdtree_parallel", xfail, thread_msg),
+    # scipy/special/tests
+    (
+        "test_exponential_integrals.py::TestExp1.test_branch_cut",
+        xfail,
+        "TODO maybe float support since +0 and -0 difference",
+    ),
+    (
+        "test_round.py::test_add_round_(up|down)",
+        xfail,
+        "TODO small floating point difference, maybe due to lack of floating point "
+        "support for controlling rounding, see "
+        "https://github.com/WebAssembly/design/issues/1384",
+    ),
+    (
+        # This test is skipped for PyPy as well, maybe for a related reason?,
+        # see
+        # https://github.com/conda-forge/scipy-feedstock/pull/196#issuecomment-979317832
+        "test_distributions.py::TestBeta.test_boost_eval_issue_14606",
+        skip,
+        "TODO C++ exception that causes a Pyodide fatal error",
+    ),
+    (
+        "test_kdeoth.py::test_kde_[12]d",
+        xfail,
+        todo_genuine_difference_msg,
+    ),
+    (
+        "test_multivariate.py::TestMultivariateT.test_cdf_against_generic_integrators",
+        skip,
+        "TODO tplquad integration does not seem to converge",
+    ),
+    (
+        "test_multivariate.py::TestCovariance.test_mvn_with_covariance_cdf.+Precision-size1",
+        xfail,
+        "TODO small floating point difference 6e-7 relative diff instead of 1e-7",
+    ),
+    (
+        "test_multivariate.py::TestMultivariateNormal.test_logcdf_default_values",
+        xfail,
+        todo_genuine_difference_msg,
+    ),
+    (
+        "test_multivariate.py::TestMultivariateNormal.test_broadcasting",
+        xfail,
+        todo_genuine_difference_msg,
+    ),
+    (
+        "test_multivariate.py::TestMultivariateNormal.test_normal_1D",
+        xfail,
+        todo_genuine_difference_msg,
+    ),
+    (
+        "test_multivariate.py::TestMultivariateNormal.test_R_values",
+        xfail,
+        todo_genuine_difference_msg,
+    ),
+    (
+        "test_multivariate.py::TestMultivariateNormal.test_cdf_with_lower_limit",
+        xfail,
+        todo_genuine_difference_msg,
+    ),
+    (
+        "test_multivariate.py::TestMultivariateT.test_cdf_against_multivariate_normal",
+        xfail,
+        todo_genuine_difference_msg,
+    ),
+    ("test_qmc.py::TestVDC.test_van_der_corput", xfail, thread_msg),
+    ("test_qmc.py::TestHalton.test_workers", xfail, thread_msg),
+    ("test_qmc.py::TestUtils.test_discrepancy_parallel", xfail, thread_msg),
+    (
+        "test_qmc.py::TestMultivariateNormalQMC.test_validations",
+        xfail,
+        "TODO did not raise maybe no floating point exception support?",
+    ),
+    (
+        "test_qmc.py::TestMultivariateNormalQMC.test_MultivariateNormalQMCDegenerate",
+        xfail,
+        todo_genuine_difference_msg,
+    ),
+    ("test_sampling.py::test_threading_behaviour", xfail, thread_msg),
+    ("test_stats.py::TestMGCStat.test_workers", xfail, process_msg),
+    (
+        "test_stats.py::TestKSTwoSamples.testLargeBoth",
+        skip,
+        "TODO test taking > 5 minutes after scipy 1.10.1 update",
+    ),
+    (
+        "test_stats.py::TestKSTwoSamples.test_some_code_paths",
+        xfail,
+        "TODO did not raise maybe no floating point exception support?",
+    ),
+    (
+        "test_stats.py::TestGeometricStandardDeviation.test_raises_value_error",
+        xfail,
+        "TODO did not raise maybe no floating point exception support?",
+    ),
+    (
+        "test_stats.py::TestBrunnerMunzel.test_brunnermunzel_normal_dist",
+        xfail,
+        fp_exception_msg,
+    ),
+]
+
+
+def pytest_collection_modifyitems(config, items):
+    for item in items:
+        path, line, name = item.reportinfo()
+        path = str(path)
+        full_name = f"{path}::{name}"
+        for pattern, mark, reason in tests_to_mark:
+            if re.search(pattern, full_name):
+                # print(full_name)
+                item.add_marker(mark(reason=reason))

packages/scipy/scipy-pytest.js

@@ -0,0 +1,84 @@
+const { opendir } = require("node:fs/promises");
+const { loadPyodide } = require("pyodide");
+
+async function main() {
+  let exit_code = 0;
+  try {
+    global.pyodide = await loadPyodide();
+    let pyodide = global.pyodide;
+    const FS = pyodide.FS;
+    const NODEFS = FS.filesystems.NODEFS;
+
+    let mountDir = "/mnt";
+    pyodide.FS.mkdir(mountDir);
+    pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: "." }, mountDir);
+
+    // Copy pytest-specific files dir if they exist
+    await pyodide.runPythonAsync(`
+       import shutil
+       import os
+
+       pytest_filenames = ["/mnt/conftest.py", "/mnt/pytest.ini"]
+
+       for filename in pytest_filenames:
+           if os.path.exists(filename):
+               shutil.copy(filename, ".")
+
+       conftest_filename = "/mnt/conftest.py"
+       if os.path.exists(conftest_filename):
+           shutil.copy(conftest_filename, ".")
+    `);
+
+    await pyodide.loadPackage(["micropip"]);
+    await pyodide.runPythonAsync(`
+       import micropip
+
+       await micropip.install('scipy')
+
+       try:
+           await micropip.install('scipy-tests')
+       except ValueError:
+           print('Hoping scipy tests are included in the scipy wheel')
+
+       pkg_list = micropip.list()
+       print(pkg_list)
+    `);
+
+    // XXX: some Fortran test modules are removed in Pyodide through a patch
+    // https://github.com/pyodide/pyodide/blob/main/packages/scipy/patches/0008-Remove-test-modules-that-fails-to-build.patch
+    // In order to avoid import errors during test discovery, we delete the
+    // problematic files. There seems to be no simpler way to do this with
+    // pytest, in particular --ignore-glob still imports the ignored file for
+    // some reason.
+    await pyodide.runPythonAsync(`
+      from pathlib import Path
+
+      import scipy.io.tests
+      path = Path(scipy.io.tests.__file__).parent / "test_fortran.py"
+      os.unlink(path)
+
+      import scipy.integrate.tests
+      path = Path(scipy.integrate.tests.__file__).parent / "test_odeint_jac.py"
+      os.unlink(path)
+    `);
+
+    await pyodide.runPythonAsync(
+      "import micropip; micropip.install(['pytest', 'hypothesis', 'pooch', 'lzma'])",
+    );
+    let pytest = pyodide.pyimport("pytest");
+    let args = process.argv.slice(2);
+    console.log("pytest args:", args);
+    exit_code = pytest.main(pyodide.toPy(args));
+  } catch (e) {
+    console.error(e);
+    // Arbitrary exit code here. I have seen this code reached instead of a
+    // Pyodide fatal error sometimes (I guess kind of similar to a random
+    // Python error). When there is a Pyodide fatal error we don't end up here
+    // somehow, and the exit code is 7
+    exit_code = 66;
+  } finally {
+    process.exit(exit_code);
+  }
+}
+
+main();

tools/check_build_trigger.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+set -x
+
+COMMIT_MSG=$(git log --no-merges -1 --oneline)
+
+# The scipy tests will be triggered on push or on pull_request when the commit
+# message contains "[scipy]"
+if [[ "$GITHUB_EVENT_NAME" == push ||
+      "$COMMIT_MSG" =~ \[scipy\] ]]; then
+    echo "trigger=true" >> "$GITHUB_OUTPUT"
+fi

tools/codespell_ignore_words.txt

@@ -14,4 +14,4 @@ te
 oint
 conveniant
 atmost
-COO
+coo