Run core tests in Safari (#2578)

Co-authored-by: Gyeongjae Choi <def6488@gmail.com>

.circleci/config.yml

@@ -15,6 +15,9 @@ defaults: &defaults
       # (it's not the case otherwise)
       CCACHE_DIR: /root/.ccache/
 
+orbs:
+  macos: circleci/macos@2.2.0
+
 jobs:
   test-docs:
     <<: *defaults
@@ -279,6 +282,62 @@ jobs:
       - store_test_results:
           path: test-results
 
+  test-main-macos:
+    parameters:
+      test-params:
+        description: The tests to run.
+        type: string
+      cache-dir:
+        description: pytest-cache-dir.
+        type: string
+        default: ""
+    resource_class: medium
+    macos:
+      xcode: 13.3.1
+
+    working_directory: ~/repo
+    steps:
+      - attach_workspace:
+          at: .
+      # The standard way of enabling safaridriver no longer works for Safari 14+
+      # https://blog.bytesguy.com/enabling-remote-automation-in-safari-14
+      - macos/add-safari-permissions
+      - run:
+          name: install miniforge
+          command: |
+            curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh
+            bash Miniforge3-MacOSX-x86_64.sh -b && rm -f Miniforge3-MacOSX-x86_64.sh
+            ~/miniforge3/bin/conda create -n pyodide python=3.10 -y
+      - run:
+          name: install dependencies
+          command: |
+            export PATH="$HOME/miniforge3/bin:$PATH"
+            conda create -n pyodide python=3.10 -y
+            source activate pyodide
+            python -m pip install -r requirements.txt
+            python -m pip install pytest-pyodide
+      - run:
+          name: test safari
+          command: |
+            export PATH="$HOME/miniforge3/bin:$PATH"
+            source activate pyodide
+            mkdir test-results
+            pip install pytest-pyodide
+            if [ -z "<< parameters.cache-dir >>" ]; then
+              export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)"
+            else
+              export CACHE_DIR=".test_cache/<< parameters.cache-dir >>"
+            fi
+            echo "pytest cache dir: $CACHE_DIR"
+            tools/pytest_wrapper.py \
+              --junitxml=test-results/junit.xml \
+              --verbose \
+              --durations 50 \
+              << parameters.test-params >> \
+              -o cache_dir=$CACHE_DIR
+      - store_test_results:
+          path: test-results
+
   benchmark-stack-size:
     <<: *defaults
     steps:
@@ -557,6 +616,15 @@ workflows:
             tags:
               only: /.*/
 
+      - test-main-macos:
+          name: test-core-safari
+          test-params: --runtime safari src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
+          requires:
+            - build-core
+          filters:
+            tags:
+              only: /.*/
+
       - test-main:
           name: test-core-chrome-webworker
           test-params: --runtime chrome src/tests/test_webworker.py

benchmark/benchmark.py

@@ -191,7 +191,8 @@ def main():
         for browser_name, cls in browser_cls:
             try:
                 t0 = time()
-                selenium = cls(port, script_timeout=timeout)
+                selenium = cls(port)
+                selenium.set_script_timeout(timeout)
                 result[browser_name] = time() - t0
             finally:
                 selenium.driver.quit()
@@ -203,7 +204,8 @@ def main():
         for package_name in ["numpy", "pandas", "matplotlib"]:
             result = {"native": float("NaN")}
             for browser_name, cls in browser_cls:
-                selenium = cls(port, script_timeout=timeout)
+                selenium = cls(port)
+                selenium.set_script_timeout(timeout)
                 try:
                     t0 = time()
                     selenium.load_package(package_name)
@@ -219,7 +221,8 @@ def main():
             try:
                 # instantiate browsers for each benchmark to prevent side effects
                 for browser_name, cls in browser_cls:
-                    selenium_backends[browser_name] = cls(port, script_timeout=timeout)
+                    selenium_backends[browser_name] = cls(port)
+                    selenium_backends[browser_name].set_script_timeout(timeout)
                     # pre-load numpy, matplotlib and pandas for the selenium instance used in benchmarks
                     selenium_backends[browser_name].load_package(
                         ["numpy", "matplotlib", "pandas"]

docs/project/changelog.md

@@ -38,6 +38,8 @@ substitutions:
   be passed as command line arguments to the Python interpreter at start up.
   {pr}`3021`
 
+- {{ Enhancement }} The core test suite is now run in Safari {pr}`2578`.
+
 - {{ Fix }} It works again to use `loadPyodide` with a relative URL as
   `indexURL` (this was a regression in v0.21.2).
   {pr}`3077`

src/tests/python_tests.yaml

@@ -782,7 +782,8 @@
 - test_unpack
 - test_unpack_ex
 - test_unparse
-- test_urllib
+- test_urllib:
+    xfail-safari: crashes the runtime
 - test_urllib2:
     skip:
       - test_http_body_pipe # fork

src/tests/test_cmdline_runner.py

@@ -7,7 +7,9 @@ import pytest
 import pyodide
 from pyodide_build.common import emscripten_version, get_pyodide_root
 
-mark = pytest.mark.xfail_browsers(chrome="node only", firefox="node only")
+mark = pytest.mark.xfail_browsers(
+    chrome="node only", firefox="node only", safari="node only"
+)
 
 pyodide_root = get_pyodide_root()
 script_path = pyodide_root / "tools/python"

src/tests/test_console.py

@@ -298,8 +298,12 @@ async def test_console_imports(selenium):
     assert await get_result("pytz.utc.zone") == "UTC"
 
 
-def test_console_html(console_html_fixture):
-    selenium = console_html_fixture
+@pytest.mark.xfail_browsers(node="Not available in node")
+def test_console_html(selenium):
+    selenium.goto(
+        f"http://{selenium.server_hostname}:{selenium.server_port}/console.html"
+    )
+    selenium.javascript_setup()
     selenium.run_js(
         """
         await window.console_ready;

src/tests/test_jsproxy.py

@@ -885,7 +885,7 @@ def test_mixins_errors_2(selenium):
                     yield e
 
             from pyodide.ffi import JsException
-            msg = "^TypeError:.* is not a function$"
+            msg = "^TypeError:.* is not a function.*"
             with raises(JsException, match=msg):
                 next(c)
             with raises(JsException, match=msg):

src/tests/test_pyodide.py

@@ -737,6 +737,7 @@ def test_restore_state(selenium):
     )
 
 
+@pytest.mark.xfail_browsers(safari="TODO: traceback is not the same on Safari")
 @pytest.mark.skip_refcount_check
 def test_fatal_error(selenium_standalone):
     assert selenium_standalone.run_js(
@@ -829,6 +830,7 @@ def test_reentrant_error(selenium):
     assert caught
 
 
+@pytest.mark.xfail_browsers(safari="TODO: traceback is not exactly the same on Safari")
 def test_js_stackframes(selenium):
     res = selenium.run_js(
         """
@@ -1301,7 +1303,7 @@ def test_args(selenium_standalone_noload):
     )
 
 
-@pytest.mark.xfail_browsers(chrome="Node only", firefox="Node only")
+@pytest.mark.xfail_browsers(chrome="Node only", firefox="Node only", safari="Node only")
 def test_relative_index_url(selenium, tmp_path):
     tmp_dir = Path(tmp_path)
     import subprocess

src/tests/test_pyodide_http.py

@@ -82,7 +82,10 @@ def test_pyfetch_set_valid_credentials_value(selenium, httpserver):
     )
 
 
-@pytest.mark.xfail_browsers(node="XMLHttpRequest is not available in node")
+@pytest.mark.xfail_browsers(
+    node="XMLHttpRequest is not available in node",
+    safari="raises TypeError: exceptions must derive from BaseException",
+)
 def test_pyfetch_coors_error(selenium, httpserver):
     httpserver.expect_request("/data").respond_with_data(
         b"HELLO",

tools/pytest_wrapper.py

@@ -26,7 +26,7 @@ def cache_dir(args: list[str]) -> None:
 
 if __name__ == "__main__":
     try:
-        subprocess.run(["pytest"] + args, check=True)
+        subprocess.run([sys.executable, "-m", "pytest"] + args, check=True)
         sys.exit(0)
     except subprocess.CalledProcessError:
         pass
@@ -46,6 +46,6 @@ if __name__ == "__main__":
     print("Rerunning failed tests sequentially")
     remove_num_threads_option(args)
     try:
-        subprocess.run(["pytest", "--lf"] + args, check=True)
+        subprocess.run([sys.executable, "-m", "pytest", "--lf"] + args, check=True)
     except subprocess.CalledProcessError:
         sys.exit(1)