diff --git a/.circleci/config.yml b/.circleci/config.yml index 71a1d352e..2c0baba46 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,6 +15,9 @@ defaults: &defaults # (it's not the case otherwise) CCACHE_DIR: /root/.ccache/ +orbs: + macos: circleci/macos@2.2.0 + jobs: test-docs: <<: *defaults @@ -279,6 +282,62 @@ jobs: - store_test_results: path: test-results + test-main-macos: + parameters: + test-params: + description: The tests to run. + type: string + cache-dir: + description: pytest-cache-dir. + type: string + default: "" + resource_class: medium + macos: + xcode: 13.3.1 + + working_directory: ~/repo + steps: + - attach_workspace: + at: . + # The standard way of enabling safaridriver no longer works for Safari 14+ + # https://blog.bytesguy.com/enabling-remote-automation-in-safari-14 + - macos/add-safari-permissions + - run: + name: install miniforge + command: | + curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh + bash Miniforge3-MacOSX-x86_64.sh -b && rm -f Miniforge3-MacOSX-x86_64.sh + ~/miniforge3/bin/conda create -n pyodide python=3.10 -y + - run: + name: install dependencies + command: | + export PATH="$HOME/miniforge3/bin:$PATH" + conda create -n pyodide python=3.10 -y + source activate pyodide + python -m pip install -r requirements.txt + python -m pip install pytest-pyodide + - run: + name: test safari + command: | + export PATH="$HOME/miniforge3/bin:$PATH" + source activate pyodide + mkdir test-results + pip install pytest-pyodide + if [ -z "<< parameters.cache-dir >>" ]; then + export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)" + else + export CACHE_DIR=".test_cache/<< parameters.cache-dir >>" + fi + echo "pytest cache dir: $CACHE_DIR" + tools/pytest_wrapper.py \ + --junitxml=test-results/junit.xml \ + --verbose \ + --durations 50 \ + << parameters.test-params >> \ + -o cache_dir=$CACHE_DIR + - store_test_results: + path: test-results + benchmark-stack-size: <<: *defaults steps: @@ -557,6 +616,15 @@ workflows: tags: only: /.*/ + - test-main-macos: + name: test-core-safari + test-params: --runtime safari src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ + requires: + - build-core + filters: + tags: + only: /.*/ + - test-main: name: test-core-chrome-webworker test-params: --runtime chrome src/tests/test_webworker.py diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index c4e335472..4ee35af7c 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -191,7 +191,8 @@ def main(): for browser_name, cls in browser_cls: try: t0 = time() - selenium = cls(port, script_timeout=timeout) + selenium = cls(port) + selenium.set_script_timeout(timeout) result[browser_name] = time() - t0 finally: selenium.driver.quit() @@ -203,7 +204,8 @@ def main(): for package_name in ["numpy", "pandas", "matplotlib"]: result = {"native": float("NaN")} for browser_name, cls in browser_cls: - selenium = cls(port, script_timeout=timeout) + selenium = cls(port) + selenium.set_script_timeout(timeout) try: t0 = time() selenium.load_package(package_name) @@ -219,7 +221,8 @@ def main(): try: # instantiate browsers for each benchmark to prevent side effects for browser_name, cls in browser_cls: - selenium_backends[browser_name] = cls(port, script_timeout=timeout) + selenium_backends[browser_name] = cls(port) + selenium_backends[browser_name].set_script_timeout(timeout) # pre-load numpy, matplotlib and pandas for the selenium instance used in benchmarks selenium_backends[browser_name].load_package( ["numpy", "matplotlib", "pandas"] diff --git a/docs/project/changelog.md b/docs/project/changelog.md index e2220abec..57930df83 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -38,6 +38,8 @@ substitutions: be passed as command line arguments to the Python interpreter at start up. {pr}`3021` +- {{ Enhancement }} The core test suite is now run in Safari {pr}`2578`. + - {{ Fix }} It works again to use `loadPyodide` with a relative URL as `indexURL` (this was a regression in v0.21.2). {pr}`3077` diff --git a/src/tests/python_tests.yaml b/src/tests/python_tests.yaml index c3a5af41f..0539886ae 100644 --- a/src/tests/python_tests.yaml +++ b/src/tests/python_tests.yaml @@ -782,7 +782,8 @@ - test_unpack - test_unpack_ex - test_unparse -- test_urllib +- test_urllib: + xfail-safari: crashes the runtime - test_urllib2: skip: - test_http_body_pipe # fork diff --git a/src/tests/test_cmdline_runner.py b/src/tests/test_cmdline_runner.py index e8875e6c7..e46440057 100644 --- a/src/tests/test_cmdline_runner.py +++ b/src/tests/test_cmdline_runner.py @@ -7,7 +7,9 @@ import pytest import pyodide from pyodide_build.common import emscripten_version, get_pyodide_root -mark = pytest.mark.xfail_browsers(chrome="node only", firefox="node only") +mark = pytest.mark.xfail_browsers( + chrome="node only", firefox="node only", safari="node only" +) pyodide_root = get_pyodide_root() script_path = pyodide_root / "tools/python" diff --git a/src/tests/test_console.py b/src/tests/test_console.py index f92c7c469..8ecc4afb5 100644 --- a/src/tests/test_console.py +++ b/src/tests/test_console.py @@ -298,8 +298,12 @@ async def test_console_imports(selenium): assert await get_result("pytz.utc.zone") == "UTC" -def test_console_html(console_html_fixture): - selenium = console_html_fixture +@pytest.mark.xfail_browsers(node="Not available in node") +def test_console_html(selenium): + selenium.goto( + f"http://{selenium.server_hostname}:{selenium.server_port}/console.html" + ) + selenium.javascript_setup() selenium.run_js( """ await window.console_ready; diff --git a/src/tests/test_jsproxy.py b/src/tests/test_jsproxy.py index af6a103d2..075ace323 100644 --- a/src/tests/test_jsproxy.py +++ b/src/tests/test_jsproxy.py @@ -885,7 +885,7 @@ def test_mixins_errors_2(selenium): yield e from pyodide.ffi import JsException - msg = "^TypeError:.* is not a function$" + msg = "^TypeError:.* is not a function.*" with raises(JsException, match=msg): next(c) with raises(JsException, match=msg): diff --git a/src/tests/test_pyodide.py b/src/tests/test_pyodide.py index 1b47826ee..c512089f3 100644 --- a/src/tests/test_pyodide.py +++ b/src/tests/test_pyodide.py @@ -737,6 +737,7 @@ def test_restore_state(selenium): ) +@pytest.mark.xfail_browsers(safari="TODO: traceback is not the same on Safari") @pytest.mark.skip_refcount_check def test_fatal_error(selenium_standalone): assert selenium_standalone.run_js( @@ -829,6 +830,7 @@ def test_reentrant_error(selenium): assert caught +@pytest.mark.xfail_browsers(safari="TODO: traceback is not exactly the same on Safari") def test_js_stackframes(selenium): res = selenium.run_js( """ @@ -1301,7 +1303,7 @@ def test_args(selenium_standalone_noload): ) -@pytest.mark.xfail_browsers(chrome="Node only", firefox="Node only") +@pytest.mark.xfail_browsers(chrome="Node only", firefox="Node only", safari="Node only") def test_relative_index_url(selenium, tmp_path): tmp_dir = Path(tmp_path) import subprocess diff --git a/src/tests/test_pyodide_http.py b/src/tests/test_pyodide_http.py index 9ebc7a086..dfa5fc79c 100644 --- a/src/tests/test_pyodide_http.py +++ b/src/tests/test_pyodide_http.py @@ -82,7 +82,10 @@ def test_pyfetch_set_valid_credentials_value(selenium, httpserver): ) -@pytest.mark.xfail_browsers(node="XMLHttpRequest is not available in node") +@pytest.mark.xfail_browsers( + node="XMLHttpRequest is not available in node", + safari="raises TypeError: exceptions must derive from BaseException", +) def test_pyfetch_coors_error(selenium, httpserver): httpserver.expect_request("/data").respond_with_data( b"HELLO", diff --git a/tools/pytest_wrapper.py b/tools/pytest_wrapper.py index 8dac3c0e2..0946e4f78 100755 --- a/tools/pytest_wrapper.py +++ b/tools/pytest_wrapper.py @@ -26,7 +26,7 @@ def cache_dir(args: list[str]) -> None: if __name__ == "__main__": try: - subprocess.run(["pytest"] + args, check=True) + subprocess.run([sys.executable, "-m", "pytest"] + args, check=True) sys.exit(0) except subprocess.CalledProcessError: pass @@ -46,6 +46,6 @@ if __name__ == "__main__": print("Rerunning failed tests sequentially") remove_num_threads_option(args) try: - subprocess.run(["pytest", "--lf"] + args, check=True) + subprocess.run([sys.executable, "-m", "pytest", "--lf"] + args, check=True) except subprocess.CalledProcessError: sys.exit(1)