TST/CI test dependent selenium WebDriver timeout (#1441)

conftest.py

@@ -22,15 +22,21 @@ BUILD_PATH = ROOT_PATH / "build"
 sys.path.append(str(ROOT_PATH))
 
 from pyodide_build._fixes import _selenium_is_connectable  # noqa: E402
+from pyodide_build.testing import set_webdriver_script_timeout, parse_driver_timeout
 
-try:
-    import selenium.webdriver.common.utils  # noqa: E402
 
-    # XXX: Temporary fix for ConnectionError in selenium
+def _monkeypatch_selenium():
+    try:
+        import selenium.webdriver.common.utils  # noqa: E402
 
-    selenium.webdriver.common.utils.is_connectable = _selenium_is_connectable
-except ModuleNotFoundError:
-    pass
+        # XXX: Temporary fix for ConnectionError in selenium
+
+        selenium.webdriver.common.utils.is_connectable = _selenium_is_connectable
+    except ModuleNotFoundError:
+        pass
+
+
+_monkeypatch_selenium()
 
 
 def pytest_addoption(parser):
@@ -87,6 +93,7 @@ class SeleniumWrapper:
         server_log=None,
         build_dir=None,
         load_pyodide=True,
+        script_timeout=20,
     ):
         if build_dir is None:
             build_dir = BUILD_PATH
@@ -106,7 +113,8 @@ class SeleniumWrapper:
         if load_pyodide:
             self.run_js("await loadPyodide({ indexURL : './'});")
             self.save_state()
-        self.driver.set_script_timeout(20)
+        self.script_timeout = script_timeout
+        self.driver.set_script_timeout(script_timeout)
 
     @property
     def logs(self):
@@ -330,19 +338,25 @@ def selenium_common(request, web_server_main, load_pyodide=True):
 @pytest.fixture(params=["firefox", "chrome"], scope="function")
 def selenium_standalone(request, web_server_main):
     with selenium_common(request, web_server_main) as selenium:
-        try:
-            yield selenium
-        finally:
-            print(selenium.logs)
+        with set_webdriver_script_timeout(
+            selenium, script_timeout=parse_driver_timeout(request)
+        ):
+            try:
+                yield selenium
+            finally:
+                print(selenium.logs)
 
 
 @pytest.fixture(params=["firefox", "chrome"], scope="function")
 def selenium_webworker_standalone(request, web_server_main):
     with selenium_common(request, web_server_main, load_pyodide=False) as selenium:
-        try:
-            yield selenium
-        finally:
-            print(selenium.logs)
+        with set_webdriver_script_timeout(
+            selenium, script_timeout=parse_driver_timeout(request)
+        ):
+            try:
+                yield selenium
+            finally:
+                print(selenium.logs)
 
 
 # selenium instance cached at the module level
@@ -352,9 +366,11 @@ def selenium_module_scope(request, web_server_main):
         yield selenium
 
 
-# We want one version of this decorated as a function-scope fixture and one
-# version decorated as a context manager.
-def selenium_per_function(selenium_module_scope):
+# Hypothesis is unhappy with function scope fixtures. Instead, use the
+# module scope fixture `selenium_module_scope` and use:
+# `with selenium_context_manager(selenium_module_scope) as selenium`
+@contextlib.contextmanager
+def selenium_context_manager(selenium_module_scope):
     try:
         selenium_module_scope.clean_logs()
         yield selenium_module_scope
@@ -362,11 +378,13 @@ def selenium_per_function(selenium_module_scope):
         print(selenium_module_scope.logs)
 
 
-selenium = pytest.fixture(selenium_per_function)
-# Hypothesis is unhappy with function scope fixtures. Instead, use the
-# module scope fixture `selenium_module_scope` and use:
-# `with selenium_context_manager(selenium_module_scope) as selenium`
-selenium_context_manager = contextlib.contextmanager(selenium_per_function)
+@pytest.fixture
+def selenium(request, selenium_module_scope):
+    with selenium_context_manager(selenium_module_scope) as selenium:
+        with set_webdriver_script_timeout(
+            selenium, script_timeout=parse_driver_timeout(request)
+        ):
+            yield selenium
 
 
 @pytest.fixture(scope="session")

packages/pandas/test_pandas.py

@@ -34,17 +34,20 @@ def generate_largish_json(n_rows: int = 91746) -> Dict:
     return data
 
 
+@pytest.mark.driver_timeout(30)
 def test_pandas(selenium, request):
     selenium.load_package("pandas")
     assert len(selenium.run("import pandas\ndir(pandas)")) == 142
 
 
+@pytest.mark.driver_timeout(30)
 def test_extra_import(selenium, request):
 
     selenium.load_package("pandas")
     selenium.run("from pandas import Series, DataFrame, Panel")
 
 
+@pytest.mark.driver_timeout(40)
 @pytest.mark.skip_refcount_check
 def test_load_largish_file(selenium_standalone, request, httpserver):
     selenium = selenium_standalone

packages/pywavelets/test_pywt.py

@@ -1,6 +1,9 @@
+import pytest
+
 from pyodide_build.testing import run_in_pyodide
 
 
+@pytest.mark.driver_timeout(30)
 @run_in_pyodide(packages=["pywavelets"])
 def test_pywt():
     import pywt

packages/scikit-image/test_skimage.py

@@ -1,6 +1,9 @@
+import pytest
+
 from pyodide_build.testing import run_in_pyodide
 
 
+@pytest.mark.driver_timeout(40)
 @run_in_pyodide(packages=["scikit-image"])
 def test_skimage():
     import numpy as np

packages/scikit-learn/test_scikit-learn.py

@@ -1,6 +1,7 @@
 import pytest
 
 
+@pytest.mark.driver_timeout(40)
 def test_scikit_learn(selenium_standalone, request):
     selenium = selenium_standalone
     selenium.load_package("scikit-learn")

packages/scipy/test_scipy.py

@@ -3,6 +3,7 @@ from textwrap import dedent
 import pytest
 
 
+@pytest.mark.driver_timeout(40)
 def test_scipy_linalg(selenium_standalone, request):
     selenium = selenium_standalone
 
@@ -29,6 +30,7 @@ def test_scipy_linalg(selenium_standalone, request):
     selenium.run(cmd)
 
 
+@pytest.mark.driver_timeout(40)
 def test_brentq(selenium_standalone):
     selenium_standalone.load_package("scipy")
     selenium_standalone.run("from scipy.optimize import brentq")

packages/test_common.py

@@ -42,6 +42,7 @@ def test_parse_package(name):
 
 
 @pytest.mark.skip_refcount_check
+@pytest.mark.driver_timeout(40)
 @pytest.mark.parametrize("name", registered_packages())
 def test_import(name, selenium_standalone):
     # check that we can parse the meta.yaml
@@ -69,7 +70,6 @@ def test_import(name, selenium_standalone):
         ))
         """
     )
-    loaded_packages = []
     for import_name in meta.get("test", {}).get("imports", []):
         selenium_standalone.run_async("import %s" % import_name)
         # Make sure that even after importing, there are no additional .pyc

pyodide_build/testing.py

@@ -1,6 +1,7 @@
 import pytest
 import inspect
-from typing import Optional, List, Callable
+from typing import Optional, List, Callable, Union
+import contextlib
 
 
 def run_in_pyodide(
@@ -82,3 +83,31 @@ def run_in_pyodide(
         return decorator(_function)
     else:
         return decorator
+
+
+@contextlib.contextmanager
+def set_webdriver_script_timeout(selenium, script_timeout: Optional[Union[int, float]]):
+    """Set selenium script timeout
+
+    Parameters
+    ----------
+    selenum : SeleniumWrapper
+       a SeleniumWrapper wrapper instance
+    script_timeout : int | float
+       value of the timeout in seconds
+    """
+    if script_timeout is not None:
+        selenium.driver.set_script_timeout(script_timeout)
+    yield
+    # revert to the initial value
+    if script_timeout is not None:
+        selenium.driver.set_script_timeout(selenium.script_timeout)
+
+
+def parse_driver_timeout(request) -> Optional[Union[int, float]]:
+    """Parse driver timeout value from pytest request object"""
+    mark = request.node.get_closest_marker("driver_timeout")
+    if mark is None:
+        return None
+    else:
+        return mark.args[0]

pyodide_build/tests/test_testing.py

@@ -0,0 +1,19 @@
+from pyodide_build.testing import set_webdriver_script_timeout
+
+
+class _MockDriver:
+    def set_script_timeout(self, value):
+        self._timeout = value
+
+
+class _MockSelenium:
+    script_timeout = 2
+    driver = _MockDriver()
+
+
+def test_set_webdriver_script_timeout():
+    selenium = _MockSelenium()
+    assert not hasattr(selenium.driver, "_timeout")
+    with set_webdriver_script_timeout(selenium, script_timeout=10):
+        assert selenium.driver._timeout == 10
+    assert selenium.driver._timeout == 2

setup.cfg

@@ -3,6 +3,7 @@ norecursedirs = build cpython emsdk/
 addopts = --doctest-modules
 markers =
     skip_refcount_check: Dont run refcount checks
+    driver_timeout: Set script timeout in WebDriver
 
 [bumpversion]
 current_version = 0.16.1