diff --git a/.circleci/config.yml b/.circleci/config.yml index c2cc58967..503375ee0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ jobs: sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8 sudo update-alternatives --set gcc /usr/bin/gcc-8 - sudo pip install pytest-xdist pytest-instafail selenium PyYAML pytest-rerunfailures + sudo pip install pytest-xdist pytest-instafail selenium PyYAML # Get recent version of Firefox and geckodriver wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-nightly-latest-ssl\&os\=linux64\&lang\=en-US diff --git a/Makefile b/Makefile index 4aaf771e5..dc7bfe7fa 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ build/renderedhtml.css: src/renderedhtml.less test: all build/test.html build/test_data.txt - py.test test -v --instafail + py.test test -v -r sxX --instafail build/test_data.txt: test/data.txt diff --git a/test/conftest.py b/test/conftest.py index ec30d02c5..6a724be4a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -74,6 +74,9 @@ class SeleniumWrapper: class FirefoxWrapper(SeleniumWrapper): + + browser = 'firefox' + def get_driver(self): from selenium.webdriver import Firefox from selenium.webdriver.firefox.options import Options @@ -89,6 +92,9 @@ class FirefoxWrapper(SeleniumWrapper): class ChromeWrapper(SeleniumWrapper): + + browser = 'chrome' + def get_driver(self): from selenium.webdriver import Chrome from selenium.webdriver.chrome.options import Options diff --git a/test/python_tests.txt b/test/python_tests.txt index b760c9519..4160dd7b6 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -1,6 +1,7 @@ -# Test modules with a failure reason after their name are skipped. +# Test modules with a failure reason after their name are either skipped +# or marked as a known failure in pytest. # -# Reason codes are: +# Following reason codes are skipped: # - platform-specific: This is testing something about a particular platform # that isn't relevant here # - audioop: Requires the audioop module @@ -17,11 +18,15 @@ # implementation of date/time formatting in strftime and strptime # - permissions: Issues with the test writing to the virtual filesystem # - locale: Fails due to include locale implementation. +# - nonsense: This functionality doesn't make sense in this context. Includes +# things like `pip`, `distutils` +# +# While the below reason codes are marked as a known failure: # - crash: The Python interpreter just stopped without a traceback. Will require # further investigation. This usually seems to be caused by calling into a # system function that doesn't behave as one would expect. -# - nonsense: This functionality doesn't make sense in this context. Includes -# things like `pip`, `distutils` +# - crash-chrome: Same as crash but only affecting Chrome +# - crash-firefox: Same as crash but only affecting Firefox test___all__ test___future__ @@ -101,8 +106,8 @@ test_codeop test_collections test_colorsys test_compare -test_compile -test_compileall crash +test_compile crash-chrome +test_compileall test_complex test_concurrent_futures test_configparser @@ -173,7 +178,7 @@ test_faulthandler test_fcntl test_file test_file_eintr subprocess -test_filecmp crash +test_filecmp test_fileinput test_fileio test_finalization @@ -201,7 +206,7 @@ test_genexps test_getargs2 test_getopt test_getpass permissions -test_gettext crash +test_gettext test_glob crash test_global test_grammar @@ -239,20 +244,20 @@ test_importlib.import_.test_packages test_importlib.import_.test_path test_importlib.import_.test_relative_imports test_importlib.source.test_case_sensitivity -test_importlib.source.test_file_loader crash -test_importlib.source.test_finder crash +test_importlib.source.test_file_loader +test_importlib.source.test_finder test_importlib.source.test_path_hook -test_importlib.source.test_source_encoding crash +test_importlib.source.test_source_encoding test_importlib.test_abc -test_importlib.test_api crash +test_importlib.test_api test_importlib.test_lazy test_importlib.test_locks test_importlib.test_namespace_pkgs test_importlib.test_spec -test_importlib.test_util crash +test_importlib.test_util test_importlib.test_windows platform-specific test_index -test_inspect crash +test_inspect test_int test_int_literal test_io crash @@ -305,7 +310,7 @@ test_mimetypes test_minidom test_mmap test_module -test_modulefinder crash +test_modulefinder test_msilib test_multibytecodec test_multiprocessing_fork @@ -334,8 +339,8 @@ test_pickle dbm test_pickletools dbm test_pipes platform-specific test_pkg -test_pkgimport crash -test_pkgutil crash +test_pkgimport +test_pkgutil test_platform subprocess test_plistlib test_poll subprocess @@ -352,7 +357,7 @@ test_pstats test_pty test_pulldom test_pwd crash -test_py_compile crash +test_py_compile test_pyclbr test_pydoc crash test_pyexpat @@ -370,7 +375,7 @@ test_resource test_richcmp test_rlcompleter crash test_robotparser -test_runpy crash +test_runpy test_sax test_sched test_scope @@ -402,7 +407,7 @@ test_stat test_statistics test_strftime strftime test_string -test_string_literals crash +test_string_literals test_stringprep test_strptime strftime test_strtod @@ -502,9 +507,9 @@ test_xml_etree_c test_xmlrpc networking test_xmlrpc_net test_yield_from -test_zipapp crash -test_zipfile crash +test_zipapp +test_zipfile test_zipfile64 -test_zipimport crash -test_zipimport_support crash +test_zipimport +test_zipimport_support test_zlib diff --git a/test/test_common.py b/test/test_common.py index 15344cb25..3649b09d5 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -27,8 +27,8 @@ def registered_packages_meta(): for name in packages} -UNSUPPORTED_PACKAGES = {'ChromeWrapper': ['pandas'], - 'FirefoxWrapper': []} +UNSUPPORTED_PACKAGES = {'chrome': ['pandas'], + 'firefox': []} @pytest.mark.parametrize('name', registered_packages()) @@ -36,12 +36,10 @@ def test_import(name, selenium_standalone): # check that we can parse the meta.yaml meta = common.parse_package(PKG_DIR / name / 'meta.yaml') - if name in UNSUPPORTED_PACKAGES[selenium_standalone.__class__.__name__]: + if name in UNSUPPORTED_PACKAGES[selenium_standalone.browser]: pytest.xfail( - '{} fails to load and is not supported on {}.' - .format(name, - selenium_standalone.__class__.__name__ - .replace('Wrapper', ''))) + '{} fails to load and is not supported on {}.' + .format(name, selenium_standalone.browser)) for import_name in meta.get('test', {}).get('imports', []): selenium_standalone.load_package(name) diff --git a/test/test_pandas.py b/test/test_pandas.py index 8f3360188..1328c1b13 100644 --- a/test/test_pandas.py +++ b/test/test_pandas.py @@ -1,13 +1,18 @@ import pytest -@pytest.mark.skip -def test_pandas(selenium): +def test_pandas(selenium, request): + if selenium.browser == 'chrome': + request.applymarker(pytest.mark.xfail( + run=False, reason='chrome not supported')) selenium.load_package("pandas") assert len(selenium.run("import pandas\ndir(pandas)")) == 179 -@pytest.mark.skip -def test_extra_import(selenium): +def test_extra_import(selenium, request): + if selenium.browser == 'chrome': + request.applymarker(pytest.mark.xfail( + run=False, reason='chrome not supported')) + selenium.load_package("pandas") selenium.run("from pandas import Series, DataFrame, Panel") diff --git a/test/test_python.py b/test/test_python.py index 5b2c6be0e..0e2e9d294 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -283,13 +283,20 @@ def test_open_url(selenium): "pyodide.open_url('test_data.txt').read()\n") == 'HELLO\n' -@pytest.mark.flaky(reruns=2) -def test_run_core_python_test(python_test, selenium): +def test_run_core_python_test(python_test, selenium, request): + + name, error_flags = python_test + + if ('crash' in error_flags or + 'crash-' + selenium.browser in error_flags): + pytest.xfail(reason='known failure with code "{}"' + .format(','.join(error_flags))) + selenium.load_package('test') try: selenium.run( "from test.libregrtest import main\n" - "main(['{}'], verbose=True, verbose3=True)".format(python_test)) + "main(['{}'], verbose=True, verbose3=True)".format(name)) except selenium.JavascriptException as e: assert 'SystemExit: 0' in str(e) @@ -297,17 +304,25 @@ def test_run_core_python_test(python_test, selenium): def pytest_generate_tests(metafunc): if 'python_test' in metafunc.fixturenames: test_modules = [] + test_modules_ids = [] if 'CIRCLECI' not in os.environ or True: with open( Path(__file__).parent / "python_tests.txt") as fp: for line in fp: line = line.strip() - if line.startswith('#'): + if line.startswith('#') or not line: continue - parts = line.split() - if len(parts) == 1: - test_modules.append(parts[0]) - metafunc.parametrize("python_test", test_modules) + error_flags = line.split() + name = error_flags.pop(0) + if (not error_flags + or set(error_flags).intersection( + {'crash', 'crash-chrome', 'crash-firefox'})): + test_modules.append((name, error_flags)) + # explicitly define test ids to keep + # a human readable test name in pytest + test_modules_ids.append(name) + metafunc.parametrize("python_test", test_modules, + ids=test_modules_ids) def test_recursive_repr(selenium):