From 30195bfa8a88f3a52b188f5e9f36c9ddb59a8678 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 17 Aug 2018 12:33:58 +0300 Subject: [PATCH 01/10] Marking known failures in test_run_core_python_test --- test/python_tests.txt | 15 ++++++++++----- test/test_python.py | 28 +++++++++++++++++++++------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/test/python_tests.txt b/test/python_tests.txt index b760c9519..c6e2fc27a 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -1,6 +1,7 @@ -# Test modules with a failure reason after their name are skipped. +# Test modules with a failure reason after their name are either skipped +# or marked as a known failure in pytest. # -# Reason codes are: +# Following reason codes are skipped: # - platform-specific: This is testing something about a particular platform # that isn't relevant here # - audioop: Requires the audioop module @@ -17,11 +18,15 @@ # implementation of date/time formatting in strftime and strptime # - permissions: Issues with the test writing to the virtual filesystem # - locale: Fails due to include locale implementation. +# - nonsense: This functionality doesn't make sense in this context. Includes +# things like `pip`, `distutils` +# +# While the below reason codes are marked as a known failure: # - crash: The Python interpreter just stopped without a traceback. Will require # further investigation. This usually seems to be caused by calling into a # system function that doesn't behave as one would expect. -# - nonsense: This functionality doesn't make sense in this context. Includes -# things like `pip`, `distutils` +# - crash_chrome: Same as crash but only affecting Chrome +# - crash_firefox: Same as crash but only affecting Firefox test___all__ test___future__ @@ -101,7 +106,7 @@ test_codeop test_collections test_colorsys test_compare -test_compile +test_compile crash-chrome test_compileall crash test_complex test_concurrent_futures diff --git a/test/test_python.py b/test/test_python.py index 5b2c6be0e..c79526466 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -284,12 +284,18 @@ def test_open_url(selenium): @pytest.mark.flaky(reruns=2) -def test_run_core_python_test(python_test, selenium): +def test_run_core_python_test(python_test, selenium, request): selenium.load_package('test') + + name, error_flags = python_test + if error_flags: + request.applymarker(pytest.mark.xfail( + run=False, reason='known failure with code "{}"' + .format(error_flags))) try: selenium.run( "from test.libregrtest import main\n" - "main(['{}'], verbose=True, verbose3=True)".format(python_test)) + "main(['{}'], verbose=True, verbose3=True)".format(name)) except selenium.JavascriptException as e: assert 'SystemExit: 0' in str(e) @@ -297,17 +303,25 @@ def test_run_core_python_test(python_test, selenium): def pytest_generate_tests(metafunc): if 'python_test' in metafunc.fixturenames: test_modules = [] + test_modules_ids = [] if 'CIRCLECI' not in os.environ or True: with open( Path(__file__).parent / "python_tests.txt") as fp: for line in fp: line = line.strip() - if line.startswith('#'): + if line.startswith('#') or not line: continue - parts = line.split() - if len(parts) == 1: - test_modules.append(parts[0]) - metafunc.parametrize("python_test", test_modules) + error_flags = line.split() + name = error_flags.pop(0) + if (not error_flags + or set(error_flags).intersection( + {'crash', 'crash-chrome', 'crash-firefox'})): + test_modules.append((name, error_flags)) + # explicitly define test ids to keep + # a human readable test name in pytest + test_modules_ids.append(name) + metafunc.parametrize("python_test", test_modules, + ids=test_modules_ids) def test_recursive_repr(selenium): From 22ecc3484780137f6991c07eebdf3692ab9ff107 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 17 Aug 2018 12:39:04 +0300 Subject: [PATCH 02/10] Supporting Chrome/Firefox specific failures --- test/test_python.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_python.py b/test/test_python.py index c79526466..ad0be8572 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -288,7 +288,10 @@ def test_run_core_python_test(python_test, selenium, request): selenium.load_package('test') name, error_flags = python_test - if error_flags: + driver_name = (selenium.__class__.__name__ + .replace('Wrapper', '').lower()) + if ('crash' in error_flags or + 'crash_' + driver_name in error_flags): request.applymarker(pytest.mark.xfail( run=False, reason='known failure with code "{}"' .format(error_flags))) From 02905d8ca5f85f5f7a0dfba6728514edc32c2f8d Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 17 Aug 2018 12:39:46 +0300 Subject: [PATCH 03/10] Show pytest report for skipped tests, XFAIL, XPASS --- Makefile | 2 +- test/python_tests.txt | 4 ++-- test/test_python.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 4aaf771e5..dc7bfe7fa 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ build/renderedhtml.css: src/renderedhtml.less test: all build/test.html build/test_data.txt - py.test test -v --instafail + py.test test -v -r sxX --instafail build/test_data.txt: test/data.txt diff --git a/test/python_tests.txt b/test/python_tests.txt index c6e2fc27a..678f31e81 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -25,8 +25,8 @@ # - crash: The Python interpreter just stopped without a traceback. Will require # further investigation. This usually seems to be caused by calling into a # system function that doesn't behave as one would expect. -# - crash_chrome: Same as crash but only affecting Chrome -# - crash_firefox: Same as crash but only affecting Firefox +# - crash-chrome: Same as crash but only affecting Chrome +# - crash-firefox: Same as crash but only affecting Firefox test___all__ test___future__ diff --git a/test/test_python.py b/test/test_python.py index ad0be8572..9e24feceb 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -285,16 +285,16 @@ def test_open_url(selenium): @pytest.mark.flaky(reruns=2) def test_run_core_python_test(python_test, selenium, request): - selenium.load_package('test') - name, error_flags = python_test driver_name = (selenium.__class__.__name__ .replace('Wrapper', '').lower()) if ('crash' in error_flags or - 'crash_' + driver_name in error_flags): + 'crash-' + driver_name in error_flags): request.applymarker(pytest.mark.xfail( run=False, reason='known failure with code "{}"' - .format(error_flags))) + .format(','.join(error_flags)))) + + selenium.load_package('test') try: selenium.run( "from test.libregrtest import main\n" From 7553ab4bfec6a2a96e92288c8801a5266b6422f0 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Fri, 17 Aug 2018 12:57:42 +0300 Subject: [PATCH 04/10] Mark pandas as XFAIL on chrome --- test/test_pandas.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/test_pandas.py b/test/test_pandas.py index 8f3360188..fe741bfc2 100644 --- a/test/test_pandas.py +++ b/test/test_pandas.py @@ -1,13 +1,18 @@ import pytest -@pytest.mark.skip -def test_pandas(selenium): +def test_pandas(selenium, request): + if 'chrome' in selenium.__class__.__name__.lower(): + request.applymarker(pytest.mark.xfail( + run=False, reason='chrome not supported')) selenium.load_package("pandas") assert len(selenium.run("import pandas\ndir(pandas)")) == 179 -@pytest.mark.skip -def test_extra_import(selenium): +def test_extra_import(selenium, request): + if 'chrome' in selenium.__class__.__name__.lower(): + request.applymarker(pytest.mark.xfail( + run=False, reason='chrome not supported')) + selenium.load_package("pandas") selenium.run("from pandas import Series, DataFrame, Panel") From 16ff5aaeefb7bf2f9da5394a5e8f5db14abf826c Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 20 Aug 2018 18:09:23 +0300 Subject: [PATCH 05/10] Add browser attribute in selenium fixture --- test/conftest.py | 6 ++++++ test/test_common.py | 12 +++++------- test/test_pandas.py | 4 ++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index ec30d02c5..6a724be4a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -74,6 +74,9 @@ class SeleniumWrapper: class FirefoxWrapper(SeleniumWrapper): + + browser = 'firefox' + def get_driver(self): from selenium.webdriver import Firefox from selenium.webdriver.firefox.options import Options @@ -89,6 +92,9 @@ class FirefoxWrapper(SeleniumWrapper): class ChromeWrapper(SeleniumWrapper): + + browser = 'chrome' + def get_driver(self): from selenium.webdriver import Chrome from selenium.webdriver.chrome.options import Options diff --git a/test/test_common.py b/test/test_common.py index 15344cb25..3649b09d5 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -27,8 +27,8 @@ def registered_packages_meta(): for name in packages} -UNSUPPORTED_PACKAGES = {'ChromeWrapper': ['pandas'], - 'FirefoxWrapper': []} +UNSUPPORTED_PACKAGES = {'chrome': ['pandas'], + 'firefox': []} @pytest.mark.parametrize('name', registered_packages()) @@ -36,12 +36,10 @@ def test_import(name, selenium_standalone): # check that we can parse the meta.yaml meta = common.parse_package(PKG_DIR / name / 'meta.yaml') - if name in UNSUPPORTED_PACKAGES[selenium_standalone.__class__.__name__]: + if name in UNSUPPORTED_PACKAGES[selenium_standalone.browser]: pytest.xfail( - '{} fails to load and is not supported on {}.' - .format(name, - selenium_standalone.__class__.__name__ - .replace('Wrapper', ''))) + '{} fails to load and is not supported on {}.' + .format(name, selenium_standalone.browser)) for import_name in meta.get('test', {}).get('imports', []): selenium_standalone.load_package(name) diff --git a/test/test_pandas.py b/test/test_pandas.py index fe741bfc2..403f93f74 100644 --- a/test/test_pandas.py +++ b/test/test_pandas.py @@ -2,7 +2,7 @@ import pytest def test_pandas(selenium, request): - if 'chrome' in selenium.__class__.__name__.lower(): + if selenium.browser == 'chrome': request.applymarker(pytest.mark.xfail( run=False, reason='chrome not supported')) selenium.load_package("pandas") @@ -10,7 +10,7 @@ def test_pandas(selenium, request): def test_extra_import(selenium, request): - if 'chrome' in selenium.__class__.__name__.lower(): + if selenium.brower == 'chrome': request.applymarker(pytest.mark.xfail( run=False, reason='chrome not supported')) From f0b9cf1c514aa7f38d505c125f18a6c39ea7acce Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 20 Aug 2018 18:27:25 +0300 Subject: [PATCH 06/10] Run CPython tests that used to crash and now pass --- test/python_tests.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/python_tests.txt b/test/python_tests.txt index 678f31e81..0aeca12f8 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -107,7 +107,7 @@ test_collections test_colorsys test_compare test_compile crash-chrome -test_compileall crash +test_compileall test_complex test_concurrent_futures test_configparser @@ -178,7 +178,7 @@ test_faulthandler test_fcntl test_file test_file_eintr subprocess -test_filecmp crash +test_filecmp test_fileinput test_fileio test_finalization @@ -206,7 +206,7 @@ test_genexps test_getargs2 test_getopt test_getpass permissions -test_gettext crash +test_gettext test_glob crash test_global test_grammar @@ -244,20 +244,20 @@ test_importlib.import_.test_packages test_importlib.import_.test_path test_importlib.import_.test_relative_imports test_importlib.source.test_case_sensitivity -test_importlib.source.test_file_loader crash -test_importlib.source.test_finder crash +test_importlib.source.test_file_loader +test_importlib.source.test_finder test_importlib.source.test_path_hook -test_importlib.source.test_source_encoding crash +test_importlib.source.test_source_encoding test_importlib.test_abc -test_importlib.test_api crash +test_importlib.test_api test_importlib.test_lazy test_importlib.test_locks test_importlib.test_namespace_pkgs test_importlib.test_spec -test_importlib.test_util crash +test_importlib.test_util test_importlib.test_windows platform-specific test_index -test_inspect crash +test_inspect test_int test_int_literal test_io crash From 02b61eea626f36b01addd5811e17416f852dcf7e Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 20 Aug 2018 18:38:01 +0300 Subject: [PATCH 07/10] Use selenium.browser in test_run_core_python_test --- test/test_python.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_python.py b/test/test_python.py index 9e24feceb..76df8b1c3 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -285,11 +285,11 @@ def test_open_url(selenium): @pytest.mark.flaky(reruns=2) def test_run_core_python_test(python_test, selenium, request): + name, error_flags = python_test - driver_name = (selenium.__class__.__name__ - .replace('Wrapper', '').lower()) + if ('crash' in error_flags or - 'crash-' + driver_name in error_flags): + 'crash-' + selenium.browser in error_flags): request.applymarker(pytest.mark.xfail( run=False, reason='known failure with code "{}"' .format(','.join(error_flags)))) From 396ca9cfcc8945229a518e2a1828ac8049fd758d Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 20 Aug 2018 23:44:08 +0300 Subject: [PATCH 08/10] Fix typo and more passing tests --- test/python_tests.txt | 22 +++++++++++----------- test/test_pandas.py | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/python_tests.txt b/test/python_tests.txt index 0aeca12f8..bf67e66aa 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -310,7 +310,7 @@ test_mimetypes test_minidom test_mmap test_module -test_modulefinder crash +test_modulefinder test_msilib test_multibytecodec test_multiprocessing_fork @@ -339,8 +339,8 @@ test_pickle dbm test_pickletools dbm test_pipes platform-specific test_pkg -test_pkgimport crash -test_pkgutil crash +test_pkgimport +test_pkgutil test_platform subprocess test_plistlib test_poll subprocess @@ -357,7 +357,7 @@ test_pstats test_pty test_pulldom test_pwd crash -test_py_compile crash +test_py_compile test_pyclbr test_pydoc crash test_pyexpat @@ -375,7 +375,7 @@ test_resource test_richcmp test_rlcompleter crash test_robotparser -test_runpy crash +test_runpy test_sax test_sched test_scope @@ -407,7 +407,7 @@ test_stat test_statistics test_strftime strftime test_string -test_string_literals crash +test_string_literals test_stringprep test_strptime strftime test_strtod @@ -466,7 +466,7 @@ test_types test_typing unknown test_ucn test_unary -test_unicode crash +test_unicode test_unicode_file test_unicode_file_functions test_unicode_identifiers @@ -507,9 +507,9 @@ test_xml_etree_c test_xmlrpc networking test_xmlrpc_net test_yield_from -test_zipapp crash -test_zipfile crash +test_zipapp +test_zipfile test_zipfile64 -test_zipimport crash -test_zipimport_support crash +test_zipimport +test_zipimport_support test_zlib diff --git a/test/test_pandas.py b/test/test_pandas.py index 403f93f74..1328c1b13 100644 --- a/test/test_pandas.py +++ b/test/test_pandas.py @@ -10,7 +10,7 @@ def test_pandas(selenium, request): def test_extra_import(selenium, request): - if selenium.brower == 'chrome': + if selenium.browser == 'chrome': request.applymarker(pytest.mark.xfail( run=False, reason='chrome not supported')) From e2a723de5f2e0f071ab604b6126873be3225faa1 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Wed, 22 Aug 2018 12:42:22 +0300 Subject: [PATCH 09/10] Don't run xfail tests --- test/python_tests.txt | 2 +- test/test_python.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/python_tests.txt b/test/python_tests.txt index bf67e66aa..4160dd7b6 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -466,7 +466,7 @@ test_types test_typing unknown test_ucn test_unary -test_unicode +test_unicode crash test_unicode_file test_unicode_file_functions test_unicode_identifiers diff --git a/test/test_python.py b/test/test_python.py index 76df8b1c3..9a80acfb7 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -290,9 +290,8 @@ def test_run_core_python_test(python_test, selenium, request): if ('crash' in error_flags or 'crash-' + selenium.browser in error_flags): - request.applymarker(pytest.mark.xfail( - run=False, reason='known failure with code "{}"' - .format(','.join(error_flags)))) + pytest.xfail(reason='known failure with code "{}"' + .format(','.join(error_flags))) selenium.load_package('test') try: From 0b752777a24d4aaad0acf9e0848cc0f5429c884a Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Wed, 22 Aug 2018 12:43:07 +0300 Subject: [PATCH 10/10] Don't use pytest-rerunfailures --- .circleci/config.yml | 2 +- test/test_python.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c2cc58967..503375ee0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ jobs: sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8 sudo update-alternatives --set gcc /usr/bin/gcc-8 - sudo pip install pytest-xdist pytest-instafail selenium PyYAML pytest-rerunfailures + sudo pip install pytest-xdist pytest-instafail selenium PyYAML # Get recent version of Firefox and geckodriver wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-nightly-latest-ssl\&os\=linux64\&lang\=en-US diff --git a/test/test_python.py b/test/test_python.py index 9a80acfb7..0e2e9d294 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -283,7 +283,6 @@ def test_open_url(selenium): "pyodide.open_url('test_data.txt').read()\n") == 'HELLO\n' -@pytest.mark.flaky(reruns=2) def test_run_core_python_test(python_test, selenium, request): name, error_flags = python_test