from pathlib import Path import pytest def test_cpython_core(python_test, selenium, request): name, error_flags = python_test # keep only flags related to the current browser flags_to_remove = ["firefox", "chrome"] flags_to_remove.remove(selenium.browser) for flag in flags_to_remove: if "crash-" + flag in error_flags: error_flags.remove("crash-" + flag) if any(flag.startswith("segfault") for flag in error_flags): pytest.skip('known segfault with code: "{}"'.format(",".join(error_flags))) if error_flags: if request.config.option.run_xfail: request.applymarker( pytest.mark.xfail( run=False, reason='known failure with code "{}"'.format(",".join(error_flags)), ) ) else: pytest.xfail('known failure with code "{}"'.format(",".join(error_flags))) selenium.load_package("test") try: selenium.run( """ from test.libregrtest import main try: main(['{}'], verbose=True, verbose3=True) except SystemExit as e: if e.code != 0: raise RuntimeError(f'Failed with code: {{e.code}}') """.format( name ) ) except selenium.JavascriptException: print(selenium.logs) raise def pytest_generate_tests(metafunc): if "python_test" in metafunc.fixturenames: test_modules = [] test_modules_ids = [] with open(Path(__file__).parent / "python_tests.txt") as fp: for line in fp: line = line.strip() if line.startswith("#") or not line: continue error_flags = line.split() name = error_flags.pop(0) test_modules.append((name, error_flags)) # explicitly define test ids to keep # a human readable test name in pytest test_modules_ids.append(name) metafunc.parametrize("python_test", test_modules, ids=test_modules_ids)