diff --git a/cpython/pyconfig.undefs.h b/cpython/pyconfig.undefs.h index 68a4d0c57..3dc5eec20 100644 --- a/cpython/pyconfig.undefs.h +++ b/cpython/pyconfig.undefs.h @@ -4,3 +4,28 @@ #undef HAVE_SOCKETPAIR #undef HAVE_UTIMENSAT #undef HAVE_SIGACTION + +/* Untested syscalls in emscripten */ +#undef HAVE_OPENAT +#undef HAVE_MKDIRAT +#undef HAVE_FCHOWNAT +#undef HAVE_RENAMEAT +#undef HAVE_LINKAT +#undef HAVE_SYMLINKAT +#undef HAVE_READLINKAT +#undef HAVE_FCHMODAT +#undef HAVE_DUP3 + +/* Syscalls not implemented in emscripten */ +#undef HAVE_PREADV +#undef HAVE_PWRITEV +#undef HAVE_PIPE2 +#undef HAVE_NICE + +/* Syscalls that resulted in a segfault */ +#undef HAVE_UTIMENSAT +#undef HAVE_SYS_SOCKET_H +#undef HAVE_SYS_IOCTL_H + +/* Unsupported functionality */ +#undef HAVE_PTHREAD_H diff --git a/test/conftest.py b/test/conftest.py index 5aaa29636..b4a973148 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -33,6 +33,9 @@ try: group.addoption( '--build-dir', action="store", default=BUILD_PATH, help="Path to the build directory") + group.addoption( + '--run-xfail', action="store_true", + help="If provided, tests marked as xfail will be run") except ImportError: pytest = None diff --git a/test/python_tests.txt b/test/python_tests.txt index d1d708daf..adfebf6a0 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -1,7 +1,11 @@ # Test modules with a failure reason after their name are either skipped # or marked as a known failure in pytest. # -# Following reason codes are skipped: +# Following reason codes are skipped, as they lead to segfaults: +# - segfault-: segfault in the corresponding system call. +# +# While the below reason codes are marked as a known failure. By default, they +# are also skipped. To run them, provide --run-xfail argument to pytest, # - platform-specific: This is testing something about a particular platform # that isn't relevant here # - async: relies on async @@ -24,8 +28,6 @@ # - fs: Fails due to virtual filesystem issues. # - nonsense: This functionality doesn't make sense in this context. Includes # things like `pip`, `distutils` -# -# While the below reason codes are marked as a known failure: # - crash: The Python interpreter just stopped without a traceback. Will require # further investigation. This usually seems to be caused by calling into a # system function that doesn't behave as one would expect. @@ -50,19 +52,19 @@ test_asyncio.test_base_events async test_asyncio.test_events async test_asyncio.test_futures async test_asyncio.test_locks async -test_asyncio.test_pep492 async -test_asyncio.test_proactor_events async -test_asyncio.test_queues async -test_asyncio.test_selector_events async -test_asyncio.test_sslproto async +test_asyncio.test_pep492 +test_asyncio.test_proactor_events +test_asyncio.test_queues +test_asyncio.test_selector_events async segfault-socketcall +test_asyncio.test_sslproto test_asyncio.test_streams async test_asyncio.test_subprocess async test_asyncio.test_tasks async -test_asyncio.test_transports async +test_asyncio.test_transports test_asyncio.test_unix_events async -test_asyncio.test_windows_events async -test_asyncio.test_windows_utils async -test_asyncore bad ioctl syscall async +test_asyncio.test_windows_events +test_asyncio.test_windows_utils +test_asyncore bad ioctl syscall async test_atexit test_audioop audioop test_augassign @@ -92,7 +94,7 @@ test_class test_cmath test_cmd test_cmd_line -test_cmd_line_script subprocess +test_cmd_line_script test_code test_code_module test_codeccallbacks @@ -131,8 +133,8 @@ test_csv test_ctypes test_curses test_dataclasses -test_datetime strftime -test_dbm permissions +test_datetime +test_dbm test_dbm_dumb permissions test_dbm_gnu test_dbm_ndbm @@ -158,7 +160,7 @@ test_dummy_thread test_dummy_threading test_dynamic test_dynamicclassattribute -test_eintr platform-specific +test_eintr test_email.test__encoded_words test_email.test__header_value_parser test_email.test_asian_codecs @@ -201,7 +203,7 @@ test_fractions test_frame test_frozen test_fstring -test_ftplib syscall 21537 +test_ftplib socket test_funcattrs test_functools threading test_future @@ -232,7 +234,7 @@ test_html test_htmlparser test_http_cookiejar test_http_cookies -test_httplib socket +test_httplib socket segfault-socketcall test_httpservers threading test_idle test_imaplib socket @@ -256,7 +258,7 @@ test_importlib.import_.test_packages test_importlib.import_.test_path test_importlib.import_.test_relative_imports test_importlib.source.test_case_sensitivity -test_importlib.source.test_file_loader unittest has no attribute mock +test_importlib.source.test_file_loader test_importlib.source.test_finder test_importlib.source.test_path_hook test_importlib.source.test_source_encoding @@ -271,12 +273,12 @@ test_importlib.test_read test_importlib.test_resource test_importlib.test_spec test_importlib.test_util -test_importlib.test_windows platform-specific +test_importlib.test_windows test_index test_inspect test_int test_int_literal -test_io crash +test_io segfault-unknown test_ioctl test_ipaddress test_isinstance @@ -353,15 +355,15 @@ test_peepholer test_pickle dbm test_pickletools dbm test_pipes platform-specific -test_pkg unknown +test_pkg test_pkgimport test_pkgutil test_platform subprocess test_plistlib test_poll subprocess test_popen subprocess -test_poplib bad ioctl syscall 21537 -test_posix crash +test_poplib bad ioctl socket +test_posix segfault-fstatfs64 test_posixpath crash test_pow test_pprint @@ -396,8 +398,8 @@ test_sched threading test_scope test_script_helper test_secrets -test_select networking -test_selectors networking +test_select networking segfault-newselect +test_selectors networking segfault-newselect test_set test_setcomps test_shelve @@ -432,7 +434,7 @@ test_structseq test_subclassinit test_subprocess test_sunau audioop -test_sundry nonsense +test_sundry test_super test_support multiprocessing test_symbol @@ -478,10 +480,10 @@ test_tuple test_turtle test_typechecks test_types -test_typing unknown +test_typing test_ucn test_unary -test_unicode floating point +test_unicode test_unicode_file test_unicode_file_functions test_unicode_identifiers @@ -490,7 +492,7 @@ test_unittest os.kill test_univnewlines test_unpack test_unpack_ex -test_urllib crash +test_urllib segfault-socketcall test_urllib2 subprocess test_urllib2_localnet socket test_urllib2net diff --git a/test/test_python.py b/test/test_python.py index 9435ca9e2..7503badc6 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -351,14 +351,29 @@ def test_open_url_cgi(selenium): """) == 'HELLO\n' -def test_run_core_python_test(python_test, selenium, request): +def test_cpython_core(python_test, selenium, request): name, error_flags = python_test - if ('crash' in error_flags or - 'crash-' + selenium.browser in error_flags): - pytest.xfail(reason='known failure with code "{}"' - .format(','.join(error_flags))) + # keep only flags related to the current browser + flags_to_remove = ['firefox', 'chrome'] + flags_to_remove.remove(selenium.browser) + for flag in flags_to_remove: + if 'crash-' + flag in error_flags: + error_flags.remove('crash-' + flag) + + if any(flag.startswith('segfault') for flag in error_flags): + pytest.skip('known segfault with code: "{}"' + .format(','.join(error_flags))) + + if error_flags: + if request.config.option.run_xfail: + request.applymarker(pytest.mark.xfail( + run=False, reason='known failure with code "{}"' + .format(','.join(error_flags)))) + else: + pytest.xfail('known failure with code "{}"' + .format(','.join(error_flags))) selenium.load_package('test') try: @@ -388,13 +403,10 @@ def pytest_generate_tests(metafunc): continue error_flags = line.split() name = error_flags.pop(0) - if (not error_flags - or set(error_flags).intersection( - {'crash', 'crash-chrome', 'crash-firefox'})): - test_modules.append((name, error_flags)) - # explicitly define test ids to keep - # a human readable test name in pytest - test_modules_ids.append(name) + test_modules.append((name, error_flags)) + # explicitly define test ids to keep + # a human readable test name in pytest + test_modules_ids.append(name) metafunc.parametrize("python_test", test_modules, ids=test_modules_ids)