From b86101a58d8af871589587ab086b43700e88c8d4 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 1 Oct 2018 12:14:15 +0200 Subject: [PATCH 1/8] Disable untested/unsupported syscalls --- cpython/pyconfig.undefs.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cpython/pyconfig.undefs.h b/cpython/pyconfig.undefs.h index 68a4d0c57..27f7981bb 100644 --- a/cpython/pyconfig.undefs.h +++ b/cpython/pyconfig.undefs.h @@ -4,3 +4,28 @@ #undef HAVE_SOCKETPAIR #undef HAVE_UTIMENSAT #undef HAVE_SIGACTION + +/* Untested syscalls in emscripten */ +#undef HAVE_OPENAT +#undef HAVE_MKDIRAT +#undef HAVE_FCHOWNAT +#undef HAVE_RENAMEAT +#undef HAVE_LINKAT +#undef HAVE_SYMLINKAT +#undef HAVE_READLINKAT +#undef HAVE_FCHMODAT +#undef HAVE_DUP3 + +/* Syscalls not implemented in emscripten */ +#undef HAVE_PREADV +#undef HAVE_PWRITEV +#undef HAVE_PIPE2 +#undef HAVE_NICE + +/* Syscalls that resulted in a segfault */ +#undef HAVE_UTIMENSAT +#undef HAVE_SYS_IOCTL_H +#undef HAVE_SYS_SOCKET_H + +/* Unsupported functionality */ +#undef HAVE_PTHREAD_H From 641f3e7b1a4cbb0ac68cd6b473f5c69dc70d3f01 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 1 Oct 2018 13:43:11 +0200 Subject: [PATCH 2/8] Skip tests that segfault, run the rest --- test/python_tests.txt | 25 ++++++++++++++----------- test/test_python.py | 32 ++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/test/python_tests.txt b/test/python_tests.txt index d1d708daf..f2ac39dce 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -1,7 +1,12 @@ # Test modules with a failure reason after their name are either skipped # or marked as a known failure in pytest. # -# Following reason codes are skipped: +# Following reason codes are skipped, whithout running them, as they lead to +# segfaults: +# - segfault-: segfault in the corresponding system call. +# +# While the below reason codes are marked as a known failure, and are still +# executed: # - platform-specific: This is testing something about a particular platform # that isn't relevant here # - async: relies on async @@ -24,8 +29,6 @@ # - fs: Fails due to virtual filesystem issues. # - nonsense: This functionality doesn't make sense in this context. Includes # things like `pip`, `distutils` -# -# While the below reason codes are marked as a known failure: # - crash: The Python interpreter just stopped without a traceback. Will require # further investigation. This usually seems to be caused by calling into a # system function that doesn't behave as one would expect. @@ -53,7 +56,7 @@ test_asyncio.test_locks async test_asyncio.test_pep492 async test_asyncio.test_proactor_events async test_asyncio.test_queues async -test_asyncio.test_selector_events async +test_asyncio.test_selector_events async segfault-socketcall test_asyncio.test_sslproto async test_asyncio.test_streams async test_asyncio.test_subprocess async @@ -201,7 +204,7 @@ test_fractions test_frame test_frozen test_fstring -test_ftplib syscall 21537 +test_ftplib socket test_funcattrs test_functools threading test_future @@ -232,7 +235,7 @@ test_html test_htmlparser test_http_cookiejar test_http_cookies -test_httplib socket +test_httplib socket segfault-socketcall test_httpservers threading test_idle test_imaplib socket @@ -360,8 +363,8 @@ test_platform subprocess test_plistlib test_poll subprocess test_popen subprocess -test_poplib bad ioctl syscall 21537 -test_posix crash +test_poplib bad ioctl socket +test_posix segfault-fstatfs64 test_posixpath crash test_pow test_pprint @@ -396,8 +399,8 @@ test_sched threading test_scope test_script_helper test_secrets -test_select networking -test_selectors networking +test_select networking segfault-newselect +test_selectors networking segfault-newselect test_set test_setcomps test_shelve @@ -490,7 +493,7 @@ test_unittest os.kill test_univnewlines test_unpack test_unpack_ex -test_urllib crash +test_urllib segfault-socketcall test_urllib2 subprocess test_urllib2_localnet socket test_urllib2net diff --git a/test/test_python.py b/test/test_python.py index 00cff7e3a..7456cc665 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -336,14 +336,25 @@ def test_open_url_cgi(selenium): """) == 'HELLO\n' -def test_run_core_python_test(python_test, selenium, request): +def test_cpython_core(python_test, selenium, request): name, error_flags = python_test - if ('crash' in error_flags or - 'crash-' + selenium.browser in error_flags): - pytest.xfail(reason='known failure with code "{}"' - .format(','.join(error_flags))) + # keep only flags related to the current browser + flags_to_remove = ['firefox', 'chrome'] + flags_to_remove.remove(selenium.browser) + for flag in flags_to_remove: + if 'crash-' + flag in error_flags: + error_flags.remove('crash-' + flag) + + if any(flag.startswith('segfault') for flag in error_flags): + pytest.skip('known segfault with code: "{}"' + .format(','.join(error_flags))) + + if error_flags: + request.applymarker(pytest.mark.xfail( + run=False, reason='known failure with code "{}"' + .format(','.join(error_flags)))) selenium.load_package('test') try: @@ -373,13 +384,10 @@ def pytest_generate_tests(metafunc): continue error_flags = line.split() name = error_flags.pop(0) - if (not error_flags - or set(error_flags).intersection( - {'crash', 'crash-chrome', 'crash-firefox'})): - test_modules.append((name, error_flags)) - # explicitly define test ids to keep - # a human readable test name in pytest - test_modules_ids.append(name) + test_modules.append((name, error_flags)) + # explicitly define test ids to keep + # a human readable test name in pytest + test_modules_ids.append(name) metafunc.parametrize("python_test", test_modules, ids=test_modules_ids) From 3b302d7cedc5c8aed862f2ff8906095945d772e1 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 1 Oct 2018 15:10:13 +0200 Subject: [PATCH 3/8] Update passing tests --- cpython/pyconfig.undefs.h | 1 - test/python_tests.txt | 32 ++++++++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/cpython/pyconfig.undefs.h b/cpython/pyconfig.undefs.h index 27f7981bb..064322c86 100644 --- a/cpython/pyconfig.undefs.h +++ b/cpython/pyconfig.undefs.h @@ -24,7 +24,6 @@ /* Syscalls that resulted in a segfault */ #undef HAVE_UTIMENSAT -#undef HAVE_SYS_IOCTL_H #undef HAVE_SYS_SOCKET_H /* Unsupported functionality */ diff --git a/test/python_tests.txt b/test/python_tests.txt index f2ac39dce..6822601a7 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -53,18 +53,18 @@ test_asyncio.test_base_events async test_asyncio.test_events async test_asyncio.test_futures async test_asyncio.test_locks async -test_asyncio.test_pep492 async -test_asyncio.test_proactor_events async -test_asyncio.test_queues async +test_asyncio.test_pep492 +test_asyncio.test_proactor_events +test_asyncio.test_queues test_asyncio.test_selector_events async segfault-socketcall -test_asyncio.test_sslproto async +test_asyncio.test_sslproto test_asyncio.test_streams async test_asyncio.test_subprocess async test_asyncio.test_tasks async -test_asyncio.test_transports async +test_asyncio.test_transports test_asyncio.test_unix_events async -test_asyncio.test_windows_events async -test_asyncio.test_windows_utils async +test_asyncio.test_windows_events +test_asyncio.test_windows_utils test_asyncore bad ioctl syscall async test_atexit test_audioop audioop @@ -134,8 +134,8 @@ test_csv test_ctypes test_curses test_dataclasses -test_datetime strftime -test_dbm permissions +test_datetime +test_dbm test_dbm_dumb permissions test_dbm_gnu test_dbm_ndbm @@ -161,7 +161,7 @@ test_dummy_thread test_dummy_threading test_dynamic test_dynamicclassattribute -test_eintr platform-specific +test_eintr test_email.test__encoded_words test_email.test__header_value_parser test_email.test_asian_codecs @@ -259,7 +259,7 @@ test_importlib.import_.test_packages test_importlib.import_.test_path test_importlib.import_.test_relative_imports test_importlib.source.test_case_sensitivity -test_importlib.source.test_file_loader unittest has no attribute mock +test_importlib.source.test_file_loader test_importlib.source.test_finder test_importlib.source.test_path_hook test_importlib.source.test_source_encoding @@ -274,7 +274,7 @@ test_importlib.test_read test_importlib.test_resource test_importlib.test_spec test_importlib.test_util -test_importlib.test_windows platform-specific +test_importlib.test_windows test_index test_inspect test_int @@ -356,7 +356,7 @@ test_peepholer test_pickle dbm test_pickletools dbm test_pipes platform-specific -test_pkg unknown +test_pkg test_pkgimport test_pkgutil test_platform subprocess @@ -435,7 +435,7 @@ test_structseq test_subclassinit test_subprocess test_sunau audioop -test_sundry nonsense +test_sundry test_super test_support multiprocessing test_symbol @@ -481,10 +481,10 @@ test_tuple test_turtle test_typechecks test_types -test_typing unknown +test_typing test_ucn test_unary -test_unicode floating point +test_unicode test_unicode_file test_unicode_file_functions test_unicode_identifiers From 2fd3d2ec3f6d5177112e22b456b2f2f0a3722b08 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 1 Oct 2018 17:01:37 +0200 Subject: [PATCH 4/8] Skip test_asyncore --- .circleci/config.yml | 4 ++-- Makefile | 1 + Makefile.envs | 1 + test/python_tests.txt | 6 +++--- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 707c3e315..710220a92 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,7 +98,7 @@ jobs: source pyodide-env/bin/activate export PATH=$PWD/firefox:$PATH - pytest test -v -k firefox + pytest test -vs -k firefox test-chrome: <<: *defaults @@ -114,7 +114,7 @@ jobs: sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types" source pyodide-env/bin/activate - pytest test -v -k chrome + pytest test -vs -k chrome benchmark: <<: *defaults diff --git a/Makefile b/Makefile index 819be81ff..f25ca8304 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ LDFLAGS=\ --memory-init-file 0 \ -s "BINARYEN_TRAP_MODE='clamp'" \ -s TEXTDECODER=0 \ + -s ASSERTIONS=1 \ -s LZ4=1 SIX_ROOT=six/six-1.11.0/build/lib diff --git a/Makefile.envs b/Makefile.envs index d3d28f6d4..ab94e2c74 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -22,4 +22,5 @@ export SIDE_LDFLAGS=\ -s SIDE_MODULE=1 \ -s WASM=1 \ -s "BINARYEN_TRAP_MODE='clamp'" \ + -s ASSERTIONS=1 \ --memory-init-file 0 diff --git a/test/python_tests.txt b/test/python_tests.txt index 6822601a7..84703e5b5 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -64,8 +64,8 @@ test_asyncio.test_tasks async test_asyncio.test_transports test_asyncio.test_unix_events async test_asyncio.test_windows_events -test_asyncio.test_windows_utils -test_asyncore bad ioctl syscall async +test_asyncio.test_windows_utils +test_asyncore bad ioctl syscall async segfault test_atexit test_audioop audioop test_augassign @@ -204,7 +204,7 @@ test_fractions test_frame test_frozen test_fstring -test_ftplib socket +test_ftplib socket segfault-ioctl test_funcattrs test_functools threading test_future From 436eba4b5fb65ebf0fa003d4710a77311670a614 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Tue, 2 Oct 2018 15:20:04 +0200 Subject: [PATCH 5/8] undef HAVE_SYS_IOCTL_H --- cpython/pyconfig.undefs.h | 1 + test/python_tests.txt | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cpython/pyconfig.undefs.h b/cpython/pyconfig.undefs.h index 064322c86..3dc5eec20 100644 --- a/cpython/pyconfig.undefs.h +++ b/cpython/pyconfig.undefs.h @@ -25,6 +25,7 @@ /* Syscalls that resulted in a segfault */ #undef HAVE_UTIMENSAT #undef HAVE_SYS_SOCKET_H +#undef HAVE_SYS_IOCTL_H /* Unsupported functionality */ #undef HAVE_PTHREAD_H diff --git a/test/python_tests.txt b/test/python_tests.txt index 84703e5b5..88f447acf 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -65,7 +65,7 @@ test_asyncio.test_transports test_asyncio.test_unix_events async test_asyncio.test_windows_events test_asyncio.test_windows_utils -test_asyncore bad ioctl syscall async segfault +test_asyncore bad ioctl syscall async test_atexit test_audioop audioop test_augassign @@ -204,7 +204,7 @@ test_fractions test_frame test_frozen test_fstring -test_ftplib socket segfault-ioctl +test_ftplib socket test_funcattrs test_functools threading test_future @@ -280,7 +280,7 @@ test_inspect test_int test_int_literal test_io crash -test_ioctl +test_ioctl segfault test_ipaddress test_isinstance test_iter From 456fe6162ff551e2a98d17cef10cc93d22dbcf1c Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Wed, 3 Oct 2018 14:00:38 +0200 Subject: [PATCH 6/8] test_io segfaults but without producing any messages --- .circleci/config.yml | 4 ++-- Makefile | 1 - Makefile.envs | 1 - test/python_tests.txt | 8 ++++---- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 710220a92..707c3e315 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,7 +98,7 @@ jobs: source pyodide-env/bin/activate export PATH=$PWD/firefox:$PATH - pytest test -vs -k firefox + pytest test -v -k firefox test-chrome: <<: *defaults @@ -114,7 +114,7 @@ jobs: sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types" source pyodide-env/bin/activate - pytest test -vs -k chrome + pytest test -v -k chrome benchmark: <<: *defaults diff --git a/Makefile b/Makefile index f25ca8304..819be81ff 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,6 @@ LDFLAGS=\ --memory-init-file 0 \ -s "BINARYEN_TRAP_MODE='clamp'" \ -s TEXTDECODER=0 \ - -s ASSERTIONS=1 \ -s LZ4=1 SIX_ROOT=six/six-1.11.0/build/lib diff --git a/Makefile.envs b/Makefile.envs index ab94e2c74..d3d28f6d4 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -22,5 +22,4 @@ export SIDE_LDFLAGS=\ -s SIDE_MODULE=1 \ -s WASM=1 \ -s "BINARYEN_TRAP_MODE='clamp'" \ - -s ASSERTIONS=1 \ --memory-init-file 0 diff --git a/test/python_tests.txt b/test/python_tests.txt index 88f447acf..d599e4a6c 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -95,7 +95,7 @@ test_class test_cmath test_cmd test_cmd_line -test_cmd_line_script subprocess +test_cmd_line_script test_code test_code_module test_codeccallbacks @@ -279,8 +279,8 @@ test_index test_inspect test_int test_int_literal -test_io crash -test_ioctl segfault +test_io segfault-unknown +test_ioctl test_ipaddress test_isinstance test_iter @@ -391,7 +391,7 @@ test_repl subprocess test_reprlib test_resource test_richcmp -test_rlcompleter crash +test_rlcompleter test_robotparser socket test_runpy test_sax From 5728cfdeedee7c12779c1d965cfa1710f7750e66 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Wed, 3 Oct 2018 15:16:03 +0200 Subject: [PATCH 7/8] Mark test_rlcompleter as xfail (again) --- test/python_tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/python_tests.txt b/test/python_tests.txt index d599e4a6c..2888d57ea 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -391,7 +391,7 @@ test_repl subprocess test_reprlib test_resource test_richcmp -test_rlcompleter +test_rlcompleter crash test_robotparser socket test_runpy test_sax From c9b39e08fc478157fc14fd8a72dace40c6910de4 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Wed, 3 Oct 2018 16:36:45 +0200 Subject: [PATCH 8/8] Add pytest --run-xfail run option --- test/conftest.py | 3 +++ test/python_tests.txt | 7 +++---- test/test_python.py | 10 +++++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 877b501f2..74aadd474 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -24,6 +24,9 @@ try: group.addoption( '--build-dir', action="store", default=BUILD_PATH, help="Path to the build directory") + group.addoption( + '--run-xfail', action="store_true", + help="If provided, tests marked as xfail will be run") except ImportError: pytest = None diff --git a/test/python_tests.txt b/test/python_tests.txt index 2888d57ea..adfebf6a0 100644 --- a/test/python_tests.txt +++ b/test/python_tests.txt @@ -1,12 +1,11 @@ # Test modules with a failure reason after their name are either skipped # or marked as a known failure in pytest. # -# Following reason codes are skipped, whithout running them, as they lead to -# segfaults: +# Following reason codes are skipped, as they lead to segfaults: # - segfault-: segfault in the corresponding system call. # -# While the below reason codes are marked as a known failure, and are still -# executed: +# While the below reason codes are marked as a known failure. By default, they +# are also skipped. To run them, provide --run-xfail argument to pytest, # - platform-specific: This is testing something about a particular platform # that isn't relevant here # - async: relies on async diff --git a/test/test_python.py b/test/test_python.py index 7456cc665..c5ba4b191 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -352,9 +352,13 @@ def test_cpython_core(python_test, selenium, request): .format(','.join(error_flags))) if error_flags: - request.applymarker(pytest.mark.xfail( - run=False, reason='known failure with code "{}"' - .format(','.join(error_flags)))) + if request.config.option.run_xfail: + request.applymarker(pytest.mark.xfail( + run=False, reason='known failure with code "{}"' + .format(','.join(error_flags)))) + else: + pytest.xfail('known failure with code "{}"' + .format(','.join(error_flags))) selenium.load_package('test') try: