From 641f3e7b1a4cbb0ac68cd6b473f5c69dc70d3f01 Mon Sep 17 00:00:00 2001
From: Roman Yurchak <rth.yurchak@pm.me>
Date: Mon, 1 Oct 2018 13:43:11 +0200
Subject: [PATCH] Skip tests that segfault, run the rest

---
 test/python_tests.txt | 25 ++++++++++++++-----------
 test/test_python.py   | 32 ++++++++++++++++++++------------
 2 files changed, 34 insertions(+), 23 deletions(-)

diff --git a/test/python_tests.txt b/test/python_tests.txt
index d1d708daf..f2ac39dce 100644
--- a/test/python_tests.txt
+++ b/test/python_tests.txt
@@ -1,7 +1,12 @@
 # Test modules with a failure reason after their name are either skipped
 # or marked as a known failure in pytest.
 #
-# Following reason codes are skipped:
+# Following reason codes are skipped, whithout running them, as they lead to
+# segfaults:
+# - segfault-<syscall>: segfault in the corresponding system call.
+#
+# While the below reason codes are marked as a known failure, and are still
+# executed:
 # - platform-specific: This is testing something about a particular platform
 #   that isn't relevant here
 # - async: relies on async
@@ -24,8 +29,6 @@
 # - fs: Fails due to virtual filesystem issues.
 # - nonsense: This functionality doesn't make sense in this context. Includes
 #   things like `pip`, `distutils`
-#
-# While the below reason codes are marked as a known failure:
 # - crash: The Python interpreter just stopped without a traceback. Will require
 #   further investigation. This usually seems to be caused by calling into a
 #   system function that doesn't behave as one would expect.
@@ -53,7 +56,7 @@ test_asyncio.test_locks       async
 test_asyncio.test_pep492      async
 test_asyncio.test_proactor_events async
 test_asyncio.test_queues          async
-test_asyncio.test_selector_events async
+test_asyncio.test_selector_events async segfault-socketcall
 test_asyncio.test_sslproto        async
 test_asyncio.test_streams         async
 test_asyncio.test_subprocess      async
@@ -201,7 +204,7 @@ test_fractions
 test_frame
 test_frozen
 test_fstring
-test_ftplib syscall 21537
+test_ftplib socket
 test_funcattrs
 test_functools  threading
 test_future
@@ -232,7 +235,7 @@ test_html
 test_htmlparser
 test_http_cookiejar
 test_http_cookies
-test_httplib  socket
+test_httplib  socket segfault-socketcall
 test_httpservers  threading
 test_idle
 test_imaplib  socket
@@ -360,8 +363,8 @@ test_platform   subprocess
 test_plistlib
 test_poll subprocess
 test_popen  subprocess
-test_poplib bad ioctl syscall 21537
-test_posix  crash
+test_poplib bad ioctl socket
+test_posix  segfault-fstatfs64
 test_posixpath  crash
 test_pow
 test_pprint
@@ -396,8 +399,8 @@ test_sched  threading
 test_scope
 test_script_helper
 test_secrets
-test_select networking
-test_selectors  networking
+test_select networking  segfault-newselect
+test_selectors  networking segfault-newselect
 test_set
 test_setcomps
 test_shelve
@@ -490,7 +493,7 @@ test_unittest os.kill
 test_univnewlines
 test_unpack
 test_unpack_ex
-test_urllib crash
+test_urllib segfault-socketcall
 test_urllib2  subprocess
 test_urllib2_localnet socket
 test_urllib2net
diff --git a/test/test_python.py b/test/test_python.py
index 00cff7e3a..7456cc665 100644
--- a/test/test_python.py
+++ b/test/test_python.py
@@ -336,14 +336,25 @@ def test_open_url_cgi(selenium):
         """) == 'HELLO\n'
 
 
-def test_run_core_python_test(python_test, selenium, request):
+def test_cpython_core(python_test, selenium, request):
 
     name, error_flags = python_test
 
-    if ('crash' in error_flags or
-            'crash-' + selenium.browser in error_flags):
-        pytest.xfail(reason='known failure with code "{}"'
-                            .format(','.join(error_flags)))
+    # keep only flags related to the current browser
+    flags_to_remove = ['firefox', 'chrome']
+    flags_to_remove.remove(selenium.browser)
+    for flag in flags_to_remove:
+        if 'crash-' + flag in error_flags:
+            error_flags.remove('crash-' + flag)
+
+    if any(flag.startswith('segfault') for flag in error_flags):
+        pytest.skip('known segfault with code: "{}"'
+                    .format(','.join(error_flags)))
+
+    if error_flags:
+        request.applymarker(pytest.mark.xfail(
+            run=False, reason='known failure with code "{}"'
+                              .format(','.join(error_flags))))
 
     selenium.load_package('test')
     try:
@@ -373,13 +384,10 @@ def pytest_generate_tests(metafunc):
                     continue
                 error_flags = line.split()
                 name = error_flags.pop(0)
-                if (not error_flags
-                    or set(error_flags).intersection(
-                            {'crash', 'crash-chrome', 'crash-firefox'})):
-                        test_modules.append((name, error_flags))
-                        # explicitly define test ids to keep
-                        # a human readable test name in pytest
-                        test_modules_ids.append(name)
+                test_modules.append((name, error_flags))
+                # explicitly define test ids to keep
+                # a human readable test name in pytest
+                test_modules_ids.append(name)
         metafunc.parametrize("python_test", test_modules,
                              ids=test_modules_ids)