Add pytest --run-xfail run option

test/conftest.py

@@ -24,6 +24,9 @@ try:
         group.addoption(
             '--build-dir', action="store", default=BUILD_PATH,
             help="Path to the build directory")
+        group.addoption(
+            '--run-xfail', action="store_true",
+            help="If provided, tests marked as xfail will be run")
 
 except ImportError:
     pytest = None

test/python_tests.txt

@@ -1,12 +1,11 @@
 # Test modules with a failure reason after their name are either skipped
 # or marked as a known failure in pytest.
 #
-# Following reason codes are skipped, whithout running them, as they lead to
-# segfaults:
+# Following reason codes are skipped, as they lead to segfaults:
 # - segfault-<syscall>: segfault in the corresponding system call.
 #
-# While the below reason codes are marked as a known failure, and are still
-# executed:
+# While the below reason codes are marked as a known failure. By default, they
+# are also skipped. To run them, provide --run-xfail argument to pytest,
 # - platform-specific: This is testing something about a particular platform
 #   that isn't relevant here
 # - async: relies on async

test/test_python.py

@@ -352,9 +352,13 @@ def test_cpython_core(python_test, selenium, request):
                     .format(','.join(error_flags)))
 
     if error_flags:
-        request.applymarker(pytest.mark.xfail(
-            run=False, reason='known failure with code "{}"'
-                              .format(','.join(error_flags))))
+        if request.config.option.run_xfail:
+            request.applymarker(pytest.mark.xfail(
+                run=False, reason='known failure with code "{}"'
+                                  .format(','.join(error_flags))))
+        else:
+            pytest.xfail('known failure with code "{}"'
+                         .format(','.join(error_flags)))
 
     selenium.load_package('test')
     try: