Merge pull request #199 from rth/disable-syscalls

Disable unsupported/untested syscalls in pyconfig.h

cpython/pyconfig.undefs.h

@@ -4,3 +4,28 @@
 #undef HAVE_SOCKETPAIR
 #undef HAVE_UTIMENSAT
 #undef HAVE_SIGACTION
+
+/* Untested syscalls in emscripten */
+#undef HAVE_OPENAT
+#undef HAVE_MKDIRAT
+#undef HAVE_FCHOWNAT
+#undef HAVE_RENAMEAT
+#undef HAVE_LINKAT
+#undef HAVE_SYMLINKAT
+#undef HAVE_READLINKAT
+#undef HAVE_FCHMODAT
+#undef HAVE_DUP3
+
+/* Syscalls not implemented in emscripten */
+#undef HAVE_PREADV
+#undef HAVE_PWRITEV
+#undef HAVE_PIPE2
+#undef HAVE_NICE
+
+/* Syscalls that resulted in a segfault */
+#undef HAVE_UTIMENSAT
+#undef HAVE_SYS_SOCKET_H
+#undef HAVE_SYS_IOCTL_H
+
+/* Unsupported functionality */
+#undef HAVE_PTHREAD_H

test/conftest.py

@@ -33,6 +33,9 @@ try:
         group.addoption(
             '--build-dir', action="store", default=BUILD_PATH,
             help="Path to the build directory")
+        group.addoption(
+            '--run-xfail', action="store_true",
+            help="If provided, tests marked as xfail will be run")
 
 except ImportError:
     pytest = None

test/python_tests.txt

@@ -1,7 +1,11 @@
 # Test modules with a failure reason after their name are either skipped
 # or marked as a known failure in pytest.
 #
-# Following reason codes are skipped:
+# Following reason codes are skipped, as they lead to segfaults:
+# - segfault-<syscall>: segfault in the corresponding system call.
+#
+# While the below reason codes are marked as a known failure. By default, they
+# are also skipped. To run them, provide --run-xfail argument to pytest,
 # - platform-specific: This is testing something about a particular platform
 #   that isn't relevant here
 # - async: relies on async
@@ -24,8 +28,6 @@
 # - fs: Fails due to virtual filesystem issues.
 # - nonsense: This functionality doesn't make sense in this context. Includes
 #   things like `pip`, `distutils`
-#
-# While the below reason codes are marked as a known failure:
 # - crash: The Python interpreter just stopped without a traceback. Will require
 #   further investigation. This usually seems to be caused by calling into a
 #   system function that doesn't behave as one would expect.
@@ -50,19 +52,19 @@ test_asyncio.test_base_events async
 test_asyncio.test_events      async
 test_asyncio.test_futures     async
 test_asyncio.test_locks       async
-test_asyncio.test_pep492      async
-test_asyncio.test_proactor_events async
-test_asyncio.test_queues          async
-test_asyncio.test_selector_events async
-test_asyncio.test_sslproto        async
+test_asyncio.test_pep492
+test_asyncio.test_proactor_events
+test_asyncio.test_queues
+test_asyncio.test_selector_events async segfault-socketcall
+test_asyncio.test_sslproto
 test_asyncio.test_streams         async
 test_asyncio.test_subprocess      async
 test_asyncio.test_tasks           async
-test_asyncio.test_transports      async
+test_asyncio.test_transports
 test_asyncio.test_unix_events     async
-test_asyncio.test_windows_events  async
-test_asyncio.test_windows_utils   async
-test_asyncore bad ioctl syscall   async
+test_asyncio.test_windows_events
+test_asyncio.test_windows_utils   
+test_asyncore bad ioctl syscall async
 test_atexit
 test_audioop  audioop
 test_augassign
@@ -92,7 +94,7 @@ test_class
 test_cmath
 test_cmd
 test_cmd_line
-test_cmd_line_script  subprocess
+test_cmd_line_script
 test_code
 test_code_module
 test_codeccallbacks
@@ -131,8 +133,8 @@ test_csv
 test_ctypes
 test_curses
 test_dataclasses
-test_datetime strftime
-test_dbm      permissions
+test_datetime
+test_dbm
 test_dbm_dumb permissions
 test_dbm_gnu
 test_dbm_ndbm
@@ -158,7 +160,7 @@ test_dummy_thread
 test_dummy_threading
 test_dynamic
 test_dynamicclassattribute
-test_eintr  platform-specific
+test_eintr
 test_email.test__encoded_words
 test_email.test__header_value_parser
 test_email.test_asian_codecs
@@ -201,7 +203,7 @@ test_fractions
 test_frame
 test_frozen
 test_fstring
-test_ftplib syscall 21537
+test_ftplib socket
 test_funcattrs
 test_functools  threading
 test_future
@@ -232,7 +234,7 @@ test_html
 test_htmlparser
 test_http_cookiejar
 test_http_cookies
-test_httplib  socket
+test_httplib  socket segfault-socketcall
 test_httpservers  threading
 test_idle
 test_imaplib  socket
@@ -256,7 +258,7 @@ test_importlib.import_.test_packages
 test_importlib.import_.test_path
 test_importlib.import_.test_relative_imports
 test_importlib.source.test_case_sensitivity
-test_importlib.source.test_file_loader  unittest has no attribute mock
+test_importlib.source.test_file_loader
 test_importlib.source.test_finder
 test_importlib.source.test_path_hook
 test_importlib.source.test_source_encoding
@@ -271,12 +273,12 @@ test_importlib.test_read
 test_importlib.test_resource
 test_importlib.test_spec
 test_importlib.test_util
-test_importlib.test_windows platform-specific
+test_importlib.test_windows
 test_index
 test_inspect
 test_int
 test_int_literal
-test_io  crash
+test_io  segfault-unknown
 test_ioctl
 test_ipaddress
 test_isinstance
@@ -353,15 +355,15 @@ test_peepholer
 test_pickle dbm
 test_pickletools  dbm
 test_pipes        platform-specific
-test_pkg          unknown
+test_pkg
 test_pkgimport
 test_pkgutil
 test_platform   subprocess
 test_plistlib
 test_poll subprocess
 test_popen  subprocess
-test_poplib bad ioctl syscall 21537
-test_posix  crash
+test_poplib bad ioctl socket
+test_posix  segfault-fstatfs64
 test_posixpath  crash
 test_pow
 test_pprint
@@ -396,8 +398,8 @@ test_sched  threading
 test_scope
 test_script_helper
 test_secrets
-test_select networking
-test_selectors  networking
+test_select networking  segfault-newselect
+test_selectors  networking segfault-newselect
 test_set
 test_setcomps
 test_shelve
@@ -432,7 +434,7 @@ test_structseq
 test_subclassinit
 test_subprocess
 test_sunau  audioop
-test_sundry nonsense
+test_sundry
 test_super
 test_support  multiprocessing
 test_symbol
@@ -478,10 +480,10 @@ test_tuple
 test_turtle
 test_typechecks
 test_types
-test_typing unknown
+test_typing
 test_ucn
 test_unary
-test_unicode  floating point
+test_unicode
 test_unicode_file
 test_unicode_file_functions
 test_unicode_identifiers
@@ -490,7 +492,7 @@ test_unittest os.kill
 test_univnewlines
 test_unpack
 test_unpack_ex
-test_urllib crash
+test_urllib segfault-socketcall
 test_urllib2  subprocess
 test_urllib2_localnet socket
 test_urllib2net

test/test_python.py

@@ -351,14 +351,29 @@ def test_open_url_cgi(selenium):
         """) == 'HELLO\n'
 
 
-def test_run_core_python_test(python_test, selenium, request):
+def test_cpython_core(python_test, selenium, request):
 
     name, error_flags = python_test
 
-    if ('crash' in error_flags or
-            'crash-' + selenium.browser in error_flags):
-        pytest.xfail(reason='known failure with code "{}"'
-                            .format(','.join(error_flags)))
+    # keep only flags related to the current browser
+    flags_to_remove = ['firefox', 'chrome']
+    flags_to_remove.remove(selenium.browser)
+    for flag in flags_to_remove:
+        if 'crash-' + flag in error_flags:
+            error_flags.remove('crash-' + flag)
+
+    if any(flag.startswith('segfault') for flag in error_flags):
+        pytest.skip('known segfault with code: "{}"'
+                    .format(','.join(error_flags)))
+
+    if error_flags:
+        if request.config.option.run_xfail:
+            request.applymarker(pytest.mark.xfail(
+                run=False, reason='known failure with code "{}"'
+                                  .format(','.join(error_flags))))
+        else:
+            pytest.xfail('known failure with code "{}"'
+                         .format(','.join(error_flags)))
 
     selenium.load_package('test')
     try:
@@ -388,13 +403,10 @@ def pytest_generate_tests(metafunc):
                     continue
                 error_flags = line.split()
                 name = error_flags.pop(0)
-                if (not error_flags
-                    or set(error_flags).intersection(
-                            {'crash', 'crash-chrome', 'crash-firefox'})):
-                        test_modules.append((name, error_flags))
-                        # explicitly define test ids to keep
-                        # a human readable test name in pytest
-                        test_modules_ids.append(name)
+                test_modules.append((name, error_flags))
+                # explicitly define test ids to keep
+                # a human readable test name in pytest
+                test_modules_ids.append(name)
         metafunc.parametrize("python_test", test_modules,
                              ids=test_modules_ids)