diff --git a/Makefile b/Makefile index 29fd241a5..e1e66cce1 100644 --- a/Makefile +++ b/Makefile @@ -185,7 +185,8 @@ TEST_EXTENSIONS= \ _testcapi.so \ _testbuffer.so \ _testimportmultiple.so \ - _testmultiphase.so + _testmultiphase.so \ + _ctypes_test.so TEST_MODULE_CFLAGS= $(SIDE_MODULE_CFLAGS) -I Include/ -I . # TODO: also include test directories included in other stdlib modules @@ -196,6 +197,7 @@ build/test.tar: $(CPYTHONLIB) node_modules/.installed cd $(CPYTHONBUILD) && emcc $(TEST_MODULE_CFLAGS) -c Modules/_testbuffer.c -o Modules/_testbuffer.o cd $(CPYTHONBUILD) && emcc $(TEST_MODULE_CFLAGS) -c Modules/_testimportmultiple.c -o Modules/_testimportmultiple.o cd $(CPYTHONBUILD) && emcc $(TEST_MODULE_CFLAGS) -c Modules/_testmultiphase.c -o Modules/_testmultiphase.o + cd $(CPYTHONBUILD) && emcc $(TEST_MODULE_CFLAGS) -c Modules/_ctypes/_ctypes_test.c -o Modules/_ctypes_test.o for testname in $(TEST_EXTENSIONS); do \ cd $(CPYTHONBUILD) && \ @@ -204,7 +206,7 @@ build/test.tar: $(CPYTHONLIB) node_modules/.installed done cd $(CPYTHONLIB) && tar -h --exclude=__pycache__ -cf $(PYODIDE_ROOT)/build/test.tar \ - test $(TEST_EXTENSIONS) + test $(TEST_EXTENSIONS) unittest/test sqlite3/test ctypes/test cd $(CPYTHONLIB) && rm $(TEST_EXTENSIONS) diff --git a/cpython/Setup.local b/cpython/Setup.local index f86ff0f9f..368c87dfe 100644 --- a/cpython/Setup.local +++ b/cpython/Setup.local @@ -21,7 +21,6 @@ _csv _csv.c CTYPES_FLAGS=-DHAVE_FFI_PREP_CIF_VAR=1 -DHAVE_FFI_PREP_CLOSURE_LOC=1 -DHAVE_FFI_CLOSURE_ALLOC=1 _ctypes _ctypes/_ctypes.c _ctypes/callbacks.c _ctypes/callproc.c _ctypes/cfield.c _ctypes/stgdict.c $(CTYPES_FLAGS) -_ctypes_test _ctypes/_ctypes_test.c unicodedata unicodedata.c _pickle _pickle.c diff --git a/emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch b/emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch index 77019aae7..a898126fd 100644 --- a/emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch +++ b/emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch @@ -1,7 +1,7 @@ From 4e80e74b9bab27b583d350df3fbb0a444f56a178 Mon Sep 17 00:00:00 2001 From: Hood Date: Thu, 24 Jun 2021 04:08:02 -0700 -Subject: [PATCH 1/8] Throw away errors in minify_wasm_js +Subject: [PATCH 1/7] Throw away errors in minify_wasm_js --- emcc.py | 13 ++++++++----- diff --git a/emsdk/patches/0002-Add-support-for-preload-file-in-Node.js-11785.patch b/emsdk/patches/0002-Add-support-for-preload-file-in-Node.js-11785.patch index 79a6c34a2..6d3406b1c 100644 --- a/emsdk/patches/0002-Add-support-for-preload-file-in-Node.js-11785.patch +++ b/emsdk/patches/0002-Add-support-for-preload-file-in-Node.js-11785.patch @@ -1,7 +1,7 @@ From ac2fc760dd5eda8f999bcf4c40490ebe4442304f Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Thu, 3 Jun 2021 09:42:51 -0700 -Subject: [PATCH 2/8] Add support for `--preload-file` in Node.js (#11785) +Subject: [PATCH 2/7] Add support for `--preload-file` in Node.js (#11785) Note: this is included in emscripten 2.O.24 --- diff --git a/emsdk/patches/0003-Fix-dup.patch b/emsdk/patches/0003-Fix-dup.patch index b129844f7..fdc239218 100644 --- a/emsdk/patches/0003-Fix-dup.patch +++ b/emsdk/patches/0003-Fix-dup.patch @@ -1,7 +1,7 @@ From 64ad57cc02d9ce10ed71ba1a35e38c88369570c1 Mon Sep 17 00:00:00 2001 From: Hood Date: Wed, 8 Sep 2021 17:49:15 -0700 -Subject: [PATCH 3/8] Fix dup +Subject: [PATCH 3/7] Fix dup This fixes two problems with the `dup` system calls: 1. `dup` expects that every file descriptor has a corresponding file (so pipes and (https://github.com/emscripten-core/emscripten/issues/14640) diff --git a/emsdk/patches/0004-Fix-side-module-exception-handling.patch b/emsdk/patches/0004-Fix-side-module-exception-handling.patch index f556043f8..58f0b4c9f 100644 --- a/emsdk/patches/0004-Fix-side-module-exception-handling.patch +++ b/emsdk/patches/0004-Fix-side-module-exception-handling.patch @@ -1,7 +1,7 @@ From a07a638594b7fcee4034d1fdae57276004955638 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 15 Feb 2022 23:27:03 -0500 -Subject: [PATCH 4/8] Fix side module exception handling +Subject: [PATCH 4/7] Fix side module exception handling See https://github.com/emscripten-core/emscripten/pull/16309 diff --git a/emsdk/patches/0005-Update-upstream-URL-for-libjpeg.-NFC-13869.patch b/emsdk/patches/0005-Update-upstream-URL-for-libjpeg.-NFC-13869.patch index ca1942f19..7b0888231 100644 --- a/emsdk/patches/0005-Update-upstream-URL-for-libjpeg.-NFC-13869.patch +++ b/emsdk/patches/0005-Update-upstream-URL-for-libjpeg.-NFC-13869.patch @@ -1,7 +1,7 @@ From 3b8024dd9ebe6c1161c99c9cc96ab20c8912eb4d Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Mon, 12 Apr 2021 09:27:04 -0700 -Subject: [PATCH 5/8] Update upstream URL for libjpeg. NFC (#13869) +Subject: [PATCH 5/7] Update upstream URL for libjpeg. NFC (#13869) The old URL seems to be generating `Forbidden!`. diff --git a/emsdk/patches/0008-Fix-pipe-close-operation-so-that-it-doesn-t-break-th.patch b/emsdk/patches/0006-Fix-pipe-close-operation-so-that-it-doesn-t-break-th.patch similarity index 87% rename from emsdk/patches/0008-Fix-pipe-close-operation-so-that-it-doesn-t-break-th.patch rename to emsdk/patches/0006-Fix-pipe-close-operation-so-that-it-doesn-t-break-th.patch index 50eb291c5..655e47c2f 100644 --- a/emsdk/patches/0008-Fix-pipe-close-operation-so-that-it-doesn-t-break-th.patch +++ b/emsdk/patches/0006-Fix-pipe-close-operation-so-that-it-doesn-t-break-th.patch @@ -1,7 +1,7 @@ -From c77f3ef38584de5f41ba11b6821a905493100c47 Mon Sep 17 00:00:00 2001 +From 9dd481f6aed10fe4e6048b15c87b5bc91f225074 Mon Sep 17 00:00:00 2001 From: Hood Date: Sat, 17 Jul 2021 16:54:40 -0700 -Subject: [PATCH 8/8] Fix pipe close operation so that it doesn't break the +Subject: [PATCH 6/7] Fix pipe close operation so that it doesn't break the other side of the pipe --- diff --git a/emsdk/patches/0007-Fix-lookupPath-when-applied-to-a-symlink-loop.patch b/emsdk/patches/0007-Fix-lookupPath-when-applied-to-a-symlink-loop.patch new file mode 100644 index 000000000..4e5b40ba8 --- /dev/null +++ b/emsdk/patches/0007-Fix-lookupPath-when-applied-to-a-symlink-loop.patch @@ -0,0 +1,31 @@ +From c1120e713b28110945baeffdbd924aeac2ea4f26 Mon Sep 17 00:00:00 2001 +From: Hood Chatham +Date: Wed, 2 Mar 2022 13:44:14 -0800 +Subject: [PATCH 7/7] Fix lookupPath when applied to a symlink loop + +The following code leads to an infinite loop in lookupPath: + +FS.symlink("linkX/inside","/linkX"); +FS.lookupPath("/linkX", {follow:true}); + +This patch fixes it. +--- + src/library_fs.js | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/library_fs.js b/src/library_fs.js +index 519fab972..ef91655f2 100644 +--- a/src/library_fs.js ++++ b/src/library_fs.js +@@ -169,7 +169,7 @@ FS.staticInit();` + + var link = FS.readlink(current_path); + current_path = PATH_FS.resolve(PATH.dirname(current_path), link); + +- var lookup = FS.lookupPath(current_path, { recurse_count: opts.recurse_count }); ++ var lookup = FS.lookupPath(current_path, { recurse_count: opts.recurse_count + 1 }); + current = lookup.node; + + if (count++ > 40) { // limit max consecutive symlinks to 40 (SYMLOOP_MAX). +-- +2.25.1 + diff --git a/src/tests/make_test_list.py b/src/tests/make_test_list.py index 91c9901e3..c12fb9237 100644 --- a/src/tests/make_test_list.py +++ b/src/tests/make_test_list.py @@ -3,82 +3,41 @@ Generate a list of test modules in the CPython distribution. """ import os + +import ruamel.yaml + +yaml = ruamel.yaml.YAML() + from pathlib import Path from sys import version_info -TEST_DIR = ( - Path(__file__).parents[2] / "cpython/installs" +PYODIDE_ROOT = Path(__file__).parents[2] +LIB_DIR = ( + PYODIDE_ROOT / "cpython/installs" f"/python-{version_info.major}.{version_info.minor}.{version_info.micro}" - f"/lib/python{version_info.major}.{version_info.minor}/test/" + f"/lib/python{version_info.major}.{version_info.minor}" ) - -explanation = """\ -# This list is generated with test/make_test_list.py script, which needs -# to be re-run after each CPython update. -# -# Test modules with a failure reason after their name are either skipped -# or marked as a known failure in pytest. -# -# Following reason codes are skipped, as they lead to segfaults: -# - segfault-: segfault in the corresponding system call. -# -# While the below reason codes are marked as a known failure. By default, they -# are also skipped. To run them, provide --run-xfail argument to pytest, -# - platform-specific: This is testing something about a particular platform -# that isn't relevant here -# - async: relies on async -# - floating point: Failures caused by floating-point differences -# - threading: Failures due to lack of a threading implementation -# - subprocess: Failures caused by no subprocess module. Some of these are -# because the underlying functionality depends on subprocess, and others are -# just a side-effect of the way the test is written. The latter should -# probably be marked as "skip" or rearchitected so we don't have to skip the -# whole module. -# - networking: Fails because it tests low-level networking. -# - dbm: Failures due to no dbm module -# - strftime: Failures due to differences / shortcomings in WebAssembly's -# implementation of date/time formatting in strftime and strptime -# - permissions: Issues with the test writing to the virtual filesystem -# - locale: Fails due to limitations in the included locale implementation. -# - multiprocessing: Fails due to no multiprocessing implementation. -# - fs: Fails due to virtual filesystem issues. -# - nonsense: This functionality doesn't make sense in this context. Includes -# things like `pip`, `distutils` -# - crash: The Python interpreter just stopped without a traceback. Will require -# further investigation. This usually seems to be caused by calling into a -# system function that doesn't behave as one would expect. -# - crash-chrome: Same as crash but only affecting Chrome -# - crash-firefox: Same as crash but only affecting Firefox - -""" +PYTHON_TESTS_YAML = Path(__file__).parent / "python_tests.yaml" -def collect_old_error_flags(): - old_error_flags = {} +def get_old_yaml(): try: - with open(Path(__file__).parent / "python_tests.txt") as fp: - for line in fp: - line = line.strip() - if line.startswith("#") or not line: - continue - error_flags = line.split() - name = error_flags.pop(0) - if error_flags: - old_error_flags[name] = error_flags + with open(PYTHON_TESTS_YAML) as fp: + result = yaml.load(fp) except FileNotFoundError: - pass - return old_error_flags + result = yaml.seq() + return result -def collect_tests(base_dir): +def collect_tests(base_dir: Path) -> set: """Collect CPython unit tests""" # Note: this functionality is somewhat equivalent to pytest test # collection. - tests = [] + tests = set() for root, _dirs, files in os.walk(base_dir): - root = Path(root).relative_to(base_dir) + root = str(Path(root).relative_to(base_dir)) if str(root) == ".": root = "" @@ -86,22 +45,33 @@ def collect_tests(base_dir): root = ".".join(str(root).split("/")) + "." for filename in files: - filename = Path(filename) - if str(filename).startswith("test_") and filename.suffix == ".py": - tests.append(root + filename.stem) + p = Path(filename) + if filename.startswith("test_") and p.suffix == ".py": + tests.add(root + p.stem) - tests.sort() return tests +def get_test_name(test) -> str: + if isinstance(test, dict): + name = next(iter(test.keys())) + else: + name = test + return name + + +def update_tests(doc_group, tests): + for idx, test in enumerate(list(doc_group)): + if get_test_name(test) not in tests: + del doc_group[idx] + + for idx, test in enumerate(sorted(tests)): + if idx == len(doc_group) or get_test_name(doc_group[idx]) != test: + doc_group.insert(idx, test) + + if __name__ == "__main__": - old_error_flags = collect_old_error_flags() - tests = collect_tests(TEST_DIR) - with open("python_tests.txt", "w") as fp: - fp.write(explanation) - for test in tests: - error_flags = " ".join(old_error_flags.get(test, [])) - line = test - if error_flags: - line += " " + error_flags - fp.write(line + "\n") + doc = get_old_yaml() + update_tests(doc, collect_tests(LIB_DIR / "test")) + + yaml.dump(doc, PYTHON_TESTS_YAML) diff --git a/src/tests/python_tests.txt b/src/tests/python_tests.txt deleted file mode 100644 index 2b9e0dbcb..000000000 --- a/src/tests/python_tests.txt +++ /dev/null @@ -1,568 +0,0 @@ -# This list is generated with test/make_test_list.py script, which needs -# to be re-run after each CPython update. -# -# Test modules with a failure reason after their name are either skipped -# or marked as a known failure in pytest. -# -# Following reason codes are skipped, as they lead to segfaults: -# - segfault-: segfault in the corresponding system call. -# -# While the below reason codes are marked as a known failure. By default, they -# are also skipped. To run them, provide --run-xfail argument to pytest, -# - platform-specific: This is testing something about a particular platform -# that isn't relevant here -# - async: relies on async -# - floating point: Failures caused by floating-point differences -# - threading: Failures due to lack of a threading implementation -# - subprocess: Failures caused by no subprocess module. Some of these are -# because the underlying functionality depends on subprocess, and others are -# just a side-effect of the way the test is written. The latter should -# probably be marked as "skip" or rearchitected so we don't have to skip the -# whole module. -# - networking: Fails because it tests low-level networking. -# - dbm: Failures due to no dbm module -# - strftime: Failures due to differences / shortcomings in WebAssembly's -# implementation of date/time formatting in strftime and strptime -# - permissions: Issues with the test writing to the virtual filesystem -# - locale: Fails due to limitations in the included locale implementation. -# - multiprocessing: Fails due to no multiprocessing implementation. -# - fs: Fails due to virtual filesystem issues. -# - nonsense: This functionality doesn't make sense in this context. Includes -# things like `pip`, `distutils` -# - crash: The Python interpreter just stopped without a traceback. Will require -# further investigation. This usually seems to be caused by calling into a -# system function that doesn't behave as one would expect. -# - crash-chrome: Same as crash but only affecting Chrome -# - crash-firefox: Same as crash but only affecting Firefox - -test___all__ multiprocessing -test___future__ -test__locale locale -test__opcode -test__osx_support platform-specific -test__xxsubinterpreters hits Py_FatalError("not the last thread") inside Py_EndInterpreter -test_abc -test_abstract_numbers -test_aifc -test_argparse slow, sometimes times out -test_array -test_asdl_parser -test_ast -test_asyncgen async -test_asynchat async -test_asyncio.test_asyncio_waitfor async -test_asyncio.test_base_events async -test_asyncio.test_buffered_proto async -test_asyncio.test_context async -test_asyncio.test_events async -test_asyncio.test_futures async -test_asyncio.test_futures2 async -test_asyncio.test_locks async -test_asyncio.test_pep492 async -test_asyncio.test_proactor_events async -test_asyncio.test_protocols -test_asyncio.test_queues async -test_asyncio.test_runners async -test_asyncio.test_selector_events async segfault-socketcall -test_asyncio.test_sendfile async -test_asyncio.test_server async -test_asyncio.test_sock_lowlevel async -test_asyncio.test_sslproto -test_asyncio.test_streams async -test_asyncio.test_subprocess async -test_asyncio.test_tasks async -test_asyncio.test_threads async -test_asyncio.test_transports -test_asyncio.test_unix_events async -test_asyncio.test_windows_events -test_asyncio.test_windows_utils -test_asyncore bad ioctl syscall async -test_atexit subprocess -test_audioop -test_audit subprocess -test_augassign -test_base64 -test_baseexception -test_bdb -test_bigaddrspace -test_bigmem -test_binascii -test_binhex -test_binop -test_bisect -test_bool -test_buffer -test_bufio -test_builtin os.get_inheritable -test_bytes -test_bz2 threading -test_c_locale_coercion -test_calendar -test_call -test_capi hangs -test_cgi -test_cgitb -test_charmapcodec -test_check_c_globals -test_class -test_clinic -test_cmath -test_cmd -test_cmd_line -test_cmd_line_script -test_code -test_code_module -test_codeccallbacks -test_codecencodings_cn -test_codecencodings_hk -test_codecencodings_iso2022 -test_codecencodings_jp -test_codecencodings_kr -test_codecencodings_tw -test_codecmaps_cn -test_codecmaps_hk -test_codecmaps_jp -test_codecmaps_kr -test_codecmaps_tw -test_codecs -test_codeop -test_collections -test_colorsys -test_compare -test_compile -test_compileall multiprocessing -test_complex -test_concurrent_futures -test_configparser -test_contains -test_context threads -test_contextlib -test_contextlib_async async -test_copy -test_copyreg dbm -test_coroutines async -test_cprofile -test_crashers -test_crypt -test_csv -test_ctypes -test_curses -test_dataclasses -test_datetime strftime -test_dbm dbm -test_dbm_dumb permissions -test_dbm_gnu dbm -test_dbm_ndbm dbm -test_decimal floating point -test_decorators -test_defaultdict -test_deque -test_descr -test_descrtut -test_devpoll -test_dict -test_dict_version -test_dictcomps -test_dictviews -test_difflib -test_dis -test_distutils nonsense -test_doctest subprocess -test_doctest2 -test_docxmlrpc socket -test_dtrace platform -test_dynamic -test_dynamicclassattribute -test_eintr subprocess -test_email.test__encoded_words -test_email.test__header_value_parser -test_email.test_asian_codecs -test_email.test_contentmanager -test_email.test_defect_handling -test_email.test_email threading -test_email.test_generator -test_email.test_headerregistry -test_email.test_inversion -test_email.test_message -test_email.test_parser -test_email.test_pickleable -test_email.test_policy -test_email.test_utils -test_embed subprocess -test_ensurepip nonsense -test_enum threading -test_enumerate -test_eof -test_epoll -test_errno -test_exception_hierarchy -test_exception_variations -test_exceptions BaseException_dealloc and BaseException_clear enter an infinite recurse -test_extcall -test_faulthandler subprocess -test_fcntl -test_file -test_file_eintr subprocess -test_filecmp -test_fileinput -test_fileio -test_finalization -test_float -test_flufl -test_fnmatch -test_subprocess1 threading -test_format -test_fractions -test_frame -test_frozen -test_fstring -test_ftplib socket -test_funcattrs -test_functools threading -test_future -test_future3 -test_future4 -test_future5 -test_gc threads -test_gdb -test_generator_stop -test_generators keyboard interrupt handling differences -test_genericalias multiprocessing -test_genericclass -test_genericpath permissions -test_genexps -test_getargs2 -test_getopt -test_getpass -test_gettext -test_glob fs -test_global -test_grammar -test_graphlib -test_grp -test_gzip -test_hash -test_hashlib threading -test_heapq -test_hmac -test_html -test_htmlparser -test_http_cookiejar -test_http_cookies -test_httplib socket segfault-socketcall -test_httpservers threading -test_idle -test_imaplib socket -test_imghdr -test_imp -test_importlib.builtin.test_finder -test_importlib.builtin.test_loader -test_importlib.extension.test_case_sensitivity -test_importlib.extension.test_finder -test_importlib.extension.test_loader fs -test_importlib.extension.test_path_hook -test_importlib.frozen.test_finder -test_importlib.frozen.test_loader -test_importlib.import_.test___loader__ -test_importlib.import_.test___package__ -test_importlib.import_.test_api -test_importlib.import_.test_caching -test_importlib.import_.test_fromlist -test_importlib.import_.test_meta_path -test_importlib.import_.test_packages -test_importlib.import_.test_path -test_importlib.import_.test_relative_imports -test_importlib.source.test_case_sensitivity -test_importlib.source.test_file_loader -test_importlib.source.test_finder -test_importlib.source.test_path_hook -test_importlib.source.test_source_encoding -test_importlib.test_abc -test_importlib.test_api -test_importlib.test_files -test_importlib.test_lazy -test_importlib.test_locks threading -test_importlib.test_main -test_importlib.test_metadata_api -test_importlib.test_namespace_pkgs -test_importlib.test_open -test_importlib.test_path -test_importlib.test_pkg_import -test_importlib.test_read -test_importlib.test_reader -test_importlib.test_resource -test_importlib.test_spec -test_importlib.test_threaded_import threading -test_importlib.test_util -test_importlib.test_windows -test_importlib.test_zip -test_index -test_inspect async -test_int -test_int_literal -test_interpreters threads -test_io segfault-unknown in test_large_file_ops -test_ioctl -test_ipaddress -test_isinstance crash-chrome -test_iter -test_iterlen -test_itertools -test_json.test_decode -test_json.test_default -test_json.test_dump -test_json.test_encode_basestring_ascii -test_json.test_enum -test_json.test_fail -test_json.test_float -test_json.test_indent -test_json.test_pass1 -test_json.test_pass2 -test_json.test_pass3 -test_json.test_recursion -test_json.test_scanstring -test_json.test_separators -test_json.test_speedups -test_json.test_tool subprocess -test_json.test_unicode -test_keyword -test_keywordonlyarg -test_kqueue -test_largefile RangeError: Array buffer allocation failed -test_lib2to3 nonsense -test_linecache -test_list -test_listcomps -test_lltrace -test_locale locale -test_logging networking -test_long -test_longexp -test_lzma -test_mailbox fs -test_mailcap not sure why it fails... -test_marshal -test_math -test_memoryio -test_memoryview -test_metaclass -test_mimetypes -test_minidom -test_mmap crash -test_module -test_modulefinder -test_msilib -test_multibytecodec -test_multiprocessing_subprocess subprocess -test_multiprocessing_subprocessserver subprocess -test_multiprocessing_main_handling -test_multiprocessing_spawn -test_named_expressions -test_netrc -test_nis -test_nntplib -test_ntpath platform -test_numeric_tower -test_opcache -test_opcodes -test_openpty platform-specific -test_operator -test_optparse -test_ordered_dict -test_os mmap -test_ossaudiodev -test_osx_env -test_pathlib fs -test_patma -test_pdb subprocess -test_peepholer -test_peg_generator.test_c_parser -test_peg_generator.test_first_sets -test_peg_generator.test_grammar_validator -test_peg_generator.test_pegen -test_pickle dbm -test_picklebuffer -test_pickletools dbm -test_pipes -test_pkg -test_pkgutil -test_platform subprocess -test_plistlib -test_poll subprocess -test_popen subprocess -test_poplib bad ioctl socket -test_positional_only_arg -test_posix segfault-fstatfs64 -test_posixpath crash -test_pow -test_pprint -test_print -test_profile -test_property -test_pstats -test_pty -test_pulldom -test_pwd -test_py_compile subprocess -test_pyclbr -test_pydoc threading -test_pyexpat -test_queue threading -test_quopri subprocess -test_raise -test_random subprocess -test_range -test_re locale -test_readline -test_regrtest subprocess -test_repl subprocess -test_reprlib -test_resource -test_richcmp -test_rlcompleter -test_robotparser socket -test_runpy subprocess -test_sax -test_sched subprocess -test_scope -test_script_helper -test_secrets -test_select networking segfault-newselect -test_selectors networking segfault-newselect -test_set -test_setcomps -test_shelve dbm -test_shlex -test_shutil crash -test_signal emscripten signals implementation isn't very complete -test_site TypeError: unhashable type: 'pyodide.JsProxy' -test_slice -test_smtpd -test_smtplib bad ioctl syscall 21537 -test_smtpnet -test_sndhdr -test_socket networking -test_socketserver networking -test_sort -test_source_encoding subprocess -test_spwd -test_sqlite threading -test_ssl -test_startfile -test_stat -test_statistics -test_strftime strftime -test_string -test_string_literals -test_stringprep -test_strptime strftime -test_strtod -test_struct -test_structmembers -test_structseq -test_subclassinit -test_subprocess subprocess -test_sunau -test_sundry -test_super -test_support socket -test_symtable -test_syntax -test_sys subprocess -test_sys_setprofile -test_sys_settrace async -test_sysconfig nonsense -test_syslog -test_tabnanny -test_tarfile crash -test_tcl -test_telnetlib bad ioctl syscall 21537 -test_tempfile fs -test_textwrap -test_thread threading -test_threadedtempfile threading -test_threading threading -test_threading_local threading -test_threadsignals threading -test_time strftime -test_timeit -test_timeout -test_tix -test_tk -test_tokenize -test_tools.test_fixcid -test_tools.test_gprof2html -test_tools.test_i18n -test_tools.test_lll -test_tools.test_md5sum -test_tools.test_pathfix -test_tools.test_pdeps -test_tools.test_pindent -test_tools.test_reindent -test_tools.test_sundry -test_trace slow, sometimes times out -test_traceback subprocess -test_tracemalloc -test_ttk_guionly -test_ttk_textonly -test_tuple -test_turtle -test_type_annotations -test_type_comments -test_typechecks -test_types -test_typing -test_ucn -test_unary -test_unicode -test_unicode_file fs MEMFS doesn't work as expected with unencodable file names -test_unicode_file_functions -test_unicode_identifiers -test_unicodedata -test_unittest os.kill -test_univnewlines -test_unpack -test_unpack_ex -test_unparse -test_urllib -test_urllib2 subprocess -test_urllib2_localnet socket -test_urllib2net -test_urllib_response -test_urllibnet -test_urlparse -test_userdict -test_userlist -test_userstring -test_utf8_mode -test_utf8source -test_uu -test_uuid subprocess -test_venv nonsense -test_wait3 threading -test_wait4 threading -test_wave -test_weakref threading -test_weakset -test_webbrowser replaced -test_winconsoleio -test_winreg -test_winsound -test_with -test_wsgiref -test_xdrlib -test_xml_dom_minicompat -test_xml_etree -test_xml_etree_c -test_xmlrpc networking -test_xmlrpc_net -test_xxlimited -test_xxtestfuzz -test_yield_from -test_zipapp -test_zipfile fs -test_zipfile64 -test_zipimport -test_zipimport_support -test_zlib -test_zoneinfo.test_zoneinfo diff --git a/src/tests/python_tests.yaml b/src/tests/python_tests.yaml new file mode 100644 index 000000000..16eb9d3f3 --- /dev/null +++ b/src/tests/python_tests.yaml @@ -0,0 +1,848 @@ +# This list is generated with test/make_test_list.py script, which needs +# to be re-run after each CPython update. + +# Test modules with a failure reason after their name are either skipped +# or marked as a known failure in pytest. + +# Following reason codes are skipped, as they lead to segfaults: +# - segfault-: segfault in the corresponding system call. + +# While the below reason codes are marked as a known failure. By default, they +# are also skipped. To run them, provide --run-xfail argument to pytest, +# - platform-specific: This is testing something about a particular platform +# that isn't relevant here +# - async: relies on async +# - floating point: Failures caused by floating-point differences +# - threading: Failures due to lack of a threading implementation +# - subprocess: Failures caused by no subprocess module. Some of these are +# because the underlying functionality depends on subprocess, and others are +# just a side-effect of the way the test is written. The latter should +# probably be marked as "skip" or rearchitected so we don't have to skip the +# whole module. +# - networking: Fails because it tests low-level networking. +# - dbm: Failures due to no dbm module +# - strftime: Failures due to differences / shortcomings in WebAssembly's +# implementation of date/time formatting in strftime and strptime +# - permissions: Issues with the test writing to the virtual filesystem +# - locale: Fails due to limitations in the included locale implementation. +# - multiprocessing: Fails due to no multiprocessing implementation. +# - fs: Fails due to virtual filesystem issues. +# - nonsense: This functionality doesn't make sense in this context. Includes +# things like `pip`, `distutils` +# - crash: The Python interpreter just stopped without a traceback. Will require +# further investigation. This usually seems to be caused by calling into a +# system function that doesn't behave as one would expect. +# - crash-chrome: Same as crash but only affecting Chrome +# - crash-firefox: Same as crash but only affecting Firefox +- test___all__: + xfail: multiprocessing +- test___future__ +- test__locale: + xfail: locale +- test__opcode +- test__osx_support: + xfail: platform-specific +- test__xxsubinterpreters: + xfail: hits Py_FatalError("not the last thread") inside Py_EndInterpreter +- test_abc +- test_abstract_numbers +- test_aifc +- test_argparse: + xfail: slow, sometimes times out +- test_array +- test_asdl_parser +- test_ast +- test_asyncgen: + xfail: async +- test_asynchat: + xfail: async +- test_asyncio.test_asyncio_waitfor: + xfail: async +- test_asyncio.test_base_events: + xfail: async +- test_asyncio.test_buffered_proto: + xfail: async +- test_asyncio.test_context: + xfail: async +- test_asyncio.test_events: + xfail: async +- test_asyncio.test_futures: + xfail: async +- test_asyncio.test_futures2: + xfail: async +- test_asyncio.test_locks: + xfail: async +- test_asyncio.test_pep492: + xfail: async +- test_asyncio.test_proactor_events: + xfail: async +- test_asyncio.test_protocols +- test_asyncio.test_queues: + xfail: async +- test_asyncio.test_runners: + xfail: async +- test_asyncio.test_selector_events: + xfail: async +- test_asyncio.test_sendfile: + xfail: async +- test_asyncio.test_server: + xfail: async +- test_asyncio.test_sock_lowlevel: + xfail: async +- test_asyncio.test_sslproto +- test_asyncio.test_streams: + xfail: async +- test_asyncio.test_subprocess: + xfail: async +- test_asyncio.test_tasks: + xfail: async +- test_asyncio.test_threads: + xfail: async +- test_asyncio.test_transports +- test_asyncio.test_unix_events: + xfail: async +- test_asyncio.test_windows_events +- test_asyncio.test_windows_utils +- test_asyncore: + xfail: async +- test_atexit: + skip: + - test_general # fork +- test_audioop +- test_audit: + xfail: all tests fork +- test_augassign +- test_base64 +- test_baseexception +- test_bdb +- test_bigaddrspace +- test_bigmem +- test_binascii +- test_binhex +- test_binop +- test_bisect +- test_bool +- test_buffer +- test_bufio +- test_builtin: + skip: + - test_open_non_inheritable + - test_compile_top_level_await +- test_bytes +- test_bz2: + skip: + - testThreading +- test_c_locale_coercion +- test_calendar +- test_call +- test_capi: + xfail: hangs +- test_cgi +- test_cgitb +- test_charmapcodec +- test_check_c_globals +- test_class +- test_clinic +- test_cmath +- test_cmd +- test_cmd_line +- test_cmd_line_script +- test_code +- test_code_module +- test_codeccallbacks +- test_codecencodings_cn +- test_codecencodings_hk +- test_codecencodings_iso2022 +- test_codecencodings_jp +- test_codecencodings_kr +- test_codecencodings_tw +- test_codecmaps_cn +- test_codecmaps_hk +- test_codecmaps_jp +- test_codecmaps_kr +- test_codecmaps_tw +- test_codecs +- test_codeop +- test_collections +- test_colorsys +- test_compare +- test_compile +- test_compileall: + xfail: multiprocessing +- test_complex +- test_concurrent_futures +- test_configparser +- test_contains +- test_context: + skip: + - test_context_threads_1 +- test_contextlib +- test_contextlib_async: + xfail: async +- test_copy +- test_copyreg: + xfail: dbm +- test_coroutines: + xfail: async +- test_cprofile +- test_crashers +- test_crypt +- test_csv +- test_ctypes: + skip: + - test_callback_too_many_args + - test_callback_large_struct + - CFunctions + - test_struct_return_2H + - test_frozentable + - FunctionTestCase + - test_an_integer + - test_optimizeflag + - test_pass_by_value_in_register + - test_struct_by_value + - test_longlong_callbacks + - test_longdouble + - test_longlong + - test_ulonglong + +- test_curses +- test_dataclasses +- test_datetime: + xfail: strftime +- test_dbm: + xfail: dbm +- test_dbm_dumb: + xfail: permissions +- test_dbm_gnu: + xfail: dbm +- test_dbm_ndbm: + xfail: dbm +- test_decimal: + xfail-chrome: times out + skip: + - test_context_subclassing # floating point + - test_none_args # Some context issue? + - test_threading +- test_decorators +- test_defaultdict +- test_deque +- test_descr +- test_descrtut +- test_devpoll +- test_dict +- test_dict_version +- test_dictcomps +- test_dictviews +- test_difflib +- test_dis +- test_distutils: + # error while loading tests ModuleNotFoundError: No module named '_osx_support' + xfail: nonsense +- test_doctest: + xfail: subprocess +- test_doctest2 +- test_docxmlrpc: + xfail: socket +- test_dtrace: + xfail: fork +- test_dynamic +- test_dynamicclassattribute +- test_eintr: + xfail: fork +- test_email.test__encoded_words +- test_email.test__header_value_parser +- test_email.test_asian_codecs +- test_email.test_contentmanager +- test_email.test_defect_handling +- test_email.test_email: + skip: + - test_make_msgid_collisions # fork +- test_email.test_generator +- test_email.test_headerregistry +- test_email.test_inversion +- test_email.test_message +- test_email.test_parser +- test_email.test_pickleable +- test_email.test_policy +- test_email.test_utils +- test_embed: + skip: + - test_set_config # fork +- test_ensurepip: + xfail: nonsense +- test_enum: + skip: + - test_unique_composite # fork +- test_enumerate +- test_eof +- test_epoll +- test_errno +- test_exception_hierarchy +- test_exception_variations +- test_exceptions: + xfail: BaseException_dealloc and BaseException_clear enter an infinite recurse +- test_extcall +- test_faulthandler: + skip: + # fork + - test_disabled_by_default + - test_sys_xoptions + - test_env_var +- test_fcntl +- test_file +- test_file_eintr: + xfail: fork +- test_filecmp +- test_fileinput +- test_fileio +- test_finalization +- test_float +- test_flufl +- test_fnmatch +- test_fork1: + xfail: threading +- test_format +- test_fractions +- test_frame +- test_frozen +- test_fstring +- test_ftplib: + xfail: socket +- test_funcattrs +- test_functools: + skip: + - "*threaded*" +- test_future +- test_future3 +- test_future4 +- test_future5 +- test_gc: + skip: + - test_garbage_at_shutdown + - test_trashcan_threads +- test_gdb +- test_generator_stop +- test_generators +- test_genericalias: + xfail: multiprocessing +- test_genericclass +- test_genericpath: + skip: + - test_samefile_on_link + - test_samestat_on_link + - test_exists_fd +- test_genexps +- test_getargs2 +- test_getopt +- test_getpass +- test_gettext +- test_glob: + skip: + # RecursionError: maximum recursion depth exceeded while calling a Python object + - test_selflink +- test_global +- test_grammar +- test_graphlib +- test_grp +- test_gzip +- test_hash +- test_hashlib: + skip: + - test_threaded_hashing +- test_heapq +- test_hmac +- test_html +- test_htmlparser +- test_http_cookiejar +- test_http_cookies +- test_httplib: + skip: + # sockets + - test_response_fileno + - testHTTPConnectionSourceAddress + - testTimeoutAttribute +- test_httpservers: + xfail: threads +- test_idle +- test_imaplib: + xfail: socket +- test_imghdr +- test_imp +- test_importlib.builtin.test_finder +- test_importlib.builtin.test_loader +- test_importlib.extension.test_case_sensitivity +- test_importlib.extension.test_finder +- test_importlib.extension.test_loader +- test_importlib.extension.test_path_hook +- test_importlib.frozen.test_finder +- test_importlib.frozen.test_loader +- test_importlib.import_.test___loader__ +- test_importlib.import_.test___package__ +- test_importlib.import_.test_api +- test_importlib.import_.test_caching +- test_importlib.import_.test_fromlist +- test_importlib.import_.test_meta_path +- test_importlib.import_.test_packages +- test_importlib.import_.test_path +- test_importlib.import_.test_relative_imports +- test_importlib.source.test_case_sensitivity +- test_importlib.source.test_file_loader +- test_importlib.source.test_finder +- test_importlib.source.test_path_hook +- test_importlib.source.test_source_encoding +- test_importlib.test_abc +- test_importlib.test_api +- test_importlib.test_files +- test_importlib.test_lazy +- test_importlib.test_locks: + xfail: threading +- test_importlib.test_main +- test_importlib.test_metadata_api +- test_importlib.test_namespace_pkgs +- test_importlib.test_open +- test_importlib.test_path +- test_importlib.test_pkg_import +- test_importlib.test_read +- test_importlib.test_reader +- test_importlib.test_resource +- test_importlib.test_spec +- test_importlib.test_threaded_import: + xfail: threading +- test_importlib.test_util +- test_importlib.test_windows +- test_importlib.test_zip +- test_index +- test_inspect: + skip: + - test_nested_class_definition_inside_async_function +- test_int +- test_int_literal +- test_interpreters: + skip: + # threads + - test_main + - "*thread*" +- test_io: + xfail: segfault-outofmemory in test_large_file_ops ("Array buffer allocation failed") + # The remaining test_io tests seem to hang too +- test_ioctl +- test_ipaddress +- test_isinstance +- test_iter +- test_iterlen +- test_itertools +- test_json.test_decode +- test_json.test_default +- test_json.test_dump +- test_json.test_encode_basestring_ascii +- test_json.test_enum +- test_json.test_fail +- test_json.test_float +- test_json.test_indent +- test_json.test_pass1 +- test_json.test_pass2 +- test_json.test_pass3 +- test_json.test_recursion +- test_json.test_scanstring +- test_json.test_separators +- test_json.test_speedups +- test_json.test_tool: + xfail: subprocess +- test_json.test_unicode +- test_keyword +- test_keywordonlyarg +- test_kqueue +- test_largefile: + xfail: segfault-outofmemory ("Array buffer allocation failed") +- test_lib2to3: + xfail: nonsense +- test_linecache +- test_list +- test_listcomps +- test_lltrace +- test_locale: + skip: + - "*diacritic*" +- test_logging: + xfail: threading +- test_long +- test_longexp +- test_lzma +- test_mailbox: + xfail: fs +- test_mailcap: + skip: + - test_test # not sure why it fails... +- test_marshal +- test_math +- test_memoryio +- test_memoryview +- test_metaclass +- test_mimetypes +- test_minidom +- test_mmap: + skip: + - "*large*" # segfault-outofmemory Invalid typed array length: 6442450944" + # resizing mmap doesn't work (raises [Errno 48] Out of memory) + - test_basic + - test_offset + - test_resize_past_pos +- test_module +- test_modulefinder +- test_msilib +- test_multibytecodec +- test_multiprocessing_fork +- test_multiprocessing_forkserver +- test_multiprocessing_main_handling +- test_multiprocessing_spawn +- test_named_expressions +- test_netrc +- test_nis +- test_nntplib +- test_ntpath: + # same set of failures as in test_posixpath + skip: + - test_exists_fd + - test_samestat_on_link + - test_samefile_on_link + - test_sameopenfile + - test_nonascii_abspath +- test_numeric_tower +- test_opcache +- test_opcodes +- test_openpty: + xfail: platform-specific +- test_operator +- test_optparse +- test_ordered_dict +- test_os: + xfail: | + segfault-eventfd; if we screen out the eventfd tests, we see 52 errors, + 22 failures out of 312 tests +- test_ossaudiodev +- test_osx_env +- test_pathlib: + skip: + - test_hardlink_to + - test_home + - test_link_to + - test_is_socket_true + - test_readlink + - test_open_mode + - test_touch_mode +- test_patma +- test_pdb: + xfail: 21/51 use subprocess +- test_peepholer +- test_peg_generator.test_c_parser +- test_peg_generator.test_first_sets +- test_peg_generator.test_grammar_validator +- test_peg_generator.test_pegen +- test_pickle: + xfail: dbm +- test_picklebuffer +- test_pickletools: + xfail: dbm +- test_pipes +- test_pkg +- test_pkgutil +- test_platform: + skip: + - test_architecture_via_symlink # fork +- test_plistlib +- test_poll: + xfail: subprocess +- test_popen: + xfail: subprocess +- test_poplib: + xfail: sockets +- test_positional_only_arg +- test_posix: + xfail: missing pwd module +- test_posixpath: + skip: + - test_exists_fd + - test_samestat_on_link + - test_samefile_on_link + - test_nonascii_abspath +- test_pow +- test_pprint +- test_print +- test_profile +- test_property +- test_pstats +- test_pty +- test_pulldom +- test_pwd +- test_py_compile: + skip: + - test_stdin # fork +- test_pyclbr +- test_pydoc: + skip: + - test_server # fork + - test_synopsis_sourceless # expects __pycache__ +- test_pyexpat +- test_queue: + # More than half of these make a thread + xfail: threads +- test_quopri: + skip: + - test_scriptdecode # fork + - test_scriptencode # fork +- test_raise +- test_random: + skip: + - test_after_fork +- test_range +- test_re: + skip: + - "*locale*" +- test_readline +- test_regrtest: + xfail: All tests fork +- test_repl: + xfail: All tests fork +- test_reprlib +- test_resource +- test_richcmp +- test_rlcompleter +- test_robotparser: + xfail: socket +- test_runpy: + skip: + - test_pymain_run* # fork +- test_sax +- test_sched: + skip: + - "*concurrent" # threads +- test_scope +- test_script_helper +- test_secrets +- test_select: + skip: + - test_select # fork +- test_selectors: + xfail: networking +- test_set +- test_setcomps +- test_shelve: + xfail: dbm +- test_shlex +- test_shutil: + skip: + - test_copy2_symlinks + - test_copy_symlinks + - test_copymode_symlink_to_symlink + - test_copystat_symlinks + - test_dont_copy_file_onto_link_to_itself + - test_copytree_dangling_symlinks + - test_copytree_symlink_dir + - test_copytree_symlinks + - test_copyfile_nonexistent_dir + +- test_signal: + xfail: emscripten signals implementation isn't very complete +- test_site: + xfail: "TypeError: unhashable type: 'pyodide.JsProxy'" +- test_slice +- test_smtpd +- test_smtplib: + xfail: sockets +- test_smtpnet +- test_sndhdr +- test_socket: + xfail: networking +- test_socketserver: + xfail: networking +- test_sort +- test_source_encoding: + skip: + - test_20731 +- test_spwd +- test_sqlite: + skip: + - "*ThreadTests*" + - test_ctx_mgr_rollback_if_commit_failed +- test_ssl +- test_startfile +- test_stat +- test_statistics +- test_strftime: + skip: + - test_strftime # strftime has some bugs +- test_string +- test_string_literals +- test_stringprep +- test_strptime: + skip: + - test_week_of_year_and_day_of_week_calculation + - test_timezone +- test_strtod +- test_struct +- test_structmembers +- test_structseq +- test_subclassinit +- test_subprocess: + xfail: subprocess +- test_sunau +- test_sundry +- test_super +- test_support: + xfail: about half the tests fork +- test_symtable +- test_syntax +- test_sys: + xfail: | + self.assertEqual(os.path.abspath(sys.executable), sys.executable) fails with '/tmp/test_python_42æ' != '' +- test_sys_setprofile +- test_sys_settrace: + xfail: async +- test_sysconfig: + xfail: nonsense +- test_syslog +- test_tabnanny +- test_tarfile: + skip: + - test_file_mode + - "test_extractall*" + - test_link_size + - test_dereference_hardlink + - test_add_hardlink + - test_add_twice +- test_tcl +- test_telnetlib: + xfail: 7/19 fail with sockets +- test_tempfile: + skip: + - test_non_directory # setup failure Errno 2: Permission Denied + - test_process_awareness # fork + - test_truncate_with_size_parameter # setup failure FileNotFoundError: [Errno 44] No such file or directory + - test_noinherit # self.assertEqual(os.get_inheritable(file.fd), False) ==> AssertionError: True != False +- test_textwrap +- test_thread: + xfail: threading +- test_threadedtempfile: + xfail: threading +- test_threading: + xfail: threading +- test_threading_local: + xfail: threading +- test_threadsignals: + xfail: threading +- test_time: + skip: + - test_thread_time # setup failure: OSError: [Errno 28] Invalid argument + - test_time_ns_type # setup failure: OSError: [Errno 28] Invalid argument + - test_localtime_timezone # pyodide.asm.js:4222 AssertionError: 'Pacific Standard Time' != 'Pacific S' + - test_4dyear # '%4Y' != '0001' + - test_ctime # '1941' != '-100' + - test_monotonic # self.assertTrue(0.450 <= dt) fails +- test_timeit +- test_timeout +- test_tix +- test_tk +- test_tokenize +- test_tools.test_fixcid +- test_tools.test_gprof2html +- test_tools.test_i18n +- test_tools.test_lll +- test_tools.test_md5sum +- test_tools.test_pathfix +- test_tools.test_pdeps +- test_tools.test_pindent +- test_tools.test_reindent +- test_tools.test_sundry +- test_trace: + xfail: slow, sometimes times out +- test_traceback: + skip: + - test_encoded_file # fork +- test_tracemalloc +- test_ttk_guionly +- test_ttk_textonly +- test_tuple: + xfail-chrome: times out +- test_turtle +- test_type_annotations +- test_type_comments +- test_typechecks +- test_types +- test_typing +- test_ucn +- test_unary +- test_unicode +- test_unicode_file: + xfail: fs MEMFS doesn't work as expected with unencodable file names +- test_unicode_file_functions +- test_unicode_identifiers +- test_unicodedata +- test_unittest: + skip: + - "*async*" + - "*Interrupt*" + - "*Handler*" + # os.kill + - testTwoResults + - test_warnings + - testRemoveResult + # fork + - testSelectedTestNamesFunctionalTest +- test_univnewlines +- test_unpack +- test_unpack_ex +- test_unparse +- test_urllib +- test_urllib2: + skip: + - test_http_body_pipe # fork +- test_urllib2_localnet: + xfail: socket +- test_urllib2net +- test_urllib_response +- test_urllibnet +- test_urlparse +- test_userdict +- test_userlist +- test_userstring +- test_utf8_mode +- test_utf8source +- test_uu +- test_uuid: + skip: + - testIssue8621 +- test_venv: + xfail: nonsense +- test_wait3: + xfail: threading +- test_wait4: + xfail: threading +- test_wave +- test_weakref: + xfail: slow, sometimes times out +- test_weakset +- test_webbrowser: + xfail: replaced +- test_winconsoleio +- test_winreg +- test_winsound +- test_with +- test_wsgiref +- test_xdrlib +- test_xml_dom_minicompat +- test_xml_etree +- test_xml_etree_c +- test_xmlrpc: + xfail: networking +- test_xmlrpc_net +- test_xxlimited +- test_xxtestfuzz +- test_yield_from +- test_zipapp +- test_zipfile: + xfail-chrome: times out +- test_zipfile64 +- test_zipimport +- test_zipimport_support +- test_zlib +- test_zoneinfo.test_zoneinfo diff --git a/src/tests/test_core_python.py b/src/tests/test_core_python.py index d8085539c..098a8134e 100644 --- a/src/tests/test_core_python.py +++ b/src/tests/test_core_python.py @@ -1,50 +1,77 @@ +from functools import cache from pathlib import Path +from textwrap import dedent +from typing import Any import pytest +import yaml +from yaml import CLoader as Loader from pyodide_build.common import UNVENDORED_STDLIB_MODULES -def test_cpython_core(python_test, selenium, request): - - name, error_flags = python_test - +def filter_info(info: dict[str, Any], browser: str) -> dict[str, Any]: + info = dict(info) # keep only flags related to the current browser flags_to_remove = ["firefox", "chrome", "node"] - flags_to_remove.remove(selenium.browser) - for flag in flags_to_remove: - if "crash-" + flag in error_flags: - error_flags.remove("crash-" + flag) + flags_to_remove.remove(browser) + for browser in flags_to_remove: + for key in list(info.keys()): + if key.endswith(browser): + del info[key] + return info - if any(flag.startswith("segfault") for flag in error_flags): - pytest.skip('known segfault: "{}"'.format(" ".join(error_flags))) - if error_flags: +def possibly_skip_test(request, info: dict[str, Any]) -> dict[str, Any]: + for reason in ( + reason for (flag, reason) in info.items() if flag.startswith("segfault") + ): + pytest.skip(f"known segfault: {reason}") + + for reason in ( + reason for [flag, reason] in info.items() if flag.startswith("xfail") + ): if request.config.option.run_xfail: request.applymarker( pytest.mark.xfail( run=False, - reason='known failure: "{}"'.format(" ".join(error_flags)), + reason=f"known failure: {reason}", ) ) else: - pytest.xfail('known failure: "{}"'.format(" ".join(error_flags))) + pytest.xfail(f"known failure: {reason}") + return info + + +def test_cpython_core(main_test, selenium, request): + [name, info] = main_test + info = filter_info(info, selenium.browser) + possibly_skip_test(request, info) + + ignore_tests = info.get("skip", []) + if not isinstance(ignore_tests, list): + raise Exception("Invalid python_tests.yaml entry: 'skip' should be a list") selenium.load_package(list(UNVENDORED_STDLIB_MODULES)) try: selenium.run( - """ - from test.libregrtest import main - import unittest.case + dedent( + f""" + import platform + from test import libregrtest + + platform.platform(aliased=True) + import _testcapi + if hasattr(_testcapi, "raise_SIGINT_then_send_None"): + # This uses raise() which doesn't work. + del _testcapi.raise_SIGINT_then_send_None + try: - main(['{}'], verbose=True, verbose3=True) + libregrtest.main(["{name}"], ignore_tests={ignore_tests}, verbose=True, verbose3=True) except SystemExit as e: if e.code != 0: - raise RuntimeError(f'Failed with code: {{e.code}}') - except unittest.case.SkipTest: - pass - """.format( - name + raise RuntimeError(f"Failed with code: {{e.code}}") + """ ) ) except selenium.JavascriptException: @@ -52,19 +79,24 @@ def test_cpython_core(python_test, selenium, request): raise +def get_test_info(test) -> tuple[str, dict[str, Any]]: + if isinstance(test, dict): + (name, info) = next(iter(test.items())) + else: + name = test + info = {} + return (name, info) + + +@cache +def get_tests() -> list[tuple[str, dict[str, Any]]]: + with open(Path(__file__).parent / "python_tests.yaml") as file: + data = yaml.load(file, Loader) + + return [get_test_info(test) for test in data] + + def pytest_generate_tests(metafunc): - if "python_test" in metafunc.fixturenames: - test_modules = [] - test_modules_ids = [] - with open(Path(__file__).parent / "python_tests.txt") as fp: - for line in fp: - line = line.strip() - if line.startswith("#") or not line: - continue - error_flags = line.split() - name = error_flags.pop(0) - test_modules.append((name, error_flags)) - # explicitly define test ids to keep - # a human readable test name in pytest - test_modules_ids.append(name) - metafunc.parametrize("python_test", test_modules, ids=test_modules_ids) + if "main_test" in metafunc.fixturenames: + tests = get_tests() + metafunc.parametrize("main_test", tests, ids=[t[0] for t in tests])