Merge pull request #79 from mdboom/codecs

Obsoletes #22.  Include all codecs, since removing tests imp. size

.circleci/config.yml

@@ -23,7 +23,7 @@ jobs:
             sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8
             sudo update-alternatives --set gcc /usr/bin/gcc-8
 
-            sudo pip install pytest-xdist selenium PyYAML
+            sudo pip install pytest-xdist pytest-instafail selenium PyYAML
 
             # Get recent version of Firefox and geckodriver
             wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-nightly-latest-ssl\&os\=linux64\&lang\=en-US

Makefile

@@ -98,7 +98,7 @@ build/renderedhtml.css: src/renderedhtml.less
 
 
 test: all build/test.html build/test_data.txt
-	py.test test -v
+	py.test test -v --instafail
 
 
 build/test_data.txt: test/data.txt
@@ -154,7 +154,6 @@ root/.built: \
 	( \
 		cd root/lib/python$(PYMINOR); \
 		rm -fr `cat ../../../remove_modules.txt`; \
-		rm encodings/mac_*.py; \
 		rm -fr test; \
 		find . -name "*.wasm.pre" -type f -delete ; \
 		find -type d -name __pycache__ -prune -exec rm -rf {} \; \

README.md

@@ -37,7 +37,7 @@ Additional build prerequisites are:
 
 Install the following dependencies into the default Python installation:
 
-   `pip install pytest selenium`
+   `pip install pytest selenium pytest-instafail`
 
 Install [geckodriver](https://github.com/mozilla/geckodriver/releases) somewhere
 on your `PATH`.

cpython/Setup.local

@@ -36,3 +36,12 @@ _md5 md5module.c
 _blake2 _blake2/blake2module.c _blake2/blake2b_impl.c ../../host/Python-3.6.4/Modules/_blake2/blake2s_impl.c
 
 #future_builtins future_builtins.c
+
+_multibytecodec cjkcodecs/multibytecodec.c
+
+_codecs_cn cjkcodecs/_codecs_cn.c
+_codecs_hk cjkcodecs/_codecs_hk.c
+_codecs_iso2022 cjkcodecs/_codecs_iso2022.c
+_codecs_jp cjkcodecs/_codecs_jp.c
+_codecs_kr cjkcodecs/_codecs_kr.c
+_codecs_tw cjkcodecs/_codecs_tw.c

cpython/patches/dont-test-undecodable-filenames.patch

@@ -0,0 +1,13 @@
+diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
+index ddcd2cc387..f422f2feb6 100644
+--- a/Lib/test/support/__init__.py
++++ b/Lib/test/support/__init__.py
+@@ -880,6 +880,8 @@ if os.name == 'nt':
+                   'Unicode filename tests may not be effective'
+                   % (TESTFN_UNENCODABLE, TESTFN_ENCODING))
+             TESTFN_UNENCODABLE = None
++elif sys.platform == 'emscripten':
++    pass
+ # Mac OS X denies unencodable filenames (invalid utf-8)
+ elif sys.platform != 'darwin':
+     try:

test/python_tests.txt

@@ -11,11 +11,6 @@
 #   probably be marked as "skip" or rearchitected so we don't have to skip the
 #   whole module.
 # - networking: Fails because it tests low-level networking.
-# - unknown encoding: Most of the larger and more rare encodings are excluded
-#   from the build for size reasons. Python code should just use Unicode, and to
-#   the extent that obscure codecs are needed, they should be available on the
-#   Javascript side. Long term plan is to transparently call over to Javascript
-#   for this stuff.
 # - dbm: Failures due to no dbm module
 # - _lsprof: Failures due to no _lsprof module
 # - strftime: Failures due to differences / shortcomings in WebAssembly's
@@ -90,18 +85,18 @@ test_cmd_line_script  subprocess
 test_code
 test_code_module
 test_codeccallbacks
-test_codecencodings_cn  unknown encoding
-test_codecencodings_hk  unknown encoding
-test_codecencodings_iso2022 unknown encoding
-test_codecencodings_jp      unknown encoding
-test_codecencodings_kr  unknown encoding
-test_codecencodings_tw  unknown encoding
+test_codecencodings_cn
+test_codecencodings_hk
+test_codecencodings_iso2022
+test_codecencodings_jp
+test_codecencodings_kr
+test_codecencodings_tw
 test_codecmaps_cn
 test_codecmaps_hk
 test_codecmaps_jp
 test_codecmaps_kr
 test_codecmaps_tw
-test_codecs unknown encoding
+test_codecs
 test_codeop
 test_collections
 test_colorsys
@@ -155,9 +150,9 @@ test_email.test__header_value_parser
 test_email.test_asian_codecs
 test_email.test_contentmanager
 test_email.test_defect_handling
-test_email.test_email unknown encoding
+test_email.test_email
 test_email.test_generator
-test_email.test_headerregistry  unknown encoding
+test_email.test_headerregistry
 test_email.test_inversion
 test_email.test_message
 test_email.test_parser
@@ -252,7 +247,7 @@ test_importlib.test_abc
 test_importlib.test_api crash
 test_importlib.test_lazy
 test_importlib.test_locks
-test_importlib.test_namespace_pkgs  unknown encoding
+test_importlib.test_namespace_pkgs
 test_importlib.test_spec
 test_importlib.test_util  crash
 test_importlib.test_windows platform-specific
@@ -300,7 +295,7 @@ test_lzma
 test_macpath  platform-specific
 test_macurl2path
 test_mailbox  crash
-test_mailcap  unknown
+test_mailcap  nonsense
 test_marshal
 test_math floating-point
 test_memoryio
@@ -312,7 +307,7 @@ test_mmap
 test_module
 test_modulefinder crash
 test_msilib
-test_multibytecodec unknown codec
+test_multibytecodec
 test_multiprocessing_fork
 test_multiprocessing_forkserver
 test_multiprocessing_main_handling
@@ -366,7 +361,7 @@ test_quopri subprocess
 test_raise
 test_random
 test_range
-test_re unknown codec
+test_re locale
 test_readline
 test_regrtest subprocess
 test_repl     subprocess
@@ -398,7 +393,7 @@ test_sndhdr audioop
 test_socket
 test_socketserver
 test_sort
-test_source_encoding  subprocess, unknown encoding
+test_source_encoding  subprocess,
 test_spwd
 test_sqlite
 test_ssl
@@ -467,7 +462,7 @@ test_typing unknown
 test_ucn
 test_unary
 test_unicode  crash
-test_unicode_file unknown codec
+test_unicode_file
 test_unicode_file_functions
 test_unicode_identifiers
 test_unicodedata
@@ -502,8 +497,8 @@ test_with
 test_wsgiref
 test_xdrlib
 test_xml_dom_minicompat
-test_xml_etree  unknown encoding
-test_xml_etree_c        unknown encoding
+test_xml_etree
+test_xml_etree_c
 test_xmlrpc networking
 test_xmlrpc_net
 test_yield_from

test/test_python.py

@@ -287,7 +287,7 @@ def test_run_core_python_test(python_test, selenium):
 def pytest_generate_tests(metafunc):
     if 'python_test' in metafunc.fixturenames:
         test_modules = []
-        if 'CIRCLECI' not in os.environ:
+        if 'CIRCLECI' not in os.environ or True:
             with open(
                     str(pathlib.Path(__file__).parents[0] /
                         "python_tests.txt")) as fp: