diff --git a/Makefile b/Makefile index 5b14bf5b9..6fbffedaf 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,6 @@ LIBXML=packages/libxml/libxml2-2.9.10/.libs/libxml2.a LIBXSLT=packages/libxslt/libxslt-1.1.33/libxslt/.libs/libxslt.a LIBICONV=packages/libiconv/libiconv-1.16/lib/.libs/libiconv.a ZLIB=packages/zlib/zlib-1.2.11/lib/libz.a -LZ4LIB=packages/lz4/lz4-1.8.3/lib/liblz4.a CLAPACK=packages/CLAPACK/CLAPACK-WA/lapack_WA.bc PYODIDE_EMCC=$(PYODIDE_ROOT)/ccache/emcc @@ -29,7 +28,6 @@ LDFLAGS=\ -O2 \ -s MODULARIZE=1 \ $(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \ - $(LZ4LIB) \ -s "BINARYEN_METHOD='native-wasm'" \ -s TOTAL_MEMORY=10485760 \ -s ALLOW_MEMORY_GROWTH=1 \ @@ -160,7 +158,6 @@ clean: make -C packages/six clean make -C packages/jedi clean make -C packages/parso clean - make -C packages/lz4 clean make -C packages/libxslt clean make -C packages/libxml clean make -C packages/libiconv clean @@ -172,7 +169,7 @@ clean-all: clean make -C cpython clean rm -fr cpython/build -%.bc: %.c $(CPYTHONLIB) $(LZ4LIB) +%.bc: %.c $(CPYTHONLIB) $(CC) -o $@ -c $< $(CFLAGS) -Isrc/type_conversion/ @@ -246,12 +243,6 @@ $(CPYTHONLIB): emsdk/emsdk/.complete $(PYODIDE_EMCC) $(PYODIDE_CXX) date +"[%F %T] done building cpython..." -$(LZ4LIB): - date +"[%F %T] Building lz4..." - make -C packages/lz4 - date +"[%F %T] done building lz4." - - $(LIBXML): $(CPYTHONLIB) $(ZLIB) date +"[%F %T] Building libxml..." make -C packages/libxml diff --git a/docs/changelog.md b/docs/changelog.md index fdf47351f..bab059f7c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -25,6 +25,8 @@ - Updated default `--ldflags` argument to `pyodide_build` scripts to equal what pyodide actually uses. - Drop support for serving .wasm files with incorrect mime type. +- Replace C lz4 implementation with (upstream) javascript implementation. + [#851](https://github.com/iodide-project/pyodide/pull/851) ## Version 0.15.0 *May 19, 2020* diff --git a/emsdk/patches/lz4_c.patch b/emsdk/patches/lz4_c.patch deleted file mode 100644 index 2dc4ba881..000000000 --- a/emsdk/patches/lz4_c.patch +++ /dev/null @@ -1,102 +0,0 @@ -diff --git a/emsdk/fastcomp/emscripten/src/library_lz4.js b/emsdk/fastcomp/emscripten/src/library_lz4.js -index 4c3f583b7..5291002a4 100644 ---- a/emsdk/fastcomp/emscripten/src/library_lz4.js -+++ b/emsdk/fastcomp/emscripten/src/library_lz4.js -@@ -5,26 +5,14 @@ mergeInto(LibraryManager.library, { - DIR_MODE: {{{ cDefine('S_IFDIR') }}} | 511 /* 0777 */, - FILE_MODE: {{{ cDefine('S_IFREG') }}} | 511 /* 0777 */, - CHUNK_SIZE: -1, -- codec: null, - init: function() { -- if (LZ4.codec) return; -- LZ4.codec = (function() { -- {{{ read('mini-lz4.js') }}}; -- return MiniLZ4; -- })(); -- LZ4.CHUNK_SIZE = LZ4.codec.CHUNK_SIZE; -+ LZ4.CHUNK_SIZE = 2048; - }, - loadPackage: function (pack) { - LZ4.init(); - var compressedData = pack['compressedData']; -- if (!compressedData) compressedData = LZ4.codec.compressPackage(pack['data']); -+ // if (!compressedData) compressedData = LZ4.codec.compressPackage(pack['data']); - assert(compressedData.cachedIndexes.length === compressedData.cachedChunks.length); -- for (var i = 0; i < compressedData.cachedIndexes.length; i++) { -- compressedData.cachedIndexes[i] = -1; -- compressedData.cachedChunks[i] = compressedData.data.subarray(compressedData.cachedOffset + i*LZ4.CHUNK_SIZE, -- compressedData.cachedOffset + (i+1)*LZ4.CHUNK_SIZE); -- assert(compressedData.cachedChunks[i].length === LZ4.CHUNK_SIZE); -- } - pack['metadata'].files.forEach(function(file) { - var dir = PATH.dirname(file.filename); - var name = PATH.basename(file.filename); -@@ -36,6 +24,12 @@ mergeInto(LibraryManager.library, { - end: file.end, - }); - }); -+ compressedData.buf = Module['_malloc'](LZ4.CHUNK_SIZE); -+ for (var i = 0; i < compressedData.cachedIndexes.length; i++) { -+ compressedData.cachedIndexes[i] = -1; -+ compressedData.cachedChunks[i] = Module['_malloc'](LZ4.CHUNK_SIZE); -+ assert(compressedData.cachedChunks[i] !== null) -+ } - }, - createNode: function (parent, name, mode, dev, contents, mtime) { - var node = FS.createNode(parent, name, mode); -@@ -112,6 +106,7 @@ mergeInto(LibraryManager.library, { - //console.log('LZ4 read ' + [offset, length, position]); - length = Math.min(length, stream.node.size - position); - if (length <= 0) return 0; -+ - var contents = stream.node.contents; - var compressedData = contents.compressedData; - var written = 0; -@@ -122,11 +117,14 @@ mergeInto(LibraryManager.library, { - var chunkIndex = Math.floor(start / LZ4.CHUNK_SIZE); - var compressedStart = compressedData.offsets[chunkIndex]; - var compressedSize = compressedData.sizes[chunkIndex]; -+ var startInChunk = start % LZ4.CHUNK_SIZE; -+ var endInChunk = Math.min(startInChunk + desired, LZ4.CHUNK_SIZE); - var currChunk; - if (compressedData.successes[chunkIndex]) { - var found = compressedData.cachedIndexes.indexOf(chunkIndex); - if (found >= 0) { - currChunk = compressedData.cachedChunks[found]; -+ buffer.set(Module.HEAPU8.subarray(currChunk + startInChunk, currChunk + endInChunk), offset + written); - } else { - // decompress the chunk - compressedData.cachedIndexes.pop(); -@@ -138,18 +136,19 @@ mergeInto(LibraryManager.library, { - Module['decompressedChunks'] = (Module['decompressedChunks'] || 0) + 1; - } - var compressed = compressedData.data.subarray(compressedStart, compressedStart + compressedSize); -- //var t = Date.now(); -- var originalSize = LZ4.codec.uncompress(compressed, currChunk); -- //console.log('decompress time: ' + (Date.now() - t)); -+ // var t = Date.now(); -+ // var originalSize = LZ4.codec.uncompress(compressed, currChunk); -+ Module.HEAPU8.set(compressed, compressedData.buf); -+ var originalSize = Module['_LZ4_decompress_safe'](compressedData.buf, currChunk, compressedSize, LZ4.CHUNK_SIZE); -+ // console.log('decompress time: ' + (Date.now() - t)); - if (chunkIndex < compressedData.successes.length-1) assert(originalSize === LZ4.CHUNK_SIZE); // all but the last chunk must be full-size -+ buffer.set(Module.HEAPU8.subarray(currChunk + startInChunk, currChunk + endInChunk), offset + written); - } -- } else { -+ } -+ else { - // uncompressed -- currChunk = compressedData.data.subarray(compressedStart, compressedStart + LZ4.CHUNK_SIZE); -+ buffer.set(compressedData.data.subarray(compressedStart + startInChunk, compressedStart + endInChunk), offset + written); - } -- var startInChunk = start % LZ4.CHUNK_SIZE; -- var endInChunk = Math.min(startInChunk + desired, LZ4.CHUNK_SIZE); -- buffer.set(currChunk.subarray(startInChunk, endInChunk), offset + written); - var currWritten = endInChunk - startInChunk; - written += currWritten; - } -@@ -181,4 +180,3 @@ if (LibraryManager.library['$FS__deps']) { - warn('FS does not seem to be in use (no preloaded files etc.), LZ4 will not do anything'); - } - #endif -- diff --git a/packages/lz4/Makefile b/packages/lz4/Makefile deleted file mode 100644 index e12f121f4..000000000 --- a/packages/lz4/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -PYODIDE_ROOT=$(abspath ../..) -include ../../Makefile.envs - -LZ4VERSION=1.8.3 - -ROOT=$(abspath .) - -SRC=$(ROOT)/lz4-$(LZ4VERSION) -TARBALL=$(ROOT)/downloads/lz4-$(LZ4VERSION).tgz -URL=https://github.com/lz4/lz4/archive/v$(LZ4VERSION).tar.gz - - -all: $(SRC)/lib/liblz4.a - - -clean: - -rm -fr downloads - -rm -fr $(SRC) - - -$(TARBALL): - [ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads - wget -q -O $@ $(URL) -# md5sum --quiet --check checksums || (rm $@; false) - - -$(SRC)/Makefile: $(TARBALL) - tar -C . -xf $(TARBALL) - touch $(SRC)/Makefile - - -$(SRC)/lib/liblz4.a: $(SRC)/Makefile - ( \ - cd $(SRC) ; \ - emmake make -j $${PYODIDE_JOBS:-3} ; \ - ) diff --git a/packages/lz4/checksums b/packages/lz4/checksums deleted file mode 100644 index 12ce3539f..000000000 --- a/packages/lz4/checksums +++ /dev/null @@ -1 +0,0 @@ -d5ce78f7b1b76002bbfffa6f78a5fc4e downloads/lz4-1.8.3.tgz