Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove lz4_c patch (#851)

This commit is contained in:
Dexter Chua 2020-12-13 19:41:22 +08:00 committed by GitHub
parent e8cc4e8835
commit 34026265fc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 3 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Makefile
View File

@ -11,7 +11,6 @@ LIBXML=packages/libxml/libxml2-2.9.10/.libs/libxml2.a
LIBXSLT=packages/libxslt/libxslt-1.1.33/libxslt/.libs/libxslt.a
LIBICONV=packages/libiconv/libiconv-1.16/lib/.libs/libiconv.a
ZLIB=packages/zlib/zlib-1.2.11/lib/libz.a
LZ4LIB=packages/lz4/lz4-1.8.3/lib/liblz4.a
CLAPACK=packages/CLAPACK/CLAPACK-WA/lapack_WA.bc
PYODIDE_EMCC=$(PYODIDE_ROOT)/ccache/emcc
@ -29,7 +28,6 @@ LDFLAGS=\
-O2 \
-s MODULARIZE=1 \
$(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \
$(LZ4LIB) \
-s "BINARYEN_METHOD='native-wasm'" \
-s TOTAL_MEMORY=10485760 \
-s ALLOW_MEMORY_GROWTH=1 \
@ -160,7 +158,6 @@ clean:
make -C packages/six clean
make -C packages/jedi clean
make -C packages/parso clean
make -C packages/lz4 clean
make -C packages/libxslt clean
make -C packages/libxml clean
make -C packages/libiconv clean
@ -172,7 +169,7 @@ clean-all: clean
make -C cpython clean
rm -fr cpython/build
%.bc: %.c $(CPYTHONLIB) $(LZ4LIB)
%.bc: %.c $(CPYTHONLIB)
$(CC) -o $@ -c $< $(CFLAGS) -Isrc/type_conversion/
@ -246,12 +243,6 @@ $(CPYTHONLIB): emsdk/emsdk/.complete $(PYODIDE_EMCC) $(PYODIDE_CXX)
date +"[%F %T] done building cpython..."
$(LZ4LIB):
date +"[%F %T] Building lz4..."
make -C packages/lz4
date +"[%F %T] done building lz4."
$(LIBXML): $(CPYTHONLIB) $(ZLIB)
date +"[%F %T] Building libxml..."
make -C packages/libxml

2
docs/changelog.md
View File

@ -25,6 +25,8 @@
- Updated default `--ldflags` argument to `pyodide_build` scripts to equal what
pyodide actually uses.
- Drop support for serving .wasm files with incorrect mime type.
- Replace C lz4 implementation with (upstream) javascript implementation.
[#851](https://github.com/iodide-project/pyodide/pull/851)
## Version 0.15.0
*May 19, 2020*

102
emsdk/patches/lz4_c.patch
View File

@ -1,102 +0,0 @@
diff --git a/emsdk/fastcomp/emscripten/src/library_lz4.js b/emsdk/fastcomp/emscripten/src/library_lz4.js
index 4c3f583b7..5291002a4 100644
--- a/emsdk/fastcomp/emscripten/src/library_lz4.js
+++ b/emsdk/fastcomp/emscripten/src/library_lz4.js
@@ -5,26 +5,14 @@ mergeInto(LibraryManager.library, {
DIR_MODE: {{{ cDefine('S_IFDIR') }}} | 511 /* 0777 */,
FILE_MODE: {{{ cDefine('S_IFREG') }}} | 511 /* 0777 */,
CHUNK_SIZE: -1,
- codec: null,
init: function() {
- if (LZ4.codec) return;
- LZ4.codec = (function() {
- {{{ read('mini-lz4.js') }}};
- return MiniLZ4;
- })();
- LZ4.CHUNK_SIZE = LZ4.codec.CHUNK_SIZE;
+ LZ4.CHUNK_SIZE = 2048;
},
loadPackage: function (pack) {
LZ4.init();
var compressedData = pack['compressedData'];
- if (!compressedData) compressedData = LZ4.codec.compressPackage(pack['data']);
+ // if (!compressedData) compressedData = LZ4.codec.compressPackage(pack['data']);
assert(compressedData.cachedIndexes.length === compressedData.cachedChunks.length);
- for (var i = 0; i < compressedData.cachedIndexes.length; i++) {
- compressedData.cachedIndexes[i] = -1;
- compressedData.cachedChunks[i] = compressedData.data.subarray(compressedData.cachedOffset + i*LZ4.CHUNK_SIZE,
- compressedData.cachedOffset + (i+1)*LZ4.CHUNK_SIZE);
- assert(compressedData.cachedChunks[i].length === LZ4.CHUNK_SIZE);
- }
pack['metadata'].files.forEach(function(file) {
var dir = PATH.dirname(file.filename);
var name = PATH.basename(file.filename);
@@ -36,6 +24,12 @@ mergeInto(LibraryManager.library, {
end: file.end,
});
});
+ compressedData.buf = Module['_malloc'](LZ4.CHUNK_SIZE);
+ for (var i = 0; i < compressedData.cachedIndexes.length; i++) {
+ compressedData.cachedIndexes[i] = -1;
+ compressedData.cachedChunks[i] = Module['_malloc'](LZ4.CHUNK_SIZE);
+ assert(compressedData.cachedChunks[i] !== null)
+ }
},
createNode: function (parent, name, mode, dev, contents, mtime) {
var node = FS.createNode(parent, name, mode);
@@ -112,6 +106,7 @@ mergeInto(LibraryManager.library, {
//console.log('LZ4 read ' + [offset, length, position]);
length = Math.min(length, stream.node.size - position);
if (length <= 0) return 0;
+
var contents = stream.node.contents;
var compressedData = contents.compressedData;
var written = 0;
@@ -122,11 +117,14 @@ mergeInto(LibraryManager.library, {
var chunkIndex = Math.floor(start / LZ4.CHUNK_SIZE);
var compressedStart = compressedData.offsets[chunkIndex];
var compressedSize = compressedData.sizes[chunkIndex];
+ var startInChunk = start % LZ4.CHUNK_SIZE;
+ var endInChunk = Math.min(startInChunk + desired, LZ4.CHUNK_SIZE);
var currChunk;
if (compressedData.successes[chunkIndex]) {
var found = compressedData.cachedIndexes.indexOf(chunkIndex);
if (found >= 0) {
currChunk = compressedData.cachedChunks[found];
+ buffer.set(Module.HEAPU8.subarray(currChunk + startInChunk, currChunk + endInChunk), offset + written);
} else {
// decompress the chunk
compressedData.cachedIndexes.pop();
@@ -138,18 +136,19 @@ mergeInto(LibraryManager.library, {
Module['decompressedChunks'] = (Module['decompressedChunks'] || 0) + 1;
}
var compressed = compressedData.data.subarray(compressedStart, compressedStart + compressedSize);
- //var t = Date.now();
- var originalSize = LZ4.codec.uncompress(compressed, currChunk);
- //console.log('decompress time: ' + (Date.now() - t));
+ // var t = Date.now();
+ // var originalSize = LZ4.codec.uncompress(compressed, currChunk);
+ Module.HEAPU8.set(compressed, compressedData.buf);
+ var originalSize = Module['_LZ4_decompress_safe'](compressedData.buf, currChunk, compressedSize, LZ4.CHUNK_SIZE);
+ // console.log('decompress time: ' + (Date.now() - t));
if (chunkIndex < compressedData.successes.length-1) assert(originalSize === LZ4.CHUNK_SIZE); // all but the last chunk must be full-size
+ buffer.set(Module.HEAPU8.subarray(currChunk + startInChunk, currChunk + endInChunk), offset + written);
}
- } else {
+ }
+ else {
// uncompressed
- currChunk = compressedData.data.subarray(compressedStart, compressedStart + LZ4.CHUNK_SIZE);
+ buffer.set(compressedData.data.subarray(compressedStart + startInChunk, compressedStart + endInChunk), offset + written);
}
- var startInChunk = start % LZ4.CHUNK_SIZE;
- var endInChunk = Math.min(startInChunk + desired, LZ4.CHUNK_SIZE);
- buffer.set(currChunk.subarray(startInChunk, endInChunk), offset + written);
var currWritten = endInChunk - startInChunk;
written += currWritten;
}
@@ -181,4 +180,3 @@ if (LibraryManager.library['$FS__deps']) {
warn('FS does not seem to be in use (no preloaded files etc.), LZ4 will not do anything');
}
#endif
-

36
packages/lz4/Makefile
View File

@ -1,36 +0,0 @@
PYODIDE_ROOT=$(abspath ../..)
include ../../Makefile.envs
LZ4VERSION=1.8.3
ROOT=$(abspath .)
SRC=$(ROOT)/lz4-$(LZ4VERSION)
TARBALL=$(ROOT)/downloads/lz4-$(LZ4VERSION).tgz
URL=https://github.com/lz4/lz4/archive/v$(LZ4VERSION).tar.gz
all: $(SRC)/lib/liblz4.a
clean:
-rm -fr downloads
-rm -fr $(SRC)
$(TARBALL):
[ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads
wget -q -O $@ $(URL)
# md5sum --quiet --check checksums || (rm $@; false)
$(SRC)/Makefile: $(TARBALL)
tar -C . -xf $(TARBALL)
touch $(SRC)/Makefile
$(SRC)/lib/liblz4.a: $(SRC)/Makefile
( \
cd $(SRC) ; \
emmake make -j $${PYODIDE_JOBS:-3} ; \
)

1
packages/lz4/checksums
View File

@ -1 +0,0 @@
d5ce78f7b1b76002bbfffa6f78a5fc4e downloads/lz4-1.8.3.tgz