Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Simplify library_lz4.js

This commit is contained in:
Michael Droettboom 2018-09-20 08:25:44 -04:00
parent c2e981b87e
commit 362b736c0a
2 changed files with 33 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
emsdk/Makefile
View File

@ -1,9 +1,7 @@
all: emsdk/.complete all: emsdk/.complete
# We hack the CPU_CORES, because if you use all of the cores on Circle-CI, you # We hack the CPU_CORES, because if you use all of the cores on Circle-CI, you
# run out of memory # run out of memory.
emsdk/.complete: emsdk/.complete:
if [ -d emsdk ]; then rm -rf emsdk; fi if [ -d emsdk ]; then rm -rf emsdk; fi

45
emsdk/patches/lz4_c.patch
View File

@ -1,8 +1,8 @@
diff --git a/src/library_lz4.js b/src/library_lz4.js diff --git a/emsdk/emscripten/tag-1.38.10/src/library_lz4.js b/emsdk/emscripten/tag-1.38.10/src/library_lz4.js
index 4c3f583b7..9540a64fb 100644 index 4c3f583b7..5291002a4 100644
--- a/src/library_lz4.js --- a/src/library_lz4.js
+++ b/src/library_lz4.js +++ b/src/library_lz4.js
@@ -5,26 +5,15 @@ mergeInto(LibraryManager.library, { @@ -5,26 +5,14 @@ mergeInto(LibraryManager.library, {
DIR_MODE: {{{ cDefine('S_IFDIR') }}} | 511 /* 0777 */, DIR_MODE: {{{ cDefine('S_IFDIR') }}} | 511 /* 0777 */,
FILE_MODE: {{{ cDefine('S_IFREG') }}} | 511 /* 0777 */, FILE_MODE: {{{ cDefine('S_IFREG') }}} | 511 /* 0777 */,
CHUNK_SIZE: -1, CHUNK_SIZE: -1,
@ -28,11 +28,23 @@ index 4c3f583b7..9540a64fb 100644
- compressedData.cachedOffset + (i+1)*LZ4.CHUNK_SIZE); - compressedData.cachedOffset + (i+1)*LZ4.CHUNK_SIZE);
- assert(compressedData.cachedChunks[i].length === LZ4.CHUNK_SIZE); - assert(compressedData.cachedChunks[i].length === LZ4.CHUNK_SIZE);
- } - }
+ compressedData.buf = null;
pack['metadata'].files.forEach(function(file) { pack['metadata'].files.forEach(function(file) {
var dir = PATH.dirname(file.filename); var dir = PATH.dirname(file.filename);
var name = PATH.basename(file.filename); var name = PATH.basename(file.filename);
@@ -112,8 +101,17 @@ mergeInto(LibraryManager.library, { @@ -36,6 +24,12 @@ mergeInto(LibraryManager.library, {
end: file.end,
});
});
+ compressedData.buf = Module['_malloc'](LZ4.CHUNK_SIZE);
+ for (var i = 0; i < compressedData.cachedIndexes.length; i++) {
+ compressedData.cachedIndexes[i] = -1;
+ compressedData.cachedChunks[i] = Module['_malloc'](LZ4.CHUNK_SIZE);
+ assert(compressedData.cachedChunks[i] !== null)
+ }
},
createNode: function (parent, name, mode, dev, contents, mtime) {
var node = FS.createNode(parent, name, mode);
@@ -112,8 +106,17 @@ mergeInto(LibraryManager.library, {
//console.log('LZ4 read ' + [offset, length, position]); //console.log('LZ4 read ' + [offset, length, position]);
length = Math.min(length, stream.node.size - position); length = Math.min(length, stream.node.size - position);
if (length <= 0) return 0; if (length <= 0) return 0;
@ -50,7 +62,16 @@ index 4c3f583b7..9540a64fb 100644
var written = 0; var written = 0;
while (written < length) { while (written < length) {
var start = contents.start + position + written; // start index in uncompressed data var start = contents.start + position + written; // start index in uncompressed data
@@ -138,18 +136,23 @@ mergeInto(LibraryManager.library, { @@ -122,6 +125,8 @@ mergeInto(LibraryManager.library, {
var chunkIndex = Math.floor(start / LZ4.CHUNK_SIZE);
var compressedStart = compressedData.offsets[chunkIndex];
var compressedSize = compressedData.sizes[chunkIndex];
+ var startInChunk = start % LZ4.CHUNK_SIZE;
+ var endInChunk = Math.min(startInChunk + desired, LZ4.CHUNK_SIZE);
var currChunk;
if (compressedData.successes[chunkIndex]) {
var found = compressedData.cachedIndexes.indexOf(chunkIndex);
@@ -138,18 +143,19 @@ mergeInto(LibraryManager.library, {
Module['decompressedChunks'] = (Module['decompressedChunks'] || 0) + 1; Module['decompressedChunks'] = (Module['decompressedChunks'] || 0) + 1;
} }
var compressed = compressedData.data.subarray(compressedStart, compressedStart + compressedSize); var compressed = compressedData.data.subarray(compressedStart, compressedStart + compressedSize);
@ -63,24 +84,22 @@ index 4c3f583b7..9540a64fb 100644
+ var originalSize = Module['_LZ4_decompress_safe'](compressedData.buf, currChunk, compressedSize, LZ4.CHUNK_SIZE); + var originalSize = Module['_LZ4_decompress_safe'](compressedData.buf, currChunk, compressedSize, LZ4.CHUNK_SIZE);
+ // console.log('decompress time: ' + (Date.now() - t)); + // console.log('decompress time: ' + (Date.now() - t));
if (chunkIndex < compressedData.successes.length-1) assert(originalSize === LZ4.CHUNK_SIZE); // all but the last chunk must be full-size if (chunkIndex < compressedData.successes.length-1) assert(originalSize === LZ4.CHUNK_SIZE); // all but the last chunk must be full-size
+ buffer.set(Module.HEAPU8.subarray(currChunk + startInChunk, currChunk + endInChunk), offset + written);
} }
- } else { - } else {
+ } + }
+ else { + else {
// uncompressed // uncompressed
- currChunk = compressedData.data.subarray(compressedStart, compressedStart + LZ4.CHUNK_SIZE); - currChunk = compressedData.data.subarray(compressedStart, compressedStart + LZ4.CHUNK_SIZE);
+ var compressed = compressedData.data.subarray(compressedStart, compressedStart + LZ4.CHUNK_SIZE); + buffer.set(compressedData.data.subarray(compressedStart + startInChunk, compressedStart + endInChunk), offset + written);
+ Module.HEAPU8.set(compressed, compressedData.buf);
+ currChunk = compressedData.buf;
} }
var startInChunk = start % LZ4.CHUNK_SIZE; - var startInChunk = start % LZ4.CHUNK_SIZE;
var endInChunk = Math.min(startInChunk + desired, LZ4.CHUNK_SIZE); - var endInChunk = Math.min(startInChunk + desired, LZ4.CHUNK_SIZE);
- buffer.set(currChunk.subarray(startInChunk, endInChunk), offset + written); - buffer.set(currChunk.subarray(startInChunk, endInChunk), offset + written);
+ buffer.set(Module.HEAPU8.subarray(currChunk + startInChunk, currChunk + endInChunk), offset + written);
var currWritten = endInChunk - startInChunk; var currWritten = endInChunk - startInChunk;
written += currWritten; written += currWritten;
} }
@@ -181,4 +184,3 @@ if (LibraryManager.library['$FS__deps']) { @@ -181,4 +187,3 @@ if (LibraryManager.library['$FS__deps']) {
warn('FS does not seem to be in use (no preloaded files etc.), LZ4 will not do anything'); warn('FS does not seem to be in use (no preloaded files etc.), LZ4 will not do anything');
} }
#endif #endif