From 8dc9232450266713167e6692573aa851d2f9cc56 Mon Sep 17 00:00:00 2001 From: Oleksii Shevchuk Date: Sat, 26 Nov 2016 11:20:33 +0200 Subject: [PATCH] Use LZMA instead of ZLib for resources --- client/gen_library_compressed_string.py | 7 +- client/lzma/7zTypes.h | 256 ++++++ client/lzma/LzmaDec.c | 1098 +++++++++++++++++++++++ client/lzma/LzmaDec.h | 227 +++++ client/requirements.txt | 1 + client/sources-linux/Makefile | 24 +- client/sources-linux/pupy.c | 56 +- client/sources-linux/pupy_load.c | 12 - client/sources/Makefile.linux | 21 +- client/sources/pupy.c | 59 +- pupy/packages/all/pupyimporter.py | 6 +- 11 files changed, 1717 insertions(+), 50 deletions(-) create mode 100644 client/lzma/7zTypes.h create mode 100644 client/lzma/LzmaDec.c create mode 100644 client/lzma/LzmaDec.h diff --git a/client/gen_library_compressed_string.py b/client/gen_library_compressed_string.py index 5b1c55aa..aadbb446 100644 --- a/client/gen_library_compressed_string.py +++ b/client/gen_library_compressed_string.py @@ -2,7 +2,8 @@ # -*- coding: utf-8 -*- import StringIO, zipfile, os.path, imp, sys import marshal -import zlib +import pylzma +import struct def get_encoded_library_string(): filepath=None @@ -20,7 +21,9 @@ def get_encoded_library_string(): ] ]) - return zlib.compress(marshal.dumps(modules),9) + payload = marshal.dumps(modules) + payload_len = len(payload) + return struct.pack('>I', payload_len) + pylzma.compress(payload,dictionary=24,fastBytes=255) with open(os.path.join("resources","library_compressed_string.txt"),'wb') as w: w.write(get_encoded_library_string()) diff --git a/client/lzma/7zTypes.h b/client/lzma/7zTypes.h new file mode 100644 index 00000000..903047b1 --- /dev/null +++ b/client/lzma/7zTypes.h @@ -0,0 +1,256 @@ +/* 7zTypes.h -- Basic types +2013-11-12 : Igor Pavlov : Public domain */ + +#ifndef __7Z_TYPES_H +#define __7Z_TYPES_H + +#ifdef _WIN32 +/* #include */ +#endif + +#include + +#ifndef EXTERN_C_BEGIN +#ifdef __cplusplus +#define EXTERN_C_BEGIN extern "C" { +#define EXTERN_C_END } +#else +#define EXTERN_C_BEGIN +#define EXTERN_C_END +#endif +#endif + +EXTERN_C_BEGIN + +#define SZ_OK 0 + +#define SZ_ERROR_DATA 1 +#define SZ_ERROR_MEM 2 +#define SZ_ERROR_CRC 3 +#define SZ_ERROR_UNSUPPORTED 4 +#define SZ_ERROR_PARAM 5 +#define SZ_ERROR_INPUT_EOF 6 +#define SZ_ERROR_OUTPUT_EOF 7 +#define SZ_ERROR_READ 8 +#define SZ_ERROR_WRITE 9 +#define SZ_ERROR_PROGRESS 10 +#define SZ_ERROR_FAIL 11 +#define SZ_ERROR_THREAD 12 + +#define SZ_ERROR_ARCHIVE 16 +#define SZ_ERROR_NO_ARCHIVE 17 + +typedef int SRes; + +#ifdef _WIN32 +/* typedef DWORD WRes; */ +typedef unsigned WRes; +#else +typedef int WRes; +#endif + +#ifndef RINOK +#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; } +#endif + +typedef unsigned char Byte; +typedef short Int16; +typedef unsigned short UInt16; + +#ifdef _LZMA_UINT32_IS_ULONG +typedef long Int32; +typedef unsigned long UInt32; +#else +typedef int Int32; +typedef unsigned int UInt32; +#endif + +#ifdef _SZ_NO_INT_64 + +/* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers. + NOTES: Some code will work incorrectly in that case! */ + +typedef long Int64; +typedef unsigned long UInt64; + +#else + +#if defined(_MSC_VER) || defined(__BORLANDC__) +typedef __int64 Int64; +typedef unsigned __int64 UInt64; +#define UINT64_CONST(n) n +#else +typedef long long int Int64; +typedef unsigned long long int UInt64; +#define UINT64_CONST(n) n ## ULL +#endif + +#endif + +#ifdef _LZMA_NO_SYSTEM_SIZE_T +typedef UInt32 SizeT; +#else +typedef size_t SizeT; +#endif + +typedef int Bool; +#define True 1 +#define False 0 + + +#ifdef _WIN32 +#define MY_STD_CALL __stdcall +#else +#define MY_STD_CALL +#endif + +#ifdef _MSC_VER + +#if _MSC_VER >= 1300 +#define MY_NO_INLINE __declspec(noinline) +#else +#define MY_NO_INLINE +#endif + +#define MY_CDECL __cdecl +#define MY_FAST_CALL __fastcall + +#else + +#define MY_NO_INLINE +#define MY_CDECL +#define MY_FAST_CALL + +#endif + + +/* The following interfaces use first parameter as pointer to structure */ + +typedef struct +{ + Byte (*Read)(void *p); /* reads one byte, returns 0 in case of EOF or error */ +} IByteIn; + +typedef struct +{ + void (*Write)(void *p, Byte b); +} IByteOut; + +typedef struct +{ + SRes (*Read)(void *p, void *buf, size_t *size); + /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. + (output(*size) < input(*size)) is allowed */ +} ISeqInStream; + +/* it can return SZ_ERROR_INPUT_EOF */ +SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size); +SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType); +SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf); + +typedef struct +{ + size_t (*Write)(void *p, const void *buf, size_t size); + /* Returns: result - the number of actually written bytes. + (result < size) means error */ +} ISeqOutStream; + +typedef enum +{ + SZ_SEEK_SET = 0, + SZ_SEEK_CUR = 1, + SZ_SEEK_END = 2 +} ESzSeek; + +typedef struct +{ + SRes (*Read)(void *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ + SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); +} ISeekInStream; + +typedef struct +{ + SRes (*Look)(void *p, const void **buf, size_t *size); + /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. + (output(*size) > input(*size)) is not allowed + (output(*size) < input(*size)) is allowed */ + SRes (*Skip)(void *p, size_t offset); + /* offset must be <= output(*size) of Look */ + + SRes (*Read)(void *p, void *buf, size_t *size); + /* reads directly (without buffer). It's same as ISeqInStream::Read */ + SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); +} ILookInStream; + +SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size); +SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset); + +/* reads via ILookInStream::Read */ +SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType); +SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size); + +#define LookToRead_BUF_SIZE (1 << 14) + +typedef struct +{ + ILookInStream s; + ISeekInStream *realStream; + size_t pos; + size_t size; + Byte buf[LookToRead_BUF_SIZE]; +} CLookToRead; + +void LookToRead_CreateVTable(CLookToRead *p, int lookahead); +void LookToRead_Init(CLookToRead *p); + +typedef struct +{ + ISeqInStream s; + ILookInStream *realStream; +} CSecToLook; + +void SecToLook_CreateVTable(CSecToLook *p); + +typedef struct +{ + ISeqInStream s; + ILookInStream *realStream; +} CSecToRead; + +void SecToRead_CreateVTable(CSecToRead *p); + +typedef struct +{ + SRes (*Progress)(void *p, UInt64 inSize, UInt64 outSize); + /* Returns: result. (result != SZ_OK) means break. + Value (UInt64)(Int64)-1 for size means unknown value. */ +} ICompressProgress; + +typedef struct +{ + void *(*Alloc)(void *p, size_t size); + void (*Free)(void *p, void *address); /* address can be 0 */ +} ISzAlloc; + +#define IAlloc_Alloc(p, size) (p)->Alloc((p), size) +#define IAlloc_Free(p, a) (p)->Free((p), a) + +#ifdef _WIN32 + +#define CHAR_PATH_SEPARATOR '\\' +#define WCHAR_PATH_SEPARATOR L'\\' +#define STRING_PATH_SEPARATOR "\\" +#define WSTRING_PATH_SEPARATOR L"\\" + +#else + +#define CHAR_PATH_SEPARATOR '/' +#define WCHAR_PATH_SEPARATOR L'/' +#define STRING_PATH_SEPARATOR "/" +#define WSTRING_PATH_SEPARATOR L"/" + +#endif + +EXTERN_C_END + +#endif diff --git a/client/lzma/LzmaDec.c b/client/lzma/LzmaDec.c new file mode 100644 index 00000000..7e5e6872 --- /dev/null +++ b/client/lzma/LzmaDec.c @@ -0,0 +1,1098 @@ +/* LzmaDec.c -- LZMA Decoder +2016-05-16 : Igor Pavlov : Public domain */ + +#include "LzmaDec.h" + +#include + +#define kNumTopBits 24 +#define kTopValue ((UInt32)1 << kNumTopBits) + +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) +#define kNumMoveBits 5 + +#define RC_INIT_SIZE 5 + +#define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } + +#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); +#define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); +#define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ + { UPDATE_0(p); i = (i + i); A0; } else \ + { UPDATE_1(p); i = (i + i) + 1; A1; } +#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) + +#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); } +#define TREE_DECODE(probs, limit, i) \ + { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } + +/* #define _LZMA_SIZE_OPT */ + +#ifdef _LZMA_SIZE_OPT +#define TREE_6_DECODE(probs, i) TREE_DECODE(probs, (1 << 6), i) +#else +#define TREE_6_DECODE(probs, i) \ + { i = 1; \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + i -= 0x40; } +#endif + +#define NORMAL_LITER_DEC GET_BIT(prob + symbol, symbol) +#define MATCHED_LITER_DEC \ + matchByte <<= 1; \ + bit = (matchByte & offs); \ + probLit = prob + offs + bit + symbol; \ + GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit) + +#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } + +#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define UPDATE_0_CHECK range = bound; +#define UPDATE_1_CHECK range -= bound; code -= bound; +#define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ + { UPDATE_0_CHECK; i = (i + i); A0; } else \ + { UPDATE_1_CHECK; i = (i + i) + 1; A1; } +#define GET_BIT_CHECK(p, i) GET_BIT2_CHECK(p, i, ; , ;) +#define TREE_DECODE_CHECK(probs, limit, i) \ + { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } + + +#define kNumPosBitsMax 4 +#define kNumPosStatesMax (1 << kNumPosBitsMax) + +#define kLenNumLowBits 3 +#define kLenNumLowSymbols (1 << kLenNumLowBits) +#define kLenNumMidBits 3 +#define kLenNumMidSymbols (1 << kLenNumMidBits) +#define kLenNumHighBits 8 +#define kLenNumHighSymbols (1 << kLenNumHighBits) + +#define LenChoice 0 +#define LenChoice2 (LenChoice + 1) +#define LenLow (LenChoice2 + 1) +#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) +#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define kNumLenProbs (LenHigh + kLenNumHighSymbols) + + +#define kNumStates 12 +#define kNumLitStates 7 + +#define kStartPosModelIndex 4 +#define kEndPosModelIndex 14 +#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + +#define kNumPosSlotBits 6 +#define kNumLenToPosStates 4 + +#define kNumAlignBits 4 +#define kAlignTableSize (1 << kNumAlignBits) + +#define kMatchMinLen 2 +#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) + +#define IsMatch 0 +#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +#define IsRepG0 (IsRep + kNumStates) +#define IsRepG1 (IsRepG0 + kNumStates) +#define IsRepG2 (IsRepG1 + kNumStates) +#define IsRep0Long (IsRepG2 + kNumStates) +#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) +#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) +#define LenCoder (Align + kAlignTableSize) +#define RepLenCoder (LenCoder + kNumLenProbs) +#define Literal (RepLenCoder + kNumLenProbs) + +#define LZMA_BASE_SIZE 1846 +#define LZMA_LIT_SIZE 0x300 + +#if Literal != LZMA_BASE_SIZE +StopCompilingDueBUG +#endif + +#define LzmaProps_GetNumProbs(p) (Literal + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) + +#define LZMA_DIC_MIN (1 << 12) + +/* First LZMA-symbol is always decoded. +And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization +Out: + Result: + SZ_OK - OK + SZ_ERROR_DATA - Error + p->remainLen: + < kMatchSpecLenStart : normal remain + = kMatchSpecLenStart : finished + = kMatchSpecLenStart + 1 : Flush marker (unused now) + = kMatchSpecLenStart + 2 : State Init Marker (unused now) +*/ + +static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + CLzmaProb *probs = p->probs; + + unsigned state = p->state; + UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; + unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; + unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1; + unsigned lc = p->prop.lc; + + Byte *dic = p->dic; + SizeT dicBufSize = p->dicBufSize; + SizeT dicPos = p->dicPos; + + UInt32 processedPos = p->processedPos; + UInt32 checkDicSize = p->checkDicSize; + unsigned len = 0; + + const Byte *buf = p->buf; + UInt32 range = p->range; + UInt32 code = p->code; + + do + { + CLzmaProb *prob; + UInt32 bound; + unsigned ttt; + unsigned posState = processedPos & pbMask; + + prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + IF_BIT_0(prob) + { + unsigned symbol; + UPDATE_0(prob); + prob = probs + Literal; + if (processedPos != 0 || checkDicSize != 0) + prob += ((UInt32)LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + + (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); + processedPos++; + + if (state < kNumLitStates) + { + state -= (state < 4) ? state : 3; + symbol = 1; + #ifdef _LZMA_SIZE_OPT + do { NORMAL_LITER_DEC } while (symbol < 0x100); + #else + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + #endif + } + else + { + unsigned matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + unsigned offs = 0x100; + state -= (state < 10) ? 3 : 6; + symbol = 1; + #ifdef _LZMA_SIZE_OPT + do + { + unsigned bit; + CLzmaProb *probLit; + MATCHED_LITER_DEC + } + while (symbol < 0x100); + #else + { + unsigned bit; + CLzmaProb *probLit; + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + } + #endif + } + + dic[dicPos++] = (Byte)symbol; + continue; + } + + { + UPDATE_1(prob); + prob = probs + IsRep + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + state += kNumStates; + prob = probs + LenCoder; + } + else + { + UPDATE_1(prob); + if (checkDicSize == 0 && processedPos == 0) + return SZ_ERROR_DATA; + prob = probs + IsRepG0 + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + IF_BIT_0(prob) + { + UPDATE_0(prob); + dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + dicPos++; + processedPos++; + state = state < kNumLitStates ? 9 : 11; + continue; + } + UPDATE_1(prob); + } + else + { + UInt32 distance; + UPDATE_1(prob); + prob = probs + IsRepG1 + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + distance = rep1; + } + else + { + UPDATE_1(prob); + prob = probs + IsRepG2 + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + distance = rep2; + } + else + { + UPDATE_1(prob); + distance = rep3; + rep3 = rep2; + } + rep2 = rep1; + } + rep1 = rep0; + rep0 = distance; + } + state = state < kNumLitStates ? 8 : 11; + prob = probs + RepLenCoder; + } + + #ifdef _LZMA_SIZE_OPT + { + unsigned lim, offset; + CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenLow + (posState << kLenNumLowBits); + offset = 0; + lim = (1 << kLenNumLowBits); + } + else + { + UPDATE_1(probLen); + probLen = prob + LenChoice2; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenMid + (posState << kLenNumMidBits); + offset = kLenNumLowSymbols; + lim = (1 << kLenNumMidBits); + } + else + { + UPDATE_1(probLen); + probLen = prob + LenHigh; + offset = kLenNumLowSymbols + kLenNumMidSymbols; + lim = (1 << kLenNumHighBits); + } + } + TREE_DECODE(probLen, lim, len); + len += offset; + } + #else + { + CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenLow + (posState << kLenNumLowBits); + len = 1; + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + len -= 8; + } + else + { + UPDATE_1(probLen); + probLen = prob + LenChoice2; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenMid + (posState << kLenNumMidBits); + len = 1; + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + } + else + { + UPDATE_1(probLen); + probLen = prob + LenHigh; + TREE_DECODE(probLen, (1 << kLenNumHighBits), len); + len += kLenNumLowSymbols + kLenNumMidSymbols; + } + } + } + #endif + + if (state >= kNumStates) + { + UInt32 distance; + prob = probs + PosSlot + + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); + TREE_6_DECODE(prob, distance); + if (distance >= kStartPosModelIndex) + { + unsigned posSlot = (unsigned)distance; + unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); + distance = (2 | (distance & 1)); + if (posSlot < kEndPosModelIndex) + { + distance <<= numDirectBits; + prob = probs + SpecPos + distance - posSlot - 1; + { + UInt32 mask = 1; + unsigned i = 1; + do + { + GET_BIT2(prob + i, i, ; , distance |= mask); + mask <<= 1; + } + while (--numDirectBits != 0); + } + } + else + { + numDirectBits -= kNumAlignBits; + do + { + NORMALIZE + range >>= 1; + + { + UInt32 t; + code -= range; + t = (0 - ((UInt32)code >> 31)); /* (UInt32)((Int32)code >> 31) */ + distance = (distance << 1) + (t + 1); + code += range & t; + } + /* + distance <<= 1; + if (code >= range) + { + code -= range; + distance |= 1; + } + */ + } + while (--numDirectBits != 0); + prob = probs + Align; + distance <<= kNumAlignBits; + { + unsigned i = 1; + GET_BIT2(prob + i, i, ; , distance |= 1); + GET_BIT2(prob + i, i, ; , distance |= 2); + GET_BIT2(prob + i, i, ; , distance |= 4); + GET_BIT2(prob + i, i, ; , distance |= 8); + } + if (distance == (UInt32)0xFFFFFFFF) + { + len += kMatchSpecLenStart; + state -= kNumStates; + break; + } + } + } + + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance + 1; + if (checkDicSize == 0) + { + if (distance >= processedPos) + { + p->dicPos = dicPos; + return SZ_ERROR_DATA; + } + } + else if (distance >= checkDicSize) + { + p->dicPos = dicPos; + return SZ_ERROR_DATA; + } + state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + } + + len += kMatchMinLen; + + { + SizeT rem; + unsigned curLen; + SizeT pos; + + if ((rem = limit - dicPos) == 0) + { + p->dicPos = dicPos; + return SZ_ERROR_DATA; + } + + curLen = ((rem < len) ? (unsigned)rem : len); + pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); + + processedPos += curLen; + + len -= curLen; + if (curLen <= dicBufSize - pos) + { + Byte *dest = dic + dicPos; + ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; + const Byte *lim = dest + curLen; + dicPos += curLen; + do + *(dest) = (Byte)*(dest + src); + while (++dest != lim); + } + else + { + do + { + dic[dicPos++] = dic[pos]; + if (++pos == dicBufSize) + pos = 0; + } + while (--curLen != 0); + } + } + } + } + while (dicPos < limit && buf < bufLimit); + + NORMALIZE; + + p->buf = buf; + p->range = range; + p->code = code; + p->remainLen = len; + p->dicPos = dicPos; + p->processedPos = processedPos; + p->reps[0] = rep0; + p->reps[1] = rep1; + p->reps[2] = rep2; + p->reps[3] = rep3; + p->state = state; + + return SZ_OK; +} + +static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) +{ + if (p->remainLen != 0 && p->remainLen < kMatchSpecLenStart) + { + Byte *dic = p->dic; + SizeT dicPos = p->dicPos; + SizeT dicBufSize = p->dicBufSize; + unsigned len = p->remainLen; + SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ + SizeT rem = limit - dicPos; + if (rem < len) + len = (unsigned)(rem); + + if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) + p->checkDicSize = p->prop.dicSize; + + p->processedPos += len; + p->remainLen -= len; + while (len != 0) + { + len--; + dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + dicPos++; + } + p->dicPos = dicPos; + } +} + +static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + do + { + SizeT limit2 = limit; + if (p->checkDicSize == 0) + { + UInt32 rem = p->prop.dicSize - p->processedPos; + if (limit - p->dicPos > rem) + limit2 = p->dicPos + rem; + } + + RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit)); + + if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) + p->checkDicSize = p->prop.dicSize; + + LzmaDec_WriteRem(p, limit); + } + while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); + + if (p->remainLen > kMatchSpecLenStart) + p->remainLen = kMatchSpecLenStart; + + return 0; +} + +typedef enum +{ + DUMMY_ERROR, /* unexpected end of input stream */ + DUMMY_LIT, + DUMMY_MATCH, + DUMMY_REP +} ELzmaDummy; + +static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inSize) +{ + UInt32 range = p->range; + UInt32 code = p->code; + const Byte *bufLimit = buf + inSize; + const CLzmaProb *probs = p->probs; + unsigned state = p->state; + ELzmaDummy res; + + { + const CLzmaProb *prob; + UInt32 bound; + unsigned ttt; + unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1); + + prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK + + /* if (bufLimit - buf >= 7) return DUMMY_LIT; */ + + prob = probs + Literal; + if (p->checkDicSize != 0 || p->processedPos != 0) + prob += ((UInt32)LZMA_LIT_SIZE * + ((((p->processedPos) & ((1 << (p->prop.lp)) - 1)) << p->prop.lc) + + (p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); + + if (state < kNumLitStates) + { + unsigned symbol = 1; + do { GET_BIT_CHECK(prob + symbol, symbol) } while (symbol < 0x100); + } + else + { + unsigned matchByte = p->dic[p->dicPos - p->reps[0] + + (p->dicPos < p->reps[0] ? p->dicBufSize : 0)]; + unsigned offs = 0x100; + unsigned symbol = 1; + do + { + unsigned bit; + const CLzmaProb *probLit; + matchByte <<= 1; + bit = (matchByte & offs); + probLit = prob + offs + bit + symbol; + GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit) + } + while (symbol < 0x100); + } + res = DUMMY_LIT; + } + else + { + unsigned len; + UPDATE_1_CHECK; + + prob = probs + IsRep + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + state = 0; + prob = probs + LenCoder; + res = DUMMY_MATCH; + } + else + { + UPDATE_1_CHECK; + res = DUMMY_REP; + prob = probs + IsRepG0 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + NORMALIZE_CHECK; + return DUMMY_REP; + } + else + { + UPDATE_1_CHECK; + } + } + else + { + UPDATE_1_CHECK; + prob = probs + IsRepG1 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + } + else + { + UPDATE_1_CHECK; + prob = probs + IsRepG2 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + } + else + { + UPDATE_1_CHECK; + } + } + } + state = kNumStates; + prob = probs + RepLenCoder; + } + { + unsigned limit, offset; + const CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0_CHECK(probLen) + { + UPDATE_0_CHECK; + probLen = prob + LenLow + (posState << kLenNumLowBits); + offset = 0; + limit = 1 << kLenNumLowBits; + } + else + { + UPDATE_1_CHECK; + probLen = prob + LenChoice2; + IF_BIT_0_CHECK(probLen) + { + UPDATE_0_CHECK; + probLen = prob + LenMid + (posState << kLenNumMidBits); + offset = kLenNumLowSymbols; + limit = 1 << kLenNumMidBits; + } + else + { + UPDATE_1_CHECK; + probLen = prob + LenHigh; + offset = kLenNumLowSymbols + kLenNumMidSymbols; + limit = 1 << kLenNumHighBits; + } + } + TREE_DECODE_CHECK(probLen, limit, len); + len += offset; + } + + if (state < 4) + { + unsigned posSlot; + prob = probs + PosSlot + + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + kNumPosSlotBits); + TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); + if (posSlot >= kStartPosModelIndex) + { + unsigned numDirectBits = ((posSlot >> 1) - 1); + + /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */ + + if (posSlot < kEndPosModelIndex) + { + prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; + } + else + { + numDirectBits -= kNumAlignBits; + do + { + NORMALIZE_CHECK + range >>= 1; + code -= range & (((code - range) >> 31) - 1); + /* if (code >= range) code -= range; */ + } + while (--numDirectBits != 0); + prob = probs + Align; + numDirectBits = kNumAlignBits; + } + { + unsigned i = 1; + do + { + GET_BIT_CHECK(prob + i, i); + } + while (--numDirectBits != 0); + } + } + } + } + } + NORMALIZE_CHECK; + return res; +} + + +void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) +{ + p->needFlush = 1; + p->remainLen = 0; + p->tempBufSize = 0; + + if (initDic) + { + p->processedPos = 0; + p->checkDicSize = 0; + p->needInitState = 1; + } + if (initState) + p->needInitState = 1; +} + +void LzmaDec_Init(CLzmaDec *p) +{ + p->dicPos = 0; + LzmaDec_InitDicAndState(p, True, True); +} + +static void LzmaDec_InitStateReal(CLzmaDec *p) +{ + SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); + SizeT i; + CLzmaProb *probs = p->probs; + for (i = 0; i < numProbs; i++) + probs[i] = kBitModelTotal >> 1; + p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; + p->state = 0; + p->needInitState = 0; +} + +SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, + ELzmaFinishMode finishMode, ELzmaStatus *status) +{ + SizeT inSize = *srcLen; + (*srcLen) = 0; + LzmaDec_WriteRem(p, dicLimit); + + *status = LZMA_STATUS_NOT_SPECIFIED; + + while (p->remainLen != kMatchSpecLenStart) + { + int checkEndMarkNow; + + if (p->needFlush) + { + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) + p->tempBuf[p->tempBufSize++] = *src++; + if (p->tempBufSize < RC_INIT_SIZE) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + if (p->tempBuf[0] != 0) + return SZ_ERROR_DATA; + p->code = + ((UInt32)p->tempBuf[1] << 24) + | ((UInt32)p->tempBuf[2] << 16) + | ((UInt32)p->tempBuf[3] << 8) + | ((UInt32)p->tempBuf[4]); + p->range = 0xFFFFFFFF; + p->needFlush = 0; + p->tempBufSize = 0; + } + + checkEndMarkNow = 0; + if (p->dicPos >= dicLimit) + { + if (p->remainLen == 0 && p->code == 0) + { + *status = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK; + return SZ_OK; + } + if (finishMode == LZMA_FINISH_ANY) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_OK; + } + if (p->remainLen != 0) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_ERROR_DATA; + } + checkEndMarkNow = 1; + } + + if (p->needInitState) + LzmaDec_InitStateReal(p); + + if (p->tempBufSize == 0) + { + SizeT processed; + const Byte *bufLimit; + if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) + { + int dummyRes = LzmaDec_TryDummy(p, src, inSize); + if (dummyRes == DUMMY_ERROR) + { + memcpy(p->tempBuf, src, inSize); + p->tempBufSize = (unsigned)inSize; + (*srcLen) += inSize; + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + if (checkEndMarkNow && dummyRes != DUMMY_MATCH) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_ERROR_DATA; + } + bufLimit = src; + } + else + bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX; + p->buf = src; + if (LzmaDec_DecodeReal2(p, dicLimit, bufLimit) != 0) + return SZ_ERROR_DATA; + processed = (SizeT)(p->buf - src); + (*srcLen) += processed; + src += processed; + inSize -= processed; + } + else + { + unsigned rem = p->tempBufSize, lookAhead = 0; + while (rem < LZMA_REQUIRED_INPUT_MAX && lookAhead < inSize) + p->tempBuf[rem++] = src[lookAhead++]; + p->tempBufSize = rem; + if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) + { + int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem); + if (dummyRes == DUMMY_ERROR) + { + (*srcLen) += lookAhead; + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + if (checkEndMarkNow && dummyRes != DUMMY_MATCH) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_ERROR_DATA; + } + } + p->buf = p->tempBuf; + if (LzmaDec_DecodeReal2(p, dicLimit, p->buf) != 0) + return SZ_ERROR_DATA; + + { + unsigned kkk = (unsigned)(p->buf - p->tempBuf); + if (rem < kkk) + return SZ_ERROR_FAIL; /* some internal error */ + rem -= kkk; + if (lookAhead < rem) + return SZ_ERROR_FAIL; /* some internal error */ + lookAhead -= rem; + } + (*srcLen) += lookAhead; + src += lookAhead; + inSize -= lookAhead; + p->tempBufSize = 0; + } + } + if (p->code == 0) + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA; +} + +SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) +{ + SizeT outSize = *destLen; + SizeT inSize = *srcLen; + *srcLen = *destLen = 0; + for (;;) + { + SizeT inSizeCur = inSize, outSizeCur, dicPos; + ELzmaFinishMode curFinishMode; + SRes res; + if (p->dicPos == p->dicBufSize) + p->dicPos = 0; + dicPos = p->dicPos; + if (outSize > p->dicBufSize - dicPos) + { + outSizeCur = p->dicBufSize; + curFinishMode = LZMA_FINISH_ANY; + } + else + { + outSizeCur = dicPos + outSize; + curFinishMode = finishMode; + } + + res = LzmaDec_DecodeToDic(p, outSizeCur, src, &inSizeCur, curFinishMode, status); + src += inSizeCur; + inSize -= inSizeCur; + *srcLen += inSizeCur; + outSizeCur = p->dicPos - dicPos; + memcpy(dest, p->dic + dicPos, outSizeCur); + dest += outSizeCur; + outSize -= outSizeCur; + *destLen += outSizeCur; + if (res != 0) + return res; + if (outSizeCur == 0 || outSize == 0) + return SZ_OK; + } +} + +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc) +{ + alloc->Free(alloc, p->probs); + p->probs = NULL; +} + +static void LzmaDec_FreeDict(CLzmaDec *p, ISzAlloc *alloc) +{ + alloc->Free(alloc, p->dic); + p->dic = NULL; +} + +void LzmaDec_Free(CLzmaDec *p, ISzAlloc *alloc) +{ + LzmaDec_FreeProbs(p, alloc); + LzmaDec_FreeDict(p, alloc); +} + +SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) +{ + UInt32 dicSize; + Byte d; + + if (size < LZMA_PROPS_SIZE) + return SZ_ERROR_UNSUPPORTED; + else + dicSize = data[1] | ((UInt32)data[2] << 8) | ((UInt32)data[3] << 16) | ((UInt32)data[4] << 24); + + if (dicSize < LZMA_DIC_MIN) + dicSize = LZMA_DIC_MIN; + p->dicSize = dicSize; + + d = data[0]; + if (d >= (9 * 5 * 5)) + return SZ_ERROR_UNSUPPORTED; + + p->lc = d % 9; + d /= 9; + p->pb = d / 5; + p->lp = d % 5; + + return SZ_OK; +} + +static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAlloc *alloc) +{ + UInt32 numProbs = LzmaProps_GetNumProbs(propNew); + if (!p->probs || numProbs != p->numProbs) + { + LzmaDec_FreeProbs(p, alloc); + p->probs = (CLzmaProb *)alloc->Alloc(alloc, numProbs * sizeof(CLzmaProb)); + p->numProbs = numProbs; + if (!p->probs) + return SZ_ERROR_MEM; + } + return SZ_OK; +} + +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) +{ + CLzmaProps propNew; + RINOK(LzmaProps_Decode(&propNew, props, propsSize)); + RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); + p->prop = propNew; + return SZ_OK; +} + +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) +{ + CLzmaProps propNew; + SizeT dicBufSize; + RINOK(LzmaProps_Decode(&propNew, props, propsSize)); + RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); + + { + UInt32 dictSize = propNew.dicSize; + SizeT mask = ((UInt32)1 << 12) - 1; + if (dictSize >= ((UInt32)1 << 30)) mask = ((UInt32)1 << 22) - 1; + else if (dictSize >= ((UInt32)1 << 22)) mask = ((UInt32)1 << 20) - 1;; + dicBufSize = ((SizeT)dictSize + mask) & ~mask; + if (dicBufSize < dictSize) + dicBufSize = dictSize; + } + + if (!p->dic || dicBufSize != p->dicBufSize) + { + LzmaDec_FreeDict(p, alloc); + p->dic = (Byte *)alloc->Alloc(alloc, dicBufSize); + if (!p->dic) + { + LzmaDec_FreeProbs(p, alloc); + return SZ_ERROR_MEM; + } + } + p->dicBufSize = dicBufSize; + p->prop = propNew; + return SZ_OK; +} + +SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, + ELzmaStatus *status, ISzAlloc *alloc) +{ + CLzmaDec p; + SRes res; + SizeT outSize = *destLen, inSize = *srcLen; + *destLen = *srcLen = 0; + *status = LZMA_STATUS_NOT_SPECIFIED; + if (inSize < RC_INIT_SIZE) + return SZ_ERROR_INPUT_EOF; + LzmaDec_Construct(&p); + RINOK(LzmaDec_AllocateProbs(&p, propData, propSize, alloc)); + p.dic = dest; + p.dicBufSize = outSize; + LzmaDec_Init(&p); + *srcLen = inSize; + res = LzmaDec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); + *destLen = p.dicPos; + if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) + res = SZ_ERROR_INPUT_EOF; + LzmaDec_FreeProbs(&p, alloc); + return res; +} diff --git a/client/lzma/LzmaDec.h b/client/lzma/LzmaDec.h new file mode 100644 index 00000000..2633abea --- /dev/null +++ b/client/lzma/LzmaDec.h @@ -0,0 +1,227 @@ +/* LzmaDec.h -- LZMA Decoder +2013-01-18 : Igor Pavlov : Public domain */ + +#ifndef __LZMA_DEC_H +#define __LZMA_DEC_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +/* #define _LZMA_PROB32 */ +/* _LZMA_PROB32 can increase the speed on some CPUs, + but memory usage for CLzmaDec::probs will be doubled in that case */ + +#ifdef _LZMA_PROB32 +#define CLzmaProb UInt32 +#else +#define CLzmaProb UInt16 +#endif + + +/* ---------- LZMA Properties ---------- */ + +#define LZMA_PROPS_SIZE 5 + +typedef struct _CLzmaProps +{ + unsigned lc, lp, pb; + UInt32 dicSize; +} CLzmaProps; + +/* LzmaProps_Decode - decodes properties +Returns: + SZ_OK + SZ_ERROR_UNSUPPORTED - Unsupported properties +*/ + +SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); + + +/* ---------- LZMA Decoder state ---------- */ + +/* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case. + Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */ + +#define LZMA_REQUIRED_INPUT_MAX 20 + +typedef struct +{ + CLzmaProps prop; + CLzmaProb *probs; + Byte *dic; + const Byte *buf; + UInt32 range, code; + SizeT dicPos; + SizeT dicBufSize; + UInt32 processedPos; + UInt32 checkDicSize; + unsigned state; + UInt32 reps[4]; + unsigned remainLen; + int needFlush; + int needInitState; + UInt32 numProbs; + unsigned tempBufSize; + Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; +} CLzmaDec; + +#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; } + +void LzmaDec_Init(CLzmaDec *p); + +/* There are two types of LZMA streams: + 0) Stream with end mark. That end mark adds about 6 bytes to compressed size. + 1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */ + +typedef enum +{ + LZMA_FINISH_ANY, /* finish at any point */ + LZMA_FINISH_END /* block must be finished at the end */ +} ELzmaFinishMode; + +/* ELzmaFinishMode has meaning only if the decoding reaches output limit !!! + + You must use LZMA_FINISH_END, when you know that current output buffer + covers last bytes of block. In other cases you must use LZMA_FINISH_ANY. + + If LZMA decoder sees end marker before reaching output limit, it returns SZ_OK, + and output value of destLen will be less than output buffer size limit. + You can check status result also. + + You can use multiple checks to test data integrity after full decompression: + 1) Check Result and "status" variable. + 2) Check that output(destLen) = uncompressedSize, if you know real uncompressedSize. + 3) Check that output(srcLen) = compressedSize, if you know real compressedSize. + You must use correct finish mode in that case. */ + +typedef enum +{ + LZMA_STATUS_NOT_SPECIFIED, /* use main error code instead */ + LZMA_STATUS_FINISHED_WITH_MARK, /* stream was finished with end mark. */ + LZMA_STATUS_NOT_FINISHED, /* stream was not finished */ + LZMA_STATUS_NEEDS_MORE_INPUT, /* you must provide more input bytes */ + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK /* there is probability that stream was finished without end mark */ +} ELzmaStatus; + +/* ELzmaStatus is used only as output value for function call */ + + +/* ---------- Interfaces ---------- */ + +/* There are 3 levels of interfaces: + 1) Dictionary Interface + 2) Buffer Interface + 3) One Call Interface + You can select any of these interfaces, but don't mix functions from different + groups for same object. */ + + +/* There are two variants to allocate state for Dictionary Interface: + 1) LzmaDec_Allocate / LzmaDec_Free + 2) LzmaDec_AllocateProbs / LzmaDec_FreeProbs + You can use variant 2, if you set dictionary buffer manually. + For Buffer Interface you must always use variant 1. + +LzmaDec_Allocate* can return: + SZ_OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - Unsupported properties +*/ + +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc); +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc); + +SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAlloc *alloc); +void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); + +/* ---------- Dictionary Interface ---------- */ + +/* You can use it, if you want to eliminate the overhead for data copying from + dictionary to some other external buffer. + You must work with CLzmaDec variables directly in this interface. + + STEPS: + LzmaDec_Constr() + LzmaDec_Allocate() + for (each new stream) + { + LzmaDec_Init() + while (it needs more decompression) + { + LzmaDec_DecodeToDic() + use data from CLzmaDec::dic and update CLzmaDec::dicPos + } + } + LzmaDec_Free() +*/ + +/* LzmaDec_DecodeToDic + + The decoding to internal dictionary buffer (CLzmaDec::dic). + You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!! + +finishMode: + It has meaning only if the decoding reaches output limit (dicLimit). + LZMA_FINISH_ANY - Decode just dicLimit bytes. + LZMA_FINISH_END - Stream must be finished after dicLimit. + +Returns: + SZ_OK + status: + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED + LZMA_STATUS_NEEDS_MORE_INPUT + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + SZ_ERROR_DATA - Data error +*/ + +SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, + const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); + + +/* ---------- Buffer Interface ---------- */ + +/* It's zlib-like interface. + See LzmaDec_DecodeToDic description for information about STEPS and return results, + but you must use LzmaDec_DecodeToBuf instead of LzmaDec_DecodeToDic and you don't need + to work with CLzmaDec variables manually. + +finishMode: + It has meaning only if the decoding reaches output limit (*destLen). + LZMA_FINISH_ANY - Decode just destLen bytes. + LZMA_FINISH_END - Stream must be finished after (*destLen). +*/ + +SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); + + +/* ---------- One Call Interface ---------- */ + +/* LzmaDecode + +finishMode: + It has meaning only if the decoding reaches output limit (*destLen). + LZMA_FINISH_ANY - Decode just destLen bytes. + LZMA_FINISH_END - Stream must be finished after (*destLen). + +Returns: + SZ_OK + status: + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + SZ_ERROR_DATA - Data error + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - Unsupported properties + SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). +*/ + +SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, + ELzmaStatus *status, ISzAlloc *alloc); + +EXTERN_C_END + +#endif diff --git a/client/requirements.txt b/client/requirements.txt index 59f57b2c..83b253dd 100644 --- a/client/requirements.txt +++ b/client/requirements.txt @@ -4,3 +4,4 @@ psutil pyaml rsa netaddr +pylzma diff --git a/client/sources-linux/Makefile b/client/sources-linux/Makefile index c7afd24b..ec67eba6 100644 --- a/client/sources-linux/Makefile +++ b/client/sources-linux/Makefile @@ -1,7 +1,7 @@ GZIP ?= gzip CC ?= gcc -CFLAGS := $(shell pkg-config --cflags python-2.7) -fPIC $(CFLAGS_EXTRA) +CFLAGS := $(shell pkg-config --cflags python-2.7) -I../lzma -fPIC $(CFLAGS_EXTRA) LDFLAGS := -lpthread -ldl -fPIC $(LDFLAGS_EXTRA) -Wl,-Bstatic -lz -Wl,-Bdynamic PFLAGS := -O PIE ?= -pie @@ -35,7 +35,7 @@ TEMPLATE_OUTPUT_PATH ?= ../../pupy/payload_templates/ PYOBJS := _memimporter.o Python-dynload.o pupy_load.o pupy.o COMMON_OBJS := resources_bootloader_pyc.o resources_python27_so.o \ resources_library_compressed_string_txt.o list.o tmplibrary.o daemonize.o \ - decompress.o + decompress.o LzmaDec.o ifeq ($(ARCH),64) COMMON_OBJS += linux-inject/inject-x86_64.o @@ -46,29 +46,17 @@ endif COMMON_OBJS += linux-inject/ptrace.o COMMON_OBJS += linux-inject/utils.o -ZLIB := $(shell $(PYTHON) $(PFLAGS) -c 'import zlib; print zlib.__file__ if "__file__" in zlib.__dict__ else "built-in"') - -ifneq ($(ZLIB),built-in) -COMMON_OBJS += resources_zlib_so.o -CFLAGS += -D_PYZLIB_DYNLOAD -endif - all: $(TEMPLATE_OUTPUT_PATH)/pupyx$(NAME).lin $(TEMPLATE_OUTPUT_PATH)/pupyx$(NAME).so -ifneq ($(ZLIB),built-in) -resources/zlib.so: $(ZLIB) - $(GZIP) -9 -c $< >$@ - -resources_zlib_so.c: ../gen_resource_header.py resources/zlib.so - $(PYTHON) $(PFLAGS) $+ -endif - import-tab.c import-tab.h: mktab.py $(PYTHON) $(PFLAGS) $< Python-dynload.o: Python-dynload.c import-tab.c import-tab.h $(CC) -c -o $@ $< $(CFLAGS) +LzmaDec.o: ../lzma/LzmaDec.c + $(CC) -O3 -c -o $@ $< + resources/library_compressed_string.txt: ../gen_library_compressed_string.py resources/library.zip $(PYTHON) $(PFLAGS) ../gen_library_compressed_string.py @@ -86,6 +74,7 @@ linux-inject/%.o: linux-inject/%.c resources/python27.so: $(LIBPYTHON) cp -vf $< $@.tmp + -chmod 600 $@.tmp -strip $@.tmp $(GZIP) -9 -c $@.tmp >$@ rm -f $@.tmp @@ -108,6 +97,7 @@ clean: find -name "*.pyc" | xargs rm -f find -name "*.pyo" | xargs rm -f find -name "*.o" | xargs rm -f + rm -f $(COMMON_OBJS) rm -f pupy pupy.so rm -f resources/library.zip rm -f resources/*.so diff --git a/client/sources-linux/pupy.c b/client/sources-linux/pupy.c index 59157c51..c30f8ea1 100644 --- a/client/sources-linux/pupy.c +++ b/client/sources-linux/pupy.c @@ -10,6 +10,10 @@ #include "debug.h" #include "Python-dynload.h" #include "daemonize.h" +#include +#include "tmplibrary.h" + +#include "LzmaDec.h" int linux_inject_main(int argc, char **argv); @@ -17,11 +21,57 @@ static char module_doc[] = "Builtins utilities for pupy"; extern const char resources_library_compressed_string_txt_start[]; extern const int resources_library_compressed_string_txt_size; + char pupy_config[40960]="####---PUPY_CONFIG_COMES_HERE---####\n"; //big array to have space for more config / code run at startup extern const uint32_t dwPupyArch; -static PyObject *Py_get_compressed_library_string(PyObject *self, PyObject *args) + +static void *_lzalloc(void *p, size_t size) { p = p; return malloc(size); } +static void _lzfree(void *p, void *address) { p = p; free(address); } +ISzAlloc _lzallocator = { _lzalloc, _lzfree }; + +static PyObject *Py_get_modules(PyObject *self, PyObject *args) { - return Py_BuildValue("s#", resources_library_compressed_string_txt_start, resources_library_compressed_string_txt_size); + char *uncompressed = NULL; + size_t uncompressed_size = 0; + + const Byte *wheader = resources_library_compressed_string_txt_start + sizeof(unsigned int); + const Byte *woheader = wheader + LZMA_PROPS_SIZE; + + CLzmaDec state; + ELzmaStatus status; + size_t srcLen; + int res; + + uncompressed_size = ntohl( + *((unsigned int *) resources_library_compressed_string_txt_start) + ); + + uncompressed = malloc(uncompressed_size); + if (!uncompressed) { + dprint("Allocation failed\n"); + abort(); + } + + dprint("Uncompressed library size = %d\n", uncompressed_size); + dprint("Compressed library size = %d\n", resources_library_compressed_string_txt_size); + + srcLen = resources_library_compressed_string_txt_size - sizeof(unsigned int) - LZMA_PROPS_SIZE; + + res = LzmaDecode( + uncompressed, &uncompressed_size, woheader, &srcLen, wheader, + LZMA_PROPS_SIZE, LZMA_FINISH_ANY, &status, &_lzallocator + ); + + if (res != SZ_OK) { + dprint("Decompression failed\n"); + abort(); + } + + PyObject * modules = PyMarshal_ReadObjectFromString( + uncompressed, uncompressed_size); + + free(uncompressed); + return modules; } static PyObject * @@ -162,7 +212,7 @@ static PyObject *Py_load_dll(PyObject *self, PyObject *args) static PyMethodDef methods[] = { { "get_pupy_config", Py_get_pupy_config, METH_NOARGS, "get_pupy_config() -> string" }, { "get_arch", Py_get_arch, METH_NOARGS, "get current pupy architecture (x86 or x64)" }, - { "_get_compressed_library_string", Py_get_compressed_library_string, METH_VARARGS }, + { "get_modules", Py_get_modules, METH_NOARGS, "get pupy library" }, { "reflective_inject_dll", Py_reflective_inject_dll, METH_VARARGS|METH_KEYWORDS, "reflective_inject_dll(pid, dll_buffer, isRemoteProcess64bits)\nreflectively inject a dll into a process. raise an Exception on failure" }, { "load_dll", Py_load_dll, METH_VARARGS, "load_dll(dllname, raw_dll) -> bool" }, { "ld_preload_inject_dll", Py_ld_preload_inject_dll, METH_VARARGS, "ld_preload_inject_dll(cmdline, dll_buffer, hook_exit) -> pid" }, diff --git a/client/sources-linux/pupy_load.c b/client/sources-linux/pupy_load.c index ec406d23..ac8faa02 100644 --- a/client/sources-linux/pupy_load.c +++ b/client/sources-linux/pupy_load.c @@ -23,11 +23,6 @@ extern const int resources_python27_so_size; extern const char resources_bootloader_pyc_start[]; extern const int resources_bootloader_pyc_size; -#ifdef _PYZLIB_DYNLOAD -extern const char resources_zlib_so_start[]; -extern const int resources_zlib_so_size; -#endif - extern DL_EXPORT(void) init_memimporter(void); extern DL_EXPORT(void) initpupy(void); @@ -104,13 +99,6 @@ uint32_t mainThread(int argc, char *argv[], bool so) { initpupy(); dprint("initpupy()\n"); -#ifdef _PYZLIB_DYNLOAD - dprint("load zlib\n"); - if (!import_module("initzlib", "zlib", resources_zlib_so_start, resources_zlib_so_size)) { - dprint("ZLib load failed.\n"); - } -#endif - /* We execute then in the context of '__main__' */ dprint("starting evaluating python code ...\n"); m = PyImport_AddModule("__main__"); diff --git a/client/sources/Makefile.linux b/client/sources/Makefile.linux index 2eec0ccd..2b7f9795 100644 --- a/client/sources/Makefile.linux +++ b/client/sources/Makefile.linux @@ -10,6 +10,7 @@ endif CC := $(BUILDENV)/$(ARCH)/cl.sh PYTHON := $(BUILDENV)/$(ARCH)/python.sh -O +HOST_PYTHON := python ifeq "$(ARCH)" "win64" CFLAGS:=$(CFLAGS) /DWIN_X64 /D_WIN64 /nologo @@ -26,10 +27,13 @@ LINKER_OPTS := PPARCH := $(PPARCH)d else DEBUG_ADD := -LINKER_OPTS :=/link /subsystem:windows /ENTRY:mainCRTStartup +LINKER_OPTS :=/link /subsystem:windows /ENTRY:mainCRTStartup /LTGC PPARCH := $(PPARCH) +CFLAGS := $(CFLAGS) /O1 /GL endif +CFLAGS := $(CFLAGS) /I..\\lzma + PYOBJS := \ _memimporter.obj \ MyLoadLibrary.obj \ @@ -44,7 +48,7 @@ COMMON_OBJS := \ MemoryModule.obj \ resources_library_compressed_string_txt.obj \ actctx.obj list.obj thread.obj remote_thread.obj \ - LoadLibraryR.obj resources_msvcr90_dll.obj + LoadLibraryR.obj resources_msvcr90_dll.obj LzmaDec.obj all: $(TEMPLATE_OUTPUT_PATH)/pupy$(PPARCH).exe $(TEMPLATE_OUTPUT_PATH)/pupy$(PPARCH).dll @@ -52,19 +56,19 @@ $(BUILDENV_READY): ./buildenv.sh "$(BUILDENV)" resources/library_compressed_string.txt: ../gen_library_compressed_string.py resources/library.zip $(BUILDENV_READY) - $(PYTHON) ../gen_library_compressed_string.py + $(HOST_PYTHON) ../gen_library_compressed_string.py resources/library.zip: ../build_library_zip.py $(BUILDENV_READY) $(PYTHON) ../build_library_zip.py -windows resources_library_compressed_string_txt.c: ../gen_resource_header.py resources/library_compressed_string.txt resources/library.zip $(BUILDENV_READY) - $(PYTHON) ../gen_resource_header.py resources/library_compressed_string.txt + $(HOST_PYTHON) ../gen_resource_header.py resources/library_compressed_string.txt resources/bootloader.pyc: ../../pupy/packages/all/pupyimporter.py ../../pupy/pp.py ../gen_python_bootloader.py $(BUILDENV_READY) $(PYTHON) ../gen_python_bootloader.py $(DEBUG_ADD) resources_bootloader_pyc.c: resources/bootloader.pyc ../gen_resource_header.py $(BUILDENV_READY) - $(PYTHON) ../gen_resource_header.py $< + $(HOST_PYTHON) ../gen_resource_header.py $< resources/python27.dll: $(BUILDENV)/$(ARCH)/drive_c/Python27/python27.dll $(BUILDENV_READY) cp $< $@ @@ -73,14 +77,17 @@ resources/msvcr90.dll: $(BUILDENV)/$(ARCH)/drive_c/Python27/msvcr90.dll $(BUILDE cp $< $@ resources_python27_dll.c: resources/python27.dll ../gen_resource_header.py $(BUILDENV_READY) - $(PYTHON) ../gen_resource_header.py $< + $(HOST_PYTHON) ../gen_resource_header.py $< resources_msvcr90_dll.c: resources/msvcr90.dll ../gen_resource_header.py $(BUILDENV_READY) - $(PYTHON) ../gen_resource_header.py $< + $(HOST_PYTHON) ../gen_resource_header.py $< $(PYOBJS): %.obj: %.c $(CC) /c $(CFLAGS) /I$(PYTHONPATH)\\include $< +LzmaDec.obj: ../lzma/LzmaDec.c + $(CC) /c $(CFLAGS) $< + main_exe.obj: main_exe.c $(CC) /c $(CFLAGS) $< diff --git a/client/sources/pupy.c b/client/sources/pupy.c index adb5405d..dd173c93 100644 --- a/client/sources/pupy.c +++ b/client/sources/pupy.c @@ -6,6 +6,7 @@ #include "Python-dynload.h" #include #include +#include "LzmaDec.h" #include "base_inject.h" static char module_doc[] = "Builtins utilities for pupy"; @@ -13,9 +14,58 @@ extern const char resources_library_compressed_string_txt_start[]; extern const int resources_library_compressed_string_txt_size; char pupy_config[40960]="####---PUPY_CONFIG_COMES_HERE---####\n"; //big array to have space for more config / code run at startup extern const DWORD dwPupyArch; -static PyObject *Py_get_compressed_library_string(PyObject *self, PyObject *args) + +static void *_lzalloc(void *p, size_t size) { p = p; return malloc(size); } +static void _lzfree(void *p, void *address) { p = p; free(address); } +ISzAlloc _lzallocator = { _lzalloc, _lzfree }; + +static PyObject *Py_get_modules(PyObject *self, PyObject *args) { - return Py_BuildValue("s#", resources_library_compressed_string_txt_start, resources_library_compressed_string_txt_size); + char *uncompressed = NULL; + size_t uncompressed_size = 0; + PyObject * modules; + + const Byte *wheader = resources_library_compressed_string_txt_start + sizeof(unsigned int); + const Byte *woheader = wheader + LZMA_PROPS_SIZE; + + CLzmaDec state; + ELzmaStatus status; + size_t srcLen; + int res; + + union { + unsigned int l; + unsigned char c[4]; + } x; + + x.c[3] = resources_library_compressed_string_txt_start[0]; + x.c[2] = resources_library_compressed_string_txt_start[1]; + x.c[1] = resources_library_compressed_string_txt_start[2]; + x.c[0] = resources_library_compressed_string_txt_start[3]; + + uncompressed_size = x.l; + + uncompressed = malloc(uncompressed_size); + if (!uncompressed) { + abort(); + } + + srcLen = resources_library_compressed_string_txt_size - sizeof(unsigned int) - LZMA_PROPS_SIZE; + + res = LzmaDecode( + uncompressed, &uncompressed_size, woheader, &srcLen, wheader, + LZMA_PROPS_SIZE, LZMA_FINISH_ANY, &status, &_lzallocator + ); + + if (res != SZ_OK) { + abort(); + } + + modules = PyMarshal_ReadObjectFromString( + uncompressed, uncompressed_size); + + free(uncompressed); + return modules; } static PyObject * @@ -74,21 +124,18 @@ static PyObject *Py_find_function_address(PyObject *self, PyObject *args) const char *lpDllName = NULL; const char *lpFuncName = NULL; void *address = NULL; - printf("DEBUG 0: %s %s\n", lpDllName, lpFuncName); if (PyArg_ParseTuple(args, "ss", &lpDllName, &lpFuncName)) { - printf("DEBUG: %s %s\n", lpDllName, lpFuncName); address = MyFindProcAddress(lpDllName, lpFuncName); } - printf("DEBUG 2: %s %s %p\n", lpDllName, lpFuncName, address); return PyLong_FromVoidPtr(address); } static PyMethodDef methods[] = { { "get_pupy_config", Py_get_pupy_config, METH_NOARGS, "get_pupy_config() -> string" }, { "get_arch", Py_get_arch, METH_NOARGS, "get current pupy architecture (x86 or x64)" }, - { "_get_compressed_library_string", Py_get_compressed_library_string, METH_VARARGS }, + { "get_modules", Py_get_modules, METH_NOARGS }, { "reflective_inject_dll", Py_reflective_inject_dll, METH_VARARGS|METH_KEYWORDS, "reflective_inject_dll(pid, dll_buffer, isRemoteProcess64bits)\nreflectively inject a dll into a process. raise an Exception on failure" }, { "load_dll", Py_load_dll, METH_VARARGS, "load_dll(dllname, raw_dll) -> bool" }, { "find_function_address", Py_find_function_address, METH_VARARGS, diff --git a/pupy/packages/all/pupyimporter.py b/pupy/packages/all/pupyimporter.py index 76436f86..d2f9cd8b 100644 --- a/pupy/packages/all/pupyimporter.py +++ b/pupy/packages/all/pupyimporter.py @@ -16,7 +16,7 @@ # This module uses the builtins modules pupy and _memimporter to load python modules and packages from memory, including .pyd files (windows only) # Pupy can dynamically add new modules to the modules dictionary to allow remote importing of python modules from memory ! # -import sys, imp, zlib, marshal +import sys, imp, marshal __debug = False; @@ -34,8 +34,8 @@ except ImportError: modules={} try: import pupy - if not (hasattr(pupy, 'pseudo') and pupy.pseudo): - modules = marshal.loads(zlib.decompress(pupy._get_compressed_library_string())) + if not (hasattr(pupy, 'pseudo') and pupy.pseudo) and not modules: + modules = pupy.get_modules() except ImportError: pass