Rooba
/
mitogen
mirror of https://github.com/mitogen-hq/mitogen.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

importer: warn on duplicate request, simplify preload logic 

* Children should never generate a request for a module that has already
  been sent, however there are a variety of edge cases where, e.g.
  asynchronous calls are made into unloaded modules in a set of
  children, causing those children to request modules (and deps) in a
  different order, which might break deduplication. So add a warning to
  catch when this happens, so we can figure out how to handle it.
  Meanwhile it's only a warning since in the worst case, this just adds
  needless latency.

* Don't bother treating sent packages separately, there doesn't seem to
  be any need for this (after docs are updated to match how preloading
  actually works now).
This commit is contained in:
David Wilson 2018-02-13 15:49:25 +05:45
parent 2e729e54cc
commit 2848d35aff
2 changed files with 7 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
mitogen/master.py
View File

@ -464,7 +464,7 @@ class ModuleResponder(object):
path, source, is_pkg = self._finder.get_module_source(fullname) path, source, is_pkg = self._finder.get_module_source(fullname)
if source is None: if source is None:
LOG.error('_build_tuple(%r): could not locate source', fullname) LOG.error('_build_tuple(%r): could not locate source', fullname)
tup = fullname, None, None, None, None tup = fullname, None, None, None, ()
self._cache[fullname] = tup self._cache[fullname] = tup
return tup return tup
@ -505,17 +505,16 @@ class ModuleResponder(object):
stream = self._router.stream_by_id(msg.src_id) stream = self._router.stream_by_id(msg.src_id)
fullname = msg.data fullname = msg.data
if fullname in stream.sent_modules:
LOG.warning('_on_get_module(): dup request for %r from %r',
fullname, stream)
try: try:
tup = self._build_tuple(fullname) tup = self._build_tuple(fullname)
for name in tup[4] or (): # related for name in tup[4]: # related
if name == fullname:
# Must be sent last
continue
parent_pkg, _, _ = name.partition('.') parent_pkg, _, _ = name.partition('.')
if parent_pkg != fullname and parent_pkg not in stream.sent_packages: if parent_pkg != fullname and parent_pkg not in stream.sent_modules:
# Parent hasn't been required, so don't load this guy yet. # Parent hasn't been sent, so don't load submodule yet.
continue continue
if name in stream.sent_modules: if name in stream.sent_modules:
@ -523,11 +522,7 @@ class ModuleResponder(object):
continue continue
self._send_load_module(stream, msg, name) self._send_load_module(stream, msg, name)
self._send_load_module(stream, msg, fullname) self._send_load_module(stream, msg, fullname)
if tup[1] is not None:
# It's a package, record the fact it was sent.
stream.sent_packages.add(fullname)
except Exception: except Exception:
LOG.debug('While importing %r', fullname, exc_info=True) LOG.debug('While importing %r', fullname, exc_info=True)

1
mitogen/parent.py
View File

@ -267,7 +267,6 @@ class Stream(mitogen.core.Stream):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(Stream, self).__init__(*args, **kwargs) super(Stream, self).__init__(*args, **kwargs)
self.sent_modules = set(['mitogen', 'mitogen.core']) self.sent_modules = set(['mitogen', 'mitogen.core'])
self.sent_packages = set(['mitogen'])
def construct(self, remote_name=None, python_path=None, debug=False, def construct(self, remote_name=None, python_path=None, debug=False,
profiling=False, **kwargs): profiling=False, **kwargs):