Merge remote-tracking branch 'origin/dmw'
* origin/dmw: issue #590: whoops, import missing test modules issue #590: rework ParentEnumerationMethod to recursively handle bad modules issue #627: reduce the default pool size in a child to 2. tests: add a few extra service tests. docs: some more hyperlink joy docs: more hyperlinks docs: add domainrefs plugin to make link aliases everywhere \o/ docs: link IS_DEAD in changelog docs: tweaks to better explain changelog race
This commit is contained in:
David Wilson
commit
caa2a4f498
|
|
@ -175,16 +175,10 @@ Noteworthy Differences
|
|||
|
||||
your_ssh_username = (ALL) NOPASSWD:/usr/bin/python -c*
|
||||
|
||||
* The `buildah <https://docs.ansible.com/ansible/latest/plugins/connection/buildah.html>`_,
|
||||
`docker <https://docs.ansible.com/ansible/2.6/plugins/connection/docker.html>`_,
|
||||
`jail <https://docs.ansible.com/ansible/2.6/plugins/connection/jail.html>`_,
|
||||
`kubectl <https://docs.ansible.com/ansible/2.6/plugins/connection/kubectl.html>`_,
|
||||
`local <https://docs.ansible.com/ansible/2.6/plugins/connection/local.html>`_,
|
||||
`lxc <https://docs.ansible.com/ansible/2.6/plugins/connection/lxc.html>`_,
|
||||
`lxd <https://docs.ansible.com/ansible/2.6/plugins/connection/lxd.html>`_,
|
||||
and `ssh <https://docs.ansible.com/ansible/2.6/plugins/connection/ssh.html>`_
|
||||
built-in connection types are supported, along with Mitogen-specific
|
||||
:ref:`machinectl <machinectl>`, :ref:`mitogen_doas <doas>`,
|
||||
* The :ans:conn:`~buildah`, :ans:conn:`~docker`, :ans:conn:`~jail`,
|
||||
:ans:conn:`~kubectl`, :ans:conn:`~local`, :ans:conn:`~lxd`, and
|
||||
:ans:conn:`~ssh` built-in connection types are supported, along with
|
||||
Mitogen-specific :ref:`machinectl <machinectl>`, :ref:`mitogen_doas <doas>`,
|
||||
:ref:`mitogen_su <su>`, :ref:`mitogen_sudo <sudo>`, and :ref:`setns <setns>`
|
||||
types. File bugs to register interest in others.
|
||||
|
||||
|
|
@ -199,16 +193,14 @@ Noteworthy Differences
|
|||
artificial serialization, causing slowdown equivalent to `task_duration *
|
||||
num_targets`. This will be addressed soon.
|
||||
|
||||
* The Ansible 2.7 `reboot
|
||||
<https://docs.ansible.com/ansible/latest/modules/reboot_module.html>`_ module
|
||||
may require a ``pre_reboot_delay`` on systemd hosts, as insufficient time
|
||||
exists for the reboot command's exit status to be reported before necessary
|
||||
processes are torn down.
|
||||
* The Ansible 2.7 :ans:mod:`reboot` may require a ``pre_reboot_delay`` on
|
||||
systemd hosts, as insufficient time exists for the reboot command's exit
|
||||
status to be reported before necessary processes are torn down.
|
||||
|
||||
* On OS X when a SSH password is specified and the default connection type of
|
||||
``smart`` is used, Ansible may select the Paramiko plug-in rather than
|
||||
Mitogen. If you specify a password on OS X, ensure ``connection: ssh``
|
||||
appears in your playbook, ``ansible.cfg``, or as ``-c ssh`` on the
|
||||
:ans:conn:`~smart` is used, Ansible may select the :ans:conn:`paramiko_ssh`
|
||||
rather than Mitogen. If you specify a password on OS X, ensure ``connection:
|
||||
ssh`` appears in your playbook, ``ansible.cfg``, or as ``-c ssh`` on the
|
||||
command-line.
|
||||
|
||||
* Ansible permits up to ``forks`` connections to be setup in parallel, whereas
|
||||
|
|
@ -345,19 +337,12 @@ command line, or as host and group variables.
|
|||
File Transfer
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
Normally `sftp(1)`_ or `scp(1)`_ are used to copy files by the
|
||||
`assemble <http://docs.ansible.com/ansible/latest/modules/assemble_module.html>`_,
|
||||
`copy <http://docs.ansible.com/ansible/latest/modules/copy_module.html>`_,
|
||||
`patch <http://docs.ansible.com/ansible/latest/modules/patch_module.html>`_,
|
||||
`script <http://docs.ansible.com/ansible/latest/modules/script_module.html>`_,
|
||||
`template <http://docs.ansible.com/ansible/latest/modules/template_module.html>`_, and
|
||||
`unarchive <http://docs.ansible.com/ansible/latest/modules/unarchive_module.html>`_
|
||||
actions, or when uploading modules with pipelining disabled. With Mitogen
|
||||
copies are implemented natively using the same interpreters, connection tree,
|
||||
and routed message bus that carries RPCs.
|
||||
|
||||
.. _scp(1): https://linux.die.net/man/1/scp
|
||||
.. _sftp(1): https://linux.die.net/man/1/sftp
|
||||
Normally :linux:man1:`sftp` or :linux:man1:`scp` are used to copy files by the
|
||||
:ans:mod:`~assemble`, :ans:mod:`~aws_s3`, :ans:mod:`~copy`, :ans:mod:`~patch`,
|
||||
:ans:mod:`~script`, :ans:mod:`~template`, :ans:mod:`~unarchive`, and
|
||||
:ans:mod:`~uri` actions, or when uploading modules with pipelining disabled.
|
||||
With Mitogen copies are implemented natively using the same interpreters,
|
||||
connection tree, and routed message bus that carries RPCs.
|
||||
|
||||
This permits direct streaming between endpoints regardless of execution
|
||||
environment, without necessitating temporary copies in intermediary accounts or
|
||||
|
|
@ -373,15 +358,15 @@ Safety
|
|||
^^^^^^
|
||||
|
||||
Transfers proceed to a hidden file in the destination directory, with content
|
||||
and metadata synced using `fsync(2) <https://linux.die.net/man/2/fsync>`_ prior
|
||||
to rename over any existing file. This ensures the file remains consistent at
|
||||
all times, in the event of a crash, or when overlapping `ansible-playbook` runs
|
||||
deploy differing file contents.
|
||||
and metadata synced using :linux:man2:`fsync` prior to rename over any existing
|
||||
file. This ensures the file remains consistent at all times, in the event of a
|
||||
crash, or when overlapping `ansible-playbook` runs deploy differing file
|
||||
contents.
|
||||
|
||||
The `sftp(1)`_ and `scp(1)`_ tools may cause undetected data corruption
|
||||
in the form of truncated files, or files containing intermingled data segments
|
||||
from overlapping runs. As part of normal operation, both tools expose a window
|
||||
where readers may observe inconsistent file contents.
|
||||
The :linux:man1:`sftp` and :linux:man1:`scp` tools may cause undetected data
|
||||
corruption in the form of truncated files, or files containing intermingled
|
||||
data segments from overlapping runs. As part of normal operation, both tools
|
||||
expose a window where readers may observe inconsistent file contents.
|
||||
|
||||
|
||||
Performance
|
||||
|
|
@ -499,11 +484,11 @@ Ansible may:
|
|||
* Create a directory owned by the SSH user either under ``remote_tmp``, or a
|
||||
system-default directory,
|
||||
* Upload action dependencies such as non-new style modules or rendered
|
||||
templates to that directory via `sftp(1)`_ or `scp(1)`_.
|
||||
templates to that directory via :linux:man1:`sftp` or :linux:man1:`scp`.
|
||||
* Attempt to modify the directory's access control list to grant access to the
|
||||
target user using `setfacl(1) <https://linux.die.net/man/1/setfacl>`_,
|
||||
requiring that tool to be installed and a supported filesystem to be in use,
|
||||
or for the ``allow_world_readable_tmpfiles`` setting to be :data:`True`.
|
||||
target user using :linux:man1:`setfacl`, requiring that tool to be installed
|
||||
and a supported filesystem to be in use, or for the
|
||||
``allow_world_readable_tmpfiles`` setting to be :data:`True`.
|
||||
* Create a directory owned by the target user either under ``remote_tmp``, or
|
||||
a system-default directory, if a new-style module needs a temporary directory
|
||||
and one was not previously created for a supporting file earlier in the
|
||||
|
|
@ -569,9 +554,9 @@ in regular Ansible:
|
|||
operations relating to modifying the directory to support cross-account
|
||||
access are avoided.
|
||||
|
||||
* An explicit work-around is included to avoid the `copy` and `template`
|
||||
actions needlessly triggering a round-trip to set their temporary file as
|
||||
executable.
|
||||
* An explicit work-around is included to avoid the :ans:mod:`~copy` and
|
||||
:ans:mod:`~template` actions needlessly triggering a round-trip to set their
|
||||
temporary file as executable.
|
||||
|
||||
* During task shutdown, it is not necessary to wait to learn if the target has
|
||||
succeeded in deleting a temporary directory, since any error that may occur
|
||||
|
|
@ -601,10 +586,10 @@ DNS Resolution
|
|||
^^^^^^^^^^^^^^
|
||||
|
||||
Modifications to ``/etc/resolv.conf`` cause the glibc resolver configuration to
|
||||
be reloaded via `res_init(3) <https://linux.die.net/man/3/res_init>`_. This
|
||||
isn't necessary on some Linux distributions carrying glibc patches to
|
||||
automatically check ``/etc/resolv.conf`` periodically, however it is necessary
|
||||
on at least Debian and BSD derivatives.
|
||||
be reloaded via :linux:man3:`res_init`. This isn't necessary on some Linux
|
||||
distributions carrying glibc patches to automatically check
|
||||
``/etc/resolv.conf`` periodically, however it is necessary on at least Debian
|
||||
and BSD derivatives.
|
||||
|
||||
|
||||
``/etc/environment``
|
||||
|
|
@ -728,9 +713,7 @@ configuration of each task.
|
|||
Buildah
|
||||
~~~~~~~
|
||||
|
||||
Like `buildah
|
||||
<https://docs.ansible.com/ansible/2.6/plugins/connection/buildah.html>`_ except
|
||||
connection delegation is supported.
|
||||
Like the :ans:conn:`buildah` except connection delegation is supported.
|
||||
|
||||
* ``ansible_host``: Name of Buildah container (default: inventory hostname).
|
||||
* ``ansible_user``: Name of user within the container to execute as.
|
||||
|
|
@ -771,9 +754,7 @@ When used as the ``mitogen_doas`` connection method:
|
|||
Docker
|
||||
~~~~~~
|
||||
|
||||
Like `docker
|
||||
<https://docs.ansible.com/ansible/2.6/plugins/connection/docker.html>`_ except
|
||||
connection delegation is supported.
|
||||
Like the :ans:conn:`docker` except connection delegation is supported.
|
||||
|
||||
* ``ansible_host``: Name of Docker container (default: inventory hostname).
|
||||
* ``ansible_user``: Name of user within the container to execute as.
|
||||
|
|
@ -789,9 +770,7 @@ connection delegation is supported.
|
|||
FreeBSD Jail
|
||||
~~~~~~~~~~~~
|
||||
|
||||
Like `jail
|
||||
<https://docs.ansible.com/ansible/2.6/plugins/connection/jail.html>`_ except
|
||||
connection delegation is supported.
|
||||
Like the :ans:conn:`jail` except connection delegation is supported.
|
||||
|
||||
* ``ansible_host``: Name of jail (default: inventory hostname).
|
||||
* ``ansible_user``: Name of user within the jail to execute as.
|
||||
|
|
@ -807,9 +786,7 @@ connection delegation is supported.
|
|||
Kubernetes Pod
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
Like `kubectl
|
||||
<https://docs.ansible.com/ansible/2.6/plugins/connection/kubectl.html>`_ except
|
||||
connection delegation is supported.
|
||||
Like the :ans:conn:`kubectl` except connection delegation is supported.
|
||||
|
||||
* ``ansible_host``: Name of pod (default: inventory hostname).
|
||||
* ``ansible_user``: Name of user to authenticate to API as.
|
||||
|
|
@ -823,9 +800,7 @@ connection delegation is supported.
|
|||
Local
|
||||
~~~~~
|
||||
|
||||
Like `local
|
||||
<https://docs.ansible.com/ansible/2.6/plugins/connection/local.html>`_ except
|
||||
connection delegation is supported.
|
||||
Like the :ans:conn:`local` except connection delegation is supported.
|
||||
|
||||
* ``ansible_python_interpreter``
|
||||
|
||||
|
|
@ -852,10 +827,9 @@ additional differences exist that may break existing playbooks.
|
|||
LXC
|
||||
~~~
|
||||
|
||||
Connect to classic LXC containers, like `lxc
|
||||
<https://docs.ansible.com/ansible/2.6/plugins/connection/lxc.html>`_ except
|
||||
connection delegation is supported, and ``lxc-attach`` is always used rather
|
||||
than the LXC Python bindings, as is usual with ``lxc``.
|
||||
Connect to classic LXC containers, like the :ans:conn:`lxc` except connection
|
||||
delegation is supported, and ``lxc-attach`` is always used rather than the LXC
|
||||
Python bindings, as is usual with ``lxc``.
|
||||
|
||||
* ``ansible_python_interpreter``
|
||||
* ``ansible_host``: Name of LXC container (default: inventory hostname).
|
||||
|
|
@ -873,10 +847,9 @@ than the LXC Python bindings, as is usual with ``lxc``.
|
|||
LXD
|
||||
~~~
|
||||
|
||||
Connect to modern LXD containers, like `lxd
|
||||
<https://docs.ansible.com/ansible/2.6/plugins/connection/lxd.html>`_ except
|
||||
connection delegation is supported. The ``lxc`` command must be available on
|
||||
the host machine.
|
||||
Connect to modern LXD containers, like the :ans:conn:`lxd` except connection
|
||||
delegation is supported. The ``lxc`` command must be available on the host
|
||||
machine.
|
||||
|
||||
* ``ansible_python_interpreter``
|
||||
* ``ansible_host``: Name of LXC container (default: inventory hostname).
|
||||
|
|
@ -1001,8 +974,7 @@ When used as the ``mitogen_sudo`` connection method:
|
|||
SSH
|
||||
~~~
|
||||
|
||||
Like `ssh <https://docs.ansible.com/ansible/2.6/plugins/connection/ssh.html>`_
|
||||
except connection delegation is supported.
|
||||
Like the :ans:conn:`ssh` except connection delegation is supported.
|
||||
|
||||
* ``ansible_ssh_timeout``
|
||||
* ``ansible_host``, ``ansible_ssh_host``
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
46
docs/conf.py
46
docs/conf.py
|
|
@ -2,13 +2,14 @@ import os
|
|||
import sys
|
||||
|
||||
sys.path.append('..')
|
||||
sys.path.append('.')
|
||||
import mitogen
|
||||
VERSION = '%s.%s.%s' % mitogen.__version__
|
||||
|
||||
author = u'Network Genomics'
|
||||
copyright = u'2019, Network Genomics'
|
||||
exclude_patterns = ['_build', '.venv']
|
||||
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput']
|
||||
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput', 'domainrefs']
|
||||
html_show_copyright = False
|
||||
html_show_sourcelink = False
|
||||
html_show_sphinx = False
|
||||
|
|
@ -36,6 +37,49 @@ templates_path = ['_templates']
|
|||
todo_include_todos = False
|
||||
version = VERSION
|
||||
|
||||
domainrefs = {
|
||||
'gh:commit': {
|
||||
'text': '%s',
|
||||
'url': 'https://github.com/dw/mitogen/commit/%s',
|
||||
},
|
||||
'gh:issue': {
|
||||
'text': '#%s',
|
||||
'url': 'https://github.com/dw/mitogen/issues/%s',
|
||||
},
|
||||
'gh:pull': {
|
||||
'text': '#%s',
|
||||
'url': 'https://github.com/dw/mitogen/pull/%s',
|
||||
},
|
||||
'ans:mod': {
|
||||
'text': '%s Module',
|
||||
'url': 'https://docs.ansible.com/ansible/latest/modules/%s_module.html',
|
||||
},
|
||||
'ans:conn': {
|
||||
'text': '%s Connection Plug-in',
|
||||
'url': 'https://docs.ansible.com/ansible/latest/plugins/connection/%s.html',
|
||||
},
|
||||
'freebsd:man2': {
|
||||
'text': '%s(2)',
|
||||
'url': 'https://www.freebsd.org/cgi/man.cgi?query=%s',
|
||||
},
|
||||
'linux:man1': {
|
||||
'text': '%s(1)',
|
||||
'url': 'http://man7.org/linux/man-pages/man1/%s.1.html',
|
||||
},
|
||||
'linux:man2': {
|
||||
'text': '%s(2)',
|
||||
'url': 'http://man7.org/linux/man-pages/man2/%s.2.html',
|
||||
},
|
||||
'linux:man3': {
|
||||
'text': '%s(3)',
|
||||
'url': 'http://man7.org/linux/man-pages/man3/%s.3.html',
|
||||
},
|
||||
'linux:man7': {
|
||||
'text': '%s(7)',
|
||||
'url': 'http://man7.org/linux/man-pages/man7/%s.7.html',
|
||||
},
|
||||
}
|
||||
|
||||
rst_epilog = """
|
||||
|
||||
.. |mitogen_version| replace:: %(VERSION)s
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
|
||||
import functools
|
||||
import re
|
||||
|
||||
import docutils.nodes
|
||||
import docutils.utils
|
||||
|
||||
|
||||
CUSTOM_RE = re.compile('(.*) <(.*)>')
|
||||
|
||||
|
||||
def role(config, role, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
template = 'https://docs.ansible.com/ansible/latest/modules/%s_module.html'
|
||||
|
||||
match = CUSTOM_RE.match(text)
|
||||
if match: # "custom text <real link>"
|
||||
title = match.group(1)
|
||||
text = match.group(2)
|
||||
elif text.startswith('~'): # brief
|
||||
text = text[1:]
|
||||
title = config.get('brief', '%s') % (
|
||||
docutils.utils.unescape(text),
|
||||
)
|
||||
else:
|
||||
title = config.get('text', '%s') % (
|
||||
docutils.utils.unescape(text),
|
||||
)
|
||||
|
||||
node = docutils.nodes.reference(
|
||||
rawsource=rawtext,
|
||||
text=title,
|
||||
refuri=config['url'] % (text,),
|
||||
**options
|
||||
)
|
||||
|
||||
return [node], []
|
||||
|
||||
|
||||
def setup(app):
|
||||
for name, info in app.config._raw_config['domainrefs'].items():
|
||||
app.add_role(name, functools.partial(role, info))
|
||||
|
|
@ -434,8 +434,9 @@ also listen on the following handles:
|
|||
|
||||
Receives `target_id` integer from downstream, verifies a route exists to
|
||||
`target_id` via the stream on which the message was received, removes that
|
||||
route from its local table, then propagates the message upward towards its
|
||||
own parent.
|
||||
route from its local table, triggers the ``disconnect`` signal on any
|
||||
:class:`mitogen.core.Context` instance in the local process, then
|
||||
propagates the message upward towards its own parent.
|
||||
|
||||
.. currentmodule:: mitogen.core
|
||||
.. data:: DETACHING
|
||||
|
|
@ -629,7 +630,8 @@ The `auth_id` field is separate from `src_id` in order to support granting
|
|||
privilege to contexts that do not follow the tree's natural trust chain. This
|
||||
supports cases where siblings are permitted to execute code on one another, or
|
||||
where isolated processes can connect to a listener and communicate with an
|
||||
already established established tree.
|
||||
already established established tree, such as where a :mod:`mitogen.unix`
|
||||
client receives the same privilege as the process it connects to.
|
||||
|
||||
|
||||
Differences Between Master And Child Brokers
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ and efficient low-level API on which tools like `Salt`_, `Ansible`_, or
|
|||
`Fabric`_, ultimately it is not intended for direct use by consumer software.
|
||||
|
||||
.. _Salt: https://docs.saltstack.com/en/latest/
|
||||
.. _Ansible: http://docs.ansible.com/
|
||||
.. _Fabric: http://www.fabfile.org/
|
||||
.. _Ansible: https://docs.ansible.com/
|
||||
.. _Fabric: https://www.fabfile.org/
|
||||
|
||||
The focus is to centralize and perfect the intricate dance required to run
|
||||
Python code safely and efficiently on a remote machine, while **avoiding
|
||||
|
|
@ -132,7 +132,7 @@ any tool such as `py2exe`_ that correctly implement the protocols in PEP-302,
|
|||
allowing truly single file applications to run across multiple machines without
|
||||
further effort.
|
||||
|
||||
.. _py2exe: http://www.py2exe.org/
|
||||
.. _py2exe: https://www.py2exe.org/
|
||||
|
||||
Common sources of import latency and bandwidth consumption are mitigated:
|
||||
|
||||
|
|
|
|||
|
|
@ -3600,6 +3600,8 @@ class Dispatcher(object):
|
|||
mode, any exception that occurs is recorded, and causes all subsequent
|
||||
calls with the same `chain_id` to fail with the same exception.
|
||||
"""
|
||||
_service_recv = None
|
||||
|
||||
def __repr__(self):
|
||||
return 'Dispatcher'
|
||||
|
||||
|
|
|
|||
|
|
@ -73,8 +73,8 @@ def reset_logging_framework():
|
|||
threads in the parent may have been using the logging package at the moment
|
||||
of fork.
|
||||
|
||||
It is not possible to solve this problem in general; see
|
||||
https://github.com/dw/mitogen/issues/150 for a full discussion.
|
||||
It is not possible to solve this problem in general; see :gh:issue:`150`
|
||||
for a full discussion.
|
||||
"""
|
||||
logging._lock = threading.RLock()
|
||||
|
||||
|
|
|
|||
|
|
@ -535,18 +535,20 @@ class SysModulesMethod(FinderMethod):
|
|||
Find `fullname` using its :data:`__file__` attribute.
|
||||
"""
|
||||
module = sys.modules.get(fullname)
|
||||
LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module)
|
||||
if getattr(module, '__name__', None) != fullname:
|
||||
LOG.debug('sys.modules[%r].__name__ does not match %r, assuming '
|
||||
'this is a hacky module alias and ignoring it',
|
||||
fullname, fullname)
|
||||
return
|
||||
|
||||
if not isinstance(module, types.ModuleType):
|
||||
LOG.debug('%r: sys.modules[%r] absent or not a regular module',
|
||||
self, fullname)
|
||||
return
|
||||
|
||||
LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module)
|
||||
alleged_name = getattr(module, '__name__', None)
|
||||
if alleged_name != fullname:
|
||||
LOG.debug('sys.modules[%r].__name__ is incorrect, assuming '
|
||||
'this is a hacky module alias and ignoring it. '
|
||||
'Got %r, module object: %r',
|
||||
fullname, alleged_name, module)
|
||||
return
|
||||
|
||||
path = _py_filename(getattr(module, '__file__', ''))
|
||||
if not path:
|
||||
return
|
||||
|
|
@ -573,43 +575,57 @@ class SysModulesMethod(FinderMethod):
|
|||
class ParentEnumerationMethod(FinderMethod):
|
||||
"""
|
||||
Attempt to fetch source code by examining the module's (hopefully less
|
||||
insane) parent package. Required for older versions of
|
||||
:mod:`ansible.compat.six`, :mod:`plumbum.colors`, and Ansible 2.8
|
||||
:mod:`ansible.module_utils.distro`.
|
||||
insane) parent package, and if no insane parents exist, simply use
|
||||
:mod:`sys.path` to search for it from scratch on the filesystem using the
|
||||
normal Python lookup mechanism.
|
||||
|
||||
This is required for older versions of :mod:`ansible.compat.six`,
|
||||
:mod:`plumbum.colors`, Ansible 2.8 :mod:`ansible.module_utils.distro` and
|
||||
its submodule :mod:`ansible.module_utils.distro._distro`.
|
||||
|
||||
When some package dynamically replaces itself in :data:`sys.modules`, but
|
||||
only conditionally according to some program logic, it is possible that
|
||||
children may attempt to load modules and subpackages from it that can no
|
||||
longer be resolved by examining a (corrupted) parent.
|
||||
|
||||
For cases like :mod:`ansible.module_utils.distro`, this must handle cases
|
||||
where a package transmuted itself into a totally unrelated module during
|
||||
import and vice versa.
|
||||
import and vice versa, where :data:`sys.modules` is replaced with junk that
|
||||
makes it impossible to discover the loaded module using the in-memory
|
||||
module object or any parent package's :data:`__path__`, since they have all
|
||||
been overwritten. Some men just want to watch the world burn.
|
||||
"""
|
||||
def find(self, fullname):
|
||||
def _find_sane_parent(self, fullname):
|
||||
"""
|
||||
See implementation for a description of how this works.
|
||||
Iteratively search :data:`sys.modules` for the least indirect parent of
|
||||
`fullname` that is loaded and contains a :data:`__path__` attribute.
|
||||
|
||||
:return:
|
||||
`(parent_name, path, modpath)` tuple, where:
|
||||
|
||||
* `modname`: canonical name of the found package, or the empty
|
||||
string if none is found.
|
||||
* `search_path`: :data:`__path__` attribute of the least
|
||||
indirect parent found, or :data:`None` if no indirect parent
|
||||
was found.
|
||||
* `modpath`: list of module name components leading from `path`
|
||||
to the target module.
|
||||
"""
|
||||
if fullname not in sys.modules:
|
||||
# Don't attempt this unless a module really exists in sys.modules,
|
||||
# else we could return junk.
|
||||
return
|
||||
path = None
|
||||
modpath = []
|
||||
while True:
|
||||
pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
|
||||
modpath.insert(0, modname)
|
||||
if not pkgname:
|
||||
return [], None, modpath
|
||||
|
||||
pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
|
||||
pkg = sys.modules.get(pkgname)
|
||||
if pkg is None or not hasattr(pkg, '__file__'):
|
||||
LOG.debug('%r: %r is not a package or lacks __file__ attribute',
|
||||
self, pkgname)
|
||||
return
|
||||
pkg = sys.modules.get(pkgname)
|
||||
path = getattr(pkg, '__path__', None)
|
||||
if pkg and path:
|
||||
return pkgname.split('.'), path, modpath
|
||||
|
||||
pkg_path = [os.path.dirname(pkg.__file__)]
|
||||
try:
|
||||
fp, path, (suffix, _, kind) = imp.find_module(modname, pkg_path)
|
||||
except ImportError:
|
||||
e = sys.exc_info()[1]
|
||||
LOG.debug('%r: imp.find_module(%r, %r) -> %s',
|
||||
self, modname, [pkg_path], e)
|
||||
return None
|
||||
|
||||
if kind == imp.PKG_DIRECTORY:
|
||||
return self._found_package(fullname, path)
|
||||
else:
|
||||
return self._found_module(fullname, path, fp)
|
||||
LOG.debug('%r: %r lacks __path__ attribute', self, pkgname)
|
||||
fullname = pkgname
|
||||
|
||||
def _found_package(self, fullname, path):
|
||||
path = os.path.join(path, '__init__.py')
|
||||
|
|
@ -638,6 +654,47 @@ class ParentEnumerationMethod(FinderMethod):
|
|||
source = source.encode('utf-8')
|
||||
return path, source, is_pkg
|
||||
|
||||
def _find_one_component(self, modname, search_path):
|
||||
try:
|
||||
#fp, path, (suffix, _, kind) = imp.find_module(modname, search_path)
|
||||
return imp.find_module(modname, search_path)
|
||||
except ImportError:
|
||||
e = sys.exc_info()[1]
|
||||
LOG.debug('%r: imp.find_module(%r, %r) -> %s',
|
||||
self, modname, [search_path], e)
|
||||
return None
|
||||
|
||||
def find(self, fullname):
|
||||
"""
|
||||
See implementation for a description of how this works.
|
||||
"""
|
||||
#if fullname not in sys.modules:
|
||||
# Don't attempt this unless a module really exists in sys.modules,
|
||||
# else we could return junk.
|
||||
#return
|
||||
|
||||
fullname = to_text(fullname)
|
||||
modname, search_path, modpath = self._find_sane_parent(fullname)
|
||||
while True:
|
||||
tup = self._find_one_component(modpath.pop(0), search_path)
|
||||
if tup is None:
|
||||
return None
|
||||
|
||||
fp, path, (suffix, _, kind) = tup
|
||||
if modpath:
|
||||
# Still more components to descent. Result must be a package
|
||||
if fp:
|
||||
fp.close()
|
||||
if kind != imp.PKG_DIRECTORY:
|
||||
LOG.debug('%r: %r appears to be child of non-package %r',
|
||||
self, fullname, path)
|
||||
return None
|
||||
search_path = [path]
|
||||
elif kind == imp.PKG_DIRECTORY:
|
||||
return self._found_package(fullname, path)
|
||||
else:
|
||||
return self._found_module(fullname, path, fp)
|
||||
|
||||
|
||||
class ModuleFinder(object):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -901,9 +901,9 @@ class CallSpec(object):
|
|||
|
||||
class PollPoller(mitogen.core.Poller):
|
||||
"""
|
||||
Poller based on the POSIX poll(2) interface. Not available on some versions
|
||||
of OS X, otherwise it is the preferred poller for small FD counts, as there
|
||||
is no setup/teardown/configuration system call overhead.
|
||||
Poller based on the POSIX :linux:man2:`poll` interface. Not available on
|
||||
some versions of OS X, otherwise it is the preferred poller for small FD
|
||||
counts, as there is no setup/teardown/configuration system call overhead.
|
||||
"""
|
||||
SUPPORTED = hasattr(select, 'poll')
|
||||
_repr = 'PollPoller()'
|
||||
|
|
@ -949,7 +949,7 @@ class PollPoller(mitogen.core.Poller):
|
|||
|
||||
class KqueuePoller(mitogen.core.Poller):
|
||||
"""
|
||||
Poller based on the FreeBSD/Darwin kqueue(2) interface.
|
||||
Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface.
|
||||
"""
|
||||
SUPPORTED = hasattr(select, 'kqueue')
|
||||
_repr = 'KqueuePoller()'
|
||||
|
|
@ -1027,7 +1027,7 @@ class KqueuePoller(mitogen.core.Poller):
|
|||
|
||||
class EpollPoller(mitogen.core.Poller):
|
||||
"""
|
||||
Poller based on the Linux epoll(2) interface.
|
||||
Poller based on the Linux :linux:man2:`epoll` interface.
|
||||
"""
|
||||
SUPPORTED = hasattr(select, 'epoll')
|
||||
_repr = 'EpollPoller()'
|
||||
|
|
|
|||
|
|
@ -55,7 +55,6 @@ except NameError:
|
|||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_POOL_SIZE = 16
|
||||
_pool = None
|
||||
_pool_pid = None
|
||||
#: Serialize pool construction.
|
||||
|
|
@ -80,7 +79,7 @@ def get_or_create_pool(size=None, router=None):
|
|||
global _pool_pid
|
||||
|
||||
my_pid = os.getpid()
|
||||
if _pool is None or my_pid != _pool_pid:
|
||||
if _pool is None or _pool.closed or my_pid != _pool_pid:
|
||||
# Avoid acquiring heavily contended lock if possible.
|
||||
_pool_lock.acquire()
|
||||
try:
|
||||
|
|
@ -88,7 +87,7 @@ def get_or_create_pool(size=None, router=None):
|
|||
_pool = Pool(
|
||||
router,
|
||||
services=[],
|
||||
size=size or DEFAULT_POOL_SIZE,
|
||||
size=size or 2,
|
||||
overwrite=True,
|
||||
recv=mitogen.core.Dispatcher._service_recv,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
# #590: a subpackage that turns itself into a module from elsewhere on sys.path.
|
||||
I_AM = "the subpackage that was replaced with a system module"
|
||||
import sys
|
||||
import system_distro
|
||||
sys.modules[__name__] = system_distro
|
||||
|
|
@ -0,0 +1 @@
|
|||
I_AM = "the module inside the replaced subpackage"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
# #590: a system module that replaces some subpackage
|
||||
I_AM = "the system module that replaced the subpackage"
|
||||
|
|
@ -180,6 +180,7 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase):
|
|||
'pkg_like_ansible.module_utils.distro._distro'
|
||||
)
|
||||
|
||||
# ensure we can resolve the subpackage.
|
||||
path, src, is_pkg = self.call('pkg_like_ansible.module_utils.distro')
|
||||
modpath = os.path.join(MODS_DIR,
|
||||
'pkg_like_ansible/module_utils/distro/__init__.py')
|
||||
|
|
@ -187,6 +188,44 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase):
|
|||
self.assertEquals(src, open(modpath, 'rb').read())
|
||||
self.assertEquals(is_pkg, True)
|
||||
|
||||
# ensure we can resolve a child of the subpackage.
|
||||
path, src, is_pkg = self.call(
|
||||
'pkg_like_ansible.module_utils.distro._distro'
|
||||
)
|
||||
modpath = os.path.join(MODS_DIR,
|
||||
'pkg_like_ansible/module_utils/distro/_distro.py')
|
||||
self.assertEquals(path, modpath)
|
||||
self.assertEquals(src, open(modpath, 'rb').read())
|
||||
self.assertEquals(is_pkg, False)
|
||||
|
||||
def test_ansible_module_utils_system_distro_succeeds(self):
|
||||
# #590: a package that turns itself into a module.
|
||||
# #590: a package that turns itself into a module.
|
||||
import pkg_like_ansible.module_utils.sys_distro as d
|
||||
self.assertEquals(d.I_AM, "the system module that replaced the subpackage")
|
||||
self.assertEquals(
|
||||
sys.modules['pkg_like_ansible.module_utils.sys_distro'].__name__,
|
||||
'system_distro'
|
||||
)
|
||||
|
||||
# ensure we can resolve the subpackage.
|
||||
path, src, is_pkg = self.call('pkg_like_ansible.module_utils.sys_distro')
|
||||
modpath = os.path.join(MODS_DIR,
|
||||
'pkg_like_ansible/module_utils/sys_distro/__init__.py')
|
||||
self.assertEquals(path, modpath)
|
||||
self.assertEquals(src, open(modpath, 'rb').read())
|
||||
self.assertEquals(is_pkg, True)
|
||||
|
||||
# ensure we can resolve a child of the subpackage.
|
||||
path, src, is_pkg = self.call(
|
||||
'pkg_like_ansible.module_utils.sys_distro._distro'
|
||||
)
|
||||
modpath = os.path.join(MODS_DIR,
|
||||
'pkg_like_ansible/module_utils/sys_distro/_distro.py')
|
||||
self.assertEquals(path, modpath)
|
||||
self.assertEquals(src, open(modpath, 'rb').read())
|
||||
self.assertEquals(is_pkg, False)
|
||||
|
||||
|
||||
class ResolveRelPathTest(testlib.TestCase):
|
||||
klass = mitogen.master.ModuleFinder
|
||||
|
|
|
|||
|
|
@ -158,14 +158,15 @@ class BrokenModulesTest(testlib.TestCase):
|
|||
self.assertEquals(1, len(router._async_route.mock_calls))
|
||||
|
||||
self.assertEquals(1, responder.get_module_count)
|
||||
self.assertEquals(0, responder.good_load_module_count)
|
||||
self.assertEquals(0, responder.good_load_module_size)
|
||||
self.assertEquals(1, responder.bad_load_module_count)
|
||||
self.assertEquals(1, responder.good_load_module_count)
|
||||
self.assertEquals(0, responder.bad_load_module_count)
|
||||
|
||||
call = router._async_route.mock_calls[0]
|
||||
msg, = call[1]
|
||||
self.assertEquals(mitogen.core.LOAD_MODULE, msg.handle)
|
||||
self.assertIsInstance(msg.unpickle(), tuple)
|
||||
|
||||
tup = msg.unpickle()
|
||||
self.assertIsInstance(tup, tuple)
|
||||
|
||||
|
||||
class ForwardTest(testlib.RouterMixin, testlib.TestCase):
|
||||
|
|
|
|||
|
|
@ -15,6 +15,13 @@ class MyService(mitogen.service.Service):
|
|||
self._counter += 1
|
||||
return self._counter, id(self)
|
||||
|
||||
@mitogen.service.expose(policy=mitogen.service.AllowParents())
|
||||
@mitogen.service.arg_spec({
|
||||
'foo': int
|
||||
})
|
||||
def test_arg_spec(self, foo):
|
||||
return foo
|
||||
|
||||
@mitogen.service.expose(policy=mitogen.service.AllowParents())
|
||||
def privileged_op(self):
|
||||
return 'privileged!'
|
||||
|
|
@ -24,7 +31,6 @@ class MyService(mitogen.service.Service):
|
|||
return 'unprivileged!'
|
||||
|
||||
|
||||
|
||||
class MyService2(MyService):
|
||||
"""
|
||||
A uniquely named service that lets us test framework activation and class
|
||||
|
|
@ -36,6 +42,44 @@ def call_service_in(context, service_name, method_name):
|
|||
return context.call_service(service_name, method_name)
|
||||
|
||||
|
||||
class CallTest(testlib.RouterMixin, testlib.TestCase):
|
||||
def test_local(self):
|
||||
pool = mitogen.service.get_or_create_pool(router=self.router)
|
||||
self.assertEquals(
|
||||
'privileged!',
|
||||
mitogen.service.call(MyService, 'privileged_op')
|
||||
)
|
||||
pool.stop()
|
||||
|
||||
def test_remote_bad_arg(self):
|
||||
c1 = self.router.local()
|
||||
self.assertRaises(
|
||||
mitogen.core.CallError,
|
||||
lambda: mitogen.service.call(
|
||||
MyService.name(),
|
||||
'test_arg_spec',
|
||||
foo='x',
|
||||
call_context=c1
|
||||
)
|
||||
)
|
||||
|
||||
def test_local_unicode(self):
|
||||
pool = mitogen.service.get_or_create_pool(router=self.router)
|
||||
self.assertEquals(
|
||||
'privileged!',
|
||||
mitogen.service.call(MyService.name(), 'privileged_op')
|
||||
)
|
||||
pool.stop()
|
||||
|
||||
def test_remote(self):
|
||||
c1 = self.router.local()
|
||||
self.assertEquals(
|
||||
'privileged!',
|
||||
mitogen.service.call(MyService, 'privileged_op',
|
||||
call_context=c1)
|
||||
)
|
||||
|
||||
|
||||
class ActivationTest(testlib.RouterMixin, testlib.TestCase):
|
||||
def test_parent_can_activate(self):
|
||||
l1 = self.router.local()
|
||||
|
|
|
|||
Loading…
Reference in New Issue