add support for ansible_collections site-package (from pip ansible==2.10.0 install) + switch to ansible 2.10.0 rather than github tag

.ci/azure-pipelines.yml

@@ -3,10 +3,6 @@
 # Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
 # https://docs.microsoft.com/azure/devops/pipelines/languages/python
 
-# pointing to ansible 2.10 tag on github; ansible changed to ansible-base on pypi
-# so to avoid complicating things for now, use git hash
-# TODO: point to ansible==2.10 when it comes out so we don't need to install collections separately
-
 jobs:
 
 - job: Mac
@@ -19,12 +15,12 @@ jobs:
       Mito27_27:
         python.version: '2.7'
         MODE: mitogen
-        VER: git+https://github.com/ansible/ansible.git@v2.10.0
+        VER: 2.10.0
       # TODO: test python3, python3 tests are broken
       Ans210_27:
         python.version: '2.7'
         MODE: localhost_ansible
-        VER: git+https://github.com/ansible/ansible.git@v2.10.0
+        VER: 2.10.0
 
 
 - job: Linux
@@ -41,7 +37,7 @@ jobs:
         python.version: '2.7'
         MODE: mitogen
         DISTRO: debian
-        VER: git+https://github.com/ansible/ansible.git@v2.10.0
+        VER: 2.10.0
 
       #MitoPy27CentOS6_26:
         #python.version: '2.7'
@@ -52,13 +48,13 @@ jobs:
         python.version: '3.6'
         MODE: mitogen
         DISTRO: centos6
-        VER: git+https://github.com/ansible/ansible.git@v2.10.0
+        VER: 2.10.0
 
       Mito37Debian_27:
         python.version: '3.7'
         MODE: mitogen
         DISTRO: debian
-        VER: git+https://github.com/ansible/ansible.git@v2.10.0
+        VER: 2.10.0
 
       #Py26CentOS7:
         #python.version: '2.7'
@@ -105,9 +101,9 @@ jobs:
       Ansible_210_27:
         python.version: '2.7'
         MODE: ansible
-        VER: git+https://github.com/ansible/ansible.git@v2.10.0
+        VER: 2.10.0
 
       Ansible_210_35:
         python.version: '3.5'
         MODE: ansible
-        VER: git+https://github.com/ansible/ansible.git@v2.10.0
+        VER: 2.10.0

.travis.yml

@@ -34,45 +34,45 @@ matrix:
     # Debops tests.
     # 2.10; 3.6 -> 2.7
     - python: "3.6"
-      env: MODE=debops_common VER=git+https://github.com/ansible/ansible.git@v2.10.0
+      env: MODE=debops_common VER=2.10.0
     # 2.10; 2.7 -> 2.7
     - python: "2.7"
-      env: MODE=debops_common VER=git+https://github.com/ansible/ansible.git@v2.10.0
+      env: MODE=debops_common VER=2.10.0
 
     # Sanity check against vanilla Ansible. One job suffices.
     - python: "2.7"
-      env: MODE=ansible VER=git+https://github.com/ansible/ansible.git@v2.10.0 DISTROS=debian STRATEGY=linear
+      env: MODE=ansible VER=2.10.0 DISTROS=debian STRATEGY=linear
 
     # ansible_mitogen tests.
 
     # 2.10 -> {debian, centos6, centos7}
     - python: "3.6"
-      env: MODE=ansible VER=git+https://github.com/ansible/ansible.git@v2.10.0
+      env: MODE=ansible VER=2.10.0
     # 2.10 -> {debian, centos6, centos7}
     - python: "2.7"
-      env: MODE=ansible VER=git+https://github.com/ansible/ansible.git@v2.10.0
+      env: MODE=ansible VER=2.10.0
     # 2.10 -> {debian, centos6, centos7}
     - python: "2.6"
-      env: MODE=ansible VER=git+https://github.com/ansible/ansible.git@v2.10.0
+      env: MODE=ansible VER=2.10.0
 
     # 2.10 -> {centos5}
     - python: "2.6"
-      env: MODE=ansible DISTROS=centos5 VER=git+https://github.com/ansible/ansible.git@v2.10.0
+      env: MODE=ansible DISTROS=centos5 VER=2.10.0
 
     # Mitogen tests.
     # 2.4 -> 2.4
     - language: c
-      env: MODE=mitogen_py24 DISTROS=centos5 VER=git+https://github.com/ansible/ansible.git@v2.10.0
+      env: MODE=mitogen_py24 DISTROS=centos5 VER=2.10.0
     # 2.7 -> 2.7 -- moved to Azure
     # 2.7 -> 2.6
     #- python: "2.7"
       #env: MODE=mitogen DISTRO=centos6
     - python: "3.6"
-      env: MODE=mitogen DISTROS=centos7 VER=git+https://github.com/ansible/ansible.git@v2.10.0
+      env: MODE=mitogen DISTROS=centos7 VER=2.10.0
     # 2.6 -> 2.7
     - python: "2.6"
-      env: MODE=mitogen DISTROS=centos7 VER=git+https://github.com/ansible/ansible.git@v2.10.0
+      env: MODE=mitogen DISTROS=centos7 VER=2.10.0
     # 2.6 -> 3.5
     - python: "2.6"
-      env: MODE=mitogen DISTROS=debian-py3 VER=git+https://github.com/ansible/ansible.git@v2.10.0
+      env: MODE=mitogen DISTROS=debian-py3 VER=2.10.0
     # 3.6 -> 2.6 -- moved to Azure

mitogen/master.py

@@ -90,10 +90,10 @@ RLOG = logging.getLogger('mitogen.ctx')
 
 
 # there are some cases where modules are loaded in memory only, such as
-# ansible collections, and the module "filename" is something like __synthetic__
+# ansible collections, and the module "filename" doesn't actually exist
-# which doesn't actually exist
 SPECIAL_FILE_PATHS = {
     "__synthetic__",
+    "<ansible_synthetic_collection_package>"
 }
 
 
@@ -146,7 +146,7 @@ def is_stdlib_path(path):
     )
 
 
-def get_child_modules(path):
+def get_child_modules(path, fullname):
     """
     Return the suffixes of submodules directly neated beneath of the package
     directory at `path`.
@@ -155,12 +155,19 @@ def get_child_modules(path):
         Path to the module's source code on disk, or some PEP-302-recognized
         equivalent. Usually this is the module's ``__file__`` attribute, but
         is specified explicitly to avoid loading the module.
+    :param str fullname:
+        Name of the package we're trying to get child modules for
 
     :return:
         List of submodule name suffixes.
     """
-    it = pkgutil.iter_modules([os.path.dirname(path)])
+    mod_path = os.path.dirname(path)
-    return [to_text(name) for _, name, _ in it]
+    if mod_path != '':
+        return [to_text(name) for _, name, _ in pkgutil.iter_modules([mod_path])]
+    else:
+        # we loaded some weird package in memory, so we'll see if it has a custom loader we can use
+        loader = pkgutil.find_loader(fullname)
+        return [to_text(name) for name, _ in loader.iter_modules(None)] if loader else []
 
 
 def _looks_like_script(path):
@@ -993,7 +1000,7 @@ class ModuleResponder(object):
             self.minify_secs += mitogen.core.now() - t0
 
         if is_pkg:
-            pkg_present = get_child_modules(path)
+            pkg_present = get_child_modules(path, fullname)
             self._log.debug('%s is a package at %s with submodules %r',
                             fullname, path, pkg_present)
         else: