From db386a03503adb1ca992c5cffdf74ba1f183b393 Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Thu, 26 Mar 2020 22:08:17 -0700 Subject: [PATCH 01/15] enable ansible 2.9.6 tests --- .ci/azure-pipelines.yml | 11 +++++++++++ .travis.yml | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 66cc9356..a537b0f5 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -46,6 +46,12 @@ jobs: MODE: mitogen DISTRO: centos6 + Mito37Debian_27: + python.version: '3.7' + MODE: mitogen + DISTRO: debian + VER: 2.9.6 + #Py26CentOS7: #python.version: '2.7' #MODE: mitogen @@ -97,3 +103,8 @@ jobs: python.version: '3.5' MODE: ansible VER: 2.8.0 + + Ansible_296_37: + python.version: '3.7' + MODE: ansible + VER: 2.9.6 diff --git a/.travis.yml b/.travis.yml index 580ced0b..8e9acd21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,9 @@ matrix: include: # Debops tests. + # 2.9.6; 3.7 -> 2.7 + - python: "3.7" + env: MODE=debops_common VER=2.9.6 # 2.8.3; 3.6 -> 2.7 - python: "3.6" env: MODE=debops_common VER=2.8.3 @@ -49,6 +52,9 @@ matrix: # ansible_mitogen tests. + # 2.9.6 -> {debian, centos6, centos7} + - python: "3.7" + env: MODE=ansible VER=2.9.6 # 2.8.3 -> {debian, centos6, centos7} - python: "3.6" env: MODE=ansible VER=2.8.3 @@ -75,6 +81,8 @@ matrix: # 2.7 -> 2.6 #- python: "2.7" #env: MODE=mitogen DISTRO=centos6 + - python: "3.7" + env: MODE=mitogen DISTRO=centos7 # 2.6 -> 2.7 - python: "2.6" env: MODE=mitogen DISTRO=centos7 From 530f5e64127c532e7b3da3de26f29f00c9bb85b2 Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Fri, 24 Apr 2020 14:01:44 -0700 Subject: [PATCH 02/15] read in ansible installation version always --- .ci/ansible_install.py | 3 +++ .ci/localhost_ansible_install.py | 3 +++ tests/ansible/requirements.txt | 2 -- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index 86e57096..4e03c003 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -13,6 +13,9 @@ batches = [ ] ] +# separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml +batches.append("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) + batches.extend( ['docker pull %s' % (ci_lib.image_for_distro(distro),)] for distro in ci_lib.DISTROS diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py index 0cb47374..e3dd207e 100755 --- a/.ci/localhost_ansible_install.py +++ b/.ci/localhost_ansible_install.py @@ -13,4 +13,7 @@ batches = [ ] ] +# separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml +batches.append("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) + ci_lib.run_batches(batches) diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt index 3fd6a0ca..c0386cd8 100644 --- a/tests/ansible/requirements.txt +++ b/tests/ansible/requirements.txt @@ -1,5 +1,3 @@ -ansible==2.8.8; python_version >= '2.7' -ansible<2.7; python_version < '2.7' paramiko==2.3.2 # Last 2.6-compat version. hdrhistogram==0.6.1 PyYAML==3.11; python_version < '2.7' From e0666edaacd938ceedd6ad74e8bc28200ab7997b Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Fri, 24 Apr 2020 14:10:00 -0700 Subject: [PATCH 03/15] fix formatting --- .ci/ansible_install.py | 5 ++--- .ci/localhost_ansible_install.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index 4e03c003..4f9d5dff 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -6,16 +6,15 @@ batches = [ [ # Must be installed separately, as PyNACL indirect requirement causes # newer version to be installed if done in a single pip run. + # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml 'pip install "pycparser<2.19" "idna<2.7"', 'pip install ' '-r tests/requirements.txt ' '-r tests/ansible/requirements.txt', + 'pip install -q ansible=={}'.format(ci_lib.ANSIBLE_VERSION) ] ] -# separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml -batches.append("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) - batches.extend( ['docker pull %s' % (ci_lib.image_for_distro(distro),)] for distro in ci_lib.DISTROS diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py index e3dd207e..f8a1dd17 100755 --- a/.ci/localhost_ansible_install.py +++ b/.ci/localhost_ansible_install.py @@ -6,14 +6,13 @@ batches = [ [ # Must be installed separately, as PyNACL indirect requirement causes # newer version to be installed if done in a single pip run. + # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml 'pip install "pycparser<2.19" "idna<2.7"', 'pip install ' '-r tests/requirements.txt ' '-r tests/ansible/requirements.txt', + 'pip install -q ansible=={}'.format(ci_lib.ANSIBLE_VERSION) ] ] -# separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml -batches.append("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) - ci_lib.run_batches(batches) From 070fde16f3b51ad5def68e996e203caaa01c648e Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Fri, 24 Apr 2020 14:22:16 -0700 Subject: [PATCH 04/15] upgrade to ubuntu bionic for python3.7 support --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8e9acd21..79e78dcf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -dist: trusty +dist: bionic notifications: email: false From 6bdcbff7b2284d783ef4ede85cb656d8f35fabf1 Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Fri, 24 Apr 2020 16:12:58 -0700 Subject: [PATCH 05/15] upgrade to ubuntu 18 might fix keyserver thing --- .ci/ansible_tests.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 017107d4..4df2dc70 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -66,10 +66,6 @@ with ci_lib.Fold('job_setup'): ci_lib.dump_file(inventory_path) if not ci_lib.exists_in_path('sshpass'): - # fix errors with apt-get update - run("sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 78BD65473CB3BD13") - run("sudo sed -i -e 's#deb https://downloads.apache.org/cassandra/debian 39x main#deb http://downloads.apache.org/cassandra/debian 39x main#g' /etc/apt/sources.list.d/cassandra.list") - run("sudo apt-get update") run("sudo apt-get install -y sshpass") From 54f1b9ca398d4888fea3520fd10a51db7141b99e Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Thu, 30 Apr 2020 12:56:02 -0700 Subject: [PATCH 06/15] bionic doesn't have python2.6... sticking with trusty and downgrading to python3.6 --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 79e78dcf..901d49df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -dist: bionic +dist: trusty notifications: email: false @@ -37,7 +37,7 @@ matrix: include: # Debops tests. # 2.9.6; 3.7 -> 2.7 - - python: "3.7" + - python: "3.6" env: MODE=debops_common VER=2.9.6 # 2.8.3; 3.6 -> 2.7 - python: "3.6" @@ -53,7 +53,7 @@ matrix: # ansible_mitogen tests. # 2.9.6 -> {debian, centos6, centos7} - - python: "3.7" + - python: "3.6" env: MODE=ansible VER=2.9.6 # 2.8.3 -> {debian, centos6, centos7} - python: "3.6" @@ -81,7 +81,7 @@ matrix: # 2.7 -> 2.6 #- python: "2.7" #env: MODE=mitogen DISTRO=centos6 - - python: "3.7" + - python: "3.6" env: MODE=mitogen DISTRO=centos7 # 2.6 -> 2.7 - python: "2.6" From 573a3c6161b90b212cd06507da1ceb7616575235 Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Thu, 30 Apr 2020 13:31:11 -0700 Subject: [PATCH 07/15] have to number format option for python2.6 --- .ci/ansible_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index 4f9d5dff..ff2d3b63 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -11,7 +11,7 @@ batches = [ 'pip install ' '-r tests/requirements.txt ' '-r tests/ansible/requirements.txt', - 'pip install -q ansible=={}'.format(ci_lib.ANSIBLE_VERSION) + 'pip install -q ansible=={0}'.format(ci_lib.ANSIBLE_VERSION) ] ] From 9b1b5c5fb6c3037717d65c5e0e62b688cf38f7cc Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Thu, 30 Apr 2020 14:22:20 -0700 Subject: [PATCH 08/15] fix comment --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 901d49df..877f9ca3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ matrix: include: # Debops tests. - # 2.9.6; 3.7 -> 2.7 + # 2.9.6; 3.6 -> 2.7 - python: "3.6" env: MODE=debops_common VER=2.9.6 # 2.8.3; 3.6 -> 2.7 From c12e3d0f92a32b6fcfa2725ee93bc8cee2f121d8 Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Thu, 30 Apr 2020 14:49:12 -0700 Subject: [PATCH 09/15] disable flaky test on py3.6 --- tests/log_handler_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/log_handler_test.py b/tests/log_handler_test.py index a2e153ed..8f4d9dd5 100644 --- a/tests/log_handler_test.py +++ b/tests/log_handler_test.py @@ -87,8 +87,8 @@ class StartupTest(testlib.RouterMixin, testlib.TestCase): self.assertTrue(expect in logs) StartupTest = unittest2.skipIf( - condition=sys.version_info < (2, 7), - reason="Message log flaky on Python < 2.7" + condition=sys.version_info < (2, 7) or sys.version_info >= (3, 6), + reason="Message log flaky on Python < 2.7 or >= 3.6" )(StartupTest) From acde13f9d63a329a6929ad07487ba5791df2213c Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Thu, 30 Apr 2020 17:53:06 -0700 Subject: [PATCH 10/15] handles a 'wait_for_connection' call right after a task caused a shutdown --- ansible_mitogen/mixins.py | 7 ++ ansible_mitogen/services.py | 6 ++ mitogen/service.py | 10 ++- tests/ansible/regression/all.yml | 1 + .../issue_655__wait_for_connection_error.yml | 85 +++++++++++++++++++ 5 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 tests/ansible/regression/issue_655__wait_for_connection_error.yml diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 50ebfabe..7672618d 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -371,6 +371,13 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): self._compute_environment_string(env) self._set_temp_file_args(module_args, wrap_async) + # there's a case where if a task shuts down the node and then immediately calls + # wait_for_connection, the `ping` test from Ansible won't pass because we lost connection + # clearing out context forces a reconnect + # see https://github.com/dw/mitogen/issues/655 and Ansible's `wait_for_connection` module for more info + if module_name == 'ping' and type(self).__name__ == 'wait_for_connection': + self._connection.context = None + self._connection._connect() result = ansible_mitogen.planner.invoke( ansible_mitogen.planner.Invocation( diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 52171903..2eb3b2e4 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -170,6 +170,12 @@ class ContextService(mitogen.service.Service): """ LOG.debug('%r.reset(%r)', self, stack) + # this could happen if we have a `shutdown -r` shell command + # and then a `wait_for_connection` right afterwards + # in this case, we have no stack to disconnect from + if not stack: + return False + l = mitogen.core.Latch() context = None with self._lock: diff --git a/mitogen/service.py b/mitogen/service.py index 6bd64eb0..3b244414 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -74,7 +74,7 @@ else: @mitogen.core.takes_router -def get_or_create_pool(size=None, router=None): +def get_or_create_pool(size=None, router=None, context=None): global _pool global _pool_pid @@ -84,6 +84,12 @@ def get_or_create_pool(size=None, router=None): _pool_lock.acquire() try: if _pool_pid != my_pid: + if router is None: + # fallback to trying to get router from context if that exists + if context is not None: + router = context.router + else: + raise ValueError("Unable to create Pool! Missing router.") _pool = Pool( router, services=[], @@ -119,7 +125,7 @@ def call(service_name, method_name, call_context=None, **kwargs): if call_context: return call_context.call_service(service_name, method_name, **kwargs) else: - pool = get_or_create_pool() + pool = get_or_create_pool(context=kwargs.get('context')) invoker = pool.get_invoker(service_name, msg=None) return getattr(invoker.service, method_name)(**kwargs) diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index 81780bb3..0d5e43cd 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -12,3 +12,4 @@ - include: issue_590__sys_modules_crap.yml - include: issue_591__setuptools_cwd_crash.yml - include: issue_615__streaming_transfer.yml +- include: issue_655__wait_for_connection_error.yml diff --git a/tests/ansible/regression/issue_655__wait_for_connection_error.yml b/tests/ansible/regression/issue_655__wait_for_connection_error.yml new file mode 100644 index 00000000..aa9472ec --- /dev/null +++ b/tests/ansible/regression/issue_655__wait_for_connection_error.yml @@ -0,0 +1,85 @@ +# https://github.com/dw/mitogen/issues/655 +# Spins up a Centos8 container and runs the wait_for_connection test inside of it +# Doing it this way because the shutdown command causes issues in our tests +# since things are ran on localhost; Azure DevOps loses connection and fails +# TODO: do we want to install docker a different way to be able to do this for other tests too +--- +# this should only run on our Mac hosts +- hosts: target + any_errors_fatal: True + gather_facts: yes + become: no + tasks: + - name: set up test container and run tests inside it + block: + - name: install deps + block: + - name: install docker + shell: | + # NOTE: for tracking purposes: https://github.com/docker/for-mac/issues/2359 + # using docker for mac CI workaround: https://github.com/drud/ddev/pull/1748/files#diff-19288f650af2dabdf1dcc5b354d1f245 + DOCKER_URL=https://download.docker.com/mac/stable/31259/Docker.dmg && + curl -O -sSL $DOCKER_URL && + open -W Docker.dmg && cp -r /Volumes/Docker/Docker.app /Applications + sudo /Applications/Docker.app/Contents/MacOS/Docker --quit-after-install --unattended && + ln -s /Applications/Docker.app/Contents/Resources/bin/docker /usr/local/bin/docker && + nohup /Applications/Docker.app/Contents/MacOS/Docker --unattended & + # wait 2 min for docker to come up + counter=0 && + while ! /usr/local/bin/docker ps 2>/dev/null ; do + if [ $counter -lt 24 ]; then + let counter=counter+1 + else + exit 1 + fi + sleep 5 + done + + # python bindings (docker_container) aren't working on this host, so gonna shell out + - name: create docker container + shell: /usr/local/bin/docker run --name testMitogen -d --rm centos:8 bash -c "sleep infinity & wait" + + - name: add container to inventory + add_host: + name: testMitogen + ansible_connection: docker + ansible_user: root + changed_when: false + environment: + PATH: /usr/local/bin/:{{ ansible_env.PATH }} + + - name: run tests + block: + # to repro the issue, will create /var/run/reboot-required + - name: create test file + file: + path: /var/run/reboot-required + state: touch + + - name: Check if reboot is required + stat: + path: /var/run/reboot-required + register: reboot_required + + - name: Reboot server + shell: sleep 2 && shutdown -r now "Ansible updates triggered" + async: 1 + poll: 0 + when: reboot_required.stat.exists == True + + - name: Wait 300 seconds for server to become available + wait_for_connection: + delay: 30 + timeout: 300 + when: reboot_required.stat.exists == True + + - name: cleanup test file + file: + path: /var/run/reboot-required + state: absent + delegate_to: testMitogen + environment: + PATH: /usr/local/bin/:{{ ansible_env.PATH }} + + - name: remove test container + shell: /usr/local/bin/docker stop testMitogen From e632310fc42f56c269d6b877b7de13b30cd29a4c Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Fri, 15 May 2020 14:55:48 -0700 Subject: [PATCH 11/15] no tmpdir to remove for old ansible versions in fetch command --- ansible_mitogen/plugins/action/mitogen_fetch.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/plugins/action/mitogen_fetch.py b/ansible_mitogen/plugins/action/mitogen_fetch.py index 1844efd8..b9eece76 100644 --- a/ansible_mitogen/plugins/action/mitogen_fetch.py +++ b/ansible_mitogen/plugins/action/mitogen_fetch.py @@ -157,6 +157,10 @@ class ActionModule(ActionBase): result.update(dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum)) finally: - self._remove_tmp_path(self._connection._shell.tmpdir) + try: + self._remove_tmp_path(self._connection._shell.tmpdir) + except AttributeError: + # .tmpdir was added to ShellModule in v2.6.0, so old versions don't have it + pass return result From 0e47280e43216b328ed55ffa88464cd7104de9ef Mon Sep 17 00:00:00 2001 From: Luiz Ribeiro Date: Sun, 7 Jun 2020 11:00:32 -0400 Subject: [PATCH 12/15] Fix mitogen_ssh_keepalive_interval documentation --- docs/ansible_detailed.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 27798faa..28197ab9 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -1009,7 +1009,7 @@ Like the :ans:conn:`ssh` except connection delegation is supported. * ``mitogen_ssh_keepalive_count``: integer count of server keepalive messages to which no reply is received before considering the SSH server dead. Defaults to 10. -* ``mitogen_ssh_keepalive_count``: integer seconds delay between keepalive +* ``mitogen_ssh_keepalive_interval``: integer seconds delay between keepalive messages. Defaults to 30. From 81076c9da81c214e4af12112c1a3fe5010e35a4a Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Sat, 30 May 2020 18:25:46 -0700 Subject: [PATCH 13/15] fixes setup module relative import fail on some pythons --- ansible_mitogen/planner.py | 28 +++++++++++++++++++++++++++- mitogen/service.py | 29 +++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 460e5be4..d070daeb 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -96,6 +96,9 @@ class Invocation(object): #: Initially ``None``, but set by :func:`invoke`. The raw source or #: binary contents of the module. self._module_source = None + #: Initially ``{}``, but set by :func:`invoke`. Optional source to send + #: to :func:`propagate_paths_and_modules` to fix Python3.5 relative import errors + self._overridden_sources = {} def get_module_source(self): if self._module_source is None: @@ -476,6 +479,7 @@ def _propagate_deps(invocation, planner, context): context=context, paths=planner.get_push_files(), modules=planner.get_module_deps(), + overridden_sources=invocation._overridden_sources ) @@ -533,6 +537,26 @@ def _get_planner(name, path, source): raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation)) +def _fix_py35(invocation, module_source): + """ + super edge case with a relative import error in Python 3.5.1-3.5.3 + in Ansible's setup module when using Mitogen + https://github.com/dw/mitogen/issues/672#issuecomment-636408833 + We replace a relative import in the setup module with the actual full file path + This works in vanilla Ansible but not in Mitogen otherwise + """ + if invocation.module_name == 'setup' and \ + invocation.module_path not in invocation._overridden_sources: + # in-memory replacement of setup module's relative import + # would check for just python3.5 and run this then but we don't know the + # target python at this time yet + module_source = module_source.replace( + b"from ...module_utils.basic import AnsibleModule", + b"from ansible.module_utils.basic import AnsibleModule" + ) + invocation._overridden_sources[invocation.module_path] = module_source + + def invoke(invocation): """ Find a Planner subclass corresponding to `invocation` and use it to invoke @@ -555,10 +579,12 @@ def invoke(invocation): invocation.module_path = mitogen.core.to_text(path) if invocation.module_path not in _planner_by_path: + module_source = invocation.get_module_source() + _fix_py35(invocation, module_source) _planner_by_path[invocation.module_path] = _get_planner( invocation.module_name, invocation.module_path, - invocation.get_module_source() + module_source ) planner = _planner_by_path[invocation.module_path](invocation) diff --git a/mitogen/service.py b/mitogen/service.py index 3b244414..69376a80 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -746,13 +746,18 @@ class PushFileService(Service): 'paths': list, 'modules': list, }) - def propagate_paths_and_modules(self, context, paths, modules): + def propagate_paths_and_modules(self, context, paths, modules, overridden_sources=None): """ One size fits all method to ensure a target context has been preloaded with a set of small files and Python modules. + + overridden_sources: optional dict containing source code to override path's source code """ for path in paths: - self.propagate_to(context, mitogen.core.to_text(path)) + overridden_source = None + if overridden_sources is not None and path in overridden_sources: + overridden_source = overridden_sources[path] + self.propagate_to(context, mitogen.core.to_text(path), overridden_source) #self.router.responder.forward_modules(context, modules) TODO @expose(policy=AllowParents()) @@ -760,14 +765,22 @@ class PushFileService(Service): 'context': mitogen.core.Context, 'path': mitogen.core.FsPathTypes, }) - def propagate_to(self, context, path): + def propagate_to(self, context, path, overridden_source=None): + """ + If the optional parameter 'overridden_source' is passed, use + that instead of the path's code as source code. This works around some bugs + of source modules such as relative imports on unsupported Python versions + """ if path not in self._cache: LOG.debug('caching small file %s', path) - fp = open(path, 'rb') - try: - self._cache[path] = mitogen.core.Blob(fp.read()) - finally: - fp.close() + if overridden_source is None: + fp = open(path, 'rb') + try: + self._cache[path] = mitogen.core.Blob(fp.read()) + finally: + fp.close() + else: + self._cache[path] = mitogen.core.Blob(overridden_source) self._forward(context, path) @expose(policy=AllowParents()) From 4d48f140a8e8e2dad878f37b4c2b565d437eb67d Mon Sep 17 00:00:00 2001 From: Nicolas Mattia Date: Fri, 26 Jun 2020 12:21:44 +0200 Subject: [PATCH 14/15] Fix typo in Ansible documentation --- docs/ansible_detailed.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 28197ab9..12d89c9c 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -9,7 +9,7 @@ Mitogen for Ansible **Mitogen for Ansible** is a completely redesigned UNIX connection layer and module runtime for `Ansible`_. Requiring minimal configuration changes, it -updates Ansible's slow and wasteful shell-centic implementation with +updates Ansible's slow and wasteful shell-centric implementation with pure-Python equivalents, invoked via highly efficient remote procedure calls to persistent interpreters tunnelled over SSH. No changes are required to target hosts. From bacc752ff261d927a2a5d4cdd668a5e984f0924c Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Thu, 11 Jun 2020 21:17:31 -0700 Subject: [PATCH 15/15] Pegs python 2 version, removes unused imports, fixes sudden ssh dir with bad perms, generates key for sudo user as well in tests --- .ci/azure-pipelines.yml | 4 ++-- .ci/localhost_ansible_tests.py | 23 +++++++++++------------ README.md | 1 - 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index a537b0f5..c23974df 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -13,10 +13,10 @@ jobs: strategy: matrix: Mito27_27: - python.version: '2.7' + python.version: '2.7.18' MODE: mitogen Ans288_27: - python.version: '2.7' + python.version: '2.7.18' MODE: localhost_ansible VER: 2.8.8 diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 888d2e44..b4d6a542 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -1,9 +1,7 @@ #!/usr/bin/env python # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen -import glob import os -import shutil import sys import ci_lib @@ -31,16 +29,17 @@ with ci_lib.Fold('job_setup'): with ci_lib.Fold('machine_prep'): - ssh_dir = os.path.expanduser('~/.ssh') - if not os.path.exists(ssh_dir): - os.makedirs(ssh_dir, int('0700', 8)) - - key_path = os.path.expanduser('~/.ssh/id_rsa') - shutil.copy(KEY_PATH, key_path) - - auth_path = os.path.expanduser('~/.ssh/authorized_keys') - os.system('ssh-keygen -y -f %s >> %s' % (key_path, auth_path)) - os.chmod(auth_path, int('0600', 8)) + # generate a new ssh key for localhost ssh + os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa") + os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys") + # also generate it for the sudo user + os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa") + os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys") + os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8)) + os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8)) + # run chmod through sudo since it's owned by root + os.system('sudo chmod 600 /var/root/.ssh') + os.system('sudo chmod 600 /var/root/.ssh/authorized_keys') if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': os.chdir(IMAGE_PREP_DIR) diff --git a/README.md b/README.md index da93a80b..c7d8b03f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ - # Mitogen