ci: Switch Continuous Integration to Tox

Also
 - Simplifies adding support for additional Ansible versions
 - Unifies Python package versioning in CI and local test environments
 - Matches Python versions tested, with those declared in setup.py
 - Expands targets covered by automated Ansible tests to
    - centos6, centos8
    - debian9, debian11
    - ubuntu1604, ubuntu2004

.ci/README.md

@@ -28,8 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `VER`: Ansible version the `_install` script should install. Default changes
-  over time.
 * `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
 * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
   name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`

.ci/ansible_install.py

@@ -3,21 +3,9 @@
 import ci_lib
 
 batches = [
-    [
-        'pip install '
-            '-r tests/requirements.txt '
-            '-r tests/ansible/requirements.txt',
-        # encoding is required for installing ansible 2.10 with pip2, otherwise we get a UnicodeDecode error
-        'LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 pip install "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION)
-    ],
     [
         'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
     ]
 ]
 
-batches[-1].extend([
-    'docker pull %s' % (ci_lib.image_for_distro(distro),)
-    for distro in ci_lib.DISTROS
-])
-
 ci_lib.run_batches(batches)

.ci/azure-pipelines-steps.yml

@@ -5,20 +5,17 @@ parameters:
   sign: false
 
 steps:
-- script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py"
-  displayName: "Run prep_azure.py"
+- task: UsePythonVersion@0
+  displayName: Install python
+  inputs:
+    versionSpec: '$(python.version)'
 
-- script: |
-    echo "##vso[task.prependpath]/tmp/venv/bin"
+- script: python -mpip install tox
+  displayName: Install tooling
 
-  displayName: activate venv
-
-- script: .ci/$(MODE)_install.py
-  displayName: "Run $(MODE)_install.py"
+- script: tox -e $(tox.env)
+  displayName: "Run tests"
   env:
     AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
     AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
     AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
-
-- script: .ci/$(MODE)_tests.py
-  displayName: "Run $(MODE)_tests.py"

.ci/azure-pipelines.yml

@@ -16,62 +16,46 @@ jobs:
   steps:
   - template: azure-pipelines-steps.yml
   pool:
+    # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md
     vmImage: macOS-10.15
   strategy:
     matrix:
       Mito27_27:
         python.version: '2.7'
-        MODE: mitogen
+        tox.env: py27-mode_mitogen
+
       # TODO: test python3, python3 tests are broken
       Ans210_27:
         python.version: '2.7'
-        MODE: localhost_ansible
-        VER: 2.10.0
+        tox.env: py27-mode_localhost-ansible2.10
 
       # NOTE: this hangs when ran in Ubuntu 18.04
       Vanilla_210_27:
         python.version: '2.7'
-        MODE: localhost_ansible
-        VER: 2.10.0
+        tox.env: py27-mode_localhost-ansible2.10
         STRATEGY: linear
         ANSIBLE_SKIP_TAGS: resource_intensive
 
 
 - job: Linux
   pool:
+    # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md
     vmImage: "Ubuntu 18.04"
   steps:
   - template: azure-pipelines-steps.yml
   strategy:
     matrix:
-      #
-      # Confirmed working
-      #
       Mito27Debian_27:
         python.version: '2.7'
-        MODE: mitogen
-        DISTRO: debian9
-
-      #MitoPy27CentOS6_26:
-        #python.version: '2.7'
-        #MODE: mitogen
-        #DISTRO: centos6
+        tox.env: py27-mode_mitogen-distro_debian9
 
       Mito36CentOS6_26:
         python.version: '3.6'
-        MODE: mitogen
-        DISTRO: centos6
-
-      Mito37Debian_27:
-        python.version: '3.7'
-        MODE: mitogen
-        DISTRO: debian9
+        tox.env: py36-mode_mitogen-distro_centos6
 
       Mito39Debian_27:
         python.version: '3.9'
-        MODE: mitogen
-        DISTRO: debian9
-        VER: 2.10.0
+        tox.env: py39-mode_mitogen-distro_debian9
 
       #Py26CentOS7:
         #python.version: '2.7'
@@ -117,15 +101,12 @@ jobs:
 
       Ansible_210_27:
         python.version: '2.7'
-        MODE: ansible
-        VER: 2.10.0
+        tox.env: py27-mode_ansible-ansible2.10
 
-      Ansible_210_35:
-        python.version: '3.5'
-        MODE: ansible
-        VER: 2.10.0
+      Ansible_210_36:
+        python.version: '3.6'
+        tox.env: py36-mode_ansible-ansible2.10
 
       Ansible_210_39:
         python.version: '3.9'
-        MODE: ansible
-        VER: 2.10.0
+        tox.env: py39-mode_ansible-ansible2.10

.ci/ci_lib.py

@@ -59,12 +59,6 @@ def have_docker():
     return proc.wait() == 0
 
 
-# -----------------
-
-# Force line buffering on stdout.
-sys.stdout = os.fdopen(1, 'w', 1)
-
-
 def _argv(s, *args):
     """Interpolate a command line using *args, return an argv style list.
 
@@ -195,13 +189,11 @@ class Fold(object):
 
 os.environ.setdefault('ANSIBLE_STRATEGY',
     os.environ.get('STRATEGY', 'mitogen_linear'))
-# Ignoreed when MODE=mitogen
-ANSIBLE_VERSION = os.environ.get('VER', '2.6.2')
 GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 # Used only when MODE=mitogen
-DISTRO = os.environ.get('DISTRO', 'debian')
+DISTRO = os.environ.get('DISTRO', 'debian9')
 # Used only when MODE=ansible
-DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split()
+DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
 TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
 BASE_PORT = 2200
 TMP = TempDir().path
@@ -254,11 +246,13 @@ def make_containers(name_prefix='', port_offset=0):
     >>> pprint.pprint(make_containers())
     [{'distro': 'debian',
       'hostname': 'localhost',
+      'image': 'public.ecr.aws/n5z0e8q9/debian-test',
       'name': 'target-debian-1',
       'port': 2201,
       'python_path': '/usr/bin/python'},
      {'distro': 'centos6',
       'hostname': 'localhost',
+      'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
       'name': 'target-centos6-2',
       'port': 2202,
       'python_path': '/usr/bin/python'}]
@@ -280,6 +274,7 @@ def make_containers(name_prefix='', port_offset=0):
         for x in range(count):
             lst.append({
                 "distro": firstbit(distro),
+                "image": image_for_distro(distro),
                 "name": name_prefix + ("target-%s-%s" % (distro, i)),
                 "hostname": docker_hostname,
                 "port": BASE_PORT + i + port_offset,
@@ -358,7 +353,7 @@ def start_containers(containers):
                 "--publish 0.0.0.0:%(port)s:22/tcp "
                 "--hostname=%(name)s "
                 "--name=%(name)s "
-                "mitogen/%(distro)s-test "
+                "%(image)s"
             % container
         ]
         for container in containers

.ci/debops_common_install.py

@@ -7,11 +7,10 @@ ci_lib.DISTROS = ['debian']
 
 ci_lib.run_batches([
     [
-        'pip install -qqq "debops[ansible]==2.1.2" "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION),
+        'pip install -qqq "debops[ansible]==2.1.2"',
     ],
     [
         'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-        'docker pull %s' % (ci_lib.image_for_distro('debian'),),
     ],
 ])
 

.ci/localhost_ansible_install.py

@@ -3,12 +3,6 @@
 import ci_lib
 
 batches = [
-    [
-        'pip install '
-            '-r tests/requirements.txt '
-            '-r tests/ansible/requirements.txt',
-        'pip install -q "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION)
-    ]
 ]
 
 ci_lib.run_batches(batches)

.ci/mitogen_install.py

@@ -3,15 +3,11 @@
 import ci_lib
 
 batches = [
-    [
-        'pip install -r tests/requirements.txt',
-    ]
 ]
 
 if ci_lib.have_docker():
     batches.append([
         'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-        'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ])
 
 

.ci/mitogen_py24_install.py

@@ -5,7 +5,6 @@ import ci_lib
 batches = [
     [
         'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-        'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ],
     [
         'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',

.ci/prep_azure.py

@@ -1,95 +0,0 @@
-#!/usr/bin/env python
-
-import os
-import sys
-
-import ci_lib
-
-batches = []
-
-if 0 and os.uname()[0] == 'Linux':
-    batches += [
-        [
-            "sudo chown `whoami`: ~",
-            "chmod u=rwx,g=rx,o= ~",
-
-            "sudo mkdir /var/run/sshd",
-            "sudo /etc/init.d/ssh start",
-
-            "mkdir -p ~/.ssh",
-            "chmod u=rwx,go= ~/.ssh",
-
-            "ssh-keyscan -H localhost >> ~/.ssh/known_hosts",
-            "chmod u=rw,go= ~/.ssh/known_hosts",
-
-            "cat tests/data/docker/mitogen__has_sudo_pubkey.key > ~/.ssh/id_rsa",
-            "chmod u=rw,go= ~/.ssh/id_rsa",
-
-            "cat tests/data/docker/mitogen__has_sudo_pubkey.key.pub > ~/.ssh/authorized_keys",
-            "chmod u=rw,go=r ~/.ssh/authorized_keys",
-        ]
-    ]
-
-# setup venv, need all python commands in 1 list to be subprocessed at the same time
-venv_steps = []
-
-need_to_fix_psycopg2 = False
-
-is_python3 = os.environ['PYTHONVERSION'].startswith('3')
-
-# @dw: The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
-# broken symlinks, incorrect permissions and missing codecs. So we use the
-# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
-# stuff into. The virtualenv can probably be removed again, but this was a
-# hard-fought battle and for now I am tired of this crap.
-if ci_lib.have_apt():
-    venv_steps.extend([
-        'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
-        'sudo add-apt-repository ppa:deadsnakes/ppa',
-        'sudo apt-get update',
-        'sudo apt-get -y install '
-            'python{pv} '
-            'python{pv}-dev '
-            'libsasl2-dev '
-            'libldap2-dev '
-            .format(pv=os.environ['PYTHONVERSION']),
-        'sudo ln -fs /usr/bin/python{pv} /usr/local/bin/python{pv}'
-        .format(pv=os.environ['PYTHONVERSION'])
-    ])
-    if is_python3:
-        venv_steps.append('sudo apt-get -y install python{pv}-venv'.format(pv=os.environ['PYTHONVERSION']))
-# TODO: somehow `Mito36CentOS6_26` has both brew and apt installed https://dev.azure.com/dw-mitogen/Mitogen/_build/results?buildId=1031&view=logs&j=7bdbcdc6-3d3e-568d-ccf8-9ddca1a9623a&t=73d379b6-4eea-540f-c97e-046a2f620483
-elif is_python3 and ci_lib.have_brew():
-    # Mac's System Integrity Protection prevents symlinking /usr/bin
-    # and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html
-    # so we'll use /usr/local/bin/python for everything
-    # /usr/local/bin/python2.7 already exists!
-    need_to_fix_psycopg2 = True
-    venv_steps.append(
-        'brew install python@{pv} postgresql'
-        .format(pv=os.environ['PYTHONVERSION'])
-    )
-
-# need wheel before building virtualenv because of bdist_wheel and setuptools deps
-venv_steps.append('/usr/local/bin/python{pv} -m pip install -U pip wheel setuptools'.format(pv=os.environ['PYTHONVERSION']))
-
-if os.environ['PYTHONVERSION'].startswith('2'):
-    venv_steps.extend([
-        '/usr/local/bin/python{pv} -m pip install -U virtualenv'.format(pv=os.environ['PYTHONVERSION']),
-        '/usr/local/bin/python{pv} -m virtualenv /tmp/venv -p /usr/local/bin/python{pv}'.format(pv=os.environ['PYTHONVERSION'])
-    ])
-else:
-    venv_steps.append('/usr/local/bin/python{pv} -m venv /tmp/venv'.format(pv=os.environ['PYTHONVERSION']))
-# fixes https://stackoverflow.com/questions/59595649/can-not-install-psycopg2-on-macos-catalina https://github.com/Azure/azure-cli/issues/12854#issuecomment-619213863
-if need_to_fix_psycopg2:
-    venv_steps.append('/tmp/venv/bin/pip3 install psycopg2==2.8.5 psycopg2-binary')
-
-venv_steps.extend([
-    # pbr is a transitive setup_requires of hdrhistogram. If it's not already
-    # installed then setuptools attempts to use easy_install, which fails.
-    '/tmp/venv/bin/pip install pbr==5.6.0',
-])
-
-batches.append(venv_steps)
-
-ci_lib.run_batches(batches)

docs/changelog.rst

@@ -21,6 +21,9 @@ To avail of fixes in an unreleased version, please download a ZIP file
 v0.3.1.dev0 (unreleased)
 ------------------------
 
+* :gh:issue:`869` Continuous Integration tests are now run with Tox
+* :gh:issue:`869` Continuous Integration tests now cover CentOS 6 & 8, Debian 9 & 11, Ubuntu 16.04 & 20.04
+
 
 v0.3.0 (2021-11-24)
 -------------------

tox.ini

@@ -23,15 +23,15 @@
 # pip --no-python-version-warning
 # pip --disable-pip-version-check
 
+# TODO distros=-py3
+
 [tox]
 envlist =
     init,
-    py{27,35,39}-mode_ansible-distros_centos,
-    py{27,35,39}-mode_ansible-distros_debian,
-    py{27,35,39}-mode_ansible-distros_ubuntu,
-    py{27,35,39}-mode_mitogen-distro_centos{6,7,8},
-    py{27,35,39}-mode_mitogen-distro_debian{9,10,11},
-    py{27,35,39}-mode_mitogen-distro_ubuntu{1604,1804,2004},
+    py{27,36,39}-mode_ansible-ansible2.10,
+    py{27,36,39}-mode_mitogen-distro_centos{6,7,8},
+    py{27,36,39}-mode_mitogen-distro_debian{9,10,11},
+    py{27,36,39}-mode_mitogen-distro_ubuntu{1604,1804,2004},
     report,
 requires =
     tox-factor
@@ -45,29 +45,37 @@ basepython =
     py37: python3.7
     py38: python3.8
     py39: python3.9
+deps =
+    -r{toxinidir}/tests/requirements.txt
+    mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt
+    ansible2.3: ansible=2.3.3.0
+    ansible2.4: ansible=2.4.6.0
+    ansible2.8: ansible=2.8.3
+    ansible2.9: ansible=2.9.6
+    ansible2.10: ansible-base<2.10.14
+    ansible2.10: ansible==2.10.0
 install_command =
     python -m pip --no-python-version-warning install {opts} {packages}
 commands_pre =
     mode_ansible: {toxinidir}/.ci/ansible_install.py
     mode_debops_common: {toxinidir}/.ci/debops_common_install.py
+    mode_localhost: {toxinidir}/.ci/localhost_ansible_install.py
     mode_mitogen: {toxinidir}/.ci/mitogen_install.py
 commands =
     mode_ansible: {toxinidir}/.ci/ansible_tests.py
     mode_debops_common: {toxinidir}/.ci/debops_common_tests.py
+    mode_localhost: {toxinidir}/.ci/localhost_ansible_tests.py
     mode_mitogen: {toxinidir}/.ci/mitogen_tests.py
 passenv =
     ANSIBLE_*
+    AWS_ACCESS_KEY_ID
+    AWS_DEFAULT_REGION
+    AWS_SECRET_ACCESS_KEY
     HOME
 setenv =
-    ANSIBLE_SKIP_TAGS = requires_local_sudo
     NOCOVERAGE_ERASE = 1
     NOCOVERAGE_REPORT = 1
-    VER=2.10.5
-    ansible2.3: VER=2.3.3.0
-    ansible2.4: VER=2.4.6.0
-    ansible2.8: VER=2.8.3
-    ansible2.9: VER=2.9.6
-    ansible2.10: VER=2.10.0
+    # Only applicable to MODE=mitogen
     distro_centos5: DISTRO=centos5
     distro_centos6: DISTRO=centos6
     distro_centos7: DISTRO=centos7
@@ -78,6 +86,7 @@ setenv =
     distro_ubuntu1604: DISTRO=ubuntu1604
     distro_ubuntu1804: DISTRO=ubuntu1804
     distro_ubuntu2004: DISTRO=ubuntu2004
+    # Note the plural, only applicable to MODE=ansible
     distros_centos: DISTROS=centos6 centos7 centos8
     distros_centos5: DISTROS=centos5
     distros_centos6: DISTROS=centos6
@@ -95,6 +104,12 @@ setenv =
     mode_debops_common: MODE=debops_common
     mode_mitogen: MODE=mitogen
     strategy_linear: STRATEGY=linear
+whitelist_externals =
+    docker
+    docker-credential-secretservice
+    echo
+    gpg2
+    pass
 
 [testenv:init]
 basepython = python3