clean up azure python version used

.ci/azure-pipelines-steps.yml

@@ -8,23 +8,7 @@ steps:
 - script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py"
   displayName: "Run prep_azure.py"
 
-# The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
-# broken symlinks, incorrect permissions and missing codecs. So we use the
-# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
-# stuff into. The virtualenv can probably be removed again, but this was a
-# hard-fought battle and for now I am tired of this crap.
 - script: |
-    # need wheel before building virtualenv because of bdist_wheel and setuptools deps
-    # Mac's System Integrity Protection prevents symlinking /usr/bin
-    # and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html
-    # the || will activate when running python3 tests
-    # TODO: get python3 tests passing
-    (sudo ln -fs /usr/bin/python$(python.version) /usr/bin/python &&
-      /usr/bin/python -m pip install -U pip wheel setuptools &&
-      /usr/bin/python -m pip install -U virtualenv &&
-      /usr/bin/python -m virtualenv /tmp/venv -p /usr/bin/python$(python.version)) ||
-    (sudo /usr/bin/python$(python.version) -m pip install -U pip wheel setuptools &&
-      /usr/bin/python$(python.version) -m venv /tmp/venv)
     echo "##vso[task.prependpath]/tmp/venv/bin"
 
   displayName: activate venv

.ci/azure-pipelines.yml

@@ -17,11 +17,11 @@ jobs:
   strategy:
     matrix:
       Mito27_27:
-        python.version: '2.7.18'
+        python.version: '2.7'
         MODE: mitogen
         VER: git+https://github.com/ansible/ansible.git@v2.10.0
-      Ans210_35:
+      Ans210_38:
-        python.version: '3.5'
+        python.version: '3.8'
         MODE: localhost_ansible
         VER: git+https://github.com/ansible/ansible.git@v2.10.0
 

.ci/ci_lib.py

@@ -49,6 +49,10 @@ def have_apt():
     proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True)
     return proc.wait() == 0
 
+def have_brew():
+    proc = subprocess.Popen('brew help >/dev/null 2>/dev/null', shell=True)
+    return proc.wait() == 0
+
 
 def have_docker():
     proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True)

.ci/prep_azure.py

@@ -30,6 +30,11 @@ if 0 and os.uname()[0] == 'Linux':
         ]
     ]
 
+# @dw: The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
+# broken symlinks, incorrect permissions and missing codecs. So we use the
+# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
+# stuff into. The virtualenv can probably be removed again, but this was a
+# hard-fought battle and for now I am tired of this crap.
 if ci_lib.have_apt():
     batches.append([
         'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
@@ -40,10 +45,34 @@ if ci_lib.have_apt():
             'python{pv}-dev '
             'libsasl2-dev '
             'libldap2-dev '
+            .format(pv=os.environ['PYTHONVERSION']),
+        'sudo ln -fs /usr/bin/python{pv} /usr/local/bin/python{pv}'
         .format(pv=os.environ['PYTHONVERSION'])
     ])
 
 
+# Mac's System Integrity Protection prevents symlinking /usr/bin
+# and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html
+# so we'll use /usr/local/bin/python for everything
+if ci_lib.have_brew():
+    batches.append([
+        'brew install python@{pv}'
+        .format(pv=os.environ['PYTHONVERSION'])
+    ])
+
+# setup venv
+# need wheel before building virtualenv because of bdist_wheel and setuptools deps
+venv_steps = ['/usr/local/bin/python{pv} -m pip install -U pip wheel setuptools']
+if os.environ['PYTHONVERSION'].startswith('2'):
+    venv_steps.extend([
+        '/usr/local/bin/python{pv} -m pip install -U virtualenv'.format(py=os.environ['PYTHONVERSION'])
+        '/usr/local/bin/python{pv} -m virtualenv /tmp/venv -p /usr/local/bin/python{pv}'.format(py=os.environ['PYTHONVERSION'])
+    ])
+else:
+    venv_steps.append('/usr/local/bin/python{pv} -m venv /tmp/venv'.format(py=os.environ['PYTHONVERSION'])
+batches.append(venv_steps)
+
+
 if ci_lib.have_docker():
     batches.extend(
         ['docker pull %s' % (ci_lib.image_for_distro(distro),)]