From 8b92e09655e31f0404b302e001c1f4f024616c57 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 25 Sep 2024 12:19:39 +0100 Subject: [PATCH] ci: Extract container registry location into variables Preperation for migrating from Azure DevOps with Amazon Elastic Container Registry (AWS ECR), to GitHub Actions with GitHub Container Registry (GHCR). DebOps tests are not currently being run, the updates to .ci/debops*.py are best effort only. --- .ci/README.md | 1 - .ci/ansible_tests.py | 2 +- .ci/ci_lib.py | 48 ++++++++++++++---------- .ci/debops_common_install.py | 3 -- .ci/debops_common_tests.py | 9 +++-- tests/image_prep/_container_finalize.yml | 2 +- tests/image_prep/group_vars/all.yml | 3 ++ tests/testlib.py | 30 +++++++++++---- tox.ini | 1 + 9 files changed, 62 insertions(+), 37 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 67a3805b..9248ac58 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2. * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` * `DISTROS`: the `ansible_` tests can run against multiple targets diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 102eda9c..3ec48dfd 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting) with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers() + containers = ci_lib.container_specs(ci_lib.DISTROS) ci_lib.start_containers(containers) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 3e716385..dfe49b97 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -27,6 +27,13 @@ os.chdir( ) ) + +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + + _print = print def print(*args, **kwargs): file = kwargs.get('file', sys.stdout) @@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) DISTRO = os.environ.get('DISTRO', 'debian9') # Used only when MODE=ansible DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() -TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) -BASE_PORT = 2200 TMP = TempDir().path @@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % ( def get_docker_hostname(): """Return the hostname where the docker daemon is running. """ + # Duplicated in testlib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -225,27 +231,34 @@ def get_docker_hostname(): return parsed.netloc.partition(':')[0] -def make_containers(name_prefix='', port_offset=0): +def container_specs( + distros, + base_port=2200, + image_template=IMAGE_TEMPLATE, + name_template='target-%(distro)s-%(index)d', +): """ >>> import pprint - >>> BASE_PORT=2200; DISTROS=['debian11', 'centos6'] - >>> pprint.pprint(make_containers()) + >>> pprint.pprint(container_specs(['debian11-py3', 'centos6'])) [{'distro': 'debian11', 'family': 'debian', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'index': 1, 'name': 'target-debian11-1', 'port': 2201, - 'python_path': '/usr/bin/python'}, + 'python_path': '/usr/bin/python3'}, {'distro': 'centos6', 'family': 'centos', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', + 'index': 2, 'name': 'target-centos6-2', 'port': 2202, 'python_path': '/usr/bin/python'}] """ docker_hostname = get_docker_hostname() + # Code duplicated in testlib.py, both should be updated together distro_pattern = re.compile(r''' (?P(?P[a-z]+)[0-9]+) (?:-(?Ppy3))? @@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0): i = 1 lst = [] - for distro in DISTROS: + for distro in distros: + # Code duplicated in testlib.py, both should be updated together d = distro_pattern.match(distro).groupdict(default=None) - distro = d['distro'] - family = d['family'] - image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) - if d['py'] == 'py3': + if d.pop('py') == 'py3': python_path = '/usr/bin/python3' else: python_path = '/usr/bin/python' - if d['count']: - count = int(count) - else: - count = 1 + count = int(d.pop('count') or '1', 10) for x in range(count): - lst.append({ - "distro": distro, "family": family, "image": image, - "name": name_prefix + ("target-%s-%s" % (distro, i)), + d['index'] = i + d.update({ + 'image': image_template % d, + 'name': name_template % d, "hostname": docker_hostname, - "port": BASE_PORT + i + port_offset, + 'port': base_port + i, "python_path": python_path, }) + lst.append(d) i += 1 return lst diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 13217133..825126c7 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -2,9 +2,6 @@ import ci_lib -# Naturally DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] - ci_lib.run_batches([ [ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index 7db8a797..b065486f 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -6,9 +6,6 @@ import sys import ci_lib -# DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT - project_dir = os.path.join(ci_lib.TMP, 'project') vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' inventory_path = 'ansible/inventory/hosts' @@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname() with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-') + containers = ci_lib.container_specs( + ['debian*2'], + base_port=2700, + name_template='debops-target-%(distro)s-%(index)d', + ) ci_lib.start_containers(containers) diff --git a/tests/image_prep/_container_finalize.yml b/tests/image_prep/_container_finalize.yml index d61d9b3b..5329fefa 100644 --- a/tests/image_prep/_container_finalize.yml +++ b/tests/image_prep/_container_finalize.yml @@ -9,7 +9,7 @@ --change 'EXPOSE 22' --change 'CMD ["/usr/sbin/sshd", "-D"]' {{ inventory_hostname }} - public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test + {{ container_image_name }} delegate_to: localhost - name: Stop containers diff --git a/tests/image_prep/group_vars/all.yml b/tests/image_prep/group_vars/all.yml index 5f182f86..91ff934d 100644 --- a/tests/image_prep/group_vars/all.yml +++ b/tests/image_prep/group_vars/all.yml @@ -4,6 +4,9 @@ common_packages: - strace - sudo +container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test" +container_registry: public.ecr.aws/n5z0e8q9 + sudo_group: MacOSX: admin Debian: sudo diff --git a/tests/testlib.py b/tests/testlib.py index a52292ce..76743e82 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -51,6 +51,12 @@ except NameError: LOG = logging.getLogger(__name__) +DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9') +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + TESTS_DIR = os.path.join(os.path.dirname(__file__)) ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') @@ -509,6 +515,7 @@ class TestCase(unittest.TestCase): def get_docker_host(): + # Duplicated in ci_lib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -549,19 +556,23 @@ class DockerizedSshDaemon(object): ] subprocess.check_output(args) - def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): - if '-' in mitogen_test_distro: - distro, _py3 = mitogen_test_distro.split('-') - else: - distro = mitogen_test_distro - _py3 = None + def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): + # Code duplicated in ci_lib.py, both should be updated together + distro_pattern = re.compile(r''' + (?P(?P[a-z]+)[0-9]+) + (?:-(?Ppy3))? + (?:\*(?P[0-9]+))? + ''', + re.VERBOSE, + ) + d = distro_pattern.match(distro).groupdict(default=None) - if _py3 == 'py3': + if d.pop('py') == 'py3': self.python_path = '/usr/bin/python3' else: self.python_path = '/usr/bin/python' - self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) + self.image = image_template % d self.start_container() self.host = self.get_host() self.port = self.get_port(self.container_name) @@ -601,6 +612,9 @@ class DockerizedSshDaemon(object): class BrokerMixin(object): broker_class = mitogen.master.Broker + + # Flag for tests that shutdown the broker themself + # e.g. unix_test.ListenerTest broker_shutdown = False def setUp(self): diff --git a/tox.ini b/tox.ini index 9820a79f..9fb31bdc 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,7 @@ commands = passenv = ANSIBLE_* HOME + MITOGEN_* # Azure DevOps, TF_BUILD is set to 'True' when running in a build task # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables TF_BUILD