diff --git a/.ci/README.md b/.ci/README.md index 67a3805b..9248ac58 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2. * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` * `DISTROS`: the `ansible_` tests can run against multiple targets diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 102eda9c..3ec48dfd 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting) with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers() + containers = ci_lib.container_specs(ci_lib.DISTROS) ci_lib.start_containers(containers) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 3e716385..dfe49b97 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -27,6 +27,13 @@ os.chdir( ) ) + +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + + _print = print def print(*args, **kwargs): file = kwargs.get('file', sys.stdout) @@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) DISTRO = os.environ.get('DISTRO', 'debian9') # Used only when MODE=ansible DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() -TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) -BASE_PORT = 2200 TMP = TempDir().path @@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % ( def get_docker_hostname(): """Return the hostname where the docker daemon is running. """ + # Duplicated in testlib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -225,27 +231,34 @@ def get_docker_hostname(): return parsed.netloc.partition(':')[0] -def make_containers(name_prefix='', port_offset=0): +def container_specs( + distros, + base_port=2200, + image_template=IMAGE_TEMPLATE, + name_template='target-%(distro)s-%(index)d', +): """ >>> import pprint - >>> BASE_PORT=2200; DISTROS=['debian11', 'centos6'] - >>> pprint.pprint(make_containers()) + >>> pprint.pprint(container_specs(['debian11-py3', 'centos6'])) [{'distro': 'debian11', 'family': 'debian', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'index': 1, 'name': 'target-debian11-1', 'port': 2201, - 'python_path': '/usr/bin/python'}, + 'python_path': '/usr/bin/python3'}, {'distro': 'centos6', 'family': 'centos', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', + 'index': 2, 'name': 'target-centos6-2', 'port': 2202, 'python_path': '/usr/bin/python'}] """ docker_hostname = get_docker_hostname() + # Code duplicated in testlib.py, both should be updated together distro_pattern = re.compile(r''' (?P(?P[a-z]+)[0-9]+) (?:-(?Ppy3))? @@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0): i = 1 lst = [] - for distro in DISTROS: + for distro in distros: + # Code duplicated in testlib.py, both should be updated together d = distro_pattern.match(distro).groupdict(default=None) - distro = d['distro'] - family = d['family'] - image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) - if d['py'] == 'py3': + if d.pop('py') == 'py3': python_path = '/usr/bin/python3' else: python_path = '/usr/bin/python' - if d['count']: - count = int(count) - else: - count = 1 + count = int(d.pop('count') or '1', 10) for x in range(count): - lst.append({ - "distro": distro, "family": family, "image": image, - "name": name_prefix + ("target-%s-%s" % (distro, i)), + d['index'] = i + d.update({ + 'image': image_template % d, + 'name': name_template % d, "hostname": docker_hostname, - "port": BASE_PORT + i + port_offset, + 'port': base_port + i, "python_path": python_path, }) + lst.append(d) i += 1 return lst diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 13217133..825126c7 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -2,9 +2,6 @@ import ci_lib -# Naturally DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] - ci_lib.run_batches([ [ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index 7db8a797..b065486f 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -6,9 +6,6 @@ import sys import ci_lib -# DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT - project_dir = os.path.join(ci_lib.TMP, 'project') vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' inventory_path = 'ansible/inventory/hosts' @@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname() with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-') + containers = ci_lib.container_specs( + ['debian*2'], + base_port=2700, + name_template='debops-target-%(distro)s-%(index)d', + ) ci_lib.start_containers(containers) diff --git a/tests/image_prep/_container_finalize.yml b/tests/image_prep/_container_finalize.yml index d61d9b3b..5329fefa 100644 --- a/tests/image_prep/_container_finalize.yml +++ b/tests/image_prep/_container_finalize.yml @@ -9,7 +9,7 @@ --change 'EXPOSE 22' --change 'CMD ["/usr/sbin/sshd", "-D"]' {{ inventory_hostname }} - public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test + {{ container_image_name }} delegate_to: localhost - name: Stop containers diff --git a/tests/image_prep/group_vars/all.yml b/tests/image_prep/group_vars/all.yml index 5f182f86..91ff934d 100644 --- a/tests/image_prep/group_vars/all.yml +++ b/tests/image_prep/group_vars/all.yml @@ -4,6 +4,9 @@ common_packages: - strace - sudo +container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test" +container_registry: public.ecr.aws/n5z0e8q9 + sudo_group: MacOSX: admin Debian: sudo diff --git a/tests/testlib.py b/tests/testlib.py index a52292ce..76743e82 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -51,6 +51,12 @@ except NameError: LOG = logging.getLogger(__name__) +DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9') +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + TESTS_DIR = os.path.join(os.path.dirname(__file__)) ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') @@ -509,6 +515,7 @@ class TestCase(unittest.TestCase): def get_docker_host(): + # Duplicated in ci_lib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -549,19 +556,23 @@ class DockerizedSshDaemon(object): ] subprocess.check_output(args) - def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): - if '-' in mitogen_test_distro: - distro, _py3 = mitogen_test_distro.split('-') - else: - distro = mitogen_test_distro - _py3 = None + def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): + # Code duplicated in ci_lib.py, both should be updated together + distro_pattern = re.compile(r''' + (?P(?P[a-z]+)[0-9]+) + (?:-(?Ppy3))? + (?:\*(?P[0-9]+))? + ''', + re.VERBOSE, + ) + d = distro_pattern.match(distro).groupdict(default=None) - if _py3 == 'py3': + if d.pop('py') == 'py3': self.python_path = '/usr/bin/python3' else: self.python_path = '/usr/bin/python' - self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) + self.image = image_template % d self.start_container() self.host = self.get_host() self.port = self.get_port(self.container_name) @@ -601,6 +612,9 @@ class DockerizedSshDaemon(object): class BrokerMixin(object): broker_class = mitogen.master.Broker + + # Flag for tests that shutdown the broker themself + # e.g. unix_test.ListenerTest broker_shutdown = False def setUp(self): diff --git a/tox.ini b/tox.ini index 9820a79f..9fb31bdc 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,7 @@ commands = passenv = ANSIBLE_* HOME + MITOGEN_* # Azure DevOps, TF_BUILD is set to 'True' when running in a build task # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables TF_BUILD