Rooba
/
mitogen
mirror of https://github.com/mitogen-hq/mitogen.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ci: Extract container registry location into variables 

Preperation for migrating from Azure DevOps with Amazon Elastic Container
Registry (AWS ECR), to GitHub Actions with GitHub Container Registry (GHCR).

DebOps tests are not currently being run, the updates to .ci/debops*.py are
best effort only.
This commit is contained in:
Alex Willmer 2024-09-25 12:19:39 +01:00
parent b926795973
commit 8b92e09655
9 changed files with 62 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.ci/README.md
View File

@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
### Environment Variables
* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
* `DISTROS`: the `ansible_` tests can run against multiple targets

2
.ci/ansible_tests.py
View File

@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting)
with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers()
containers = ci_lib.container_specs(ci_lib.DISTROS)
ci_lib.start_containers(containers)

48
.ci/ci_lib.py
View File

@ -27,6 +27,13 @@ os.chdir(
)
)
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
_print = print
def print(*args, **kwargs):
file = kwargs.get('file', sys.stdout)
@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
DISTRO = os.environ.get('DISTRO', 'debian9')
# Used only when MODE=ansible
DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
BASE_PORT = 2200
TMP = TempDir().path
@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
def get_docker_hostname():
"""Return the hostname where the docker daemon is running.
"""
# Duplicated in testlib
url = os.environ.get('DOCKER_HOST')
if url in (None, 'http+docker://localunixsocket'):
return 'localhost'
@ -225,27 +231,34 @@ def get_docker_hostname():
return parsed.netloc.partition(':')[0]
def make_containers(name_prefix='', port_offset=0):
def container_specs(
distros,
base_port=2200,
image_template=IMAGE_TEMPLATE,
name_template='target-%(distro)s-%(index)d',
):
"""
>>> import pprint
>>> BASE_PORT=2200; DISTROS=['debian11', 'centos6']
>>> pprint.pprint(make_containers())
>>> pprint.pprint(container_specs(['debian11-py3', 'centos6']))
[{'distro': 'debian11',
'family': 'debian',
'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/debian11-test',
'index': 1,
'name': 'target-debian11-1',
'port': 2201,
'python_path': '/usr/bin/python'},
'python_path': '/usr/bin/python3'},
{'distro': 'centos6',
'family': 'centos',
'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
'index': 2,
'name': 'target-centos6-2',
'port': 2202,
'python_path': '/usr/bin/python'}]
"""
docker_hostname = get_docker_hostname()
# Code duplicated in testlib.py, both should be updated together
distro_pattern = re.compile(r'''
(?P<distro>(?P<family>[a-z]+)[0-9]+)
(?:-(?P<py>py3))?
@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0):
i = 1
lst = []
for distro in DISTROS:
for distro in distros:
# Code duplicated in testlib.py, both should be updated together
d = distro_pattern.match(distro).groupdict(default=None)
distro = d['distro']
family = d['family']
image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
if d['py'] == 'py3':
if d.pop('py') == 'py3':
python_path = '/usr/bin/python3'
else:
python_path = '/usr/bin/python'
if d['count']:
count = int(count)
else:
count = 1
count = int(d.pop('count') or '1', 10)
for x in range(count):
lst.append({
"distro": distro, "family": family, "image": image,
"name": name_prefix + ("target-%s-%s" % (distro, i)),
d['index'] = i
d.update({
'image': image_template % d,
'name': name_template % d,
"hostname": docker_hostname,
"port": BASE_PORT + i + port_offset,
'port': base_port + i,
"python_path": python_path,
})
lst.append(d)
i += 1
return lst

3
.ci/debops_common_install.py
View File

@ -2,9 +2,6 @@
import ci_lib
# Naturally DebOps only supports Debian.
ci_lib.DISTROS = ['debian']
ci_lib.run_batches([
[
'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"',

9
.ci/debops_common_tests.py
View File

@ -6,9 +6,6 @@ import sys
import ci_lib
# DebOps only supports Debian.
ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
project_dir = os.path.join(ci_lib.TMP, 'project')
vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
inventory_path = 'ansible/inventory/hosts'
@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname()
with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-')
containers = ci_lib.container_specs(
['debian*2'],
base_port=2700,
name_template='debops-target-%(distro)s-%(index)d',
)
ci_lib.start_containers(containers)

2
tests/image_prep/_container_finalize.yml
View File

@ -9,7 +9,7 @@
--change 'EXPOSE 22'
--change 'CMD ["/usr/sbin/sshd", "-D"]'
{{ inventory_hostname }}
public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test
{{ container_image_name }}
delegate_to: localhost
- name: Stop containers

3
tests/image_prep/group_vars/all.yml
View File

@ -4,6 +4,9 @@ common_packages:
- strace
- sudo
container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test"
container_registry: public.ecr.aws/n5z0e8q9
sudo_group:
MacOSX: admin
Debian: sudo

30
tests/testlib.py
View File

@ -51,6 +51,12 @@ except NameError:
LOG = logging.getLogger(__name__)
DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
TESTS_DIR = os.path.join(os.path.dirname(__file__))
ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib')
ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
@ -509,6 +515,7 @@ class TestCase(unittest.TestCase):
def get_docker_host():
# Duplicated in ci_lib
url = os.environ.get('DOCKER_HOST')
if url in (None, 'http+docker://localunixsocket'):
return 'localhost'
@ -549,19 +556,23 @@ class DockerizedSshDaemon(object):
]
subprocess.check_output(args)
def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')):
if '-' in mitogen_test_distro:
distro, _py3 = mitogen_test_distro.split('-')
else:
distro = mitogen_test_distro
_py3 = None
def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE):
# Code duplicated in ci_lib.py, both should be updated together
distro_pattern = re.compile(r'''
(?P<distro>(?P<family>[a-z]+)[0-9]+)
(?:-(?P<py>py3))?
(?:\*(?P<count>[0-9]+))?
''',
re.VERBOSE,
)
d = distro_pattern.match(distro).groupdict(default=None)
if _py3 == 'py3':
if d.pop('py') == 'py3':
self.python_path = '/usr/bin/python3'
else:
self.python_path = '/usr/bin/python'
self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
self.image = image_template % d
self.start_container()
self.host = self.get_host()
self.port = self.get_port(self.container_name)
@ -601,6 +612,9 @@ class DockerizedSshDaemon(object):
class BrokerMixin(object):
broker_class = mitogen.master.Broker
# Flag for tests that shutdown the broker themself
# e.g. unix_test.ListenerTest
broker_shutdown = False
def setUp(self):

1
tox.ini
View File

@ -98,6 +98,7 @@ commands =
passenv =
ANSIBLE_*
HOME
MITOGEN_*
# Azure DevOps, TF_BUILD is set to 'True' when running in a build task
# https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables
TF_BUILD