From 7ca073cdf8b65c690db428ecb51c667fd84bb4a7 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Fri, 9 Aug 2019 20:19:16 +0100
Subject: [PATCH] issue #482: ci: add stray process checks to all jobs

List of interesting processes can probably expand more over time.
---
 .ci/ansible_tests.py       |  9 +++++
 .ci/ci_lib.py              | 73 ++++++++++++++++++++++++++++++++++++++
 .ci/debops_common_tests.py |  6 ++++
 .ci/mitogen_install.py     |  1 +
 .ci/mitogen_tests.py       |  2 ++
 5 files changed, 91 insertions(+)

diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py
index 51eab874..4df2dc70 100755
--- a/.ci/ansible_tests.py
+++ b/.ci/ansible_tests.py
@@ -20,11 +20,17 @@ def pause_if_interactive():
             signal.pause()
 
 
+interesting = ci_lib.get_interesting_procs()
+
+
 with ci_lib.Fold('unit_tests'):
     os.environ['SKIP_MITOGEN'] = '1'
     ci_lib.run('./run_tests -v')
 
 
+ci_lib.check_stray_processes(interesting)
+
+
 with ci_lib.Fold('docker_setup'):
     containers = ci_lib.make_containers()
     ci_lib.start_containers(containers)
@@ -75,4 +81,7 @@ with ci_lib.Fold('ansible'):
         pause_if_interactive()
         raise
 
+
+ci_lib.check_stray_processes(interesting, containers)
+
 pause_if_interactive()
diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py
index 34a6faba..9225723e 100644
--- a/.ci/ci_lib.py
+++ b/.ci/ci_lib.py
@@ -215,6 +215,44 @@ def make_containers(name_prefix='', port_offset=0):
     return lst
 
 
+INTERESTING_COMMS = ('python', 'ssh', 'sudo', 'su', 'doas')
+
+
+def proc_is_docker(pid):
+    try:
+        fp = open('/proc/%s/cgroup' % (pid,), 'rb')
+    except IOError:
+        return False
+
+    try:
+        return 'docker' in fp.read()
+    finally:
+        fp.close()
+
+
+def get_interesting_procs(container_name=None):
+    args = ['ps', '-a', '-x', '-oppid=', '-opid=', '-ocomm=', '-ocommand=']
+    if container_name is not None:
+        args = ['docker', 'exec', container_name] + args
+
+    out = []
+    for line in subprocess__check_output(args).splitlines():
+        ppid, pid, comm, rest = line.split(None, 3)
+        if (
+            (
+                any(comm.startswith(s) for s in INTERESTING_COMMS) or
+                'mitogen:' in rest
+            ) and
+            (
+                container_name is not None or
+                (not proc_is_docker(pid))
+            )
+        ):
+            out.append((int(pid), line))
+
+    return sorted(out)
+
+
 def start_containers(containers):
     if os.environ.get('KEEP'):
         return
@@ -236,9 +274,44 @@ def start_containers(containers):
         ]
         for container in containers
     ])
+
+    for container in containers:
+        container['interesting'] = get_interesting_procs(container['name'])
+
     return containers
 
 
+def verify_procs(hostname, old, new):
+    oldpids = set(pid for pid, _ in old)
+    if any(pid not in oldpids for pid, _ in new):
+        print('%r had stray processes running:' % (hostname,))
+        for pid, line in new:
+            if pid not in oldpids:
+                print('New process:', line)
+
+        print()
+        return False
+
+    return True
+
+
+def check_stray_processes(old, containers=None):
+    ok = True
+
+    new = get_interesting_procs()
+    if old is not None:
+        ok &= verify_procs('test host machine', old, new)
+
+    for container in containers or ():
+        ok &= verify_procs(
+            container['name'],
+            container['interesting'],
+            get_interesting_procs(container['name'])
+        )
+
+    assert ok, 'stray processes were found'
+
+
 def dump_file(path):
     print()
     print('--- %s ---' % (path,))
diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py
index 8b35de1e..e8f2907b 100755
--- a/.ci/debops_common_tests.py
+++ b/.ci/debops_common_tests.py
@@ -68,9 +68,15 @@ with ci_lib.Fold('job_setup'):
     os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
 
 
+interesting = ci_lib.get_interesting_procs()
+
 with ci_lib.Fold('first_run'):
     ci_lib.run('debops common %s', ' '.join(sys.argv[1:]))
 
+ci_lib.check_stray_processes(interesting, containers)
+
 
 with ci_lib.Fold('second_run'):
     ci_lib.run('debops common %s', ' '.join(sys.argv[1:]))
+
+ci_lib.check_stray_processes(interesting, containers)
diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py
index 72bc75e3..b8862f89 100755
--- a/.ci/mitogen_install.py
+++ b/.ci/mitogen_install.py
@@ -14,4 +14,5 @@ if ci_lib.have_docker():
         'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ])
 
+
 ci_lib.run_batches(batches)
diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py
index 36928ac9..4de94b4c 100755
--- a/.ci/mitogen_tests.py
+++ b/.ci/mitogen_tests.py
@@ -14,4 +14,6 @@ os.environ.update({
 if not ci_lib.have_docker():
     os.environ['SKIP_DOCKER_TESTS'] = '1'
 
+interesting = ci_lib.get_interesting_procs()
 ci_lib.run('./run_tests -v')
+ci_lib.check_stray_processes(interesting)