From d78117981a15a71aed91ad525fd2c17c7fc3ae7f Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Mon, 11 Mar 2024 12:02:05 +0100
Subject: [PATCH]  [3.11] gh-115421: List all test/ subdirs in Makefile, and
 test them (GH-115813)

This backports:
- GH-115813
- GH-115422

Unlike on the main branch, new directories are added to the end,
so they're a bit easier to patch out if a redistributor needs to do so.

On main & 3.12, there's a special case for `idlelib/idle_test`; on
3.11 TESTSUBDIRS has several more entries that are not in `test/`.
This backport ignores all of them (including idlelib).
(The alternative would be list them, as additions to TEST_HOME_DIR.
But that's probably too invasive; people might split stdlib up in
surprising ways.)

Co-authored-by: Nikita Sobolev <mail@sobolevn.me>
---
 Lib/test/test_tools/test_makefile.py | 67 ++++++++++++++++++++++++++++
 Makefile.pre.in                      |  6 ++-
 2 files changed, 72 insertions(+), 1 deletion(-)
 create mode 100644 Lib/test/test_tools/test_makefile.py

diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py
new file mode 100644
index 00000000000..aed8420bd05
--- /dev/null
+++ b/Lib/test/test_tools/test_makefile.py
@@ -0,0 +1,67 @@
+"""
+Tests for `Makefile`.
+"""
+
+import os
+import unittest
+from test import support
+import sysconfig
+
+MAKEFILE = sysconfig.get_makefile_filename()
+
+if not support.check_impl_detail(cpython=True):
+    raise unittest.SkipTest('cpython only')
+if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE):
+    raise unittest.SkipTest('Makefile could not be found')
+
+
+class TestMakefile(unittest.TestCase):
+    def list_test_dirs(self):
+        result = []
+        found_testsubdirs = False
+        with open(MAKEFILE, 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.startswith('TESTSUBDIRS='):
+                    found_testsubdirs = True
+                    result.append(
+                        line.removeprefix('TESTSUBDIRS=').replace(
+                            '\\', '',
+                        ).strip(),
+                    )
+                    continue
+                if found_testsubdirs:
+                    if '\t' not in line:
+                        break
+                    result.append(line.replace('\\', '').strip())
+
+        # In Python 3.11 (and lower), many test modules are not in
+        # the tests/ directory. This check ignores them.
+        result = [d for d in result if d.startswith('test/') or d == 'test']
+
+        return result
+
+    def test_makefile_test_folders(self):
+        test_dirs = self.list_test_dirs()
+
+        used = []
+        for dirpath, _, _ in os.walk(support.TEST_HOME_DIR):
+            dirname = os.path.basename(dirpath)
+            if dirname == '__pycache__':
+                continue
+
+            relpath = os.path.relpath(dirpath, support.STDLIB_DIR)
+            with self.subTest(relpath=relpath):
+                self.assertIn(
+                    relpath,
+                    test_dirs,
+                    msg=(
+                        f"{relpath!r} is not included in the Makefile's list "
+                        "of test directories to install"
+                    )
+                )
+                used.append(relpath)
+
+        # Check that there are no extra entries:
+        unique_test_dirs = set(test_dirs)
+        self.assertSetEqual(unique_test_dirs, set(used))
+        self.assertEqual(len(test_dirs), len(unique_test_dirs))
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 783479d0df2..81d4d50f82f 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -2065,7 +2065,11 @@ TESTSUBDIRS=	ctypes/test \
 		tkinter/test/test_tkinter \
 		tkinter/test/test_ttk \
 		unittest/test \
-		unittest/test/testmock
+		unittest/test/testmock \
+		test/test_concurrent_futures \
+		test/test_multiprocessing_fork \
+		test/test_multiprocessing_forkserver \
+		test/test_multiprocessing_spawn
 
 TEST_MODULES=@TEST_MODULES@
 libinstall:	all $(srcdir)/Modules/xxmodule.c