[3.11] gh-117178: Recover lazy loading of self-referential modules (GH-117179) (#117320)

Co-authored-by: Chris Markiewicz <effigies@gmail.com>

Lib/importlib/util.py

@@ -232,12 +232,11 @@ def __getattribute__(self, attr):
             # Only the first thread to get the lock should trigger the load
             # and reset the module's class. The rest can now getattr().
             if object.__getattribute__(self, '__class__') is _LazyModule:
-                # The first thread comes here multiple times as it descends the
-                # call stack. The first time, it sets is_loading and triggers
-                # exec_module(), which will access module.__dict__, module.__name__,
-                # and/or module.__spec__, reentering this method. These accesses
-                # need to be allowed to proceed without triggering the load again.
-                if loader_state['is_loading'] and attr.startswith('__') and attr.endswith('__'):
+                # Reentrant calls from the same thread must be allowed to proceed without
+                # triggering the load again.
+                # exec_module() and self-referential imports are the primary ways this can
+                # happen, but in any case we must return something to avoid deadlock.
+                if loader_state['is_loading']:
                     return object.__getattribute__(self, attr)
                 loader_state['is_loading'] = True
 

Lib/test/test_importlib/test_lazy.py

@@ -178,6 +178,24 @@ def access_module():
             # Or multiple load attempts
             self.assertEqual(loader.load_count, 1)
 
+    def test_lazy_self_referential_modules(self):
+        # Directory modules with submodules that reference the parent can attempt to access
+        # the parent module during a load. Verify that this common pattern works with lazy loading.
+        # json is a good example in the stdlib.
+        json_modules = [name for name in sys.modules if name.startswith('json')]
+        with test_util.uncache(*json_modules):
+            # Standard lazy loading, unwrapped
+            spec = util.find_spec('json')
+            loader = util.LazyLoader(spec.loader)
+            spec.loader = loader
+            module = util.module_from_spec(spec)
+            sys.modules['json'] = module
+            loader.exec_module(module)
+
+            # Trigger load with attribute lookup, ensure expected behavior
+            test_load = module.loads('{}')
+            self.assertEqual(test_load, {})
+
 
 if __name__ == '__main__':
     unittest.main()

Misc/NEWS.d/next/Library/2024-03-23-14-26-18.gh-issue-117178.vTisTG.rst

@@ -0,0 +1,2 @@
+Fix regression in lazy loading of self-referential modules, introduced in
+gh-114781.