gh-153236: Propagate lazy submodule import errors

Author: pablogsal

Include/internal/pycore_global_objects_fini_generated.h

@@ -267,6 +267,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(_filters)
         STRUCT_FOR_ID(_finalizing)
         STRUCT_FOR_ID(_find_and_load)
+        STRUCT_FOR_ID(_find_and_load_lazy_submodule)
         STRUCT_FOR_ID(_fix_up_module)
         STRUCT_FOR_ID(_flags_)
         STRUCT_FOR_ID(_get_sourcefile)

Include/internal/pycore_global_strings.h

@@ -39,8 +39,13 @@ extern PyObject * _PyImport_GetAbsName(
 // Symbol is exported for the JIT on Windows builds.
 PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate(
     PyThreadState *tstate, PyObject *lazy_import);
-extern PyObject * _PyImport_TryLoadLazySubmodule(
-    PyObject *mod_name, PyObject *attr_name);
+typedef enum {
+    _Py_LAZY_SUBMODULE_ERROR = -1,
+    _Py_LAZY_SUBMODULE_NOT_FOUND = 0,
+    _Py_LAZY_SUBMODULE_LOADED = 1,
+} _PyLazySubmoduleImportResult;
+extern _PyLazySubmoduleImportResult _PyImport_TryLoadLazySubmodule(
+    PyObject *mod_name, PyObject *attr_name, PyObject **result);
 extern PyObject * _PyImport_LazyImportModuleLevelObject(
     PyThreadState *tstate, PyObject *name, PyObject *builtins,
     PyObject *globals, PyObject *locals, PyObject *fromlist, int level);

Include/internal/pycore_import.h

@@ -1254,7 +1254,7 @@ def _sanity_check(name, package, level):
 
 _ERR_MSG_PREFIX = 'No module named '
 
-def _find_and_load_unlocked(name, import_):
+def _find_and_load_unlocked(name, import_, *, lazy_submodule=False):
     path = None
     sys.audit(
         "import",
@@ -1277,6 +1277,8 @@ def _find_and_load_unlocked(name, import_):
         try:
             path = parent_module.__path__
         except AttributeError:
+            if lazy_submodule:
+                return None
             msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package'
             raise ModuleNotFoundError(msg, name=name) from None
         parent_spec = parent_module.__spec__
@@ -1289,6 +1291,8 @@ def _find_and_load_unlocked(name, import_):
         child = name.rpartition('.')[2]
     spec = _find_spec(name, path)
     if spec is None:
+        if lazy_submodule:
+            return None
         raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name)
     else:
         if parent_spec:
@@ -1320,7 +1324,7 @@ def _find_and_load_unlocked(name, import_):
 _NEEDS_LOADING = object()
 
 
-def _find_and_load(name, import_):
+def _find_and_load(name, import_, *, lazy_submodule=False):
     """Find and load the module."""
 
     # Optimization: we avoid unneeded module locking if the module
@@ -1337,7 +1341,8 @@ def _find_and_load(name, import_):
         with lock_manager:
             module = sys.modules.get(name, _NEEDS_LOADING)
             if module is _NEEDS_LOADING:
-                return _find_and_load_unlocked(name, import_)
+                return _find_and_load_unlocked(
+                    name, import_, lazy_submodule=lazy_submodule)
 
         # Optimization: only call _bootstrap._lock_unlock_module() if
         # module.__spec__._initializing is True.
@@ -1351,7 +1356,7 @@ def _find_and_load(name, import_):
         # to preserve normal semantics: the caller gets the exception from
         # the actual import failure rather than a synthetic error.
         if sys.modules.get(name) is not module:
-            return _find_and_load(name, import_)
+            return _find_and_load(name, import_, lazy_submodule=lazy_submodule)
 
     if module is None:
         message = f'import of {name} halted; None in sys.modules'
@@ -1360,6 +1365,10 @@ def _find_and_load(name, import_):
     return module
 
 
+def _find_and_load_lazy_submodule(name, import_):
+    return _find_and_load(name, import_, lazy_submodule=True)
+
+
 def _gcd_import(name, package=None, level=0):
     """Import and return the module based on its name, the package the call is
     being made from, and the level adjustment.

Include/internal/pycore_runtime_init_generated.h

@@ -676,52 +676,35 @@ def test_lazy_modules_tracks_lazy_imports(self):
 
 @support.requires_subprocess()
 class ErrorHandlingTests(LazyImportTestCase):
-    """Tests for error handling during lazy import reification.
+    """Tests for error handling during lazy import reification."""
 
-    PEP 810: Errors during reification should show exception chaining with
-    both the lazy import definition location and the access location.
-    """
-
-    def test_import_error_shows_chained_traceback(self):
+    def test_missing_lazy_submodule_raises_attribute_error(self):
         """Accessing a nonexistent lazy submodule via parent attr raises AttributeError."""
         code = textwrap.dedent("""
-            import sys
             lazy import test.test_lazy_import.data.nonexistent_module
 
             try:
-                x = test.test_lazy_import.data.nonexistent_module
-            except AttributeError as e:
-                print("OK")
+                _ = test.test_lazy_import.data.nonexistent_module
+            except AttributeError:
+                pass
+            else:
+                raise AssertionError("AttributeError was not raised")
         """)
-        result = subprocess.run(
-            [sys.executable, "-c", code],
-            capture_output=True,
-            text=True
-        )
-        self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}")
-        self.assertIn("OK", result.stdout)
+        assert_python_ok("-c", code)
 
-    def test_attribute_error_on_from_import_shows_chained_traceback(self):
+    def test_missing_lazy_from_import_shows_chained_traceback(self):
         """Accessing missing attribute from lazy from-import should chain errors."""
-        # Tests 'lazy from module import nonexistent' behavior
         code = textwrap.dedent("""
-            import sys
             lazy from test.test_lazy_import.data.basic2 import nonexistent_name
 
             try:
-                x = nonexistent_name
+                _ = nonexistent_name
             except ImportError as e:
-                # PEP 810: Enhanced error reporting through exception chaining
                 assert e.__cause__ is not None, "Expected chained exception"
-                print("OK")
+            else:
+                raise AssertionError("ImportError was not raised")
         """)
-        result = subprocess.run(
-            [sys.executable, "-c", code],
-            capture_output=True,
-            text=True
-        )
-        self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}")
-        self.assertIn("OK", result.stdout)
+        assert_python_ok("-c", code)
 
     def test_reification_retries_on_failure(self):
         """Failed reification should allow retry on subsequent access.
@@ -731,53 +714,74 @@ def test_reification_retries_on_failure(self):
         """
         code = textwrap.dedent("""
             import sys
-            import types
 
             lazy import test.test_lazy_import.data.broken_module
 
-            # First access - should fail
             try:
-                x = test.test_lazy_import.data.broken_module
-            except AttributeError:
-                pass
+                _ = test.test_lazy_import.data.broken_module
+            except ValueError as exc:
+                assert str(exc) == "This module always fails to import", exc
+            else:
+                raise AssertionError("ValueError was not raised")
+
+            assert "test.test_lazy_import.data.broken_module" not in sys.modules
 
-            # The lazy object should still be a lazy proxy (not reified)
-            g = globals()
-            lazy_obj = g['test']
-            # The root 'test' binding should still allow retry
-            # Second access - should also fail (retry the import)
             try:
-                x = test.test_lazy_import.data.broken_module
-            except AttributeError:
-                print("OK - retry worked")
+                _ = test.test_lazy_import.data.broken_module
+            except ValueError as exc:
+                assert str(exc) == "This module always fails to import", exc
+            else:
+                raise AssertionError("ValueError was not raised")
         """)
-        result = subprocess.run(
-            [sys.executable, "-c", code],
-            capture_output=True,
-            text=True
-        )
-        self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}")
-        self.assertIn("OK", result.stdout)
+        assert_python_ok("-c", code)
 
-    def test_error_during_module_execution_propagates(self):
-        """Errors in module code during reification should propagate correctly."""
+    def test_module_not_found_during_module_execution_propagates(self):
+        code = textwrap.dedent("""
+            lazy import test.test_lazy_import.data.missing_dependency
+
+            try:
+                _ = test.test_lazy_import.data.missing_dependency
+            except ModuleNotFoundError as exc:
+                assert exc.name == "missing_dependency_for_lazy_import_test", exc.name
+                assert str(exc) == "No module named 'missing_dependency_for_lazy_import_test'", exc
+            else:
+                raise AssertionError("ModuleNotFoundError was not raised")
+        """)
+        assert_python_ok("-c", code)
+
+    def test_self_named_module_not_found_during_module_execution_propagates(self):
+        code = textwrap.dedent("""
+            lazy import test.test_lazy_import.data.self_named_module_not_found
+
+            try:
+                _ = test.test_lazy_import.data.self_named_module_not_found
+            except ModuleNotFoundError as exc:
+                assert exc.name == "test.test_lazy_import.data.self_named_module_not_found", exc.name
+                assert str(exc) == "boom", exc
+            else:
+                raise AssertionError("ModuleNotFoundError was not raised")
+        """)
+        assert_python_ok("-c", code)
+
+    def test_none_in_sys_modules_during_submodule_resolution_propagates(self):
         code = textwrap.dedent("""
             import sys
-            lazy import test.test_lazy_import.data.broken_module
+
+            sys.modules["test.test_lazy_import.data.blocked_module"] = None
+            lazy import test.test_lazy_import.data.blocked_module
 
             try:
-                _ = test.test_lazy_import.data.broken_module
-                print("FAIL - should have raised")
-            except AttributeError:
-                print("OK")
+                _ = test.test_lazy_import.data.blocked_module
+            except ModuleNotFoundError as exc:
+                assert exc.name == "test.test_lazy_import.data.blocked_module", exc.name
+                assert str(exc) == (
+                    "import of test.test_lazy_import.data.blocked_module "
+                    "halted; None in sys.modules"
+                ), exc
+            else:
+                raise AssertionError("ModuleNotFoundError was not raised")
         """)
-        result = subprocess.run(
-            [sys.executable, "-c", code],
-            capture_output=True,
-            text=True
-        )
-        self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}")
-        self.assertIn("OK", result.stdout)
+        assert_python_ok("-c", code)
 
     def test_circular_lazy_import_does_not_crash_for_gh_144727(self):
         with tempfile.TemporaryDirectory() as tmpdir:

Include/internal/pycore_unicodeobject_generated.h

@@ -0,0 +1 @@
+import missing_dependency_for_lazy_import_test

Lib/importlib/_bootstrap.py

@@ -0,0 +1 @@
+raise ModuleNotFoundError("boom", name=__name__)

Lib/test/test_lazy_import/__init__.py

@@ -0,0 +1,2 @@
+Propagate exceptions raised while importing lazy submodules instead of
+reporting them as missing attributes.