From 1c296cfca5408d89cccc504ad349dbdf7dcad83c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 6 Jul 2026 22:23:38 +0100 Subject: [PATCH] gh-153236: Propagate lazy submodule import errors --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_import.h | 9 +- .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 4 + Lib/importlib/_bootstrap.py | 17 ++- Lib/test/test_lazy_import/__init__.py | 132 +++++++++--------- .../data/missing_dependency.py | 1 + .../data/self_named_module_not_found.py | 1 + ...-00.gh-issue-153236.lazy-import-errors.rst | 2 + Objects/moduleobject.c | 11 +- Python/import.c | 70 +++++++--- 12 files changed, 154 insertions(+), 96 deletions(-) create mode 100644 Lib/test/test_lazy_import/data/missing_dependency.py create mode 100644 Lib/test/test_lazy_import/data/self_named_module_not_found.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 7d952a1e52561a6..6df1c01f151f68e 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1544,6 +1544,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_filters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load_lazy_submodule)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fix_up_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8a8bbc3b6d05bf7..873344fbdcb67b0 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -267,6 +267,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_filters) STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) + STRUCT_FOR_ID(_find_and_load_lazy_submodule) STRUCT_FOR_ID(_fix_up_module) STRUCT_FOR_ID(_flags_) STRUCT_FOR_ID(_get_sourcefile) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index a1078828afa572e..669e328c266d00f 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -39,8 +39,13 @@ extern PyObject * _PyImport_GetAbsName( // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); -extern PyObject * _PyImport_TryLoadLazySubmodule( - PyObject *mod_name, PyObject *attr_name); +typedef enum { + _Py_LAZY_SUBMODULE_ERROR = -1, + _Py_LAZY_SUBMODULE_NOT_FOUND = 0, + _Py_LAZY_SUBMODULE_LOADED = 1, +} _PyLazySubmoduleImportResult; +extern _PyLazySubmoduleImportResult _PyImport_TryLoadLazySubmodule( + PyObject *mod_name, PyObject *attr_name, PyObject **result); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 366d2d300fb4780..378f27ca17b5079 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1542,6 +1542,7 @@ extern "C" { INIT_ID(_filters), \ INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ + INIT_ID(_find_and_load_lazy_submodule), \ INIT_ID(_fix_up_module), \ INIT_ID(_flags_), \ INIT_ID(_get_sourcefile), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 00d6297432b5fca..daf6840aa47f328 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -848,6 +848,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_find_and_load_lazy_submodule); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_fix_up_module); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index eb1686a5c8217c6..081bb98ec2a9723 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1254,7 +1254,7 @@ def _sanity_check(name, package, level): _ERR_MSG_PREFIX = 'No module named ' -def _find_and_load_unlocked(name, import_): +def _find_and_load_unlocked(name, import_, *, lazy_submodule=False): path = None sys.audit( "import", @@ -1277,6 +1277,8 @@ def _find_and_load_unlocked(name, import_): try: path = parent_module.__path__ except AttributeError: + if lazy_submodule: + return None msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package' raise ModuleNotFoundError(msg, name=name) from None parent_spec = parent_module.__spec__ @@ -1289,6 +1291,8 @@ def _find_and_load_unlocked(name, import_): child = name.rpartition('.')[2] spec = _find_spec(name, path) if spec is None: + if lazy_submodule: + return None raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name) else: if parent_spec: @@ -1320,7 +1324,7 @@ def _find_and_load_unlocked(name, import_): _NEEDS_LOADING = object() -def _find_and_load(name, import_): +def _find_and_load(name, import_, *, lazy_submodule=False): """Find and load the module.""" # Optimization: we avoid unneeded module locking if the module @@ -1337,7 +1341,8 @@ def _find_and_load(name, import_): with lock_manager: module = sys.modules.get(name, _NEEDS_LOADING) if module is _NEEDS_LOADING: - return _find_and_load_unlocked(name, import_) + return _find_and_load_unlocked( + name, import_, lazy_submodule=lazy_submodule) # Optimization: only call _bootstrap._lock_unlock_module() if # module.__spec__._initializing is True. @@ -1351,7 +1356,7 @@ def _find_and_load(name, import_): # to preserve normal semantics: the caller gets the exception from # the actual import failure rather than a synthetic error. if sys.modules.get(name) is not module: - return _find_and_load(name, import_) + return _find_and_load(name, import_, lazy_submodule=lazy_submodule) if module is None: message = f'import of {name} halted; None in sys.modules' @@ -1360,6 +1365,10 @@ def _find_and_load(name, import_): return module +def _find_and_load_lazy_submodule(name, import_): + return _find_and_load(name, import_, lazy_submodule=True) + + def _gcd_import(name, package=None, level=0): """Import and return the module based on its name, the package the call is being made from, and the level adjustment. diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index cf1d3eb793a8b94..8b67852025567b6 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -676,52 +676,35 @@ def test_lazy_modules_tracks_lazy_imports(self): @support.requires_subprocess() class ErrorHandlingTests(LazyImportTestCase): - """Tests for error handling during lazy import reification. + """Tests for error handling during lazy import reification.""" - PEP 810: Errors during reification should show exception chaining with - both the lazy import definition location and the access location. - """ - - def test_import_error_shows_chained_traceback(self): + def test_missing_lazy_submodule_raises_attribute_error(self): """Accessing a nonexistent lazy submodule via parent attr raises AttributeError.""" code = textwrap.dedent(""" - import sys lazy import test.test_lazy_import.data.nonexistent_module try: - x = test.test_lazy_import.data.nonexistent_module - except AttributeError as e: - print("OK") + _ = test.test_lazy_import.data.nonexistent_module + except AttributeError: + pass + else: + raise AssertionError("AttributeError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) - def test_attribute_error_on_from_import_shows_chained_traceback(self): + def test_missing_lazy_from_import_shows_chained_traceback(self): """Accessing missing attribute from lazy from-import should chain errors.""" - # Tests 'lazy from module import nonexistent' behavior code = textwrap.dedent(""" - import sys lazy from test.test_lazy_import.data.basic2 import nonexistent_name try: - x = nonexistent_name + _ = nonexistent_name except ImportError as e: - # PEP 810: Enhanced error reporting through exception chaining assert e.__cause__ is not None, "Expected chained exception" - print("OK") + else: + raise AssertionError("ImportError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) def test_reification_retries_on_failure(self): """Failed reification should allow retry on subsequent access. @@ -731,53 +714,74 @@ def test_reification_retries_on_failure(self): """ code = textwrap.dedent(""" import sys - import types lazy import test.test_lazy_import.data.broken_module - # First access - should fail try: - x = test.test_lazy_import.data.broken_module - except AttributeError: - pass + _ = test.test_lazy_import.data.broken_module + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc + else: + raise AssertionError("ValueError was not raised") + + assert "test.test_lazy_import.data.broken_module" not in sys.modules - # The lazy object should still be a lazy proxy (not reified) - g = globals() - lazy_obj = g['test'] - # The root 'test' binding should still allow retry - # Second access - should also fail (retry the import) try: - x = test.test_lazy_import.data.broken_module - except AttributeError: - print("OK - retry worked") + _ = test.test_lazy_import.data.broken_module + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc + else: + raise AssertionError("ValueError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) - def test_error_during_module_execution_propagates(self): - """Errors in module code during reification should propagate correctly.""" + def test_module_not_found_during_module_execution_propagates(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.missing_dependency + + try: + _ = test.test_lazy_import.data.missing_dependency + except ModuleNotFoundError as exc: + assert exc.name == "missing_dependency_for_lazy_import_test", exc.name + assert str(exc) == "No module named 'missing_dependency_for_lazy_import_test'", exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_self_named_module_not_found_during_module_execution_propagates(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.self_named_module_not_found + + try: + _ = test.test_lazy_import.data.self_named_module_not_found + except ModuleNotFoundError as exc: + assert exc.name == "test.test_lazy_import.data.self_named_module_not_found", exc.name + assert str(exc) == "boom", exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_none_in_sys_modules_during_submodule_resolution_propagates(self): code = textwrap.dedent(""" import sys - lazy import test.test_lazy_import.data.broken_module + + sys.modules["test.test_lazy_import.data.blocked_module"] = None + lazy import test.test_lazy_import.data.blocked_module try: - _ = test.test_lazy_import.data.broken_module - print("FAIL - should have raised") - except AttributeError: - print("OK") + _ = test.test_lazy_import.data.blocked_module + except ModuleNotFoundError as exc: + assert exc.name == "test.test_lazy_import.data.blocked_module", exc.name + assert str(exc) == ( + "import of test.test_lazy_import.data.blocked_module " + "halted; None in sys.modules" + ), exc + else: + raise AssertionError("ModuleNotFoundError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) def test_circular_lazy_import_does_not_crash_for_gh_144727(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/Lib/test/test_lazy_import/data/missing_dependency.py b/Lib/test/test_lazy_import/data/missing_dependency.py new file mode 100644 index 000000000000000..aa2f2d0f65f73fe --- /dev/null +++ b/Lib/test/test_lazy_import/data/missing_dependency.py @@ -0,0 +1 @@ +import missing_dependency_for_lazy_import_test diff --git a/Lib/test/test_lazy_import/data/self_named_module_not_found.py b/Lib/test/test_lazy_import/data/self_named_module_not_found.py new file mode 100644 index 000000000000000..941230d5da211d8 --- /dev/null +++ b/Lib/test/test_lazy_import/data/self_named_module_not_found.py @@ -0,0 +1 @@ +raise ModuleNotFoundError("boom", name=__name__) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst new file mode 100644 index 000000000000000..c5043af0a4f80b7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst @@ -0,0 +1,2 @@ +Propagate exceptions raised while importing lazy submodules instead of +reporting them as missing attributes. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f447403ef31b43a..b8cd6025c20ba56 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1299,8 +1299,6 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } -// Check if `name` is a lazily pending submodule of module `m`. -// Returns a new reference on success, or NULL with no error set. static PyObject * try_load_lazy_submodule(PyModuleObject *m, PyObject *name) { @@ -1313,10 +1311,13 @@ try_load_lazy_submodule(PyModuleObject *m, PyObject *name) Py_DECREF(mod_name); return NULL; } - PyObject *result = _PyImport_TryLoadLazySubmodule(mod_name, name); + PyObject *result = NULL; + _PyLazySubmoduleImportResult status = + _PyImport_TryLoadLazySubmodule(mod_name, name, &result); Py_DECREF(mod_name); - if (result == NULL) { - PyErr_Clear(); + if (status != _Py_LAZY_SUBMODULE_LOADED) { + assert(status == _Py_LAZY_SUBMODULE_ERROR || + status == _Py_LAZY_SUBMODULE_NOT_FOUND); return NULL; } if (PyDict_SetItem(m->md_dict, name, result) < 0) { diff --git a/Python/import.c b/Python/import.c index 63e23e21beb1266..dc26316c6c23424 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4103,7 +4103,9 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) } static PyObject * -import_find_and_load(PyThreadState *tstate, PyObject *abs_name) +import_find_and_load_with_name(PyThreadState *tstate, PyObject *abs_name, + PyObject *find_and_load, + PyObject *not_found) { PyObject *mod = NULL; PyInterpreterState *interp = tstate->interp; @@ -4130,12 +4132,14 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) if (PyDTrace_IMPORT_FIND_LOAD_START_ENABLED()) PyDTrace_IMPORT_FIND_LOAD_START(PyUnicode_AsUTF8(abs_name)); - mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), &_Py_ID(_find_and_load), + mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), find_and_load, abs_name, IMPORT_FUNC(interp), NULL); - if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) + if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) { + int found = mod != NULL && mod != not_found; PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name), - mod != NULL); + found); + } if (import_time) { PyTime_t t2; @@ -4156,6 +4160,13 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef accumulated } +static PyObject * +import_find_and_load(PyThreadState *tstate, PyObject *abs_name) +{ + return import_find_and_load_with_name( + tstate, abs_name, &_Py_ID(_find_and_load), NULL); +} + static PyObject * get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level) @@ -4439,52 +4450,69 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, return res; } -PyObject * -_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name) +_PyLazySubmoduleImportResult +_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name, + PyObject **result) { - PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(result != NULL); + *result = NULL; + + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); if (lazy_pending == NULL) { - return NULL; + return _Py_LAZY_SUBMODULE_NOT_FOUND; } PyObject *pending_set; int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); - if (rc <= 0) { - return NULL; + if (rc < 0) { + return _Py_LAZY_SUBMODULE_ERROR; + } + if (rc == 0) { + return _Py_LAZY_SUBMODULE_NOT_FOUND; } int contains = PySet_Contains(pending_set, attr_name); - if (contains <= 0) { + if (contains < 0) { Py_DECREF(pending_set); - return NULL; + return _Py_LAZY_SUBMODULE_ERROR; + } + if (contains == 0) { + Py_DECREF(pending_set); + return _Py_LAZY_SUBMODULE_NOT_FOUND; } PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); if (full_name == NULL) { Py_DECREF(pending_set); - return NULL; + return _Py_LAZY_SUBMODULE_ERROR; } - PyObject *mod = PyImport_ImportModuleLevelObject( - full_name, NULL, NULL, NULL, 0); + PyObject *mod = import_find_and_load_with_name( + tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); if (mod == NULL) { Py_DECREF(pending_set); Py_DECREF(full_name); - return NULL; + return _Py_LAZY_SUBMODULE_ERROR; + } + if (mod == Py_None) { + Py_DECREF(mod); + Py_DECREF(pending_set); + Py_DECREF(full_name); + return _Py_LAZY_SUBMODULE_NOT_FOUND; } - Py_DECREF(mod); if (PySet_Discard(pending_set, attr_name) < 0) { + Py_DECREF(mod); Py_DECREF(pending_set); Py_DECREF(full_name); - return NULL; + return _Py_LAZY_SUBMODULE_ERROR; } Py_DECREF(pending_set); - - PyObject *submod = PyImport_GetModule(full_name); Py_DECREF(full_name); - return submod; + *result = mod; + return _Py_LAZY_SUBMODULE_LOADED; } PyObject *