From 00db886877d37c0c418194a5f831d34c5d7e9e6f Mon Sep 17 00:00:00 2001 From: Bowen Hua Date: Fri, 16 Oct 2026 00:11:28 +0800 Subject: [PATCH] gh-158597: Resolve lazy submodules with the declaration in scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pending child in `lazy_pending_submodules` is keyed only by `(parent, child)`, so a later declaration of the same name by another module replaces the earlier one. Resolving the child then copies the location out of the surviving declaration, so the chained `ImportError` points into the file of a module that merely declared the name. Key the declarations by the declaring module, and select one from the frame's globals at access time, so the child resolves with the declaration in scope where the access is written. A declaration made by another module still names the same child and imports the same thing, so it is still used when there is nothing in scope; its location is then not reported as a frame of the accessing module's traceback. The selection also decides which `__import__` runs, since `_PyLazyImport_LoadChild()` takes `lz_builtins` from the declaration it is given. An access that has a declaration of its own in scope now uses that module's hook instead of the last-registered one's — the case Yhg1s described in the gh-158521 review. An access with no declaration in scope is unchanged. --- Include/internal/pycore_lazyimportobject.h | 3 +- Lib/test/test_lazy_import/__init__.py | 65 ++++++++ ...-10-13-01-40-00.gh-issue-158597.k3QwZt.rst | 5 + Objects/lazyimportobject.c | 16 +- Python/import.c | 144 ++++++++++++++++-- 5 files changed, 217 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-01-40-00.gh-issue-158597.k3QwZt.rst diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index 2da037b0b6fb842..12d158ee4032687 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -19,7 +19,8 @@ PyAPI_FUNC(PyObject *) _PyLazyImport_New( PyObject *name, PyObject *fromlist); extern PyObject *_PyLazyImport_LoadChild( - PyThreadState *tstate, PyObject *declaration, PyObject *name); + PyThreadState *tstate, PyObject *declaration, PyObject *name, + int attributed); extern int _PyLazyImport_IsActive(PyObject *declaration); extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op); diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 31006f804fd6d51..fe8ea7dd6d3d895 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -13,6 +13,7 @@ import contextlib from test import support +from test.support import os_helper from test.support.script_helper import assert_python_ok, assert_python_failure try: @@ -938,6 +939,70 @@ def test_chained_exception_import_shows_notes(self, name): """) assert_python_ok("-c", code) + @support.requires_subprocess() + def test_pending_child_reports_accessing_module_declaration(self): + """A pending child resolves with the declaration in scope for the access. + + gh-158597: a declaration of the same name made by another module must + not replace the accessing module's own declaration. + """ + a_source = ( + "import os, sys, traceback\n" # 1 + "sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))\n" + "\n" # 3 + "lazy import xml.missing\n" # 4 + "import B\n" # 5 + "\n" # 6 + "try:\n" # 7 + " xml.missing\n" # 8 + "except ModuleNotFoundError as exc:\n" + " cause = exc.__cause__\n" + " assert cause is not None, 'expected a chained ImportError'\n" + " frame = traceback.extract_tb(cause.__traceback__)[-1]\n" + " assert frame.filename.endswith('A.py'), frame\n" + " assert frame.lineno == 4, frame\n" + "else:\n" + " raise AssertionError('ModuleNotFoundError was not raised')\n" + ) + b_source = "lazy import xml.missing\n" + with os_helper.temp_dir() as tmp: + for name, source in (('A.py', a_source), ('B.py', b_source)): + with open(os.path.join(tmp, name), 'w', encoding='utf-8') as f: + f.write(source) + assert_python_ok(os.path.join(tmp, 'A.py')) + + @support.requires_subprocess() + def test_unattributed_access_is_not_given_another_declaration(self): + """A declaration made by another module is not this module's frame. + + gh-158597: an access from a module that declared nothing must still + fail the same way, but the chained error must not point into the file + of a module that merely declared the name. + """ + with os_helper.temp_dir() as tmp: + for name in ('A', 'B'): + path = os.path.join(tmp, name + '.py') + with open(path, 'w', encoding='utf-8') as f: + f.write('lazy import xml.missing\n') + code = textwrap.dedent(f""" + import sys, traceback + sys.path.insert(0, {tmp!r}) + import A, B + import xml + try: + xml.missing + except ModuleNotFoundError as exc: + cause = exc.__cause__ + assert cause is not None, 'expected a chained ImportError' + frames = traceback.extract_tb(cause.__traceback__) + declared = [f.filename for f in frames + if f.filename.endswith(('A.py', 'B.py'))] + assert not declared, declared + else: + raise AssertionError('ModuleNotFoundError was not raised') + """) + assert_python_ok("-c", code) + def test_reification_retries_on_failure(self): """Failed reification should allow retry on subsequent access. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-01-40-00.gh-issue-158597.k3QwZt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-01-40-00.gh-issue-158597.k3QwZt.rst new file mode 100644 index 000000000000000..6a6156eb88bb503 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-01-40-00.gh-issue-158597.k3QwZt.rst @@ -0,0 +1,5 @@ +Resolve a lazily imported submodule reached through its parent's attribute +with the declaration in scope where the access is written, instead of the +declaration of whichever module registered that name last. A declaration made +by another module is no longer reported as a frame of the accessing module's +traceback. diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index aafd2ce764a1cbc..4b279479c0ff8fb 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -26,6 +26,9 @@ typedef struct { // Declaration location. PyCodeObject *lz_code; int lz_instr_offset; + // The location is the accessing module's own declaration, so it belongs in + // that module's traceback. + int lz_attributed; // The root was accessed, so its declared path requires import semantics. int lz_active; } PyLazyImportObject; @@ -73,6 +76,7 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, m->lz_code = NULL; m->lz_instr_offset = -1; + m->lz_attributed = 1; m->lz_active = 0; if (frame != NULL) { @@ -346,7 +350,11 @@ lazy_import_add_exception_cause(PyThreadState *tstate, PyLazyImportObject *lz) goto done; } _PyErr_SetRaisedException(tstate, cause); - _PyTraceback_Add(funcname, filename, lineno); + if (lz->lz_attributed) { + _PyTraceback_Add(funcname, filename, lineno); + } + // A declaration made by another module is not a frame of the traceback + // being raised here, so only the cause reports it. PyException_SetCause(exc, _PyErr_GetRaisedException(tstate)); done: @@ -570,7 +578,7 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) // Resolve a pending child as an aliased import, preserving its declaration. PyObject * _PyLazyImport_LoadChild(PyThreadState *tstate, PyObject *declaration, - PyObject *name) + PyObject *name, int attributed) { assert(PyLazyImport_CheckExact(declaration)); PyLazyImportObject *source = PyLazyImportObject_CAST(declaration); @@ -599,6 +607,7 @@ _PyLazyImport_LoadChild(PyThreadState *tstate, PyObject *declaration, PyLazyImportObject *child = PyLazyImportObject_CAST(lz); child->lz_code = (PyCodeObject *)Py_XNewRef(source->lz_code); child->lz_instr_offset = source->lz_instr_offset; + child->lz_attributed = attributed; PyObject *result = _PyImport_LoadLazyImportTstate(tstate, lz); Py_DECREF(lz); if (result != NULL && PyModule_Check(result) && @@ -606,7 +615,8 @@ _PyLazyImport_LoadChild(PyThreadState *tstate, PyObject *declaration, !PyDict_Contains(_PyModule_GetDict(result), &_Py_ID(__path__))) { // Reaching a non-package must still reject an invalid dotted import. Py_DECREF(result); - return _PyLazyImport_LoadChild(tstate, declaration, source->lz_from); + return _PyLazyImport_LoadChild(tstate, declaration, source->lz_from, + attributed); } return result; } diff --git a/Python/import.c b/Python/import.c index 88d4980991f16a6..e5007c867c95b8f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4223,12 +4223,78 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) return lazy_submodules; } +// A pending child maps either to Py_None, for a name that is pending without a +// declaration, or to a dict of the declarations recorded for it. Each +// declaration is keyed by the declaring module's __name__, or by None when that +// module has none, so that the child can be resolved with the declaration in +// scope where the access is written instead of with whichever declaration was +// registered last. +static int +add_lazy_declaration(PyObject *submodules, PyObject *child, + PyObject *declaring, PyObject *source) +{ + PyObject *decls = NULL; + if (PyDict_GetItemRef(submodules, child, &decls) < 0) { + return -1; + } + if (decls == NULL || decls == Py_None) { + Py_XDECREF(decls); + decls = PyDict_New(); + if (decls == NULL) { + return -1; + } + if (PyDict_SetItem(submodules, child, decls) < 0) { + Py_DECREF(decls); + return -1; + } + } + int err = PyDict_SetItem(decls, declaring, source); + Py_DECREF(decls); + return err; +} + +// The declaration recorded by *declaring*, as a new reference, or NULL when +// that module recorded none. *declaring* is the __name__ of the module +// performing the access, or None when it has no __name__. +static PyObject * +lazy_declaration_for(PyObject *entry, PyObject *declaring) +{ + if (entry == NULL || entry == Py_None) { + return NULL; + } + PyObject *decl = NULL; + if (PyDict_GetItemRef(entry, declaring, &decl) < 0) { + return NULL; + } + return decl; +} + +// The most recently registered declaration for a pending child, as a new +// reference, or NULL when it has none. Every declaration of a child names the +// same module, so any of them imports the same thing. This serves an access +// that has no declaration of its own in scope, and keeps the declaration that +// would have been used before declarations became keyed by their module. +static PyObject * +lazy_any_declaration(PyObject *entry) +{ + if (entry == NULL || entry == Py_None) { + return NULL; + } + Py_ssize_t pos = 0; + PyObject *value = NULL, *last = NULL; + while (PyDict_Next(entry, &pos, NULL, &value)) { + last = value; + } + return last == NULL ? NULL : Py_NewRef(last); +} + // Records all parent-child relationships in lazy_pending_submodules // for a lazily imported module name. When a parent module's attribute // is accessed, _Py_module_getattro_impl will check lazy_pending_submodules // and trigger the import. static int -register_lazy_on_parent(PyThreadState *tstate, PyObject *name, PyObject *source) +register_lazy_on_parent(PyThreadState *tstate, PyObject *name, + PyObject *source, PyObject *declaring) { PyDictObject *pending = (PyDictObject *)LAZY_PENDING_SUBMODULES(tstate->interp); @@ -4261,9 +4327,18 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, PyObject *source) PyObject *value = cached != NULL && cached != Py_None ? Py_None : source; int err = -1; if (submodules != NULL && !PyErr_Occurred()) { - err = source == Py_None ? - PyDict_SetDefaultRef(submodules, child, value, NULL) : - PyDict_SetItem(submodules, child, value); + if (source == Py_None) { + err = PyDict_SetDefaultRef(submodules, child, value, NULL); + } + else if (value == Py_None) { + err = PyDict_SetItem(submodules, child, value); + } + else { + err = add_lazy_declaration(submodules, child, + declaring != NULL ? declaring + : Py_None, + source); + } } Py_XDECREF(cached); Py_DECREF(child); @@ -4296,7 +4371,7 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *existing_module, rc = lazy_modules_add(tstate, fromname, NULL); if (rc == 0) { - rc = register_lazy_on_parent(tstate, fromname, Py_None); + rc = register_lazy_on_parent(tstate, fromname, Py_None, Py_None); } Py_DECREF(fromname); return rc; @@ -4379,6 +4454,27 @@ lazy_submodule_matches(PyObject *value, void *expected) return value == expected; } +// The __name__ of the module performing an attribute access, as a new +// reference, or None when the access cannot be attributed to a module. +static PyObject * +lazy_accessing_module_name(void) +{ + PyObject *globals = PyEval_GetGlobals(); // borrowed + if (globals == NULL || !PyDict_Check(globals)) { + return Py_NewRef(Py_None); + } + PyObject *name = NULL; + if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &name) < 0) { + PyErr_Clear(); + return Py_NewRef(Py_None); + } + if (name == NULL || !PyUnicode_Check(name)) { + Py_CLEAR(name); + return Py_NewRef(Py_None); + } + return name; +} + PyObject * _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, int suppress, int *recheck_dict) @@ -4389,7 +4485,7 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, if (PyDict_GetItemRef(mod_dict, &_Py_ID(__name__), &mod_name) <= 0) { return NULL; } - PyObject *mod = NULL, *pending_set = NULL, *source = NULL; + PyObject *mod = NULL, *pending_set = NULL, *entry = NULL, *source = NULL; if (!PyUnicode_Check(mod_name)) { goto done; } @@ -4401,18 +4497,28 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, } *recheck_dict = 1; - if (PyDict_GetItemRef(pending_set, attr_name, &source) <= 0) { + if (PyDict_GetItemRef(pending_set, attr_name, &entry) <= 0) { goto done; } + // Resolve with the declaration in scope where the access is written. One + // made by another module still imports the same child, but its location + // must not be reported as a frame of this module's traceback. + PyObject *accessing = lazy_accessing_module_name(); + source = lazy_declaration_for(entry, accessing); + Py_DECREF(accessing); + int attributed = source != NULL; + if (source == NULL) { + source = lazy_any_declaration(entry); + } PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); if (full_name == NULL) { goto done; } // Match eager from-imports: accept a partial module in an import cycle. - if (source != Py_None && !suppress && + if (source != NULL && !suppress && (_PyLazyImport_IsActive(source) || !PyDict_Contains(mod_dict, &_Py_ID(__getattr__)))) { - mod = _PyLazyImport_LoadChild(tstate, source, full_name); + mod = _PyLazyImport_LoadChild(tstate, source, full_name, attributed); } else { mod = PyImport_GetModule(full_name); @@ -4428,7 +4534,7 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, if (loaded != NULL && loaded != Py_None) { (void)_PyDict_DelItemIf(mod_dict, attr_name, lazy_submodule_matches, loaded); - (void)PyDict_SetDefaultRef(pending_set, attr_name, source, NULL); + (void)PyDict_SetDefaultRef(pending_set, attr_name, entry, NULL); } Py_XDECREF(loaded); PyErr_SetRaisedException(exc); @@ -4456,6 +4562,7 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, done: Py_XDECREF(source); + Py_XDECREF(entry); Py_XDECREF(pending_set); Py_DECREF(mod_name); return mod; @@ -4571,8 +4678,21 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, } } } - else if (register_lazy_on_parent(tstate, abs_name, res) < 0) { - goto error; + else { + // Remember which module declared this name, so that a later attribute + // access reports the declaration that is in scope for it. + PyObject *declaring = NULL; + if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &declaring) < 0) { + goto error; + } + if (declaring != NULL && !PyUnicode_Check(declaring)) { + Py_CLEAR(declaring); + } + int rc = register_lazy_on_parent(tstate, abs_name, res, declaring); + Py_XDECREF(declaring); + if (rc < 0) { + goto error; + } } goto done;