diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index 2da037b0b6fb84..12d158ee403268 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -19,7 +19,8 @@ PyAPI_FUNC(PyObject *) _PyLazyImport_New( PyObject *name, PyObject *fromlist); extern PyObject *_PyLazyImport_LoadChild( - PyThreadState *tstate, PyObject *declaration, PyObject *name); + PyThreadState *tstate, PyObject *declaration, PyObject *name, + int attributed); extern int _PyLazyImport_IsActive(PyObject *declaration); extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op); diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 31006f804fd6d5..fe8ea7dd6d3d89 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -13,6 +13,7 @@ import contextlib from test import support +from test.support import os_helper from test.support.script_helper import assert_python_ok, assert_python_failure try: @@ -938,6 +939,70 @@ def test_chained_exception_import_shows_notes(self, name): """) assert_python_ok("-c", code) + @support.requires_subprocess() + def test_pending_child_reports_accessing_module_declaration(self): + """A pending child resolves with the declaration in scope for the access. + + gh-158597: a declaration of the same name made by another module must + not replace the accessing module's own declaration. + """ + a_source = ( + "import os, sys, traceback\n" # 1 + "sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))\n" + "\n" # 3 + "lazy import xml.missing\n" # 4 + "import B\n" # 5 + "\n" # 6 + "try:\n" # 7 + " xml.missing\n" # 8 + "except ModuleNotFoundError as exc:\n" + " cause = exc.__cause__\n" + " assert cause is not None, 'expected a chained ImportError'\n" + " frame = traceback.extract_tb(cause.__traceback__)[-1]\n" + " assert frame.filename.endswith('A.py'), frame\n" + " assert frame.lineno == 4, frame\n" + "else:\n" + " raise AssertionError('ModuleNotFoundError was not raised')\n" + ) + b_source = "lazy import xml.missing\n" + with os_helper.temp_dir() as tmp: + for name, source in (('A.py', a_source), ('B.py', b_source)): + with open(os.path.join(tmp, name), 'w', encoding='utf-8') as f: + f.write(source) + assert_python_ok(os.path.join(tmp, 'A.py')) + + @support.requires_subprocess() + def test_unattributed_access_is_not_given_another_declaration(self): + """A declaration made by another module is not this module's frame. + + gh-158597: an access from a module that declared nothing must still + fail the same way, but the chained error must not point into the file + of a module that merely declared the name. + """ + with os_helper.temp_dir() as tmp: + for name in ('A', 'B'): + path = os.path.join(tmp, name + '.py') + with open(path, 'w', encoding='utf-8') as f: + f.write('lazy import xml.missing\n') + code = textwrap.dedent(f""" + import sys, traceback + sys.path.insert(0, {tmp!r}) + import A, B + import xml + try: + xml.missing + except ModuleNotFoundError as exc: + cause = exc.__cause__ + assert cause is not None, 'expected a chained ImportError' + frames = traceback.extract_tb(cause.__traceback__) + declared = [f.filename for f in frames + if f.filename.endswith(('A.py', 'B.py'))] + assert not declared, declared + else: + raise AssertionError('ModuleNotFoundError was not raised') + """) + assert_python_ok("-c", code) + def test_reification_retries_on_failure(self): """Failed reification should allow retry on subsequent access. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-01-40-00.gh-issue-158597.k3QwZt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-01-40-00.gh-issue-158597.k3QwZt.rst new file mode 100644 index 00000000000000..6a6156eb88bb50 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-01-40-00.gh-issue-158597.k3QwZt.rst @@ -0,0 +1,5 @@ +Resolve a lazily imported submodule reached through its parent's attribute +with the declaration in scope where the access is written, instead of the +declaration of whichever module registered that name last. A declaration made +by another module is no longer reported as a frame of the accessing module's +traceback. diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index aafd2ce764a1cb..4b279479c0ff8f 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -26,6 +26,9 @@ typedef struct { // Declaration location. PyCodeObject *lz_code; int lz_instr_offset; + // The location is the accessing module's own declaration, so it belongs in + // that module's traceback. + int lz_attributed; // The root was accessed, so its declared path requires import semantics. int lz_active; } PyLazyImportObject; @@ -73,6 +76,7 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, m->lz_code = NULL; m->lz_instr_offset = -1; + m->lz_attributed = 1; m->lz_active = 0; if (frame != NULL) { @@ -346,7 +350,11 @@ lazy_import_add_exception_cause(PyThreadState *tstate, PyLazyImportObject *lz) goto done; } _PyErr_SetRaisedException(tstate, cause); - _PyTraceback_Add(funcname, filename, lineno); + if (lz->lz_attributed) { + _PyTraceback_Add(funcname, filename, lineno); + } + // A declaration made by another module is not a frame of the traceback + // being raised here, so only the cause reports it. PyException_SetCause(exc, _PyErr_GetRaisedException(tstate)); done: @@ -570,7 +578,7 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) // Resolve a pending child as an aliased import, preserving its declaration. PyObject * _PyLazyImport_LoadChild(PyThreadState *tstate, PyObject *declaration, - PyObject *name) + PyObject *name, int attributed) { assert(PyLazyImport_CheckExact(declaration)); PyLazyImportObject *source = PyLazyImportObject_CAST(declaration); @@ -599,6 +607,7 @@ _PyLazyImport_LoadChild(PyThreadState *tstate, PyObject *declaration, PyLazyImportObject *child = PyLazyImportObject_CAST(lz); child->lz_code = (PyCodeObject *)Py_XNewRef(source->lz_code); child->lz_instr_offset = source->lz_instr_offset; + child->lz_attributed = attributed; PyObject *result = _PyImport_LoadLazyImportTstate(tstate, lz); Py_DECREF(lz); if (result != NULL && PyModule_Check(result) && @@ -606,7 +615,8 @@ _PyLazyImport_LoadChild(PyThreadState *tstate, PyObject *declaration, !PyDict_Contains(_PyModule_GetDict(result), &_Py_ID(__path__))) { // Reaching a non-package must still reject an invalid dotted import. Py_DECREF(result); - return _PyLazyImport_LoadChild(tstate, declaration, source->lz_from); + return _PyLazyImport_LoadChild(tstate, declaration, source->lz_from, + attributed); } return result; } diff --git a/Python/import.c b/Python/import.c index 88d4980991f16a..e5007c867c95b8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4223,12 +4223,78 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) return lazy_submodules; } +// A pending child maps either to Py_None, for a name that is pending without a +// declaration, or to a dict of the declarations recorded for it. Each +// declaration is keyed by the declaring module's __name__, or by None when that +// module has none, so that the child can be resolved with the declaration in +// scope where the access is written instead of with whichever declaration was +// registered last. +static int +add_lazy_declaration(PyObject *submodules, PyObject *child, + PyObject *declaring, PyObject *source) +{ + PyObject *decls = NULL; + if (PyDict_GetItemRef(submodules, child, &decls) < 0) { + return -1; + } + if (decls == NULL || decls == Py_None) { + Py_XDECREF(decls); + decls = PyDict_New(); + if (decls == NULL) { + return -1; + } + if (PyDict_SetItem(submodules, child, decls) < 0) { + Py_DECREF(decls); + return -1; + } + } + int err = PyDict_SetItem(decls, declaring, source); + Py_DECREF(decls); + return err; +} + +// The declaration recorded by *declaring*, as a new reference, or NULL when +// that module recorded none. *declaring* is the __name__ of the module +// performing the access, or None when it has no __name__. +static PyObject * +lazy_declaration_for(PyObject *entry, PyObject *declaring) +{ + if (entry == NULL || entry == Py_None) { + return NULL; + } + PyObject *decl = NULL; + if (PyDict_GetItemRef(entry, declaring, &decl) < 0) { + return NULL; + } + return decl; +} + +// The most recently registered declaration for a pending child, as a new +// reference, or NULL when it has none. Every declaration of a child names the +// same module, so any of them imports the same thing. This serves an access +// that has no declaration of its own in scope, and keeps the declaration that +// would have been used before declarations became keyed by their module. +static PyObject * +lazy_any_declaration(PyObject *entry) +{ + if (entry == NULL || entry == Py_None) { + return NULL; + } + Py_ssize_t pos = 0; + PyObject *value = NULL, *last = NULL; + while (PyDict_Next(entry, &pos, NULL, &value)) { + last = value; + } + return last == NULL ? NULL : Py_NewRef(last); +} + // Records all parent-child relationships in lazy_pending_submodules // for a lazily imported module name. When a parent module's attribute // is accessed, _Py_module_getattro_impl will check lazy_pending_submodules // and trigger the import. static int -register_lazy_on_parent(PyThreadState *tstate, PyObject *name, PyObject *source) +register_lazy_on_parent(PyThreadState *tstate, PyObject *name, + PyObject *source, PyObject *declaring) { PyDictObject *pending = (PyDictObject *)LAZY_PENDING_SUBMODULES(tstate->interp); @@ -4261,9 +4327,18 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, PyObject *source) PyObject *value = cached != NULL && cached != Py_None ? Py_None : source; int err = -1; if (submodules != NULL && !PyErr_Occurred()) { - err = source == Py_None ? - PyDict_SetDefaultRef(submodules, child, value, NULL) : - PyDict_SetItem(submodules, child, value); + if (source == Py_None) { + err = PyDict_SetDefaultRef(submodules, child, value, NULL); + } + else if (value == Py_None) { + err = PyDict_SetItem(submodules, child, value); + } + else { + err = add_lazy_declaration(submodules, child, + declaring != NULL ? declaring + : Py_None, + source); + } } Py_XDECREF(cached); Py_DECREF(child); @@ -4296,7 +4371,7 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *existing_module, rc = lazy_modules_add(tstate, fromname, NULL); if (rc == 0) { - rc = register_lazy_on_parent(tstate, fromname, Py_None); + rc = register_lazy_on_parent(tstate, fromname, Py_None, Py_None); } Py_DECREF(fromname); return rc; @@ -4379,6 +4454,27 @@ lazy_submodule_matches(PyObject *value, void *expected) return value == expected; } +// The __name__ of the module performing an attribute access, as a new +// reference, or None when the access cannot be attributed to a module. +static PyObject * +lazy_accessing_module_name(void) +{ + PyObject *globals = PyEval_GetGlobals(); // borrowed + if (globals == NULL || !PyDict_Check(globals)) { + return Py_NewRef(Py_None); + } + PyObject *name = NULL; + if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &name) < 0) { + PyErr_Clear(); + return Py_NewRef(Py_None); + } + if (name == NULL || !PyUnicode_Check(name)) { + Py_CLEAR(name); + return Py_NewRef(Py_None); + } + return name; +} + PyObject * _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, int suppress, int *recheck_dict) @@ -4389,7 +4485,7 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, if (PyDict_GetItemRef(mod_dict, &_Py_ID(__name__), &mod_name) <= 0) { return NULL; } - PyObject *mod = NULL, *pending_set = NULL, *source = NULL; + PyObject *mod = NULL, *pending_set = NULL, *entry = NULL, *source = NULL; if (!PyUnicode_Check(mod_name)) { goto done; } @@ -4401,18 +4497,28 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, } *recheck_dict = 1; - if (PyDict_GetItemRef(pending_set, attr_name, &source) <= 0) { + if (PyDict_GetItemRef(pending_set, attr_name, &entry) <= 0) { goto done; } + // Resolve with the declaration in scope where the access is written. One + // made by another module still imports the same child, but its location + // must not be reported as a frame of this module's traceback. + PyObject *accessing = lazy_accessing_module_name(); + source = lazy_declaration_for(entry, accessing); + Py_DECREF(accessing); + int attributed = source != NULL; + if (source == NULL) { + source = lazy_any_declaration(entry); + } PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); if (full_name == NULL) { goto done; } // Match eager from-imports: accept a partial module in an import cycle. - if (source != Py_None && !suppress && + if (source != NULL && !suppress && (_PyLazyImport_IsActive(source) || !PyDict_Contains(mod_dict, &_Py_ID(__getattr__)))) { - mod = _PyLazyImport_LoadChild(tstate, source, full_name); + mod = _PyLazyImport_LoadChild(tstate, source, full_name, attributed); } else { mod = PyImport_GetModule(full_name); @@ -4428,7 +4534,7 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, if (loaded != NULL && loaded != Py_None) { (void)_PyDict_DelItemIf(mod_dict, attr_name, lazy_submodule_matches, loaded); - (void)PyDict_SetDefaultRef(pending_set, attr_name, source, NULL); + (void)PyDict_SetDefaultRef(pending_set, attr_name, entry, NULL); } Py_XDECREF(loaded); PyErr_SetRaisedException(exc); @@ -4456,6 +4562,7 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, done: Py_XDECREF(source); + Py_XDECREF(entry); Py_XDECREF(pending_set); Py_DECREF(mod_name); return mod; @@ -4571,8 +4678,21 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, } } } - else if (register_lazy_on_parent(tstate, abs_name, res) < 0) { - goto error; + else { + // Remember which module declared this name, so that a later attribute + // access reports the declaration that is in scope for it. + PyObject *declaring = NULL; + if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &declaring) < 0) { + goto error; + } + if (declaring != NULL && !PyUnicode_Check(declaring)) { + Py_CLEAR(declaring); + } + int rc = register_lazy_on_parent(tstate, abs_name, res, declaring); + Py_XDECREF(declaring); + if (rc < 0) { + goto error; + } } goto done;