diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index a964b43ebee1746..9811254396b47f4 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -917,6 +917,13 @@ If an error occurs during module loading (such as :exc:`ImportError` or :exc:`SyntaxError`), it is raised at the point where the lazy import is first used, not at the import statement itself. +For ordinary packages, plain dotted lazy imports resolve the root first and +then import each requested submodule through its declaring namespace's current +:func:`__import__` hook. A shared package selects the declaration with matching +builtins, or the first declaring namespace if none matches. Hooks that return +objects outside :data:`sys.modules` retain full-path import behavior. +Unresolved plain-import placeholders retain their declaring namespace. + See :pep:`810` for the full specification of lazy imports. .. versionadded:: 3.15 diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index f64b65e0f890dfb..95f18324c0233e5 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -44,6 +44,10 @@ extern PyObject * _PyImport_TryLoadLazySubmodule( PyObject *module, PyObject *attr_name, int suppress, int *recheck_dict); extern int _PyImport_ClearLazySubmodule( PyThreadState *tstate, PyObject *name, int bind); +extern int _PyImport_RegisterLazySubmodules( + PyThreadState *tstate, PyObject *name, PyObject *source); +extern PyObject *_PyImport_GetLazyGroup( + PyThreadState *tstate, PyObject *root, PyObject *globals, PyObject *candidate); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index fe501be1afc59b1..9af4cfb2f3d79d8 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -359,6 +359,8 @@ struct _import_state { // Package names map to pending children: declarations for plain imports, // or None for from-import names that may be ordinary attributes. PyObject *lazy_pending_submodules; + // Weakly indexed declaration groups, owned by unresolved placeholders. + PyObject *lazy_import_groups; // Avoid pending-child work for ordinary cached imports. int has_lazy_submodules; #ifdef Py_GIL_DISABLED diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index 2da037b0b6fb842..d8bef696aef4fc7 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -19,7 +19,13 @@ PyAPI_FUNC(PyObject *) _PyLazyImport_New( PyObject *name, PyObject *fromlist); extern PyObject *_PyLazyImport_LoadChild( - PyThreadState *tstate, PyObject *declaration, PyObject *name); + PyThreadState *tstate, PyObject *declaration, PyObject *name, + PyObject *import_func); +// Borrowed references from the original import declaration. +extern PyObject *_PyLazyImport_GetBuiltins(PyObject *declaration); +// Share declaration ownership with plain imports in the same namespace. +extern PyObject *_PyLazyImport_Group( + PyThreadState *tstate, PyObject *source, PyObject *globals); extern int _PyLazyImport_IsActive(PyObject *declaration); extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op); diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index e0c5552fe11c53e..74085b27ac285fa 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -470,6 +470,272 @@ def test_siblings(self): assert {package}.{second} is sys.modules['{package}.{second}'] """) + def test_siblings_with_import_hook(self): + accesses = ('xml.dom', 'getattr(xml, "dom")', + 'getattr(xml, "dom", None)', 'hasattr(xml, "dom")', + 'helper(xml)', 'xml') + for private in (False, True): + for sibling in ('xml.etree', 'xml.missing'): + for reverse in (False, True): + statements = ['lazy import xml.dom', + f'lazy import {sibling}'] + if reverse: + statements.reverse() + for access in accesses: + code = '\n'.join(statements + [f'result = {access}']) + with self.subTest(private=private, sibling=sibling, + reverse=reverse, access=access): + self.check(f""" + import builtins, sys + original = builtins.__import__ + calls = [] + def hook(name, *args): + if name.startswith('xml'): + calls.append(name) + return original(name, *args) + def helper(package): + return package.dom + namespace = {{'helper': helper}} + if {private}: + namespace['__builtins__'] = dict( + vars(builtins), __import__=hook) + else: + builtins.__import__ = hook + exec({code!r}, namespace) + assert {sibling!r} not in calls, calls + assert {sibling!r} not in sys.modules + if {access!r} == 'xml': + assert namespace['result'] is sys.modules['xml'] + assert 'xml.dom' not in sys.modules + assert calls == ['xml'], calls + else: + result = namespace['result'] + if {access!r} == 'hasattr(xml, "dom")': + assert result is True + else: + assert result is sys.modules['xml.dom'] + assert calls == ['xml', 'xml.dom'], calls + """) + + def test_preloaded_sibling_with_import_hook(self): + for private in (False, True): + with self.subTest(private=private): + self.check(f""" + import builtins, sys, xml.etree + original = builtins.__import__ + calls = [] + def hook(name, *args): + if name.startswith('xml'): + calls.append(name) + if name == 'xml.etree': + raise AssertionError('requested an unused sibling') + return original(name, *args) + namespace = {{}} + if {private}: + namespace['__builtins__'] = dict( + vars(builtins), __import__=hook) + else: + builtins.__import__ = hook + exec('lazy import xml.dom\\nlazy import xml.etree', namespace) + exec('root = xml', namespace) + assert namespace['root'] is sys.modules['xml'] + assert 'xml.dom' not in sys.modules + assert calls == ['xml'], calls + exec('result = xml.dom', namespace) + assert namespace['result'] is sys.modules['xml.dom'] + assert calls == ['xml', 'xml.dom'], calls + """) + + def test_pending_child_preserves_fromlist(self): + for previous, fromlist in ((None, ()), ((), None)): + with self.subTest(fromlist=fromlist): + self.check(f""" + import builtins + original = builtins.__import__ + calls = [] + def hook(name, globals, locals, fromlist, level): + calls.append((name, fromlist)) + assert fromlist == {fromlist!r} + return original(name, globals, locals, fromlist, level) + namespace = {{'__builtins__': dict( + vars(builtins), __import__=hook)}} + exec("xml = __lazy_import__('xml.dom.minidom', fromlist={previous!r})\\n" + "xml = __lazy_import__('xml.dom.pulldom', fromlist={fromlist!r})", + namespace) + exec("assert xml.dom.__name__ == 'xml.dom'", namespace) + assert calls == [('xml', {fromlist!r}), + ('xml.dom', {fromlist!r})], calls + """) + + def test_cached_child_with_import_hook(self): + for access in ('xml.dom', 'getattr(xml, "dom", None)', 'helper(xml)'): + with self.subTest(access=access): + self.check(f""" + import builtins, sys, xml.dom + del xml.dom + original = builtins.__import__ + calls = [] + denied = True + def hook(name, *args): + calls.append(name) + if denied and name == 'xml.dom': + raise PermissionError('denied child') + return original(name, *args) + def helper(package): + return package.dom + namespace = {{'helper': helper, '__builtins__': dict( + vars(builtins), __import__=hook)}} + exec('lazy import xml.dom\\nlazy import xml.missing', namespace) + try: + exec({access!r}, namespace) + except PermissionError: + pass + else: + raise AssertionError('import hook was bypassed') + assert calls == ['xml', 'xml.dom'], calls + denied = False + exec('result = ' + {access!r}, namespace) + assert namespace['result'] is sys.modules['xml.dom'] + assert calls == ['xml', 'xml.dom', 'xml.dom'], calls + """) + + def test_failed_hook_preserves_namespace_declarations(self): + self.check(""" + import builtins, sys + original = builtins.__import__ + calls = [] + def second_import(name, *args, **kwargs): + if name == 'xml.dom': + calls.append('second') + raise RuntimeError('second') + return original(name, *args, **kwargs) + second = {'__builtins__': dict(vars(builtins), __import__=second_import)} + def first_import(name, *args, **kwargs): + module = original(name, *args, **kwargs) + if name == 'xml.dom': + calls.append('first') + del module.dom + del sys.modules[name] + exec('lazy import xml.dom', second) + raise RuntimeError('first') + return module + first = {'__builtins__': dict(vars(builtins), __import__=first_import)} + exec('lazy import xml.dom', first) + for namespace, message in ((first, 'first'), (first, 'first'), + (second, 'second')): + try: + exec('xml.dom', namespace) + except RuntimeError as exc: + assert str(exc) == message, exc + else: + raise AssertionError('hook did not run') + assert calls == ['first', 'first', 'second'], calls + """) + + def test_siblings_after_builtins_replacement(self): + self.check(""" + import builtins, types, xml.dom + calls = [] + def hook(name, *args): + calls.append(name) + assert name == 'xml.dom', name + return types.SimpleNamespace(dom=42) + def helper(package): + return package.dom + namespace = {'__builtins__': dict(vars(builtins), __import__=hook), + 'helper': helper} + exec('lazy import xml.dom', namespace) + namespace['__builtins__'] = vars(builtins) + exec('lazy import xml.missing', namespace) + del xml.dom + exec('result = helper(xml)', namespace) + assert namespace['result'] == 42 + assert calls == ['xml.dom'], calls + assert 'dom' not in vars(xml) + """) + + def test_live_namespace_keeps_overwritten_sibling_source(self): + for statement in ('lazy import {name}', + 'live_owners = __lazy_import__({name!r})', + "dict.__setitem__(globals(), 'live_owners', " + "__lazy_import__({name!r}))"): + code = '\n'.join(statement.format(name='live_owners.' + child) + for child in ('child', 'missing')) + with self.subTest(statement=statement): + self.check(f""" + import builtins, gc, sys, types + root = types.ModuleType('live_owners') + root.__path__ = [] + sys.modules['live_owners'] = root + def first_import(name, *args): + raise AssertionError('used the first namespace hook') + first = {{'__builtins__': dict( + vars(builtins), __import__=first_import)}} + calls = [] + def second_import(name, *args): + calls.append(name) + if name == 'live_owners': + return root + assert name == 'live_owners.child', name + return types.SimpleNamespace(child=42) + second = {{'__builtins__': dict( + vars(builtins), __import__=second_import)}} + exec('lazy import live_owners.child', first) + exec({code!r}, second) + gc.collect() + exec('root = live_owners', second) + gc.collect() + exec('result = live_owners.child', second) + assert second['result'] == 42 + assert calls == ['live_owners', 'live_owners.child'], calls + """) + + def test_discarded_namespaces_do_not_retain_all_import_hooks(self): + self.check(""" + import builtins, gc, weakref + class Payload: + pass + references = [] + for _ in range(32): + payload = Payload() + references.append(weakref.ref(payload)) + namespace = {'__builtins__': dict(vars(builtins), payload=payload)} + exec('lazy import dormant_owners.child', namespace) + del namespace, payload + gc.collect() + # Keep one fallback for accesses outside the declaring namespace. + assert sum(ref() is not None for ref in references) == 1 + """) + + def test_cached_declaration_does_not_run_spec_callbacks(self): + self.check(""" + import xml.dom + class Spec: + @property + def _initializing(self): + raise AssertionError('spec read during declaration') + xml.__spec__ = Spec() + lazy import xml.dom + """) + + def test_hook_creates_aliased_package(self): + self.check(""" + import builtins, sys, types + root = types.ModuleType('alias') + root.__path__ = [] + calls = [] + def hook(name, *args): + calls.append(name) + sys.modules['pkg'] = root + if name == 'pkg.child': + root.child = 42 + return root + namespace = {'__builtins__': dict(vars(builtins), __import__=hook)} + exec('lazy import pkg.child\\nresult = pkg.child', namespace) + assert namespace['result'] == 42 + assert calls == ['pkg', 'pkg.child'], calls + """) + def test_star_import(self): self.check(""" lazy import urllib.nonexistent @@ -2708,9 +2974,8 @@ def test_empty_fromlist_preserved_for_custom_import(self): calls = [] def import_hook(name, globals, locals, fromlist, level): - assert name == "xml.dom", name assert fromlist == (), fromlist - calls.append(fromlist) + calls.append((name, fromlist)) return module builtins.__import__ = import_hook @@ -2721,7 +2986,7 @@ def import_hook(name, globals, locals, fromlist, level): builtins.__import__ = default_import builtins.__lazy_import__ = default_lazy_import - assert calls == [(), ()], calls + assert calls == [('xml', ()), ('xml.dom', ()), ('xml.dom', ())], calls """) assert_python_ok("-c", code) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-00-15-00.gh-issue-158140.custom-hooks.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-00-15-00.gh-issue-158140.custom-hooks.rst new file mode 100644 index 000000000000000..cee809f72f4f5de --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-00-15-00.gh-issue-158140.custom-hooks.rst @@ -0,0 +1,3 @@ +Resolve lazy submodules of ordinary packages independently with custom +:func:`__import__` hooks and private builtins. Select pending declarations +from the appropriate builtins namespace. diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index bb35ddcdae1755e..daaa42c7bc27481 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_ceval.h" +#include "pycore_critical_section.h" #include "pycore_dict.h" #include "pycore_gc.h" #include "pycore_import.h" @@ -13,6 +14,7 @@ #include "pycore_pyerrors.h" #include "pycore_traceback.h" #include "pycore_tstate.h" +#include "pycore_weakref.h" typedef struct { PyObject_HEAD @@ -23,10 +25,18 @@ typedef struct { // in lz_from, and the attribute to import from (PyUnicode) it in lz_attr. PyObject *lz_from; PyObject *lz_attr; + // Public placeholders share their namespace's sibling declarations. + // Registry sources have none of these references. + PyObject *lz_declaration; + PyObject *lz_group; + PyObject *lz_namespace; + // Only the group anchor has a history, replaced under its object lock. + PyObject *lz_declarations; + PyObject *lz_weakreflist; // Declaration location. PyCodeObject *lz_code; int lz_instr_offset; - // The root was accessed, so its declared path requires import semantics. + // The original declaration's root was accessed, requiring import semantics. int lz_active; } PyLazyImportObject; @@ -70,6 +80,11 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, m->lz_builtins = Py_XNewRef(builtins); m->lz_from = Py_NewRef(name); m->lz_attr = Py_XNewRef(fromlist); + m->lz_declaration = NULL; + m->lz_group = NULL; + m->lz_namespace = NULL; + m->lz_declarations = NULL; + m->lz_weakreflist = NULL; m->lz_code = NULL; m->lz_instr_offset = -1; @@ -84,6 +99,121 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, return (PyObject *)m; } +static int +lazy_import_is_plain(PyLazyImportObject *lz) +{ + return PyUnicode_Check(lz->lz_from) && + (lz->lz_attr == NULL || + (PyTuple_Check(lz->lz_attr) && PyTuple_GET_SIZE(lz->lz_attr) == 0)); +} + +// Return the borrowed source, independent of any binding metadata. +static PyObject * +lazy_import_get_declaration(PyObject *declaration) +{ + assert(PyLazyImport_CheckExact(declaration)); + PyLazyImportObject *lz = PyLazyImportObject_CAST(declaration); + return lz->lz_declaration != NULL ? lz->lz_declaration : declaration; +} + +static PyObject * +lazy_import_group_history(PyLazyImportObject *group) +{ + PyObject *history; + Py_BEGIN_CRITICAL_SECTION(group); + history = Py_NewRef(group->lz_declarations); + Py_END_CRITICAL_SECTION(); + return history; +} + +static int +lazy_import_set_history(PyLazyImportObject *group, PyObject *previous, + PyObject *history) +{ + int stored; + Py_BEGIN_CRITICAL_SECTION(group); + stored = group->lz_declarations == previous; + if (stored) { + group->lz_declarations = Py_NewRef(history); + } + Py_END_CRITICAL_SECTION(); + if (stored) { + Py_DECREF(previous); // Release the group's old reference outside the lock. + } + return stored; +} + +PyObject * +_PyLazyImport_Group(PyThreadState *tstate, PyObject *source, PyObject *globals) +{ + assert(PyLazyImport_CheckExact(source)); + PyLazyImportObject *lz = PyLazyImportObject_CAST(source); + if (!lazy_import_is_plain(lz)) { + return Py_NewRef(source); + } + PyObject *path = PyUnicode_FromObject(lz->lz_from); + if (path == NULL) { + return NULL; + } + Py_ssize_t end = PyUnicode_GET_LENGTH(path); + Py_ssize_t dot = PyUnicode_FindChar(path, '.', 0, end, 1); + PyObject *name = dot == -2 ? NULL : + PyUnicode_Substring(path, 0, dot < 0 ? end : dot); + PyObject *binding = name == NULL ? NULL : + _PyLazyImport_New(NULL, lz->lz_builtins, lz->lz_from, lz->lz_attr); + if (name == NULL || binding == NULL) { + goto error; + } + PyLazyImportObject *wrapper = PyLazyImportObject_CAST(binding); + wrapper->lz_code = (PyCodeObject *)Py_XNewRef(lz->lz_code); + wrapper->lz_instr_offset = lz->lz_instr_offset; + wrapper->lz_declaration = Py_NewRef(source); + wrapper->lz_namespace = Py_NewRef(globals); + wrapper->lz_declarations = PyDict_New(); + if (wrapper->lz_declarations == NULL || + PyDict_SetItem(wrapper->lz_declarations, path, source) < 0) { + goto error; + } + PyObject *anchor = _PyImport_GetLazyGroup(tstate, name, globals, binding); + if (anchor == NULL) { + goto error; + } + if (anchor == binding) { + Py_DECREF(anchor); + } + else { + wrapper->lz_group = anchor; + Py_CLEAR(wrapper->lz_namespace); + Py_CLEAR(wrapper->lz_declarations); + PyLazyImportObject *group = PyLazyImportObject_CAST(anchor); + for (;;) { + PyObject *previous = lazy_import_group_history(group); + PyObject *history = PyDict_Copy(previous); + int stored = -1; + // Repeated paths move to the end, preserving declaration order. + if (history != NULL && PyDict_Pop(history, path, NULL) >= 0 && + PyDict_SetItem(history, path, source) == 0) { + stored = lazy_import_set_history(group, previous, history); + } + Py_DECREF(previous); + Py_XDECREF(history); + if (stored < 0) { + goto error; + } + if (stored) { + break; + } + } + } + goto done; +error: + Py_CLEAR(binding); +done: + Py_DECREF(path); + Py_XDECREF(name); + return binding; +} + // Reuse concrete attributes of initialized modules without waiting for imports // or resolving lazy attributes. Failed cache lookups are retried at resolution. // May return NULL with or without an exception set. @@ -156,6 +286,10 @@ lazy_import_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(m->lz_builtins); Py_VISIT(m->lz_from); Py_VISIT(m->lz_attr); + Py_VISIT(m->lz_declaration); + Py_VISIT(m->lz_group); + Py_VISIT(m->lz_namespace); + Py_VISIT(m->lz_declarations); Py_VISIT(m->lz_code); return 0; } @@ -167,6 +301,10 @@ lazy_import_clear(PyObject *op) Py_CLEAR(m->lz_builtins); Py_CLEAR(m->lz_from); Py_CLEAR(m->lz_attr); + Py_CLEAR(m->lz_declaration); + Py_CLEAR(m->lz_group); + Py_CLEAR(m->lz_namespace); + Py_CLEAR(m->lz_declarations); Py_CLEAR(m->lz_code); return 0; } @@ -175,6 +313,7 @@ static void lazy_import_dealloc(PyObject *op) { _PyObject_GC_UNTRACK(op); + FT_CLEAR_WEAKREFS(op, PyLazyImportObject_CAST(op)->lz_weakreflist); (void)lazy_import_clear(op); Py_TYPE(op)->tp_free(op); } @@ -351,14 +490,100 @@ _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op) { _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; assert(PyLazyImport_CheckExact(op)); - int active = ts->lazy_imports == NULL ? 0 : PySet_Contains(ts->lazy_imports, op); + PyObject *source = lazy_import_get_declaration(op); + int active = ts->lazy_imports == NULL ? 0 : + PySet_Contains(ts->lazy_imports, source); assert(active >= 0); // Exact placeholders use identity hashing and equality. return active; } +static int +lazy_import_activate(PyThreadState *tstate, PyObject *source) +{ + PyLazyImportObject *lz = PyLazyImportObject_CAST(source); + FT_ATOMIC_STORE_INT_RELAXED(lz->lz_active, 1); + return _PyImport_RegisterLazySubmodules(tstate, lz->lz_from, source); +} + +static int +lazy_import_activate_declarations(PyThreadState *tstate, PyLazyImportObject *lz) +{ + if (lz->lz_declaration == NULL) { + return lazy_import_activate(tstate, (PyObject *)lz); + } + PyLazyImportObject *group = lz->lz_group == NULL ? lz : + PyLazyImportObject_CAST(lz->lz_group); + PyObject *snapshot = lazy_import_group_history(group); + int err = 0, found = 0; + PyObject *path, *source; + Py_ssize_t pos = 0; + while (err == 0 && PyDict_Next(snapshot, &pos, NULL, &source)) { + found |= source == lz->lz_declaration; + err = lazy_import_activate(tstate, source); + } + if (err == 0 && !found) { + err = lazy_import_activate(tstate, lz->lz_declaration); + } + while (err == 0 && PyDict_GET_SIZE(snapshot) != 0) { + PyObject *previous = lazy_import_group_history(group); + PyObject *history = PyDict_Copy(previous); + int stored = -1; + if (history != NULL) { + pos = 0; + while (PyDict_Next(snapshot, &pos, &path, &source)) { + if (PyDict_GetItemWithError(history, path) == source && + PyDict_Pop(history, path, NULL) < 0) { + err = -1; + break; + } + } + if (err == 0) { + stored = lazy_import_set_history(group, previous, history); + } + } + Py_DECREF(previous); + Py_XDECREF(history); + if (stored != 0) { + err = stored < 0 ? -1 : 0; + break; + } + } + Py_DECREF(snapshot); + return err; +} + +// Aliases, non-packages and privately supplied objects keep full-path imports. +static int +lazy_import_needs_full(PyObject *module, PyObject *name, int custom) +{ + if (!PyModule_CheckExact(module)) { + return 1; + } + if (custom) { + PyObject *cached = PyImport_GetModule(name); + int canonical = cached == module; + Py_XDECREF(cached); + if (PyErr_Occurred()) { + return -1; + } + if (!canonical) { + return 1; + } + } + PyObject *dict = _PyModule_GetDict(module), *actual_name = NULL; + if (PyDict_GetItemRef(dict, &_Py_ID(__name__), &actual_name) < 0) { + return -1; + } + int matches = actual_name != NULL && PyUnicode_Check(actual_name) && + PyUnicode_Compare(actual_name, name) == 0; + Py_XDECREF(actual_name); + int package = PyDict_Contains(dict, &_Py_ID(__path__)); + return package < 0 ? -1 : !matches || !package; +} + static PyObject * lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, - PyObject **imported_module) + PyObject **imported_module, PyObject *import_override) { PyObject *obj = NULL; PyObject *fromlist = NULL; @@ -396,6 +621,7 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, return NULL; } _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + PyObject *resolving_key = lazy_import_get_declaration(lazy_import); if (ts->lazy_imports == NULL) { ts->lazy_imports = PySet_New(NULL); if (ts->lazy_imports == NULL) { @@ -403,7 +629,7 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, } } resolving = ts->lazy_imports; - if (PySet_Add(resolving, lazy_import) < 0) { + if (PySet_Add(resolving, resolving_key) < 0) { goto done; } @@ -425,7 +651,10 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, globals = Py_None; } - if (PyMapping_GetOptionalItem(root->lz_builtins, &_Py_ID(__import__), + if (import_override != NULL) { + import_func = Py_NewRef(import_override); + } + else if (PyMapping_GetOptionalItem(root->lz_builtins, &_Py_ID(__import__), &import_func) < 0) { goto done; } @@ -433,12 +662,10 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, PyErr_SetString(PyExc_ImportError, "__import__ not found"); goto done; } + int custom = !_PyImport_IsDefaultImportFunc(tstate->interp, import_func); PyObject *name = Py_NewRef(root->lz_from); - if (first == NULL && - (fromlist == Py_None || - (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist) == 0)) && - _PyImport_IsDefaultImportFunc(tstate->interp, import_func) && - root->lz_builtins == tstate->interp->builtins) { + int plain = first == NULL && lazy_import_is_plain(root); + if (plain) { // A plain import binds the root package. Its pending children are // imported separately when their attributes are accessed. Py_ssize_t dot = PyUnicode_FindChar( @@ -446,7 +673,6 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, if (dot >= 0) { // Preserve eager traversal through aliased or non-package parents. int regular = 1; - int complete = 1; Py_ssize_t end = PyUnicode_GET_LENGTH(name); for (Py_ssize_t i = dot; i >= 0 && regular; i = PyUnicode_FindChar(name, '.', i + 1, end, 1)) { @@ -454,7 +680,6 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, PyObject *cached = prefix == NULL ? NULL : lazy_import_get_loaded_attr(tstate, prefix, &_Py_ID(__name__)); if (cached == NULL) { - complete = 0; Py_XDECREF(prefix); break; } @@ -466,12 +691,9 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, Py_DECREF(cached); Py_DECREF(prefix); } - PyObject *loaded = lazy_import_get_loaded_attr( - tstate, name, &_Py_ID(__name__)); - if (regular && (!complete || loaded == NULL) && !PyErr_Occurred()) { + if (regular && !PyErr_Occurred()) { Py_SETREF(name, PyUnicode_Substring(name, 0, dot)); } - Py_XDECREF(loaded); if (PyErr_Occurred()) { Py_CLEAR(name); } @@ -488,31 +710,35 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, name, fromlist, _PyLong_GetZero() ); if (obj != NULL && name != root->lz_from) { - FT_ATOMIC_STORE_INT_RELAXED(root->lz_active, 1); + PyObject *source = lazy_import_get_declaration((PyObject *)root); + FT_ATOMIC_STORE_INT_RELAXED( + PyLazyImportObject_CAST(source)->lz_active, 1); FT_ATOMIC_STORE_INT_RELAXED(tstate->interp->imports.has_lazy_submodules, 1); - int package = PyModule_CheckExact(obj) ? - PyDict_Contains(_PyModule_GetDict(obj), &_Py_ID(__path__)) : 0; - if (package < 0) { + int full = lazy_import_needs_full(obj, name, custom); + if (full < 0) { Py_CLEAR(obj); } - else if (!package) { + else if (full) { Py_DECREF(obj); obj = _PyEval_ImportNameWithImport( tstate, import_func, globals, globals, root->lz_from, fromlist, _PyLong_GetZero()); } } - if (obj != NULL && name == root->lz_from && - _PyImport_IsDefaultImportFunc(tstate->interp, import_func) && + + if (obj != NULL && name == root->lz_from && !custom && _PyImport_ClearLazySubmodule(tstate, name, 0) < 0) { Py_CLEAR(obj); } - Py_DECREF(name); obj = lazy_import_resolve_result(tstate, obj); + if (obj != NULL && plain && + lazy_import_activate_declarations(tstate, root) < 0) { + Py_CLEAR(obj); + } + Py_DECREF(name); // The normal importer may publish this module on its parent. Custom // hooks retain control of their own assignments to the parent. - if (imported_module != NULL && obj != NULL && PyModule_Check(obj) && - _PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { + if (imported_module != NULL && obj != NULL && PyModule_Check(obj) && !custom) { *imported_module = Py_NewRef(obj); } if (obj != NULL && first != NULL) { @@ -538,7 +764,7 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, if (resolving != NULL) { // A failed set resize can leave the placeholder inserted. Removing by // identity also permits greenlets to finish in a different order. - if (PySet_Discard(resolving, lazy_import) < 0) { + if (PySet_Discard(resolving, resolving_key) < 0) { Py_CLEAR(obj); } if (PySet_GET_SIZE(resolving) == 0) { @@ -556,17 +782,25 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, PyObject * _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) { - return lazy_import_resolve_impl(tstate, lazy_import, NULL); + return lazy_import_resolve_impl(tstate, lazy_import, NULL, NULL); +} + +PyObject * +_PyLazyImport_GetBuiltins(PyObject *declaration) +{ + assert(PyLazyImport_CheckExact(declaration)); + return PyLazyImportObject_CAST(declaration)->lz_builtins; } // Resolve a pending child as an aliased import, preserving its declaration. PyObject * _PyLazyImport_LoadChild(PyThreadState *tstate, PyObject *declaration, - PyObject *name) + PyObject *name, PyObject *import_func) { assert(PyLazyImport_CheckExact(declaration)); PyLazyImportObject *source = PyLazyImportObject_CAST(declaration); - PyObject *lz = _PyLazyImport_New(NULL, source->lz_builtins, name, NULL); + PyObject *lz = _PyLazyImport_New(NULL, source->lz_builtins, + name, source->lz_attr); Py_ssize_t end = PyUnicode_GET_LENGTH(name); Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, 1); if (dot == -2) { @@ -591,15 +825,24 @@ _PyLazyImport_LoadChild(PyThreadState *tstate, PyObject *declaration, PyLazyImportObject *child = PyLazyImportObject_CAST(lz); child->lz_code = (PyCodeObject *)Py_XNewRef(source->lz_code); child->lz_instr_offset = source->lz_instr_offset; - PyObject *result = _PyImport_LoadLazyImportTstate(tstate, lz); + PyObject *result = lazy_import_resolve_impl(tstate, lz, NULL, import_func); Py_DECREF(lz); - if (result != NULL && PyModule_Check(result) && - PyUnicode_Compare(source->lz_from, name) != 0 && - !PyDict_Contains(_PyModule_GetDict(result), &_Py_ID(__path__))) { - // Reaching a non-package must still reject an invalid dotted import. - Py_DECREF(result); - return _PyLazyImport_LoadChild(tstate, declaration, source->lz_from); + if (result != NULL && PyUnicode_Compare(source->lz_from, name) != 0) { + int full = lazy_import_needs_full(result, name, + !_PyImport_IsDefaultImportFunc(tstate->interp, import_func)); + if (full < 0) { + Py_CLEAR(result); + } + else if (full) { + PyObject *validated = _PyLazyImport_LoadChild( + tstate, declaration, source->lz_from, import_func); + if (validated == NULL) { + Py_CLEAR(result); + } + Py_XDECREF(validated); + } } + return result; } @@ -664,7 +907,7 @@ _PyLazyImport_Reify(PyThreadState *tstate, PyObject *placeholder, { PyObject *imported_module = NULL; PyObject *value = lazy_import_resolve_impl( - tstate, placeholder, &imported_module); + tstate, placeholder, &imported_module, NULL); if (value == NULL) { Py_XDECREF(imported_module); return NULL; @@ -738,6 +981,7 @@ PyTypeObject PyLazyImport_Type = { .tp_getattro = lazy_import_getattro, .tp_traverse = lazy_import_traverse, .tp_clear = lazy_import_clear, + .tp_weaklistoffset = offsetof(PyLazyImportObject, lz_weakreflist), .tp_methods = lazy_import_methods, .tp_alloc = PyType_GenericAlloc, .tp_free = PyObject_GC_Del, diff --git a/Python/import.c b/Python/import.c index 4d3e74f45e5e366..b320e17c1f26e01 100644 --- a/Python/import.c +++ b/Python/import.c @@ -270,9 +270,17 @@ PyObject * _PyImport_InitLazyModules(PyInterpreterState *interp) { assert(LAZY_MODULES(interp) == NULL && - LAZY_PENDING_SUBMODULES(interp) == NULL); + LAZY_PENDING_SUBMODULES(interp) == NULL && + interp->imports.lazy_import_groups == NULL); LAZY_PENDING_SUBMODULES(interp) = PyDict_New(); + if (LAZY_PENDING_SUBMODULES(interp) == NULL) { + return NULL; + } + interp->imports.lazy_import_groups = PyDict_New(); + if (interp->imports.lazy_import_groups == NULL) { + return NULL; + } LAZY_MODULES(interp) = PySet_New(0); return LAZY_MODULES(interp); } @@ -282,6 +290,7 @@ _PyImport_ClearLazyModules(PyInterpreterState *interp) { Py_CLEAR(LAZY_MODULES(interp)); Py_CLEAR(LAZY_PENDING_SUBMODULES(interp)); + Py_CLEAR(interp->imports.lazy_import_groups); } int @@ -4144,7 +4153,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, // Add *name* to `sys.lazy_modules`. // Skip the registration if *name* is already imported, and fully initialized. // If *existing_mod_p* is not NULL, set it to the already imported object (or -// NULL if not found), for use by register_lazy_on_parent. +// NULL if not found), for use by _PyImport_RegisterLazySubmodules. static int lazy_modules_add(PyThreadState *tstate, PyObject *name, PyObject **existing_mod_p) @@ -4204,53 +4213,271 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) return lazy_submodules; } +static int lazy_submodule_matches(PyObject *value, void *expected); + +static PyObject * +remove_lazy_group(PyObject *args, PyObject *reference) +{ + PyObject *groups = PyTuple_GET_ITEM(args, 0); + PyObject *key = PyTuple_GET_ITEM(args, 1); + if (_PyDict_DelItemIf(groups, key, lazy_submodule_matches, reference) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +// The index owns only weak references. Public placeholders own the groups. +PyObject * +_PyImport_GetLazyGroup(PyThreadState *tstate, PyObject *root, + PyObject *globals, PyObject *candidate) +{ + static PyMethodDef remove_group = { + "remove_lazy_group", remove_lazy_group, METH_O + }; + PyObject *groups = tstate->interp->imports.lazy_import_groups; + PyObject *owner = PyLong_FromVoidPtr(globals); + PyObject *key = owner == NULL ? NULL : PyTuple_Pack(2, root, owner); + Py_XDECREF(owner); + if (key == NULL) { + return NULL; + } + PyObject *args = PyTuple_Pack(2, groups, key); + PyObject *callback = args == NULL ? NULL : PyCFunction_New(&remove_group, args); + Py_XDECREF(args); + PyObject *reference = callback == NULL ? NULL : + PyWeakref_NewRef(candidate, callback); + Py_XDECREF(callback); + PyObject *result = NULL; + while (reference != NULL) { + PyObject *previous = NULL; + int present = PyDict_SetDefaultRef(groups, key, reference, &previous); + if (present >= 0 && PyWeakref_GetRef(previous, &result) >= 0 && + result == NULL) { + // A dead group's callback may still be waiting to remove its entry. + present = _PyDict_ReplaceItemIf(groups, key, previous, reference); + if (present > 0) { + result = Py_NewRef(candidate); + } + } + Py_XDECREF(previous); + if (result != NULL || PyErr_Occurred()) { + break; + } + } + Py_XDECREF(reference); + Py_DECREF(key); + return result; +} + +// Entries contain one declaration per builtins mapping. The first owner and +// activated declarations are strong; other owners remain weak until resolved. +static PyObject * +lazy_source_get(PyObject *reference) +{ + return PyWeakref_CheckRef(reference) ? _PyWeakref_GET_REF(reference) : + Py_NewRef(reference); +} + +static Py_ssize_t +lazy_sources_size(PyObject *entry) +{ + return entry == NULL || entry == Py_None ? 0 : + (PyTuple_CheckExact(entry) ? PyTuple_GET_SIZE(entry) : 1); +} + +static PyObject * +lazy_sources_item(PyObject *entry, Py_ssize_t index) +{ + return PyTuple_CheckExact(entry) ? PyTuple_GET_ITEM(entry, index) : entry; +} + +static int +lazy_sources_append(PyObject *sources, PyObject *reference) +{ + PyObject *source = lazy_source_get(reference); + if (source == NULL) { + return 0; + } + PyObject *owner = _PyLazyImport_GetBuiltins(source); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(sources); i++) { + PyObject *old_ref = PyList_GET_ITEM(sources, i); + PyObject *old = lazy_source_get(old_ref); + if (old == NULL) { + if (PySequence_DelItem(sources, i--) < 0) { + Py_DECREF(source); + return -1; + } + continue; + } + int matches = _PyLazyImport_GetBuiltins(old) == owner; + Py_DECREF(old); + if (matches) { + PyObject *replacement = !PyWeakref_CheckRef(old_ref) || + !PyWeakref_CheckRef(reference) ? source : reference; + int err = PyList_SetItem(sources, i, Py_NewRef(replacement)); + Py_DECREF(source); + return err; + } + } + int err = PyList_Append(sources, + PyList_GET_SIZE(sources) == 0 ? source : reference); + Py_DECREF(source); + return err; +} + +static PyObject * +merge_lazy_sources(PyObject *older, PyObject *newer) +{ + Py_ssize_t count = lazy_sources_size(older); + PyObject *sources = PyList_New(count); + if (sources == NULL) { + return NULL; + } + for (Py_ssize_t i = 0; i < count; i++) { + PyList_SET_ITEM(sources, i, Py_NewRef(lazy_sources_item(older, i))); + } + for (Py_ssize_t i = 0; i < lazy_sources_size(newer); i++) { + if (lazy_sources_append(sources, lazy_sources_item(newer, i)) < 0) { + Py_DECREF(sources); + return NULL; + } + } + Py_ssize_t size = PyList_GET_SIZE(sources); + PyObject *result = size == 0 ? Py_NewRef(Py_None) : + (size == 1 ? Py_NewRef(PyList_GET_ITEM(sources, 0)) : PyList_AsTuple(sources)); + Py_DECREF(sources); + return result; +} + +// Build outside the registry lock; finalizers can register more declarations. +// Restore older snapshots without replacing declarations made by a callback. +static int +update_lazy_sources(PyObject *pending, PyObject *child, PyObject *source, + int restore) +{ + for (;;) { + PyObject *previous = NULL; + if (PyDict_GetItemRef(pending, child, &previous) < 0) { + return -1; + } + if (previous == source) { + Py_DECREF(previous); + return 0; + } + PyObject *merged = restore ? merge_lazy_sources(source, previous) : + merge_lazy_sources(previous, source); + if (merged == NULL) { + Py_XDECREF(previous); + return -1; + } + int replaced; + if (previous == NULL) { + int present = PyDict_SetDefaultRef(pending, child, merged, NULL); + replaced = present < 0 ? -1 : !present; + } + else { + replaced = _PyDict_ReplaceItemIf(pending, child, previous, merged); + } + Py_XDECREF(previous); + Py_DECREF(merged); + if (replaced != 0) { + return replaced < 0 ? -1 : 0; + } + } +} + +// A helper without its own declaration uses the first live owner's declaration. +static PyObject * +select_lazy_source(PyObject *entry, PyObject *builtins) +{ + PyObject *fallback = NULL; + for (Py_ssize_t i = 0; i < lazy_sources_size(entry); i++) { + PyObject *source = lazy_source_get(lazy_sources_item(entry, i)); + if (source == NULL) { + continue; + } + if (_PyLazyImport_GetBuiltins(source) == builtins) { + Py_XDECREF(fallback); + return source; + } + if (fallback == NULL) { + fallback = source; + } + else { + Py_DECREF(source); + } + } + return fallback == NULL ? Py_NewRef(Py_None) : fallback; +} + // Records all parent-child relationships in lazy_pending_submodules // for a lazily imported module name. When a parent module's attribute // is accessed, _Py_module_getattro_impl will check lazy_pending_submodules // and trigger the import. -static int -register_lazy_on_parent(PyThreadState *tstate, PyObject *name, PyObject *source) +int +_PyImport_RegisterLazySubmodules(PyThreadState *tstate, PyObject *name, + PyObject *source) { PyDictObject *pending = (PyDictObject *)LAZY_PENDING_SUBMODULES(tstate->interp); assert(pending != NULL); + PyObject *reference = source == Py_None || _PyLazyImport_IsActive(source) ? + Py_NewRef(source) : PyWeakref_NewRef(source, NULL); + if (reference == NULL) { + return -1; + } + int err = 0; Py_ssize_t end = PyUnicode_GET_LENGTH(name); while (true) { Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, -1); if (dot < 0) { - return dot == -1 ? 0 : -1; + err = dot == -1 ? 0 : -1; + break; } PyObject *parent = PyUnicode_Substring(name, 0, dot); if (parent == NULL) { - return -1; + err = -1; + break; } PyObject *child = PyUnicode_Substring(name, dot + 1, end); if (child == NULL) { Py_DECREF(parent); - return -1; + err = -1; + break; } PyObject *submodules = ensure_lazy_pending_submodules(pending, parent); - Py_DECREF(parent); PyObject *fullname = PyUnicode_Substring(name, 0, end); PyObject *cached = fullname == NULL ? NULL : import_get_module(tstate, fullname); - Py_XDECREF(fullname); - PyObject *value = cached != NULL && cached != Py_None ? Py_None : source; - int err = -1; - if (submodules != NULL && !PyErr_Occurred()) { - err = source == Py_None ? + PyObject *module = cached != NULL && cached != Py_None ? + import_get_module(tstate, parent) : NULL; + int bound = module != NULL && PyModule_Check(module) ? + PyDict_Contains(_PyModule_GetDict(module), child) : 0; + PyObject *value = bound > 0 ? Py_None : reference; + err = -1; + if (submodules != NULL && bound >= 0 && !PyErr_Occurred()) { + err = value == Py_None ? PyDict_SetDefaultRef(submodules, child, value, NULL) : - PyDict_SetItem(submodules, child, value); + update_lazy_sources(submodules, child, value, 0); + if (err >= 0 && bound > 0 && + source != Py_None && _PyLazyImport_IsActive(source)) { + err = _PyImport_ClearLazySubmodule(tstate, fullname, 0); + } } + Py_XDECREF(module); Py_XDECREF(cached); + Py_XDECREF(fullname); + Py_DECREF(parent); Py_DECREF(child); Py_XDECREF(submodules); if (err < 0) { - return -1; + break; } end = dot; } + Py_DECREF(reference); + return err < 0 ? -1 : 0; } static int @@ -4274,7 +4501,7 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *existing_module, rc = lazy_modules_add(tstate, fromname, NULL); if (rc == 0) { - rc = register_lazy_on_parent(tstate, fromname, Py_None); + rc = _PyImport_RegisterLazySubmodules(tstate, fromname, Py_None); } Py_DECREF(fromname); return rc; @@ -4367,11 +4594,15 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, if (PyDict_GetItemRef(mod_dict, &_Py_ID(__name__), &mod_name) <= 0) { return NULL; } - PyObject *mod = NULL, *pending_set = NULL, *source = NULL; - if (!PyUnicode_Check(mod_name)) { + PyObject *mod = NULL, *pending_set = NULL, *source = NULL, *entry = NULL; + PyObject *import_func = NULL, *key = NULL, *full_name = NULL; + PyThreadState *tstate = _PyThreadState_GET(); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + int custom = 0, resolving = 0; + attr_name = PyUnicode_FromObject(attr_name); + if (attr_name == NULL || !PyUnicode_Check(mod_name)) { goto done; } - PyThreadState *tstate = _PyThreadState_GET(); PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(tstate->interp); if (lazy_pending == NULL || PyDict_GetItemRef(lazy_pending, mod_name, &pending_set) <= 0) { @@ -4379,18 +4610,74 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, } *recheck_dict = 1; - if (PyDict_GetItemRef(pending_set, attr_name, &source) <= 0) { + if (PyDict_GetItemRef(pending_set, attr_name, &entry) <= 0) { goto done; } - PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); + source = select_lazy_source(entry, _PyEval_GetBuiltins(tstate)); + full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); if (full_name == NULL) { goto done; } + if (source == Py_None && ts->lazy_imports != NULL) { + PyObject *active; + Py_hash_t hash; + Py_ssize_t pos = 0; + while (_PySet_NextEntry(ts->lazy_imports, &pos, &active, &hash)) { + if (PyTuple_CheckExact(active) && PyTuple_GET_SIZE(active) == 2 && + PyUnicode_CheckExact(PyTuple_GET_ITEM(active, 1)) && + PyUnicode_Compare(PyTuple_GET_ITEM(active, 1), full_name) == 0) { + mod = PyImport_GetModule(full_name); + if (mod == Py_None) { + Py_CLEAR(mod); + } + goto done; + } + } + } + if (source != Py_None) { + // Guard before reading __import__: custom builtins can reenter here. + PyObject *builtins = _PyLazyImport_GetBuiltins(source); + PyObject *owner = PyLong_FromVoidPtr(builtins); + key = owner == NULL ? NULL : PyTuple_Pack(2, owner, full_name); + Py_XDECREF(owner); + if (key == NULL) { + goto done; + } + int active = ts->lazy_imports == NULL ? 0 : PySet_Contains(ts->lazy_imports, key); + if (active != 0) { + if (active > 0) { + mod = PyImport_GetModule(full_name); + if (mod == Py_None) { + Py_CLEAR(mod); + } + } + goto done; + } + if (ts->lazy_imports == NULL) { + ts->lazy_imports = PySet_New(NULL); + if (ts->lazy_imports == NULL) { + goto done; + } + } + resolving = 1; + if (PySet_Add(ts->lazy_imports, key) < 0) { + goto done; + } + if (PyMapping_GetOptionalItem(builtins, &_Py_ID(__import__), + &import_func) < 0) { + goto done; + } + if (import_func == NULL) { + PyErr_SetString(PyExc_ImportError, "__import__ not found"); + goto done; + } + custom = !_PyImport_IsDefaultImportFunc(tstate->interp, import_func); + } // Match eager from-imports: accept a partial module in an import cycle. - if (source != Py_None && !suppress && + if (source != Py_None && (custom || (!suppress && (_PyLazyImport_IsActive(source) || - !PyDict_Contains(mod_dict, &_Py_ID(__getattr__)))) { - mod = _PyLazyImport_LoadChild(tstate, source, full_name); + !PyDict_Contains(mod_dict, &_Py_ID(__getattr__)))))) { + mod = _PyLazyImport_LoadChild(tstate, source, full_name, import_func); } else { mod = PyImport_GetModule(full_name); @@ -4406,13 +4693,33 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, if (loaded != NULL && loaded != Py_None) { (void)_PyDict_DelItemIf(mod_dict, attr_name, lazy_submodule_matches, loaded); - (void)PyDict_SetDefaultRef(pending_set, attr_name, source, NULL); } + (void)update_lazy_sources(pending_set, attr_name, entry, 1); Py_XDECREF(loaded); PyErr_SetRaisedException(exc); remove_importlib_frames(tstate); } /* Keep missing children pending: a finder may provide them later. */ + else if (custom) { + // Hooks own their assignments. An unpublished result must not replace + // the shared package attribute or consume another owner's declaration. + PyObject *completed = NULL; + int err = PyDict_GetItemRef(pending_set, attr_name, &completed); + if (err >= 0) { + err = PyDict_Contains(mod_dict, attr_name); + if (err == 0) { + err = update_lazy_sources(pending_set, attr_name, entry, 1); + } + else if (err > 0 && completed != NULL) { + err = _PyDict_DelItemIf(pending_set, attr_name, + lazy_submodule_matches, completed); + } + } + Py_XDECREF(completed); + if (err < 0) { + Py_CLEAR(mod); + } + } else { PyObject *spec = NULL; int rc = mod == Py_None ? 0 : @@ -4430,9 +4737,18 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, Py_CLEAR(mod); } } - Py_DECREF(full_name); - done: + if (resolving && PySet_Discard(ts->lazy_imports, key) < 0) { + Py_CLEAR(mod); + } + if (resolving && PySet_GET_SIZE(ts->lazy_imports) == 0) { + (void)PySet_Clear(ts->lazy_imports); + } + Py_XDECREF(full_name); + Py_XDECREF(key); + Py_XDECREF(import_func); + Py_XDECREF(entry); + Py_XDECREF(attr_name); Py_XDECREF(source); Py_XDECREF(pending_set); Py_DECREF(mod_name); @@ -4549,9 +4865,10 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, } } } - else if (register_lazy_on_parent(tstate, abs_name, res) < 0) { + else if (_PyImport_RegisterLazySubmodules(tstate, abs_name, res) < 0) { goto error; } + Py_SETREF(res, _PyLazyImport_Group(tstate, res, globals)); goto done; error: @@ -4771,6 +5088,7 @@ _PyImport_ClearCore(PyInterpreterState *interp) Py_CLEAR(IMPORT_FUNC(interp)); Py_CLEAR(LAZY_IMPORT_FUNC(interp)); Py_CLEAR(interp->imports.lazy_pending_submodules); + Py_CLEAR(interp->imports.lazy_import_groups); Py_CLEAR(interp->imports.lazy_modules); Py_CLEAR(interp->imports.lazy_imports_filter); }