gh-158140: Resolve lazy submodules independently with custom import hooks

Author: pablogsal

Doc/reference/simple_stmts.rst

@@ -917,6 +917,36 @@ If an error occurs during module loading (such as :exc:`ImportError` or
 :exc:`SyntaxError`), it is raised at the point where the lazy import is first
 used, not at the import statement itself.
 
+Plain dotted lazy imports resolve the root package before their submodules,
+including when a custom :func:`__import__` hook is present. For example,
+``lazy import pkg.a`` calls the hook with ``pkg`` on first use of ``pkg``.
+Access to a pending ``pkg.a`` then calls the hook with ``pkg.a``.
+For packages stored under their declared names, other declared submodules
+remain unresolved. Existing attributes are accessed normally. Aliased imports,
+such as ``lazy import pkg.a as a``, still request the full module name.
+
+The root uses the declaring namespace's builtins. A pending child uses the
+declaration whose builtins match the namespace that accesses it. If no
+declaration matches, Python uses the first namespace that declared that child.
+Custom hooks control the attributes of objects that they supply outside
+:data:`sys.modules`.
+
+.. impl-detail::
+
+   Assigning plain lazy placeholders to the same root name may merge their
+   declaration metadata. This includes assignments from containers or namespace
+   dictionaries and need not preserve placeholder identity.
+
+   Once a root resolves, its pending children retain their declaring builtins
+   mappings. Private modules own this metadata and release it when collected.
+
+Custom import hook exceptions propagate through optional attribute access,
+including :func:`hasattr` and :func:`getattr` with a default value.
+
+Canonical modules under another name and non-packages retain full-path
+validation, so an invalid import such as ``lazy import math.pi`` still raises
+an exception.
+
 See :pep:`810` for the full specification of lazy imports.
 
 .. versionadded:: 3.15

Include/internal/pycore_import.h

@@ -44,6 +44,8 @@ extern PyObject * _PyImport_TryLoadLazySubmodule(
     PyObject *module, PyObject *attr_name, int suppress, int *recheck_dict);
 extern int _PyImport_ClearLazySubmodule(
     PyThreadState *tstate, PyObject *name, int bind);
+extern int _PyImport_RefreshLazySubmodules(
+    PyThreadState *tstate, PyObject *source, PyObject *root, PyObject *prefix);
 extern PyObject * _PyImport_LazyImportModuleLevelObject(
     PyThreadState *tstate, PyObject *name, PyObject *builtins,
     PyObject *globals, PyObject *locals, PyObject *fromlist, int level);

Include/internal/pycore_lazyimportobject.h

@@ -19,7 +19,17 @@ PyAPI_FUNC(PyObject *) _PyLazyImport_New(
     PyObject *name, PyObject *fromlist);
 
 extern PyObject *_PyLazyImport_LoadChild(
-    PyThreadState *tstate, PyObject *declaration, PyObject *name);
+    PyThreadState *tstate, PyObject *declaration, PyObject *name,
+    PyObject *import_func);
+// Borrowed references from the original import declaration.
+extern PyObject *_PyLazyImport_GetBuiltins(PyObject *declaration);
+extern PyObject *_PyLazyImport_GetName(PyObject *declaration);
+// Return a new binding that retains earlier plain imports for the same root.
+extern PyObject *_PyLazyImport_MergeBindings(
+    PyObject *value, PyObject *previous, PyObject *name);
+// Store a lazy binding in a dictionary, retaining concurrent declarations.
+PyAPI_FUNC(int) _PyLazyImport_Store(
+    PyObject *ns, PyObject *name, PyObject *value);
 extern int _PyLazyImport_IsActive(PyObject *declaration);
 
 extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op);

Include/internal/pycore_moduleobject.h

@@ -26,11 +26,16 @@ typedef int (*_Py_modexecfunc)(PyObject *);
 typedef struct {
     PyObject_HEAD
     PyObject *md_dict;
+    // Pending imports on module objects supplied outside sys.modules.
+    // An empty dictionary still identifies a module supplied privately.
+    PyObject *md_lazy_submodules;
     void *md_state;
     PyObject *md_weaklist;
     // for logging purposes after md_dict is cleared
     PyObject *md_name;
     bool md_token_is_def;  /* if true, `md_token` is the PyModuleDef */
+    // Retain global pending imports after the module leaves sys.modules.
+    uint8_t md_lazy_global_lookup;
 #ifdef Py_GIL_DISABLED
     bool md_requires_gil;
 #endif
@@ -72,6 +77,11 @@ static inline PyObject* _PyModule_GetDict(PyObject *mod) {
 }
 
 extern PyObject * _PyModule_GetFilenameObject(PyObject *);
+// Both return new references; Get returns NULL without error if absent.
+extern PyObject *_PyModule_GetLazySubmodules(PyObject *module);
+extern PyObject *_PyModule_EnsureLazySubmodules(PyObject *module);
+extern void _PyModule_EnableLazyGlobalLookup(PyObject *module);
+extern int _PyModule_HasLazyGlobalLookup(PyObject *module);
 extern Py_ssize_t _PyModule_GetFilenameUTF8(
         PyObject *module,
         char *buffer,

Include/internal/pycore_tstate.h

@@ -51,7 +51,8 @@ typedef struct _PyThreadStateImpl {
     PyObject *asyncio_running_loop; // Strong reference
     PyObject *asyncio_running_task; // Strong reference
 
-    // Reused for the thread's lifetime; owns placeholders while they resolve.
+    // Reused for the thread's lifetime. Owns resolving declarations and
+    // (builtins identity, fullname) keys for pending child imports.
     PyObject *lazy_imports;
 
     // Distinguishes between yield and return from PyEval_EvalFrame().