From 67ed53598bf82329edfd5ae599e8b36f5a275d71 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 27 Sep 2026 15:19:05 +0100 Subject: [PATCH 1/9] Move lazy resolution next to the placeholder Keep the placeholder structure private to lazyimportobject.c. Move the resolution and attribute lookup code there so it can use the structure without exposing its fields to the eval loop or import code. --- Include/internal/pycore_lazyimportobject.h | 10 - Objects/lazyimportobject.c | 265 +++++++++++++++++++++ Python/ceval.c | 37 --- Python/import.c | 214 ----------------- 4 files changed, 265 insertions(+), 261 deletions(-) diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index b81e4211b08ff3..ae88ba8c221955 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -14,16 +14,6 @@ extern "C" { PyAPI_DATA(PyTypeObject) PyLazyImport_Type; #define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type) -typedef struct { - PyObject_HEAD - PyObject *lz_builtins; - PyObject *lz_from; - PyObject *lz_attr; - // Frame information for the original import location. - PyCodeObject *lz_code; // Code object where the lazy import was created. - int lz_instr_offset; // Instruction offset where the lazy import was created. -} PyLazyImportObject; - PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import); PyAPI_FUNC(PyObject *) _PyLazyImport_New( diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index e008c076352ebb..a31dc472ad6230 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -4,10 +4,24 @@ #include "pycore_ceval.h" #include "pycore_frame.h" #include "pycore_import.h" +#include "pycore_interp.h" +#include "pycore_long.h" +#include "pycore_setobject.h" +#include "pycore_traceback.h" #include "pycore_interpframe.h" #include "pycore_lazyimportobject.h" #include "pycore_modsupport.h" +typedef struct { + PyObject_HEAD + PyObject *lz_builtins; + PyObject *lz_from; + PyObject *lz_attr; + // Frame information for the original import location. + PyCodeObject *lz_code; // Code object where the lazy import was created. + int lz_instr_offset; // Instruction offset where the lazy import was created. +} PyLazyImportObject; + #define PyLazyImportObject_CAST(op) ((PyLazyImportObject *)(op)) PyObject * @@ -51,6 +65,43 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name return (PyObject *)m; } +PyObject * +_PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *v, PyObject *name) +{ + assert(PyLazyImport_CheckExact(v)); + assert(name); + assert(PyUnicode_Check(name)); + PyObject *ret; + PyLazyImportObject *d = (PyLazyImportObject *)v; + PyObject *mod = NULL; + // Only `from a import b` can take b off an already imported a; + // `import a.b as c` has to import a.b first. + if (d->lz_attr != NULL && PyTuple_Check(d->lz_attr) && + PyTuple_GET_SIZE(d->lz_attr) > 0) { + mod = PyImport_GetModule(d->lz_from); + } + if (mod != NULL) { + // Check if the module already has the attribute, if so, resolve it + // eagerly. + if (PyModule_Check(mod)) { + PyObject *mod_dict = PyModule_GetDict(mod); + if (mod_dict != NULL) { + if (PyDict_GetItemRef(mod_dict, name, &ret) < 0) { + Py_DECREF(mod); + return NULL; + } + if (ret != NULL) { + Py_DECREF(mod); + return ret; + } + } + } + Py_DECREF(mod); + } + + return _PyLazyImport_New(frame, d->lz_builtins, v, name); +} + static int lazy_import_traverse(PyObject *op, visitproc visit, void *arg) { @@ -168,6 +219,220 @@ _PyLazyImport_GetName(PyObject *op) return lazy_import_name(lazy_import); } +// Look up, in order, the attributes recorded from the root placeholder to lz +// on the module the root's import returned. +static PyObject * +lazy_import_replay_from(PyThreadState *tstate, PyObject *mod, + PyLazyImportObject *lz) +{ + if (!PyLazyImport_CheckExact(lz->lz_from)) { + return Py_NewRef(mod); + } + PyObject *from = lazy_import_replay_from( + tstate, mod, (PyLazyImportObject *)lz->lz_from); + if (from == NULL) { + return NULL; + } + PyObject *obj = _PyEval_ImportFrom(tstate, from, lz->lz_attr); + Py_DECREF(from); + return obj; +} + +PyObject * +_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) +{ + PyObject *obj = NULL; + PyObject *fromlist = Py_None; + PyObject *import_func = NULL; + assert(lazy_import != NULL); + assert(PyLazyImport_CheckExact(lazy_import)); + + PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import; + PyInterpreterState *interp = tstate->interp; + + // Walk back to the placeholder IMPORT_NAME left, and the first lookup on it. + PyLazyImportObject *root = lz, *first = NULL; + while (PyLazyImport_CheckExact(root->lz_from)) { + first = root; + root = (PyLazyImportObject *)root->lz_from; + } + + // Acquire the global import lock to serialize reification + _PyImport_AcquireLock(interp); + + // Check if we are already importing this module, if so, then we want to + // return an error that indicates we've hit a cycle which will indicate + // the value isn't yet available. + PyObject *importing = interp->imports.lazy_importing_modules; + if (importing == NULL) { + importing = interp->imports.lazy_importing_modules = PySet_New(NULL); + if (importing == NULL) { + _PyImport_ReleaseLock(interp); + return NULL; + } + } + + assert(PyAnySet_CheckExact(importing)); + int is_loading = _PySet_Contains((PySetObject *)importing, lazy_import); + if (is_loading < 0) { + _PyImport_ReleaseLock(interp); + return NULL; + } + else if (is_loading == 1) { + PyObject *name = _PyLazyImport_GetName(lazy_import); + if (name == NULL) { + _PyImport_ReleaseLock(interp); + return NULL; + } + PyObject *errmsg = PyUnicode_FromFormat( + "cannot import name %R (most likely due to a circular import)", + name); + if (errmsg == NULL) { + Py_DECREF(name); + _PyImport_ReleaseLock(interp); + return NULL; + } + PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg, + root->lz_from, NULL); + Py_DECREF(errmsg); + Py_DECREF(name); + _PyImport_ReleaseLock(interp); + return NULL; + } + else if (PySet_Add(importing, lazy_import) < 0) { + goto error; + } + + if (root->lz_attr != NULL) { + // `from a import b, c`: import only the name being resolved. + // Keep an empty tuple intact for custom __import__ hooks. + fromlist = first && PyTuple_GET_SIZE(root->lz_attr) > 0 + ? PyTuple_Pack(1, first->lz_attr) + : Py_NewRef(root->lz_attr); + if (fromlist == NULL) { + goto error; + } + } + + PyObject *globals = PyEval_GetGlobals(); + + if (PyMapping_GetOptionalItem(root->lz_builtins, &_Py_ID(__import__), + &import_func) < 0) { + goto error; + } + if (import_func == NULL) { + PyErr_SetString(PyExc_ImportError, "__import__ not found"); + goto error; + } + obj = _PyEval_ImportNameWithImport( + tstate, import_func, globals, globals, + root->lz_from, fromlist, _PyLong_GetZero() + ); + if (obj == NULL) { + goto error; + } + + PyObject *from = obj; + obj = lazy_import_replay_from(tstate, from, lz); + Py_DECREF(from); + if (obj == NULL) { + goto error; + } + + assert(!PyLazyImport_CheckExact(obj)); + + goto ok; + +error: + Py_CLEAR(obj); + + // If an error occurred and we have frame information, add it to the + // exception. + if (PyErr_Occurred() && lz->lz_code != NULL && lz->lz_instr_offset >= 0) { + // Get the current exception - this already has the full traceback + // from the access point. + PyObject *exc = _PyErr_GetRaisedException(tstate); + + // Get import name - this can fail and set an exception. + PyObject *import_name = _PyLazyImport_GetName(lazy_import); + if (!import_name) { + // Failed to get import name, just restore original exception. + _PyErr_SetRaisedException(tstate, exc); + goto ok; + } + + // Resolve line number from instruction offset on demand. + int lineno = PyCode_Addr2Line((PyCodeObject *)lz->lz_code, + lz->lz_instr_offset*2); + + // Get strings - these can return NULL on encoding errors. + const char *filename_str = PyUnicode_AsUTF8(lz->lz_code->co_filename); + if (!filename_str) { + // Unicode conversion failed - clear error and restore original + // exception. + PyErr_Clear(); + Py_DECREF(import_name); + _PyErr_SetRaisedException(tstate, exc); + goto ok; + } + + const char *funcname_str = PyUnicode_AsUTF8(lz->lz_code->co_name); + if (!funcname_str) { + // Unicode conversion failed - clear error and restore original + // exception. + PyErr_Clear(); + Py_DECREF(import_name); + _PyErr_SetRaisedException(tstate, exc); + goto ok; + } + + // Create a cause exception showing where the lazy import was declared. + PyObject *msg = PyUnicode_FromFormat( + "lazy import of '%U' raised an exception during resolution", + import_name + ); + Py_DECREF(import_name); // Done with import_name. + + if (!msg) { + // Failed to create message - restore original exception. + _PyErr_SetRaisedException(tstate, exc); + goto ok; + } + + PyObject *cause_exc = PyObject_CallOneArg(PyExc_ImportError, msg); + Py_DECREF(msg); // Done with msg. + + if (!cause_exc) { + // Failed to create exception - restore original. + _PyErr_SetRaisedException(tstate, exc); + goto ok; + } + + // Add traceback entry for the lazy import declaration. + _PyErr_SetRaisedException(tstate, cause_exc); + _PyTraceback_Add(funcname_str, filename_str, lineno); + PyObject *cause_with_tb = _PyErr_GetRaisedException(tstate); + + // Set the cause on the original exception. + PyException_SetCause(exc, cause_with_tb); // Steals ref to cause_with_tb. + + // Restore the original exception with its full traceback. + _PyErr_SetRaisedException(tstate, exc); + } + +ok: + if (PySet_Discard(importing, lazy_import) < 0) { + Py_CLEAR(obj); + } + + // Release the global import lock. + _PyImport_ReleaseLock(interp); + + Py_XDECREF(fromlist); + Py_XDECREF(import_func); + return obj; +} + static PyObject * lazy_import_resolve(PyObject *self, PyObject *args) { diff --git a/Python/ceval.c b/Python/ceval.c index c1fbcb0611384c..41987162c360b2 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3325,43 +3325,6 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) return NULL; } -PyObject * -_PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *v, PyObject *name) -{ - assert(PyLazyImport_CheckExact(v)); - assert(name); - assert(PyUnicode_Check(name)); - PyObject *ret; - PyLazyImportObject *d = (PyLazyImportObject *)v; - PyObject *mod = NULL; - // Only `from a import b` can take b off an already imported a; - // `import a.b as c` has to import a.b first. - if (d->lz_attr != NULL && PyTuple_Check(d->lz_attr) && - PyTuple_GET_SIZE(d->lz_attr) > 0) { - mod = PyImport_GetModule(d->lz_from); - } - if (mod != NULL) { - // Check if the module already has the attribute, if so, resolve it - // eagerly. - if (PyModule_Check(mod)) { - PyObject *mod_dict = PyModule_GetDict(mod); - if (mod_dict != NULL) { - if (PyDict_GetItemRef(mod_dict, name, &ret) < 0) { - Py_DECREF(mod); - return NULL; - } - if (ret != NULL) { - Py_DECREF(mod); - return ret; - } - } - } - Py_DECREF(mod); - } - - return _PyLazyImport_New(frame, d->lz_builtins, v, name); -} - #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ "BaseException is not allowed" diff --git a/Python/import.c b/Python/import.c index 2fa63dd01833a6..40ff14c0137890 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3898,220 +3898,6 @@ _PyImport_ResolveName(PyThreadState *tstate, PyObject *name, return resolve_name(tstate, name, globals, level); } -// Look up, in order, the attributes recorded from the root placeholder to lz -// on the module the root's import returned. -static PyObject * -lazy_import_replay_from(PyThreadState *tstate, PyObject *mod, - PyLazyImportObject *lz) -{ - if (!PyLazyImport_CheckExact(lz->lz_from)) { - return Py_NewRef(mod); - } - PyObject *from = lazy_import_replay_from( - tstate, mod, (PyLazyImportObject *)lz->lz_from); - if (from == NULL) { - return NULL; - } - PyObject *obj = _PyEval_ImportFrom(tstate, from, lz->lz_attr); - Py_DECREF(from); - return obj; -} - -PyObject * -_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) -{ - PyObject *obj = NULL; - PyObject *fromlist = Py_None; - PyObject *import_func = NULL; - assert(lazy_import != NULL); - assert(PyLazyImport_CheckExact(lazy_import)); - - PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import; - PyInterpreterState *interp = tstate->interp; - - // Walk back to the placeholder IMPORT_NAME left, and the first lookup on it. - PyLazyImportObject *root = lz, *first = NULL; - while (PyLazyImport_CheckExact(root->lz_from)) { - first = root; - root = (PyLazyImportObject *)root->lz_from; - } - - // Acquire the global import lock to serialize reification - _PyImport_AcquireLock(interp); - - // Check if we are already importing this module, if so, then we want to - // return an error that indicates we've hit a cycle which will indicate - // the value isn't yet available. - PyObject *importing = interp->imports.lazy_importing_modules; - if (importing == NULL) { - importing = interp->imports.lazy_importing_modules = PySet_New(NULL); - if (importing == NULL) { - _PyImport_ReleaseLock(interp); - return NULL; - } - } - - assert(PyAnySet_CheckExact(importing)); - int is_loading = _PySet_Contains((PySetObject *)importing, lazy_import); - if (is_loading < 0) { - _PyImport_ReleaseLock(interp); - return NULL; - } - else if (is_loading == 1) { - PyObject *name = _PyLazyImport_GetName(lazy_import); - if (name == NULL) { - _PyImport_ReleaseLock(interp); - return NULL; - } - PyObject *errmsg = PyUnicode_FromFormat( - "cannot import name %R (most likely due to a circular import)", - name); - if (errmsg == NULL) { - Py_DECREF(name); - _PyImport_ReleaseLock(interp); - return NULL; - } - PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg, - root->lz_from, NULL); - Py_DECREF(errmsg); - Py_DECREF(name); - _PyImport_ReleaseLock(interp); - return NULL; - } - else if (PySet_Add(importing, lazy_import) < 0) { - goto error; - } - - if (root->lz_attr != NULL) { - // `from a import b, c`: import only the name being resolved. - // Keep an empty tuple intact for custom __import__ hooks. - fromlist = first && PyTuple_GET_SIZE(root->lz_attr) > 0 - ? PyTuple_Pack(1, first->lz_attr) - : Py_NewRef(root->lz_attr); - if (fromlist == NULL) { - goto error; - } - } - - PyObject *globals = PyEval_GetGlobals(); - - if (PyMapping_GetOptionalItem(root->lz_builtins, &_Py_ID(__import__), - &import_func) < 0) { - goto error; - } - if (import_func == NULL) { - PyErr_SetString(PyExc_ImportError, "__import__ not found"); - goto error; - } - obj = _PyEval_ImportNameWithImport( - tstate, import_func, globals, globals, - root->lz_from, fromlist, _PyLong_GetZero() - ); - if (obj == NULL) { - goto error; - } - - PyObject *from = obj; - obj = lazy_import_replay_from(tstate, from, lz); - Py_DECREF(from); - if (obj == NULL) { - goto error; - } - - assert(!PyLazyImport_CheckExact(obj)); - - goto ok; - -error: - Py_CLEAR(obj); - - // If an error occurred and we have frame information, add it to the - // exception. - if (PyErr_Occurred() && lz->lz_code != NULL && lz->lz_instr_offset >= 0) { - // Get the current exception - this already has the full traceback - // from the access point. - PyObject *exc = _PyErr_GetRaisedException(tstate); - - // Get import name - this can fail and set an exception. - PyObject *import_name = _PyLazyImport_GetName(lazy_import); - if (!import_name) { - // Failed to get import name, just restore original exception. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Resolve line number from instruction offset on demand. - int lineno = PyCode_Addr2Line((PyCodeObject *)lz->lz_code, - lz->lz_instr_offset*2); - - // Get strings - these can return NULL on encoding errors. - const char *filename_str = PyUnicode_AsUTF8(lz->lz_code->co_filename); - if (!filename_str) { - // Unicode conversion failed - clear error and restore original - // exception. - PyErr_Clear(); - Py_DECREF(import_name); - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - const char *funcname_str = PyUnicode_AsUTF8(lz->lz_code->co_name); - if (!funcname_str) { - // Unicode conversion failed - clear error and restore original - // exception. - PyErr_Clear(); - Py_DECREF(import_name); - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Create a cause exception showing where the lazy import was declared. - PyObject *msg = PyUnicode_FromFormat( - "lazy import of '%U' raised an exception during resolution", - import_name - ); - Py_DECREF(import_name); // Done with import_name. - - if (!msg) { - // Failed to create message - restore original exception. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - PyObject *cause_exc = PyObject_CallOneArg(PyExc_ImportError, msg); - Py_DECREF(msg); // Done with msg. - - if (!cause_exc) { - // Failed to create exception - restore original. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Add traceback entry for the lazy import declaration. - _PyErr_SetRaisedException(tstate, cause_exc); - _PyTraceback_Add(funcname_str, filename_str, lineno); - PyObject *cause_with_tb = _PyErr_GetRaisedException(tstate); - - // Set the cause on the original exception. - PyException_SetCause(exc, cause_with_tb); // Steals ref to cause_with_tb. - - // Restore the original exception with its full traceback. - _PyErr_SetRaisedException(tstate, exc); - } - -ok: - if (PySet_Discard(importing, lazy_import) < 0) { - Py_CLEAR(obj); - } - - // Release the global import lock. - _PyImport_ReleaseLock(interp); - - Py_XDECREF(fromlist); - Py_XDECREF(import_func); - return obj; -} - static PyObject * import_find_and_load_with_name(PyThreadState *tstate, PyObject *abs_name, PyObject *find_and_load, From ac89dd0892cba39599bb730dba33b90040436c11 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 27 Sep 2026 15:19:11 +0100 Subject: [PATCH 2/9] Track lazy resolution per thread Let importlib handle module locks instead of holding the global import lock while resolving a placeholder. Keep active placeholders in a set on the thread and remove them on every exit, including allocation failures. Use the same cycle and recursion checks when a hook returns a placeholder. Keep the source references alive until all attribute lookups finish. --- Include/internal/pycore_interp_structs.h | 5 +- Include/internal/pycore_lazyimportobject.h | 7 +- Include/internal/pycore_tstate.h | 3 + Objects/lazyimportobject.c | 336 ++++++++++----------- Python/import.c | 1 - Python/pystate.c | 4 + 6 files changed, 166 insertions(+), 190 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 6c907e0cf79894..a9c61ea6db3a8c 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -352,15 +352,12 @@ struct _import_state { PyObject *lazy_import_func; int lazy_imports_mode; PyObject *lazy_imports_filter; - PyObject *lazy_importing_modules; // The set stored in sys.lazy_modules if values that have been // lazily imported. This value is only for debugging/introspection // purposes and is not used by the runtime. PyObject *lazy_modules; // A dict mapping package names to a set of submodule names that - // have been imported lazily from packages which have been imported - // lazily. When the package is reified we need to add a - // LazyImportObject which refers to the submodule on the module. + // may need to be imported when an attribute is missing on the package. PyObject *lazy_pending_submodules; #ifdef Py_GIL_DISABLED PyMutex lazy_mutex; diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index ae88ba8c221955..4e7a437d0eff41 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -14,10 +14,11 @@ extern "C" { PyAPI_DATA(PyTypeObject) PyLazyImport_Type; #define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type) - -PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import); PyAPI_FUNC(PyObject *) _PyLazyImport_New( - struct _PyInterpreterFrame *frame, PyObject *import_func, PyObject *from, PyObject *attr); + struct _PyInterpreterFrame *frame, PyObject *builtins, + PyObject *name, PyObject *fromlist); + +extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op); #ifdef __cplusplus } diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index eb2b0c84acdc7c..58dd66b69efc9d 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -51,6 +51,9 @@ typedef struct _PyThreadStateImpl { PyObject *asyncio_running_loop; // Strong reference PyObject *asyncio_running_task; // Strong reference + // Reused for the thread's lifetime; owns placeholders while they resolve. + PyObject *lazy_imports; + // Distinguishes between yield and return from PyEval_EvalFrame(). // See gen_send_ex2() in Objects/genobject.c enum { diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index a31dc472ad6230..6371918037889b 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -2,32 +2,37 @@ #include "Python.h" #include "pycore_ceval.h" -#include "pycore_frame.h" +#include "pycore_dict.h" +#include "pycore_gc.h" #include "pycore_import.h" -#include "pycore_interp.h" -#include "pycore_long.h" -#include "pycore_setobject.h" -#include "pycore_traceback.h" #include "pycore_interpframe.h" #include "pycore_lazyimportobject.h" -#include "pycore_modsupport.h" +#include "pycore_long.h" +#include "pycore_moduleobject.h" +#include "pycore_pyerrors.h" +#include "pycore_traceback.h" +#include "pycore_tstate.h" typedef struct { PyObject_HEAD - PyObject *lz_builtins; + PyObject *lz_builtins; // Roots own the mapping; projections retain the root. + // A root stores its absolute name and original fromlist. A projection + // stores its source placeholder and the attribute to import from it. PyObject *lz_from; PyObject *lz_attr; - // Frame information for the original import location. - PyCodeObject *lz_code; // Code object where the lazy import was created. - int lz_instr_offset; // Instruction offset where the lazy import was created. + // Declaration location. + PyCodeObject *lz_code; + int lz_instr_offset; } PyLazyImportObject; #define PyLazyImportObject_CAST(op) ((PyLazyImportObject *)(op)) +static PyObject *lazy_import_name(PyLazyImportObject *m); + PyObject * -_PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name, PyObject *fromlist) +_PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, + PyObject *name, PyObject *fromlist) { - PyLazyImportObject *m; if (!name || !(PyUnicode_Check(name) || PyLazyImport_CheckExact(name))) { PyErr_SetString(PyExc_TypeError, "expected str or lazy_import for name"); return NULL; @@ -40,7 +45,11 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name "lazy_import: fromlist must be None, a string, or a tuple"); return NULL; } - m = PyObject_GC_New(PyLazyImportObject, &PyLazyImport_Type); + assert(PyLazyImport_CheckExact(name) ? builtins == NULL : builtins != NULL); + assert(!PyLazyImport_CheckExact(name) || + (fromlist != NULL && PyUnicode_Check(fromlist))); + PyLazyImportObject *m = PyObject_GC_New( + PyLazyImportObject, &PyLazyImport_Type); if (m == NULL) { return NULL; } @@ -48,17 +57,12 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name m->lz_from = Py_NewRef(name); m->lz_attr = Py_XNewRef(fromlist); - // Capture frame information for the original import location. m->lz_code = NULL; m->lz_instr_offset = -1; if (frame != NULL) { - PyCodeObject *code = _PyFrame_GetCode(frame); - if (code != NULL) { - m->lz_code = (PyCodeObject *)Py_NewRef(code); - // Calculate the instruction offset from the current frame. - m->lz_instr_offset = _PyInterpreterFrame_LASTI(frame); - } + m->lz_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + m->lz_instr_offset = _PyInterpreterFrame_LASTI(frame); } _PyObject_GC_TRACK(m); @@ -99,7 +103,7 @@ _PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObje Py_DECREF(mod); } - return _PyLazyImport_New(frame, d->lz_builtins, v, name); + return _PyLazyImport_New(frame, NULL, v, name); } static int @@ -136,23 +140,20 @@ lazy_import_dealloc(PyObject *op) static PyObject * lazy_import_getattro(PyObject *op, PyObject *name) { - PyObject *value = _PyObject_GenericGetAttrWithDict(op, name, NULL, /* suppress */1); - if (value == NULL) { - if (PyErr_Occurred()) { - // pass up non-AttributeError exception - return NULL; - } - PyObject *lz_name = _PyLazyImport_GetName(op); - if (lz_name == NULL) { - return NULL; - } - PyErr_Format(PyExc_AttributeError, - "cannot access attribute %R on unresolved lazy import %R", - name, lz_name); - Py_DECREF(lz_name); + PyObject *value = _PyObject_GenericGetAttrWithDict( + op, name, NULL, /* suppress */ 1); + if (value != NULL || PyErr_Occurred()) { + return value; + } + PyObject *lz_name = lazy_import_name(PyLazyImportObject_CAST(op)); + if (lz_name == NULL) { return NULL; } - return value; + PyErr_Format(PyExc_AttributeError, + "cannot access attribute %R on unresolved lazy import %R", + name, lz_name); + Py_DECREF(lz_name); + return NULL; } // The dotted name of the object that resolving the placeholder returns. @@ -211,12 +212,17 @@ lazy_import_repr(PyObject *op) return res; } -PyObject * -_PyLazyImport_GetName(PyObject *op) +// Consume a result, resolving a placeholder returned by an import hook or +// an attribute lookup under the same cycle and recursion checks. +static PyObject * +lazy_import_resolve_result(PyThreadState *tstate, PyObject *obj) { - PyLazyImportObject *lazy_import = PyLazyImportObject_CAST(op); - assert(PyLazyImport_CheckExact(lazy_import)); - return lazy_import_name(lazy_import); + if (obj == NULL || !PyLazyImport_CheckExact(obj)) { + return obj; + } + PyObject *result = _PyImport_LoadLazyImportTstate(tstate, obj); + Py_DECREF(obj); + return result; } // Look up, in order, the attributes recorded from the root placeholder to lz @@ -235,20 +241,70 @@ lazy_import_replay_from(PyThreadState *tstate, PyObject *mod, } PyObject *obj = _PyEval_ImportFrom(tstate, from, lz->lz_attr); Py_DECREF(from); - return obj; + return lazy_import_resolve_result(tstate, obj); +} + +// Preserve the resolution error and attach the import's declaration location. +static void +lazy_import_add_exception_cause(PyThreadState *tstate, PyLazyImportObject *lz) +{ + if (!PyErr_Occurred() || lz->lz_code == NULL || lz->lz_instr_offset < 0) { + return; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyObject *name = lazy_import_name(lz); + if (name == NULL) { + goto done; + } + int lineno = PyCode_Addr2Line(lz->lz_code, lz->lz_instr_offset * 2); + const char *filename = PyUnicode_AsUTF8(lz->lz_code->co_filename); + if (filename == NULL) { + goto done; + } + const char *funcname = PyUnicode_AsUTF8(lz->lz_code->co_name); + if (funcname == NULL) { + goto done; + } + PyObject *msg = PyUnicode_FromFormat( + "lazy import of '%U' raised an exception during resolution", name); + if (msg == NULL) { + goto done; + } + PyObject *cause = PyObject_CallOneArg(PyExc_ImportError, msg); + Py_DECREF(msg); + if (cause == NULL) { + goto done; + } + _PyErr_SetRaisedException(tstate, cause); + _PyTraceback_Add(funcname, filename, lineno); + PyException_SetCause(exc, _PyErr_GetRaisedException(tstate)); + +done: + Py_XDECREF(name); + _PyErr_SetRaisedException(tstate, exc); +} + +int +_PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op) +{ + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(PyLazyImport_CheckExact(op)); + int active = ts->lazy_imports == NULL ? 0 : PySet_Contains(ts->lazy_imports, op); + assert(active >= 0); // Exact placeholders use identity hashing and equality. + return active; } PyObject * _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) { PyObject *obj = NULL; - PyObject *fromlist = Py_None; + PyObject *fromlist = NULL; PyObject *import_func = NULL; + PyObject *resolving = NULL; assert(lazy_import != NULL); assert(PyLazyImport_CheckExact(lazy_import)); PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import; - PyInterpreterState *interp = tstate->interp; // Walk back to the placeholder IMPORT_NAME left, and the first lookup on it. PyLazyImportObject *root = lz, *first = NULL; @@ -257,179 +313,95 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) root = (PyLazyImportObject *)root->lz_from; } - // Acquire the global import lock to serialize reification - _PyImport_AcquireLock(interp); - - // Check if we are already importing this module, if so, then we want to - // return an error that indicates we've hit a cycle which will indicate - // the value isn't yet available. - PyObject *importing = interp->imports.lazy_importing_modules; - if (importing == NULL) { - importing = interp->imports.lazy_importing_modules = PySet_New(NULL); - if (importing == NULL) { - _PyImport_ReleaseLock(interp); - return NULL; - } - } - - assert(PyAnySet_CheckExact(importing)); - int is_loading = _PySet_Contains((PySetObject *)importing, lazy_import); - if (is_loading < 0) { - _PyImport_ReleaseLock(interp); - return NULL; - } - else if (is_loading == 1) { - PyObject *name = _PyLazyImport_GetName(lazy_import); + if (_PyLazyImport_IsResolving(tstate, lazy_import)) { + PyObject *name = lazy_import_name(lz); if (name == NULL) { - _PyImport_ReleaseLock(interp); return NULL; } PyObject *errmsg = PyUnicode_FromFormat( "cannot import name %R (most likely due to a circular import)", name); - if (errmsg == NULL) { - Py_DECREF(name); - _PyImport_ReleaseLock(interp); - return NULL; - } - PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg, - root->lz_from, NULL); - Py_DECREF(errmsg); Py_DECREF(name); - _PyImport_ReleaseLock(interp); + if (errmsg != NULL) { + PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg, + root->lz_from, NULL); + Py_DECREF(errmsg); + } return NULL; } - else if (PySet_Add(importing, lazy_import) < 0) { - goto error; + if (_Py_EnterRecursiveCallTstate(tstate, " while resolving a lazy import")) { + return NULL; } - - if (root->lz_attr != NULL) { - // `from a import b, c`: import only the name being resolved. - // Keep an empty tuple intact for custom __import__ hooks. - fromlist = first && PyTuple_GET_SIZE(root->lz_attr) > 0 - ? PyTuple_Pack(1, first->lz_attr) - : Py_NewRef(root->lz_attr); - if (fromlist == NULL) { - goto error; + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + if (ts->lazy_imports == NULL) { + ts->lazy_imports = PySet_New(NULL); + if (ts->lazy_imports == NULL) { + goto done; } } + resolving = ts->lazy_imports; + if (PySet_Add(resolving, lazy_import) < 0) { + goto done; + } + + // `from a import b, c`: import only the name being resolved. + // Keep an empty tuple intact for custom __import__ hooks. + if (first != NULL && root->lz_attr != NULL && + PyTuple_Check(root->lz_attr) && PyTuple_GET_SIZE(root->lz_attr) > 0) { + fromlist = PyTuple_Pack(1, first->lz_attr); + } + else { + fromlist = Py_NewRef(root->lz_attr != NULL ? root->lz_attr : Py_None); + } + if (fromlist == NULL) { + goto done; + } PyObject *globals = PyEval_GetGlobals(); + if (globals == NULL) { + globals = Py_None; + } if (PyMapping_GetOptionalItem(root->lz_builtins, &_Py_ID(__import__), &import_func) < 0) { - goto error; + goto done; } if (import_func == NULL) { PyErr_SetString(PyExc_ImportError, "__import__ not found"); - goto error; + goto done; } obj = _PyEval_ImportNameWithImport( tstate, import_func, globals, globals, root->lz_from, fromlist, _PyLong_GetZero() ); - if (obj == NULL) { - goto error; + obj = lazy_import_resolve_result(tstate, obj); + if (obj != NULL && first != NULL) { + // Keep the hook and root result alive until all attribute lookups finish. + PyObject *from = obj; + obj = lazy_import_replay_from(tstate, from, lz); + Py_DECREF(from); } - PyObject *from = obj; - obj = lazy_import_replay_from(tstate, from, lz); - Py_DECREF(from); +done: if (obj == NULL) { - goto error; + lazy_import_add_exception_cause(tstate, lz); } - - assert(!PyLazyImport_CheckExact(obj)); - - goto ok; - -error: - Py_CLEAR(obj); - - // If an error occurred and we have frame information, add it to the - // exception. - if (PyErr_Occurred() && lz->lz_code != NULL && lz->lz_instr_offset >= 0) { - // Get the current exception - this already has the full traceback - // from the access point. - PyObject *exc = _PyErr_GetRaisedException(tstate); - - // Get import name - this can fail and set an exception. - PyObject *import_name = _PyLazyImport_GetName(lazy_import); - if (!import_name) { - // Failed to get import name, just restore original exception. - _PyErr_SetRaisedException(tstate, exc); - goto ok; + assert(obj == NULL || !PyLazyImport_CheckExact(obj)); + if (resolving != NULL) { + // A failed set resize can leave the placeholder inserted. Removing by + // identity also permits greenlets to finish in a different order. + if (PySet_Discard(resolving, lazy_import) < 0) { + Py_CLEAR(obj); } - - // Resolve line number from instruction offset on demand. - int lineno = PyCode_Addr2Line((PyCodeObject *)lz->lz_code, - lz->lz_instr_offset*2); - - // Get strings - these can return NULL on encoding errors. - const char *filename_str = PyUnicode_AsUTF8(lz->lz_code->co_filename); - if (!filename_str) { - // Unicode conversion failed - clear error and restore original - // exception. - PyErr_Clear(); - Py_DECREF(import_name); - _PyErr_SetRaisedException(tstate, exc); - goto ok; + if (PySet_GET_SIZE(resolving) == 0) { + // Keep the set, but release the capacity used by deep resolutions. + (void)PySet_Clear(resolving); } - - const char *funcname_str = PyUnicode_AsUTF8(lz->lz_code->co_name); - if (!funcname_str) { - // Unicode conversion failed - clear error and restore original - // exception. - PyErr_Clear(); - Py_DECREF(import_name); - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Create a cause exception showing where the lazy import was declared. - PyObject *msg = PyUnicode_FromFormat( - "lazy import of '%U' raised an exception during resolution", - import_name - ); - Py_DECREF(import_name); // Done with import_name. - - if (!msg) { - // Failed to create message - restore original exception. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - PyObject *cause_exc = PyObject_CallOneArg(PyExc_ImportError, msg); - Py_DECREF(msg); // Done with msg. - - if (!cause_exc) { - // Failed to create exception - restore original. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Add traceback entry for the lazy import declaration. - _PyErr_SetRaisedException(tstate, cause_exc); - _PyTraceback_Add(funcname_str, filename_str, lineno); - PyObject *cause_with_tb = _PyErr_GetRaisedException(tstate); - - // Set the cause on the original exception. - PyException_SetCause(exc, cause_with_tb); // Steals ref to cause_with_tb. - - // Restore the original exception with its full traceback. - _PyErr_SetRaisedException(tstate, exc); - } - -ok: - if (PySet_Discard(importing, lazy_import) < 0) { - Py_CLEAR(obj); } - // Release the global import lock. - _PyImport_ReleaseLock(interp); - Py_XDECREF(fromlist); Py_XDECREF(import_func); + _Py_LeaveRecursiveCallTstate(tstate); return obj; } diff --git a/Python/import.c b/Python/import.c index 40ff14c0137890..4539ccaa088f50 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4651,7 +4651,6 @@ _PyImport_ClearCore(PyInterpreterState *interp) Py_CLEAR(LAZY_IMPORT_FUNC(interp)); Py_CLEAR(interp->imports.lazy_pending_submodules); Py_CLEAR(interp->imports.lazy_modules); - Py_CLEAR(interp->imports.lazy_importing_modules); Py_CLEAR(interp->imports.lazy_imports_filter); } diff --git a/Python/pystate.c b/Python/pystate.c index 737bf0f216bd34..1222c0632b7f7f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1630,6 +1630,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; + _tstate->lazy_imports = NULL; #ifdef _Py_TIER2 _tstate->jit_tracer_state = NULL; @@ -1873,6 +1874,9 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(tstate->context); + // Finalizers above may resolve imports and create this set. + Py_CLEAR(((_PyThreadStateImpl *)tstate)->lazy_imports); + #ifdef Py_GIL_DISABLED // Each thread should clear own freelists in free-threading builds. struct _Py_freelists *freelists = _Py_freelists_GET(); From 846f93497ae013ab48eced32d79222dbb31ca5c9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 27 Sep 2026 15:19:20 +0100 Subject: [PATCH 3/9] Keep the original lazy import error chain Keep an existing cause or context when resolution fails. Add the import location as a note in that case, without adding the same note twice. Errors without an existing chain still get the declaration as their cause. --- Objects/lazyimportobject.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index 6371918037889b..52fcef2e7531bf 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -265,6 +265,25 @@ lazy_import_add_exception_cause(PyThreadState *tstate, PyLazyImportObject *lz) if (funcname == NULL) { goto done; } + PyBaseExceptionObject *base_exc = (PyBaseExceptionObject *)exc; + if (base_exc->cause != NULL || base_exc->context != NULL || + base_exc->suppress_context) { + // Preserve the original chain, including `raise ... from None`. + PyObject *note = PyUnicode_FromFormat( + "lazy import of '%U' declared in %s at %s:%d", + name, funcname, filename, lineno); + if (note != NULL) { + PyObject *notes; + if (PyObject_GetOptionalAttr(exc, &_Py_ID(__notes__), ¬es) >= 0) { + if (notes == NULL || PySequence_Contains(notes, note) == 0) { + (void)_PyException_AddNote(exc, note); + } + Py_XDECREF(notes); + } + Py_DECREF(note); + } + goto done; + } PyObject *msg = PyUnicode_FromFormat( "lazy import of '%U' raised an exception during resolution", name); if (msg == NULL) { From dd01552d2a7fb2f5870e402b5ab5abf27df7fbd3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 27 Sep 2026 15:19:21 +0100 Subject: [PATCH 4/9] Avoid waiting in the lazy from shortcut Only reuse a concrete attribute from a module that has finished loading. Leave lazy attributes for resolution so module hooks still get a chance. Check the module again after reading its spec, and keep interrupts visible. --- Objects/lazyimportobject.c | 77 +++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index 52fcef2e7531bf..3d0491bc992669 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -69,40 +69,67 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, return (PyObject *)m; } +// Reuse concrete attributes of initialized modules without waiting for imports +// or resolving lazy attributes. Failed cache lookups are retried at resolution. +static PyObject * +lazy_import_get_loaded_attr(PyThreadState *tstate, PyObject *name, + PyObject *attr_name) +{ + PyObject *mod = NULL, *spec = NULL, *current = NULL, *attr = NULL; + PyObject *modules = Py_XNewRef(_PyImport_GetModules(tstate->interp)); + if (modules == NULL) { + return NULL; + } + int rc = PyMapping_GetOptionalItem(modules, name, &mod); + if (rc <= 0 || !PyModule_Check(mod)) { + goto done; + } + PyObject *dict = _PyModule_GetDict(mod); + if (PyObject_GetOptionalAttr(mod, &_Py_ID(__spec__), &spec) < 0 || + _PyModuleSpec_IsInitializing(spec) != 0) { + goto done; + } + // An initialization check can replace the module in sys.modules. + if (modules != _PyImport_GetModules(tstate->interp) || + PyMapping_GetOptionalItem(modules, name, ¤t) <= 0 || + current != mod) { + goto done; + } + if (PyDict_GetItemRef(dict, attr_name, &attr) < 0) { + goto done; + } + if (attr != NULL && PyLazyImport_CheckExact(attr)) { + Py_CLEAR(attr); + } + +done: + Py_XDECREF(current); + Py_XDECREF(spec); + Py_XDECREF(mod); + Py_DECREF(modules); + if (PyErr_ExceptionMatches(PyExc_Exception)) { + PyErr_Clear(); + } + return attr; +} + PyObject * -_PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *v, PyObject *name) +_PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, + PyObject *v, PyObject *name) { assert(PyLazyImport_CheckExact(v)); assert(name); assert(PyUnicode_Check(name)); - PyObject *ret; - PyLazyImportObject *d = (PyLazyImportObject *)v; - PyObject *mod = NULL; + PyLazyImportObject *lz = PyLazyImportObject_CAST(v); // Only `from a import b` can take b off an already imported a; // `import a.b as c` has to import a.b first. - if (d->lz_attr != NULL && PyTuple_Check(d->lz_attr) && - PyTuple_GET_SIZE(d->lz_attr) > 0) { - mod = PyImport_GetModule(d->lz_from); - } - if (mod != NULL) { - // Check if the module already has the attribute, if so, resolve it - // eagerly. - if (PyModule_Check(mod)) { - PyObject *mod_dict = PyModule_GetDict(mod); - if (mod_dict != NULL) { - if (PyDict_GetItemRef(mod_dict, name, &ret) < 0) { - Py_DECREF(mod); - return NULL; - } - if (ret != NULL) { - Py_DECREF(mod); - return ret; - } - } + if (lz->lz_attr != NULL && PyTuple_Check(lz->lz_attr) && + PyTuple_GET_SIZE(lz->lz_attr) > 0) { + PyObject *attr = lazy_import_get_loaded_attr(tstate, lz->lz_from, name); + if (attr != NULL || PyErr_Occurred()) { + return attr; } - Py_DECREF(mod); } - return _PyLazyImport_New(frame, NULL, v, name); } From b78ebc9c8927084c69e90940827633fc0b30c6f4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 27 Sep 2026 15:19:21 +0100 Subject: [PATCH 5/9] Simplify lazy import declarations Normalize the fromlist before calling the filter and check its items before registering imports. Use one cleanup path and the existing filter accessor. Pass the same five arguments to custom lazy hooks as to __import__. --- Include/internal/pycore_import.h | 2 - Python/ceval.c | 4 +- Python/clinic/import.c.h | 4 +- Python/import.c | 110 +++++++++++++------------------ 4 files changed, 49 insertions(+), 71 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 669e328c266d00..8f392e1d0b124b 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -32,8 +32,6 @@ extern int _PyImport_FixupBuiltin( PyObject *modules ); -extern PyObject * _PyImport_ResolveName( - PyThreadState *tstate, PyObject *name, PyObject *globals, int level); extern PyObject * _PyImport_GetAbsName( PyThreadState *tstate, PyObject *name, PyObject *globals, int level); // Symbol is exported for the JIT on Windows builds. diff --git a/Python/ceval.c b/Python/ceval.c index 41987162c360b2..d8bb40f72358db 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3150,8 +3150,8 @@ _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, goto error; } - PyObject *args[6] = {name, globals, locals, fromlist, level, builtins}; - res = PyObject_Vectorcall(lazy_import_func, args, 6, NULL); + PyObject *args[5] = {name, globals, locals, fromlist, level}; + res = PyObject_Vectorcall(lazy_import_func, args, 5, NULL); error: Py_XDECREF(lazy_import_func); return res; diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 2e4e178b308406..749eaebca19dc5 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -627,7 +627,7 @@ PyDoc_STRVAR(_imp__set_lazy_attributes__doc__, "_set_lazy_attributes($module, modobj, name, /)\n" "--\n" "\n" -"Sets attributes to lazy submodules on the module, as side effects."); +"Remove the resolved module name from sys.lazy_modules."); #define _IMP__SET_LAZY_ATTRIBUTES_METHODDEF \ {"_set_lazy_attributes", _PyCFunction_CAST(_imp__set_lazy_attributes), METH_FASTCALL, _imp__set_lazy_attributes__doc__}, @@ -665,4 +665,4 @@ _imp__set_lazy_attributes(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=0974db098d601372 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ae9cf67e39955555 input=a9049054013a1b77]*/ diff --git a/Python/import.c b/Python/import.c index 4539ccaa088f50..955fdf6df2e373 100644 --- a/Python/import.c +++ b/Python/import.c @@ -11,7 +11,6 @@ #include "pycore_interp.h" // struct _import_runtime_state #include "pycore_interpframe.h" #include "pycore_lazyimportobject.h" -#include "pycore_long.h" // _PyLong_GetZero #include "pycore_magic_number.h" // PYC_MAGIC_NUMBER_TOKEN #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_namespace.h" // _PyNamespace_Type @@ -25,7 +24,6 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_time.h" // _PyTime_AsMicroseconds() -#include "pycore_traceback.h" #include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() #include "pycore_weakref.h" // _PyWeakref_GET_REF() @@ -3891,13 +3889,6 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level return NULL; } -PyObject * -_PyImport_ResolveName(PyThreadState *tstate, PyObject *name, - PyObject *globals, int level) -{ - return resolve_name(tstate, name, globals, level); -} - static PyObject * import_find_and_load_with_name(PyThreadState *tstate, PyObject *abs_name, PyObject *find_and_load, @@ -3966,9 +3957,9 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) tstate, abs_name, &_Py_ID(_find_and_load), NULL); } -static PyObject * -get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals, - int level) +PyObject * +_PyImport_GetAbsName(PyThreadState *tstate, PyObject *name, PyObject *globals, + int level) { if (level > 0) { return resolve_name(tstate, name, globals, level); @@ -3980,13 +3971,6 @@ get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals, return Py_NewRef(name); } -PyObject * -_PyImport_GetAbsName(PyThreadState *tstate, PyObject *name, - PyObject *globals, int level) -{ - return get_abs_name(tstate, name, globals, level); -} - PyObject * PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, @@ -4018,7 +4002,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, goto error; } - abs_name = get_abs_name(tstate, name, globals, level); + abs_name = _PyImport_GetAbsName(tstate, name, globals, level); if (abs_name == NULL) { goto error; } @@ -4332,12 +4316,11 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, return NULL; } - PyObject *abs_name = get_abs_name(tstate, name, globals, level); + PyObject *abs_name = _PyImport_GetAbsName(tstate, name, globals, level); if (abs_name == NULL) { return NULL; } - PyInterpreterState *interp = tstate->interp; _PyInterpreterFrame *frame = _PyEval_GetFrame(); if (frame == NULL || frame->f_globals != frame->f_locals) { Py_DECREF(abs_name); @@ -4346,71 +4329,69 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, return NULL; } + PyObject *res = NULL; + if (fromlist != NULL && PyUnicode_Check(fromlist)) { + fromlist = PyTuple_Pack(1, fromlist); + if (fromlist == NULL) { + goto done; + } + } + else { + fromlist = Py_NewRef(fromlist != NULL ? fromlist : Py_None); + } + // Check if the filter disables the lazy import. - // We must hold a reference to the filter while calling it to prevent - // use-after-free if another thread replaces it via - // PyImport_SetLazyImportsFilter. - LAZY_IMPORTS_LOCK(interp); - PyObject *filter = Py_XNewRef(LAZY_IMPORTS_FILTER(interp)); - LAZY_IMPORTS_UNLOCK(interp); + PyObject *filter = PyImport_GetLazyImportsFilter(); if (filter != NULL) { PyObject *modname; if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &modname) < 0) { Py_DECREF(filter); - Py_DECREF(abs_name); - return NULL; + goto done; } if (modname == NULL) { assert(!PyErr_Occurred()); modname = Py_NewRef(Py_None); } - if (fromlist == NULL) { - assert(!PyErr_Occurred()); - fromlist = Py_None; - } PyObject *args[] = {modname, abs_name, fromlist}; - PyObject *res = PyObject_Vectorcall(filter, args, 3, NULL); + PyObject *decision = PyObject_Vectorcall(filter, args, 3, NULL); Py_DECREF(modname); Py_DECREF(filter); - if (res == NULL) { - Py_DECREF(abs_name); - return NULL; + if (decision == NULL) { + goto done; } - int is_true = PyObject_IsTrue(res); - Py_DECREF(res); + int is_true = PyObject_IsTrue(decision); + Py_DECREF(decision); if (is_true < 0) { - Py_DECREF(abs_name); - return NULL; + goto done; } if (!is_true) { - Py_DECREF(abs_name); - return PyImport_ImportModuleLevelObject( + res = PyImport_ImportModuleLevelObject( name, globals, locals, fromlist, level ); + goto done; } } - // here, 'filter' is either NULL or is equivalent to a borrowed reference - if (fromlist && PyUnicode_Check(fromlist)) { - fromlist = PyTuple_Pack(1, fromlist); - if (fromlist == NULL) { - Py_DECREF(abs_name); - return NULL; + // Validate the entire fromlist before registering any pending imports. + if (PyTuple_Check(fromlist)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { + PyObject *item = PyTuple_GET_ITEM(fromlist, i); + if (!PyUnicode_Check(item)) { + _PyErr_Format(tstate, PyExc_TypeError, + "Item in ``from list'' must be str, not %.200s", + Py_TYPE(item)->tp_name); + goto done; + } } } - else { - Py_XINCREF(fromlist); - } - PyObject *res = _PyLazyImport_New(frame, builtins, abs_name, fromlist); + res = _PyLazyImport_New(frame, builtins, abs_name, fromlist); if (res == NULL) { - Py_XDECREF(fromlist); - Py_DECREF(abs_name); - return NULL; + goto done; } // Add the module name to sys.lazy_modules set (PEP 810). @@ -4419,7 +4400,7 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, goto error; } - if (fromlist && PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist)) { + if (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist)) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { if (register_from_lazy_on_parent(tstate, abs_name, PyTuple_GET_ITEM(fromlist, i)) < 0) @@ -4432,14 +4413,13 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, goto error; } - Py_XDECREF(fromlist); - Py_DECREF(abs_name); - return res; + goto done; error: + Py_CLEAR(res); +done: Py_XDECREF(fromlist); Py_DECREF(abs_name); - Py_DECREF(res); - return NULL; + return res; } PyObject * @@ -5445,13 +5425,13 @@ _imp._set_lazy_attributes modobj: object name: unicode / -Sets attributes to lazy submodules on the module, as side effects. +Remove the resolved module name from sys.lazy_modules. [clinic start generated code]*/ static PyObject * _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, PyObject *name) -/*[clinic end generated code: output=3369bb3242b1f043 input=38ea6f30956dd7d6]*/ +/*[clinic end generated code: output=3369bb3242b1f043 input=900339e013ab2b82]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); if (PySet_Discard(LAZY_MODULES(interp), name) < 0) { From 917df44c277a86162011a11fcaf629928da1986e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 27 Sep 2026 15:19:31 +0100 Subject: [PATCH 6/9] Handle lazy module hooks and children together Resolve lazy __getattr__ and __dir__ hooks before calling them. Treat a hook already being resolved as unavailable so it can import a sibling. Bind a loaded child before removing its pending entry. Recheck the module dict when another thread may have completed the load. Reuse modules already in sys.modules during package cycles, and simplify child registration. --- Include/internal/pycore_import.h | 12 +-- Objects/moduleobject.c | 132 +++++++++++++------------ Python/import.c | 159 ++++++++++++------------------- 3 files changed, 138 insertions(+), 165 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 8f392e1d0b124b..81448e23787fb3 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -37,13 +37,11 @@ extern PyObject * _PyImport_GetAbsName( // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); -typedef enum { - _Py_LAZY_SUBMODULE_ERROR = -1, - _Py_LAZY_SUBMODULE_NOT_FOUND = 0, - _Py_LAZY_SUBMODULE_LOADED = 1, -} _PyLazySubmoduleImportResult; -extern _PyLazySubmoduleImportResult _PyImport_TryLoadLazySubmodule( - PyObject *mod_name, PyObject *attr_name, PyObject **result); +// Bind a pending child on module before removing its registration. Return a +// new reference, NULL without an exception if absent, or NULL on error. +// Set recheck_dict if a concurrent lookup may have already bound the child. +extern PyObject * _PyImport_TryLoadLazySubmodule( + PyObject *module, PyObject *attr_name, int *recheck_dict); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index b8cd6025c20ba5..18aa169ee50c80 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -7,7 +7,7 @@ #include "pycore_fileutils.h" // _Py_wgetcwd #include "pycore_import.h" // _PyImport_GetNextModuleIndex() #include "pycore_interp.h" // PyInterpreterState.importlib -#include "pycore_lazyimportobject.h" // _PyLazyImportObject_Check() +#include "pycore_lazyimportobject.h" // PyLazyImport_CheckExact() #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() @@ -1299,32 +1299,25 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } -static PyObject * -try_load_lazy_submodule(PyModuleObject *m, PyObject *name) +static int +module_get_resolved_dict_item(PyObject *dict, PyObject *name, PyObject **result) { - PyObject *mod_name; - int rc = PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name); - if (rc <= 0) { - return NULL; - } - if (!PyUnicode_Check(mod_name)) { - Py_DECREF(mod_name); - return NULL; + int rc = PyDict_GetItemRef(dict, name, result); + if (rc <= 0 || !PyLazyImport_CheckExact(*result)) { + return rc; } - PyObject *result = NULL; - _PyLazySubmoduleImportResult status = - _PyImport_TryLoadLazySubmodule(mod_name, name, &result); - Py_DECREF(mod_name); - if (status != _Py_LAZY_SUBMODULE_LOADED) { - assert(status == _Py_LAZY_SUBMODULE_ERROR || - status == _Py_LAZY_SUBMODULE_NOT_FOUND); - return NULL; + PyThreadState *tstate = PyThreadState_GET(); + if (_PyLazyImport_IsResolving(tstate, *result)) { + Py_CLEAR(*result); + return 0; } - if (PyDict_SetItem(m->md_dict, name, result) < 0) { - Py_DECREF(result); - return NULL; + PyObject *value = _PyImport_LoadLazyImportTstate(tstate, *result); + Py_SETREF(*result, value); + if (value == NULL || PyDict_SetItem(dict, name, value) < 0) { + Py_CLEAR(*result); + return -1; } - return result; + return 1; } PyObject* @@ -1333,11 +1326,52 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) // When suppress=1, this function suppresses AttributeError. PyObject *attr, *mod_name, *getattr; attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); + if (attr == NULL) { + if (suppress == 1) { + if (PyErr_Occurred()) { + // pass up non-AttributeError exception + return NULL; + } + } + else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + // pass up non-AttributeError exception + return NULL; + } + PyErr_Clear(); + } + assert(m->md_dict != NULL); + int recheck_dict; + attr = _PyImport_TryLoadLazySubmodule((PyObject *)m, name, &recheck_dict); + if (attr != NULL || PyErr_Occurred()) { + return attr; + } + // A concurrent load may have bound the child and removed its pending + // registration after our initial dictionary lookup. + if (recheck_dict) { + if (Py_TYPE(m) != &PyModule_Type) { + PyObject *descr = _PyType_LookupRef(Py_TYPE(m), name); + if (descr != NULL) { + // Preserve the fallback after a descriptor raised. + recheck_dict = Py_TYPE(descr)->tp_descr_get == NULL; + Py_DECREF(descr); + } + } + if (recheck_dict && + PyDict_GetItemRef(m->md_dict, name, &attr) < 0) { + return NULL; + } + } + } if (attr) { if (PyLazyImport_CheckExact(attr)) { // gh-144957: Module __getattr__ should get a chance to provide // the attribute before resolving a lazy import placeholder. - if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + // Resolving __getattr__ itself must not invoke the hook. + getattr = NULL; + if (!_PyUnicode_EqualToASCIIString(name, "__getattr__") && + module_get_resolved_dict_item( + m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { Py_DECREF(attr); return NULL; } @@ -1354,17 +1388,14 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) } PyErr_Clear(); } - PyObject *new_value = _PyImport_LoadLazyImportTstate( - PyThreadState_GET(), attr); + PyThreadState *tstate = PyThreadState_GET(); + if (suppress && _PyLazyImport_IsResolving(tstate, attr)) { + // Only direct reentry means this attribute is unavailable. + Py_DECREF(attr); + return NULL; + } + PyObject *new_value = _PyImport_LoadLazyImportTstate(tstate, attr); if (new_value == NULL) { - if (suppress && - PyErr_ExceptionMatches(PyExc_ImportCycleError)) { - // ImportCycleError is raised when a lazy object tries - // to import itself. In this case, the error should not - // propagate to the caller and instead treated as if the - // attribute doesn't exist. - PyErr_Clear(); - } Py_DECREF(attr); return NULL; } @@ -1377,28 +1408,8 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) } return attr; } - if (suppress == 1) { - if (PyErr_Occurred()) { - // pass up non-AttributeError exception - return NULL; - } - } - else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - // pass up non-AttributeError exception - return NULL; - } - PyErr_Clear(); - } - assert(m->md_dict != NULL); - attr = try_load_lazy_submodule(m, name); - if (attr != NULL) { - return attr; - } - if (PyErr_Occurred()) { - return NULL; - } - if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + if (module_get_resolved_dict_item( + m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { return NULL; } if (getattr) { @@ -1589,11 +1600,14 @@ module_dir(PyObject *self, PyObject *args) if (dict != NULL) { if (PyDict_Check(dict)) { - PyObject *dirfunc = PyDict_GetItemWithError(dict, &_Py_ID(__dir__)); - if (dirfunc) { + PyObject *dirfunc; + int rc = module_get_resolved_dict_item( + dict, &_Py_ID(__dir__), &dirfunc); + if (rc > 0) { result = _PyObject_CallNoArgs(dirfunc); + Py_DECREF(dirfunc); } - else if (!PyErr_Occurred()) { + else if (rc == 0) { result = PyDict_Keys(dict); } } diff --git a/Python/import.c b/Python/import.c index 955fdf6df2e373..576b8d47639fa4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4128,7 +4128,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } -// ensure we have the set for the parent module name in sys.lazy_modules. +// Ensure a set of pending submodule names exists for the parent. // Returns a new reference. static PyObject * ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) @@ -4157,59 +4157,35 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) static int register_lazy_on_parent(PyThreadState *tstate, PyObject *name) { - int ret = -1; - PyObject *parent = NULL; - PyObject *child = NULL; - - PyInterpreterState *interp = tstate->interp; - PyObject *lazy_pending_submodules = LAZY_PENDING_SUBMODULES(interp); - assert(lazy_pending_submodules != NULL); - - Py_INCREF(name); + PyDictObject *pending = + (PyDictObject *)LAZY_PENDING_SUBMODULES(tstate->interp); + assert(pending != NULL); + Py_ssize_t end = PyUnicode_GET_LENGTH(name); while (true) { - Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, - PyUnicode_GET_LENGTH(name), -1); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, -1); if (dot < 0) { - PyObject *lazy_submodules = ensure_lazy_pending_submodules( - (PyDictObject *)lazy_pending_submodules, name); - if (lazy_submodules == NULL) { - goto done; - } - Py_DECREF(lazy_submodules); - ret = 0; - goto done; + return dot == -1 ? 0 : -1; } - parent = PyUnicode_Substring(name, 0, dot); + PyObject *parent = PyUnicode_Substring(name, 0, dot); if (parent == NULL) { - goto done; + return -1; } - Py_XDECREF(child); - child = PyUnicode_Substring(name, dot + 1, PyUnicode_GET_LENGTH(name)); + PyObject *child = PyUnicode_Substring(name, dot + 1, end); if (child == NULL) { - goto done; - } - - PyObject *lazy_submodules = ensure_lazy_pending_submodules( - (PyDictObject *)lazy_pending_submodules, parent); - if (lazy_submodules == NULL) { - goto done; + Py_DECREF(parent); + return -1; } - if (PySet_Add(lazy_submodules, child) < 0) { - Py_DECREF(lazy_submodules); - goto done; + PyObject *submodules = ensure_lazy_pending_submodules(pending, parent); + Py_DECREF(parent); + int err = submodules == NULL ? -1 : PySet_Add(submodules, child); + Py_DECREF(child); + Py_XDECREF(submodules); + if (err < 0) { + return -1; } - Py_DECREF(lazy_submodules); - - Py_SETREF(name, parent); - parent = NULL; + end = dot; } - -done: - Py_XDECREF(child); - Py_XDECREF(parent); - Py_XDECREF(name); - return ret; } static int @@ -4223,80 +4199,65 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, // Add the module name to sys.lazy_modules set (PEP 810). PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - if (PySet_Add(lazy_modules, fromname) < 0) { - Py_DECREF(fromname); - return -1; + int res = PySet_Add(lazy_modules, fromname); + if (res == 0) { + res = register_lazy_on_parent(tstate, fromname); } - - int res = register_lazy_on_parent(tstate, fromname); Py_DECREF(fromname); return res; } -_PyLazySubmoduleImportResult -_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name, - PyObject **result) +PyObject * +_PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, + int *recheck_dict) { - assert(result != NULL); - *result = NULL; - - PyThreadState *tstate = _PyThreadState_GET(); - PyInterpreterState *interp = tstate->interp; - PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); - if (lazy_pending == NULL) { - return _Py_LAZY_SUBMODULE_NOT_FOUND; + *recheck_dict = 0; + PyObject *mod_dict = _PyModule_GetDict(module); + PyObject *mod_name; + if (PyDict_GetItemRef(mod_dict, &_Py_ID(__name__), &mod_name) <= 0) { + return NULL; } - - PyObject *pending_set; - int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); - if (rc < 0) { - return _Py_LAZY_SUBMODULE_ERROR; + PyObject *mod = NULL, *pending_set = NULL; + if (!PyUnicode_Check(mod_name)) { + goto done; } - if (rc == 0) { - return _Py_LAZY_SUBMODULE_NOT_FOUND; + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(tstate->interp); + if (lazy_pending == NULL || + PyDict_GetItemRef(lazy_pending, mod_name, &pending_set) <= 0) { + goto done; } - int contains = PySet_Contains(pending_set, attr_name); - if (contains < 0) { - Py_DECREF(pending_set); - return _Py_LAZY_SUBMODULE_ERROR; - } - if (contains == 0) { - Py_DECREF(pending_set); - return _Py_LAZY_SUBMODULE_NOT_FOUND; + *recheck_dict = 1; + if (PySet_Contains(pending_set, attr_name) <= 0) { + goto done; } - PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); if (full_name == NULL) { - Py_DECREF(pending_set); - return _Py_LAZY_SUBMODULE_ERROR; + goto done; } - - PyObject *mod = import_find_and_load_with_name( - tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); + // Match eager from-imports: accept a partial module in an import cycle. + mod = PyImport_GetModule(full_name); + if ((mod == NULL && !PyErr_Occurred()) || mod == Py_None) { + Py_XDECREF(mod); + mod = import_find_and_load_with_name( + tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); + } + Py_DECREF(full_name); if (mod == NULL) { - Py_DECREF(pending_set); - Py_DECREF(full_name); remove_importlib_frames(tstate); - return _Py_LAZY_SUBMODULE_ERROR; } - if (mod == Py_None) { - Py_DECREF(mod); - Py_DECREF(pending_set); - Py_DECREF(full_name); - return _Py_LAZY_SUBMODULE_NOT_FOUND; + /* Keep missing children pending: a finder may provide them later. */ + else if (mod == Py_None || + PyDict_SetItem(mod_dict, attr_name, mod) < 0 || + PySet_Discard(pending_set, attr_name) < 0) { + Py_CLEAR(mod); } - if (PySet_Discard(pending_set, attr_name) < 0) { - Py_DECREF(mod); - Py_DECREF(pending_set); - Py_DECREF(full_name); - return _Py_LAZY_SUBMODULE_ERROR; - } - Py_DECREF(pending_set); - Py_DECREF(full_name); - *result = mod; - return _Py_LAZY_SUBMODULE_LOADED; +done: + Py_XDECREF(pending_set); + Py_DECREF(mod_name); + return mod; } PyObject * From 4dc64c864178c326ad98cd57039787dba5f0fb61 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 27 Sep 2026 15:19:31 +0100 Subject: [PATCH 7/9] Share lazy import rebinding Remember the namespace where lookup found the placeholder and use one helper to resolve and replace it. Only replace a binding that still holds the same placeholder, preserving assignments and deletions during import. Check and replace ordinary dict entries atomically. Keep the mapping protocol for custom namespaces and allow reads from readonly namespaces. Reuse the global lookup helper in the eval loop and remove the duplicate. --- Include/internal/pycore_dict.h | 9 +- Include/internal/pycore_lazyimportobject.h | 6 + Include/internal/pycore_opcode_metadata.h | 3 +- Include/internal/pycore_uop_metadata.h | 15 +++ Modules/_testinternalcapi/test_cases.c.h | 136 +++++---------------- Objects/dictobject.c | 116 +++++++++--------- Objects/lazyimportobject.c | 35 ++++++ Objects/moduleobject.c | 19 +-- Python/bytecodes.c | 83 +++---------- Python/ceval.c | 50 ++++---- Python/executor_cases.c.h | 97 ++++++++++----- Python/generated_cases.c.h | 136 +++++---------------- Python/optimizer_cases.c.h | 7 +- 13 files changed, 302 insertions(+), 410 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 7032b61d7654b8..81f1de3a5be865 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -23,6 +23,11 @@ PyAPI_FUNC(int) _PyDict_DelItemIf(PyObject *mp, PyObject *key, int (*predicate)(PyObject *value, void *arg), void *arg); +// Atomically replace an existing value if it is expected (by identity). +// Return 1 if replaced, 0 if absent or different, or -1 on error. +extern int _PyDict_ReplaceItemIf(PyObject *dict, PyObject *key, + PyObject *expected, PyObject *replacement); + // "KnownHash" variants // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, @@ -147,8 +152,8 @@ PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyOb */ PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); -PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); -PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); +// Return the borrowed source dictionary, or NULL if absent or on error. +PyAPI_FUNC(PyObject *) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); // Loads the __builtins__ object from the globals dict. Returns a new reference. extern PyObject *_PyDict_LoadBuiltinsFromGlobals(PyObject *globals); diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index 4e7a437d0eff41..4431a3764818fe 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -20,6 +20,12 @@ PyAPI_FUNC(PyObject *) _PyLazyImport_New( extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op); +// Resolve a placeholder and replace its binding only if it is unchanged. +// namespace is the source captured during lookup, before resolution runs. +PyAPI_FUNC(PyObject *) _PyLazyImport_Reify( + PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *ns); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 38bccdd48c0f86..d3546119bc9ee0 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1253,7 +1253,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1476,6 +1476,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { _LOAD_FAST_CHECK, OPARG_SIMPLE, 0 } } }, [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _LOAD_FAST, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_DEREF, OPARG_SIMPLE, 0 } } }, + [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, OPARG_SIMPLE, 0 } } }, [LOAD_GLOBAL] = { .nuops = 2, .uops = { { _LOAD_GLOBAL, OPARG_SIMPLE, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 3 } } }, [LOAD_GLOBAL_BUILTIN] = { .nuops = 4, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_BUILTINS, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, OPERAND1_1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 4 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 4, .uops = { { _NOP, OPARG_SIMPLE, 1 }, { _LOAD_GLOBAL_MODULE, 1, 1 }, { _LOAD_GLOBAL_MODULE, OPERAND1_1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 4 } } }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f464effe23bc0b..325b811472f8f8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -190,6 +190,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG, + [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL_CONDITIONAL] = HAS_ARG_FLAG, @@ -1840,6 +1841,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_LOAD_FROM_DICT_OR_GLOBALS] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_FROM_DICT_OR_GLOBALS_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_LOAD_NAME] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -4306,6 +4316,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_LOCALS_r01] = _LOAD_LOCALS, [_LOAD_LOCALS_r12] = _LOAD_LOCALS, [_LOAD_LOCALS_r23] = _LOAD_LOCALS, + [_LOAD_FROM_DICT_OR_GLOBALS_r11] = _LOAD_FROM_DICT_OR_GLOBALS, [_LOAD_NAME_r01] = _LOAD_NAME, [_LOAD_GLOBAL_r00] = _LOAD_GLOBAL, [_PUSH_NULL_CONDITIONAL_r00] = _PUSH_NULL_CONDITIONAL, @@ -5810,6 +5821,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_FAST_CHECK_r23] = "_LOAD_FAST_CHECK_r23", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", [_LOAD_FROM_DICT_OR_DEREF_r11] = "_LOAD_FROM_DICT_OR_DEREF_r11", + [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", + [_LOAD_FROM_DICT_OR_GLOBALS_r11] = "_LOAD_FROM_DICT_OR_GLOBALS_r11", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", [_LOAD_GLOBAL_r00] = "_LOAD_GLOBAL_r00", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", @@ -6476,6 +6489,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_LOCALS: return 0; + case _LOAD_FROM_DICT_OR_GLOBALS: + return 1; case _LOAD_NAME: return 0; case _LOAD_GLOBAL: diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index a04770bd83b176..245b0793eea02b 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -9907,6 +9907,16 @@ JUMP_TO_LABEL(error); } } + else if (PyLazyImport_CheckExact(value_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + _PyFrame_StackPointerInvalidate(frame); + if (value_o == NULL) { + JUMP_TO_LABEL(error); + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9932,11 +9942,21 @@ _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); int err; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); _PyFrame_StackPointerInvalidate(frame); + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + err = -1; + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9947,81 +9967,17 @@ JUMP_TO_LABEL(error); } if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - } - JUMP_TO_LABEL(error); - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } - } - else { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(v)) { + JUMP_TO_LABEL(error); } } - v = PyStackRef_FromPyObjectSteal(v_o); + else { + v = PyStackRef_FromPyObjectSteal(v_o); + } stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10275,38 +10231,6 @@ if (v_o == NULL) { JUMP_TO_LABEL(error); } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - if (l_v == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(l_v); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 15377ac083c2b2..57874de6ee7497 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2009,6 +2009,31 @@ _PyDict_InsertSplitValue(PyDictObject *mp, PyObject *key, PyObject *value, Py_ss ASSERT_CONSISTENT(mp); } +// Replace a value at an existing entry. Steals the new value reference. +static void +replace_value(PyDictObject *mp, PyObject *key, Py_ssize_t ix, + PyObject *old_value, PyObject *value) +{ + if (old_value != value) { + _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); + assert(old_value != NULL); + if (DK_IS_UNICODE(mp->ma_keys)) { + if (_PyDict_HasSplitTable(mp)) { + STORE_SPLIT_VALUE(mp, ix, value); + } + else { + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); + } + } + else { + PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); + } + } + Py_DECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ +} + /* Internal routine to insert a new item into the table. Used both by the internal resize routine and by the public insert routine. @@ -2056,24 +2081,7 @@ insertdict(PyDictObject *mp, return 0; } - if (old_value != value) { - _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); - assert(old_value != NULL); - if (DK_IS_UNICODE(mp->ma_keys)) { - if (_PyDict_HasSplitTable(mp)) { - STORE_SPLIT_VALUE(mp, ix, value); - } - else { - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; - STORE_VALUE(ep, value); - } - } - else { - PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; - STORE_VALUE(ep, value); - } - } - Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ + replace_value(mp, key, ix, old_value, value); ASSERT_CONSISTENT(mp); Py_DECREF(key); return 0; @@ -2700,44 +2708,7 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) return rv; } -/* Fast version of global value lookup (LOAD_GLOBAL). - * Lookup in globals, then builtins. - * - * - * - * - * Raise an exception and return NULL if an error occurred (ex: computing the - * key hash failed, key comparison failed, ...). Return NULL if the key doesn't - * exist. Return the value if the key exists. - * - * Returns a new reference. - */ PyObject * -_PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) -{ - Py_ssize_t ix; - Py_hash_t hash; - PyObject *value; - - hash = _PyObject_HashDictKey(key); - if (hash == -1) { - return NULL; - } - - /* namespace 1: globals */ - ix = _Py_dict_lookup_threadsafe(globals, key, hash, &value); - if (ix == DKIX_ERROR) - return NULL; - if (ix != DKIX_EMPTY && value != NULL) - return value; - - /* namespace 2: builtins */ - ix = _Py_dict_lookup_threadsafe(builtins, key, hash, &value); - assert(ix >= 0 || value == NULL); - return value; -} - -void _PyDict_LoadGlobalStackRef(PyDictObject *globals, PyDictObject *builtins, PyObject *key, _PyStackRef *res) { Py_ssize_t ix; @@ -2746,21 +2717,22 @@ _PyDict_LoadGlobalStackRef(PyDictObject *globals, PyDictObject *builtins, PyObje hash = _PyObject_HashDictKey(key); if (hash == -1) { *res = PyStackRef_NULL; - return; + return NULL; } /* namespace 1: globals */ ix = _Py_dict_lookup_threadsafe_stackref(globals, key, hash, res); if (ix == DKIX_ERROR) { - return; + return NULL; } if (ix != DKIX_EMPTY && !PyStackRef_IsNull(*res)) { - return; + return (PyObject *)globals; } /* namespace 2: builtins */ ix = _Py_dict_lookup_threadsafe_stackref(builtins, key, hash, res); assert(ix >= 0 || PyStackRef_IsNull(*res)); + return PyStackRef_IsNull(*res) ? NULL : (PyObject *)builtins; } PyObject * @@ -3105,6 +3077,34 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, return res; } +int +_PyDict_ReplaceItemIf(PyObject *op, PyObject *key, + PyObject *expected, PyObject *replacement) +{ + assert(PyDict_Check(op)); + assert(expected != NULL && replacement != NULL); + Py_hash_t hash = PyObject_Hash(key); + if (hash == -1) { + return -1; + } + int result = 0; + Py_BEGIN_CRITICAL_SECTION(op); + PyDictObject *mp = (PyDictObject *)op; + PyObject *current; + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, ¤t); + if (ix == DKIX_ERROR) { + result = -1; + } + else if (current == expected) { + // Do not look up the key again: equality can execute Python code. + replace_value(mp, key, ix, current, Py_NewRef(replacement)); + ASSERT_CONSISTENT(mp); + result = 1; + } + Py_END_CRITICAL_SECTION(); + return result; +} + static void clear_embedded_values(PyDictValues *values, Py_ssize_t nentries) { diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index 3d0491bc992669..4bf5d4ba4b44d1 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -451,6 +451,41 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) return obj; } +PyObject * +_PyLazyImport_Reify(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace) +{ + PyObject *value = _PyImport_LoadLazyImportTstate(tstate, placeholder); + if (value == NULL) { + return NULL; + } + int rc; + if (PyDict_CheckExact(namespace)) { + rc = _PyDict_ReplaceItemIf(namespace, name, placeholder, value); + } + else if (Py_TYPE(namespace)->tp_as_mapping == NULL || + Py_TYPE(namespace)->tp_as_mapping->mp_ass_subscript == NULL) { + // Read-only namespaces can resolve a value without caching it. + return value; + } + else { + // Custom namespaces retain their mapping protocol. Atomic replacement + // is only available for exact dictionaries. + PyObject *current; + rc = PyMapping_GetOptionalItem(namespace, name, ¤t); + if (rc > 0) { + if (current == placeholder) { + rc = PyObject_SetItem(namespace, name, value); + } + Py_DECREF(current); + } + } + if (rc < 0) { + Py_CLEAR(value); + } + return value; +} + static PyObject * lazy_import_resolve(PyObject *self, PyObject *args) { diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 18aa169ee50c80..57df195b7f3201 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1311,13 +1311,9 @@ module_get_resolved_dict_item(PyObject *dict, PyObject *name, PyObject **result) Py_CLEAR(*result); return 0; } - PyObject *value = _PyImport_LoadLazyImportTstate(tstate, *result); + PyObject *value = _PyLazyImport_Reify(tstate, *result, name, dict); Py_SETREF(*result, value); - if (value == NULL || PyDict_SetItem(dict, name, value) < 0) { - Py_CLEAR(*result); - return -1; - } - return 1; + return value == NULL ? -1 : 1; } PyObject* @@ -1394,15 +1390,8 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) Py_DECREF(attr); return NULL; } - PyObject *new_value = _PyImport_LoadLazyImportTstate(tstate, attr); - if (new_value == NULL) { - Py_DECREF(attr); - return NULL; - } - - if (PyDict_SetItem(m->md_dict, name, new_value) < 0) { - Py_CLEAR(new_value); - } + PyObject *new_value = _PyLazyImport_Reify( + tstate, attr, name, m->md_dict); Py_DECREF(attr); return new_value; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fdfd5a0e609046..a425b572bb95be 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2193,80 +2193,30 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); int err; - PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); - + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); + if (v_o == NULL) { + err = -1; + } + } PyStackRef_CLOSE(mod_or_class_dict); ERROR_IF(err < 0); if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - ERROR_NO_POP(); - } - - if (PyLazyImport_CheckExact(v_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - Py_SETREF(v_o, l_v); - ERROR_IF(v_o == NULL); - } - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - ERROR_IF(err < 0); - if (v_o == NULL) { - /* namespace 2: builtins */ - v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - ERROR_IF(err < 0); - if (v_o == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - ERROR_IF(true); - } - } - if (PyLazyImport_CheckExact(v_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - Py_SETREF(v_o, l_v); - ERROR_IF(v_o == NULL); - } - } + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); + ERROR_IF(PyStackRef_IsNull(v)); + } + else { + v = PyStackRef_FromPyObjectSteal(v_o); } - v = PyStackRef_FromPyObjectSteal(v_o); } inst(LOAD_NAME, (-- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); ERROR_IF(v_o == NULL); - if (PyLazyImport_CheckExact(v_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - // cannot early-decref v_o as it may cause a side-effect on l_v - if (l_v == NULL) { - Py_DECREF(v_o); - ERROR_IF(true); - } - int err = PyDict_SetItem(GLOBALS(), name, l_v); - if (err < 0) { - Py_DECREF(v_o); - Py_DECREF(l_v); - ERROR_IF(true); - } - Py_SETREF(v_o, l_v); - } - v = PyStackRef_FromPyObjectSteal(v_o); } @@ -2428,6 +2378,13 @@ dummy_func( ERROR_NO_POP(); } } + else if (PyLazyImport_CheckExact(value_o)) { + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + if (value_o == NULL) { + ERROR_NO_POP(); + } + } PyStackRef_CLOSE(class_dict_st); value = PyStackRef_FromPyObjectSteal(value_o); } diff --git a/Python/ceval.c b/Python/ceval.c index d8bb40f72358db..b90de789accd44 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3590,13 +3590,13 @@ _PyEval_GetANext(PyObject *aiter) void _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name, _PyStackRef *writeto) { + PyObject *namespace = globals; if (PyAnyDict_CheckExact(globals) && PyAnyDict_CheckExact(builtins)) { - _PyDict_LoadGlobalStackRef((PyDictObject *)globals, - (PyDictObject *)builtins, - name, writeto); + namespace = _PyDict_LoadGlobalStackRef((PyDictObject *)globals, + (PyDictObject *)builtins, + name, writeto); if (PyStackRef_IsNull(*writeto) && !PyErr_Occurred()) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ + // A missing key does not set an exception in the dictionary helper. _PyEval_FormatExcCheckArg(PyThreadState_GET(), PyExc_NameError, NAME_ERROR_MSG, name); } @@ -3611,6 +3611,7 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name } if (res == NULL) { /* namespace 2: builtins */ + namespace = builtins; if (PyMapping_GetOptionalItem(builtins, name, &res) < 0) { *writeto = PyStackRef_NULL; return; @@ -3628,20 +3629,10 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name PyObject *res_o = PyStackRef_AsPyObjectBorrow(*writeto); if (res_o != NULL && PyLazyImport_CheckExact(res_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(PyThreadState_GET(), res_o); + PyObject *l_v = _PyLazyImport_Reify( + PyThreadState_GET(), res_o, name, namespace); PyStackRef_CLOSE(writeto[0]); - if (l_v == NULL) { - assert(PyErr_Occurred()); - *writeto = PyStackRef_NULL; - return; - } - int err = PyDict_SetItem(globals, name, l_v); - if (err < 0) { - Py_DECREF(l_v); - *writeto = PyStackRef_NULL; - return; - } - *writeto = PyStackRef_FromPyObjectSteal(l_v); + *writeto = l_v == NULL ? PyStackRef_NULL : PyStackRef_FromPyObjectSteal(l_v); } } @@ -3672,32 +3663,39 @@ _PyEval_GetAwaitable(PyObject *iterable, int oparg) PyObject * _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name) { - PyObject *value; - if (frame->f_locals == NULL) { + PyObject *namespace = frame->f_locals; + if (namespace == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); return NULL; } - if (PyMapping_GetOptionalItem(frame->f_locals, name, &value) < 0) { + if (PyMapping_GetOptionalItem(namespace, name, &value) < 0) { return NULL; } if (value != NULL) { - return value; + goto found; } - if (PyDict_GetItemRef(frame->f_globals, name, &value) < 0) { + namespace = frame->f_globals; + if (PyDict_GetItemRef(namespace, name, &value) < 0) { return NULL; } if (value != NULL) { - return value; + goto found; } - if (PyMapping_GetOptionalItem(frame->f_builtins, name, &value) < 0) { + namespace = frame->f_builtins; + if (PyMapping_GetOptionalItem(namespace, name, &value) < 0) { return NULL; } if (value == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); + return NULL; + } +found: + if (PyLazyImport_CheckExact(value)) { + Py_SETREF(value, _PyLazyImport_Reify(tstate, value, name, namespace)); } return value; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e3dc2e781a3beb..f34ec0431633e4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -10183,55 +10183,77 @@ break; } - /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - - case _LOAD_NAME_r01: { - CHECK_CURRENT_CACHED_VALUES(0); + case _LOAD_FROM_DICT_OR_GLOBALS_r11: { + CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef mod_or_class_dict; _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); + mod_or_class_dict = _stack_item_0; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); + int err; + stack_pointer[0] = mod_or_class_dict; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (PyLazyImport_CheckExact(v_o)) { + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); _PyFrame_StackPointerInvalidate(frame); - if (l_v == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + if (v_o == NULL) { + err = -1; } + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(mod_or_class_dict); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (v_o == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(l_v); - _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(v)) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); + } + else { + v = PyStackRef_FromPyObjectSteal(v_o); + } + _tos_cache0 = v; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _LOAD_NAME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); } v = PyStackRef_FromPyObjectSteal(v_o); _tos_cache0 = v; @@ -10603,6 +10625,17 @@ JUMP_TO_ERROR(); } } + else if (PyLazyImport_CheckExact(value_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + _PyFrame_StackPointerInvalidate(frame); + if (value_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 48f76ccf59f8bc..84b6a541f2c5bf 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9905,6 +9905,16 @@ JUMP_TO_LABEL(error); } } + else if (PyLazyImport_CheckExact(value_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + _PyFrame_StackPointerInvalidate(frame); + if (value_o == NULL) { + JUMP_TO_LABEL(error); + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9930,11 +9940,21 @@ _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); int err; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); _PyFrame_StackPointerInvalidate(frame); + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + err = -1; + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9945,81 +9965,17 @@ JUMP_TO_LABEL(error); } if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - } - JUMP_TO_LABEL(error); - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } - } - else { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(v)) { + JUMP_TO_LABEL(error); } } - v = PyStackRef_FromPyObjectSteal(v_o); + else { + v = PyStackRef_FromPyObjectSteal(v_o); + } stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10273,38 +10229,6 @@ if (v_o == NULL) { JUMP_TO_LABEL(error); } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - if (l_v == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(l_v); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 21f275f27cafe0..89423d6c6900af 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2176,7 +2176,12 @@ break; } - /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ + case _LOAD_FROM_DICT_OR_GLOBALS: { + JitOptRef v; + v = sym_new_not_null(ctx); + stack_pointer[-1] = v; + break; + } case _LOAD_NAME: { JitOptRef v; From ad34faea95935c9cedfb940930e3c377322a4e09 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 27 Sep 2026 15:47:18 +0100 Subject: [PATCH 8/9] Keep imported attributes on their parent package Loading a child can replace the parent binding with the module before we fetch the imported attribute. Keep the module returned by the normal importer so the shared helper can replace it in its actual parent namespace. Other values and deletions still prevent replacement, and custom hooks keep control of their assignments. --- Include/internal/pycore_lazyimportobject.h | 3 +- Objects/lazyimportobject.c | 74 +++++++++++++++++++++- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index 4431a3764818fe..c3f8417c089316 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -20,7 +20,8 @@ PyAPI_FUNC(PyObject *) _PyLazyImport_New( extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op); -// Resolve a placeholder and replace its binding only if it is unchanged. +// Resolve a placeholder and replace its binding if it is unchanged or holds +// the child module published by the normal importer during resolution. // namespace is the source captured during lookup, before resolution runs. PyAPI_FUNC(PyObject *) _PyLazyImport_Reify( PyThreadState *tstate, PyObject *placeholder, diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index 4bf5d4ba4b44d1..72624b746fe64c 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -340,8 +340,9 @@ _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op) return active; } -PyObject * -_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) +static PyObject * +lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, + PyObject **imported_module) { PyObject *obj = NULL; PyObject *fromlist = NULL; @@ -421,6 +422,12 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) root->lz_from, fromlist, _PyLong_GetZero() ); obj = lazy_import_resolve_result(tstate, obj); + // The normal importer may publish this module on its parent. Custom + // hooks retain control of their own assignments to the parent. + if (imported_module != NULL && obj != NULL && PyModule_Check(obj) && + _PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { + *imported_module = Py_NewRef(obj); + } if (obj != NULL && first != NULL) { // Keep the hook and root result alive until all attribute lookups finish. PyObject *from = obj; @@ -451,21 +458,81 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) return obj; } +PyObject * +_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) +{ + return lazy_import_resolve_impl(tstate, lazy_import, NULL); +} + +// Loading pkg.child can replace a placeholder in pkg.child with the module +// before a from-import retrieves the value that belongs in that binding. +static int +lazy_import_replace_child(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace, + PyObject *child, PyObject *value) +{ + PyLazyImportObject *root = (PyLazyImportObject *)placeholder; + if (!PyLazyImport_CheckExact(root->lz_from)) { + return 0; + } + while (PyLazyImport_CheckExact(root->lz_from)) { + root = (PyLazyImportObject *)root->lz_from; + } + Py_ssize_t end = PyUnicode_GET_LENGTH(root->lz_from); + Py_ssize_t dot = PyUnicode_FindChar(root->lz_from, '.', 0, end, -1); + if (dot < 0) { + return dot == -1 ? 0 : -1; + } + if (end - dot - 1 != PyUnicode_GET_LENGTH(name)) { + return 0; + } + int matches = PyUnicode_Tailmatch(root->lz_from, name, dot + 1, end, 1); + if (matches <= 0) { + return matches; + } + PyObject *parent_name = PyUnicode_Substring(root->lz_from, 0, dot); + if (parent_name == NULL) { + return -1; + } + PyObject *modules = Py_XNewRef(_PyImport_GetModules(tstate->interp)); + PyObject *parent = NULL; + int rc = 0; + if (modules != NULL) { + rc = PyMapping_GetOptionalItem(modules, parent_name, &parent); + if (rc > 0 && PyModule_Check(parent) && + _PyModule_GetDict(parent) == namespace) { + rc = _PyDict_ReplaceItemIf(namespace, name, child, value); + } + } + Py_XDECREF(parent); + Py_XDECREF(modules); + Py_DECREF(parent_name); + return rc; +} + PyObject * _PyLazyImport_Reify(PyThreadState *tstate, PyObject *placeholder, PyObject *name, PyObject *namespace) { - PyObject *value = _PyImport_LoadLazyImportTstate(tstate, placeholder); + PyObject *imported_module = NULL; + PyObject *value = lazy_import_resolve_impl( + tstate, placeholder, &imported_module); if (value == NULL) { + Py_XDECREF(imported_module); return NULL; } int rc; if (PyDict_CheckExact(namespace)) { rc = _PyDict_ReplaceItemIf(namespace, name, placeholder, value); + if (rc == 0 && imported_module != NULL) { + rc = lazy_import_replace_child( + tstate, placeholder, name, namespace, imported_module, value); + } } else if (Py_TYPE(namespace)->tp_as_mapping == NULL || Py_TYPE(namespace)->tp_as_mapping->mp_ass_subscript == NULL) { // Read-only namespaces can resolve a value without caching it. + Py_XDECREF(imported_module); return value; } else { @@ -483,6 +550,7 @@ _PyLazyImport_Reify(PyThreadState *tstate, PyObject *placeholder, if (rc < 0) { Py_CLEAR(value); } + Py_XDECREF(imported_module); return value; } From 662dfea6b8559cde6904c6bd710b57a49ecacde4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 30 Sep 2026 17:21:08 +0100 Subject: [PATCH 9/9] gh-158140: Resolve lazily imported sibling submodules independently --- Include/internal/pycore_import.h | 3 + Include/internal/pycore_interp_structs.h | 7 +- Include/internal/pycore_lazyimportobject.h | 8 + Lib/test/test_lazy_import/__init__.py | 811 ++++++++++++++++++ ...-09-30-15-00-00.gh-issue-158140.v5Aj2R.rst | 4 + Objects/lazyimportobject.c | 369 +++++++- Objects/moduleobject.c | 40 +- Python/import.c | 216 ++++- 8 files changed, 1404 insertions(+), 54 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-15-00-00.gh-issue-158140.v5Aj2R.rst diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 81448e23787fb3..8075ce66e2e335 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -40,6 +40,9 @@ PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( // Bind a pending child on module before removing its registration. Return a // new reference, NULL without an exception if absent, or NULL on error. // Set recheck_dict if a concurrent lookup may have already bound the child. +extern PyObject *_PyImport_LoadLazySubmodule(PyThreadState *tstate, PyObject *name); +extern PyObject *_PyImport_GetLazySubmodules(PyThreadState *, PyObject *); +extern int _PyImport_ClearLazySubmodules(PyThreadState *, PyObject *); extern PyObject * _PyImport_TryLoadLazySubmodule( PyObject *module, PyObject *attr_name, int *recheck_dict); extern PyObject * _PyImport_LazyImportModuleLevelObject( diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index a9c61ea6db3a8c..2692ac41cd6db4 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -356,9 +356,12 @@ struct _import_state { // lazily imported. This value is only for debugging/introspection // purposes and is not used by the runtime. PyObject *lazy_modules; - // A dict mapping package names to a set of submodule names that - // may need to be imported when an attribute is missing on the package. + // A dict mapping package names to dicts of pending child names. + // Values are declarations, True for completed imports, or False for + // optional from-import attributes. PyObject *lazy_pending_submodules; + // Cached imports may need to replace a published lazy binding. + int has_lazy_submodules; #ifdef Py_GIL_DISABLED PyMutex lazy_mutex; #endif diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index c3f8417c089316..8b2abf4bb5c1f9 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -18,6 +18,9 @@ PyAPI_FUNC(PyObject *) _PyLazyImport_New( struct _PyInterpreterFrame *frame, PyObject *builtins, PyObject *name, PyObject *fromlist); +extern int _PyLazyImport_IsSubmodule(PyObject *op); +extern int _PyLazyImport_IsStrictSubmodule(PyObject *op); + extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op); // Resolve a placeholder and replace its binding if it is unchanged or holds @@ -27,6 +30,11 @@ PyAPI_FUNC(PyObject *) _PyLazyImport_Reify( PyThreadState *tstate, PyObject *placeholder, PyObject *name, PyObject *ns); +// Optional module lookups preserve from-import and __getattr__ fallbacks. +extern PyObject *_PyLazyImport_ReifyOptional( + PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *ns); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 5026c9670d81d2..a6c560bd105428 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -451,6 +451,801 @@ def test_lazy_with_from(self): class PackageTests(LazyImportTestCase): """Tests for lazy imports with packages.""" + @support.requires_subprocess() + def test_lazy_import_sibling_submodules(self): + for package, siblings in ( + ("test.test_lazy_import.data.pkg", ("b", "bar")), + ("test.test_lazy_import.data.metasyntactic.foo", ("ack", "bar")), + ): + for first, second in (siblings, siblings[::-1]): + with self.subTest(package=package, first=first): + code = textwrap.dedent(f""" + import sys + lazy import {package}.{first} + lazy import {package}.{second} + + def get_first(): + return {package}.{first} + + for _ in range(30): + assert get_first() is sys.modules['{package}.{first}'] + assert '{package}.{second}' not in sys.modules + assert {package}.{second} is sys.modules['{package}.{second}'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_siblings_through_module_attribute(self): + code = textwrap.dedent(""" + import sys + import types + namespace = types.ModuleType('lazy_siblings') + exec('lazy import xml.dom\\nlazy import xml.sax', namespace.__dict__) + assert namespace.xml.dom is sys.modules['xml.dom'] + assert 'xml.sax' not in sys.modules + assert namespace.xml.sax is sys.modules['xml.sax'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_blocked_sibling_stays_deferred(self): + code = textwrap.dedent(""" + import sys + sys.modules['xml.sax'] = None + lazy import xml.dom + lazy import xml.sax + + assert xml.dom is sys.modules['xml.dom'] + assert sys.modules['xml.sax'] is None + for _ in range(2): + try: + xml.sax + except ModuleNotFoundError as exc: + assert exc.name == 'xml.sax' + assert 'None in sys.modules' in str(exc), exc + else: + raise AssertionError('blocked sibling imported') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_parent_without_submodules(self): + code = textwrap.dedent(""" + import sys + lazy import test.test_lazy_import.data.pkg.b + lazy import test.test_lazy_import.data.pkg.bar + + assert test.__name__ == 'test' + assert 'test.test_lazy_import.data.pkg' not in sys.modules + assert test.test_lazy_import.data.pkg.__name__ == 'test.test_lazy_import.data.pkg' + assert 'test.test_lazy_import.data.pkg.b' not in sys.modules + assert 'test.test_lazy_import.data.pkg.bar' not in sys.modules + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_descendant_after_eager_parent(self): + code = textwrap.dedent(""" + import importlib + import sys + lazy import xml.dom.minidom + importlib.import_module('xml.dom') + + assert xml is sys.modules['xml'] + assert 'xml.dom.minidom' not in sys.modules + assert xml.dom.minidom is sys.modules['xml.dom.minidom'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_missing_nested_submodule(self): + package = 'test.test_lazy_import.data.metasyntactic.foo' + for preload in (False, True): + with self.subTest(preload=preload): + code = f"import {package}\n" if preload else "" + code += textwrap.dedent(f""" + lazy import {package}.nonexistent + + for _ in range(2): + try: + {package}.nonexistent + except ModuleNotFoundError as exc: + assert exc.name == '{package}.nonexistent' + assert exc.__cause__ is not None + else: + raise AssertionError('nonexistent module imported') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_sibling_error_declaration(self): + package = 'test.test_lazy_import.data.pkg' + for first, second in (('nonexistent', 'b'), ('b', 'nonexistent')): + with self.subTest(first=first): + declarations = ( + f'lazy import {package}.{first}\n' + f'lazy import {package}.{second}\n' + ) + code = textwrap.dedent(f""" + import traceback + exec(compile({declarations!r}, 'lazy_declarations.py', 'exec')) + try: + {package}.nonexistent + except ModuleNotFoundError as exc: + assert exc.name == '{package}.nonexistent' + assert exc.__cause__ is not None + origin = traceback.extract_tb(exc.__cause__.__traceback__)[-1] + assert origin.filename == 'lazy_declarations.py', origin + assert origin.lineno == {1 if first == 'nonexistent' else 2}, origin + else: + raise AssertionError('nonexistent module imported') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_preserves_assigned_package_attribute(self): + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg as package + package.b = 42 + lazy import test.test_lazy_import.data.pkg.b + + assert test.test_lazy_import.data.pkg.b == 42 + assert 'test.test_lazy_import.data.pkg.b' not in sys.modules + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_children_use_current_import_hook(self): + code = textwrap.dedent(""" + import builtins + import types + lazy import xml.dom.minidom + assert xml.__name__ == 'xml' + + child = types.ModuleType('xml.dom') + child.__path__ = [] + value = object() + calls = [] + original = builtins.__import__ + + def hook(name, *args): + calls.append(name) + if name == 'xml.dom': + xml.dom = child + elif name == 'xml.dom.minidom': + child.minidom = value + else: + raise AssertionError(name) + return xml + + builtins.__import__ = hook + try: + assert xml.dom.minidom is value + finally: + builtins.__import__ = original + assert calls == ['xml.dom', 'xml.dom.minidom'], calls + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_children_use_declaring_namespace_builtins(self): + code = textwrap.dedent(""" + import builtins + import types + import xml + + first = types.ModuleType('first') + second = types.ModuleType('second') + first.__dict__['__builtins__'] = builtins.__dict__.copy() + second.__dict__['__builtins__'] = builtins.__dict__.copy() + exec('lazy import xml.dom', first.__dict__) + exec('lazy import xml.sax', second.__dict__) + assert second.xml is xml + + def denied(*args): + raise AssertionError('used another namespace import hook') + + second.__dict__['__builtins__']['__import__'] = denied + assert first.xml.dom.__name__ == 'xml.dom' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_same_child_uses_declaring_namespace_builtins(self): + code = textwrap.dedent(""" + import builtins + import types + + first = types.ModuleType('first') + second = types.ModuleType('second') + first.__dict__['__builtins__'] = builtins.__dict__.copy() + second.__dict__['__builtins__'] = builtins.__dict__.copy() + exec('lazy import xml.dom', first.__dict__) + exec('lazy import xml.dom', second.__dict__) + + def denied(name, *args): + if name == 'xml.dom': + raise AssertionError('used another namespace import hook') + return builtins.__import__(name, *args) + + second.__dict__['__builtins__']['__import__'] = denied + assert first.xml.dom.__name__ == 'xml.dom' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_shared_parent_uses_declaring_namespace_builtins(self): + for resolve_first in (False, True): + with self.subTest(resolve_first=resolve_first): + code = textwrap.dedent(f""" + import builtins + import types + + first = types.ModuleType('first') + second = types.ModuleType('second') + first.__dict__['__builtins__'] = builtins.__dict__.copy() + second.__dict__['__builtins__'] = builtins.__dict__.copy() + exec('lazy import xml.dom.minidom', first.__dict__) + exec('lazy import xml.dom.pulldom', second.__dict__) + if {resolve_first}: + assert first.xml.__name__ == 'xml' + assert second.xml.__name__ == 'xml' + + def denied(*args): + raise AssertionError('used another namespace import hook') + + second.__dict__['__builtins__']['__import__'] = denied + assert first.xml.dom.minidom.__name__ == 'xml.dom.minidom' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_resolved_lazy_import_releases_namespace_builtins(self): + for mode in ('lazy', 'cached', 'eager'): + with self.subTest(mode=mode): + code = textwrap.dedent(f""" + import builtins + import gc + import importlib + import weakref + + class Payload: + pass + + payload = Payload() + payload_ref = weakref.ref(payload) + namespace_builtins = dict(vars(builtins), payload=payload) + namespace = {{'__builtins__': namespace_builtins}} + if {mode == 'cached'}: + importlib.import_module('xml.dom') + exec('lazy import xml.dom', namespace) + if {mode == 'eager'}: + importlib.import_module('xml.dom') + else: + exec('xml.dom', namespace) + + namespace.clear() + del namespace_builtins, payload + gc.collect() + assert payload_ref() is None + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_with_spec_initializing_error(self): + for eager in (False, True): + with self.subTest(eager=eager): + code = textwrap.dedent(f""" + import importlib + import sys + import warnings + from importlib.machinery import ModuleSpec + + class Spec: + @property + def _initializing(self): + raise RuntimeError('initializing lookup failed') + + class Loader: + def find_spec(self, name, path=None, target=None): + if name == 'xml.child': + return ModuleSpec(name, self) + + def create_module(self, spec): + return None + + def exec_module(self, module): + module.__spec__ = Spec() + + sys.meta_path.insert(0, Loader()) + warnings.simplefilter('error', ImportWarning) + lazy import xml.child + if {eager}: + child = importlib.import_module('xml.child') + else: + child = xml.child + assert child is sys.modules['xml.child'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_invalid_lazy_import_preserves_plain_import(self): + for separate_namespaces in (False, True): + with self.subTest(separate_namespaces=separate_namespaces): + if separate_namespaces: + code = textwrap.dedent(""" + import types + first = types.ModuleType('first') + second = types.ModuleType('second') + exec('lazy import math.pi', first.__dict__) + exec('lazy import math', second.__dict__) + assert second.math.sin(0) == 0 + """) + else: + code = textwrap.dedent(""" + lazy import math.pi + lazy import math + assert math.sin(0) == 0 + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_nested_nonpackage(self): + module = 'test.test_lazy_import.data.pkg.b' + for name in ('foo', 'nonexistent'): + with self.subTest(name=name): + code = textwrap.dedent(f""" + lazy import {module}.{name} + + for _ in range(2): + try: + {module}.{name} + except ModuleNotFoundError as exc: + assert exc.name == '{module}.{name}', exc.name + assert "'{module}' is not a package" in str(exc), exc + else: + raise AssertionError('imported from a nonpackage') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_nested_nonpackage_subclass(self): + code = textwrap.dedent(""" + import types + import xml.dom.minidom as leaf + + class Module(types.ModuleType): + pass + + leaf.__class__ = Module + lazy import xml.dom.minidom.Node + try: + xml.dom.minidom.Node + except ModuleNotFoundError as exc: + assert exc.name == 'xml.dom.minidom.Node', exc.name + else: + raise AssertionError('imported from a nonpackage') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_loaded_nonpackage_subclass(self): + code = textwrap.dedent(""" + import sys + import types + import xml + from importlib.machinery import ModuleSpec + + class Module(types.ModuleType): + pass + + class Loader: + def find_spec(self, name, path=None, target=None): + if name == 'xml.child': + return ModuleSpec(name, self) + + def create_module(self, spec): + return Module(spec.name) + + def exec_module(self, module): + module.value = 42 + + sys.meta_path.insert(0, Loader()) + lazy import xml.child.value + try: + xml.child.value + except ModuleNotFoundError as exc: + assert exc.name == 'xml.child.value', exc.name + else: + raise AssertionError('imported from a nonpackage') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_eager_import_after_invalid_lazy_import(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.pkg.b.foo + try: + test.test_lazy_import.data.pkg.b.foo + except ModuleNotFoundError: + pass + else: + raise AssertionError('imported from a nonpackage') + + import test.test_lazy_import.data.pkg.b + assert callable(test.test_lazy_import.data.pkg.b.foo) + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_eager_import_replaces_lazy_binding_to_cached_child(self): + code = textwrap.dedent(""" + import xml.dom.minidom as child + import xml.dom as parent + del parent.minidom + + lazy import xml.dom.minidom.Node + assert xml.__name__ == 'xml' + import xml.dom.minidom + assert xml.dom.minidom is child + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_invalid_lazy_import_preserves_sibling_from_import(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.pkg.b.foo + lazy import test.test_lazy_import.data.pkg.bar + test.test_lazy_import.data.pkg.bar + + from test.test_lazy_import.data.pkg import b + assert b.foo() == 'foo' + assert test.test_lazy_import.data.pkg.b.foo() == 'foo' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_submodule_ignores_package_getattr(self): + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg as package + original = package.__getattr__ + + def hook(name): + if name == 'b': + return 42 + return original(name) + + package.__getattr__ = hook + lazy import test.test_lazy_import.data.pkg.b + assert test.test_lazy_import.data.pkg.b is sys.modules['test.test_lazy_import.data.pkg.b'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_missing_submodule_ignores_package_getattr(self): + package = 'test.test_lazy_import.data.pkg' + for name in ('dynamic_attr', 'raising_attr'): + with self.subTest(name=name): + code = textwrap.dedent(f""" + lazy import {package}.{name} + + for _ in range(2): + try: + {package}.{name} + except ModuleNotFoundError as exc: + assert exc.name == '{package}.{name}', exc.name + else: + raise AssertionError('nonexistent module imported') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_lazy_import_preserves_dynamic_from_import(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.pkg.dynamic_attr + from test.test_lazy_import.data.pkg import dynamic_attr + assert dynamic_attr == 'from_getattr' + + lazy from test.test_lazy_import.data.pkg import dynamic_attr as other + assert other == 'from_getattr' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_lazy_import_preserves_wrapped_import_hook(self): + code = textwrap.dedent(""" + import builtins + original = builtins.__import__ + builtins.__import__ = lambda *args, **kwargs: original(*args, **kwargs) + + lazy import test.test_lazy_import.data.pkg.dynamic_attr + lazy import test.test_lazy_import.data.pkg.bar + test.test_lazy_import.data.pkg.bar + from test.test_lazy_import.data.pkg import dynamic_attr + assert dynamic_attr == 'from_getattr' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_lazy_import_with_removed_import_hook(self): + code = textwrap.dedent(""" + import builtins + import types + namespace = types.ModuleType('namespace') + namespace.__dict__['__builtins__'] = builtins.__dict__.copy() + exec('lazy import test.test_lazy_import.data.pkg.dynamic_attr\\n' + 'lazy import test.test_lazy_import.data.pkg.bar', namespace.__dict__) + namespace.test.test_lazy_import.data.pkg.bar + del namespace.__dict__['__builtins__']['__import__'] + + from test.test_lazy_import.data.pkg import dynamic_attr + assert dynamic_attr == 'from_getattr' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_lazy_import_preserves_dynamic_attribute(self): + package = 'test.test_lazy_import.data.pkg' + for name in ('dynamic_attr', 'b'): + with self.subTest(name=name): + code = textwrap.dedent(f""" + import sys + lazy import {package}.{name} + lazy import {package}.bar + {package}.bar + package = sys.modules['{package}'] + original = package.__getattr__ + + def hook(attr): + if attr == 'b': + return 42 + return original(attr) + + package.__getattr__ = hook + value = package.{name} + if {name == 'b'}: + assert value is sys.modules['{package}.b'] + else: + assert value == 'from_getattr', value + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_published_child_preserves_dynamic_from_import(self): + for prefix in ('', 'lazy '): + with self.subTest(prefix=prefix): + code = textwrap.dedent(f""" + import xml as package + package.__getattr__ = lambda name: 42 + lazy import xml.missing + lazy import xml + assert xml is package + + {prefix}from xml import missing + assert missing == 42 + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_published_child_preserves_loader_error(self): + for prefix in ('', 'lazy '): + with self.subTest(prefix=prefix): + code = textwrap.dedent(f""" + import sys + import xml as package + from importlib.machinery import ModuleSpec + + failure = ModuleNotFoundError('loader failed', name='xml.broken') + package.__getattr__ = lambda name: 42 + + class Loader: + calls = 0 + + def create_module(self, spec): + return None + + def exec_module(self, module): + self.calls += 1 + raise failure + + def find_spec(self, name, path=None, target=None): + if name == 'xml.broken': + return ModuleSpec(name, self) + + loader = Loader() + sys.meta_path.insert(0, loader) + lazy import xml.broken + lazy import xml + assert xml is package + + try: + exec('{prefix}from xml import broken') + broken + except ModuleNotFoundError as exc: + assert exc is failure, exc + else: + raise AssertionError('suppressed loader exception') + assert loader.calls == 1, loader.calls + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_aliased_package(self): + code = textwrap.dedent(""" + import sys + import xml + sys.modules['alias_xml'] = xml + lazy import alias_xml.dom + assert alias_xml is xml + assert alias_xml.dom is sys.modules['alias_xml.dom'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_aliased_package_siblings(self): + code = textwrap.dedent(""" + import sys + import xml + sys.modules['alias_xml'] = xml + lazy import alias_xml.dom + lazy import alias_xml.sax + + assert alias_xml is xml + assert 'alias_xml.dom' not in sys.modules + assert 'alias_xml.sax' not in sys.modules + assert alias_xml.dom is sys.modules['alias_xml.dom'] + assert 'alias_xml.sax' not in sys.modules + assert alias_xml.sax is sys.modules['alias_xml.sax'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_cached_intermediate_alias(self): + code = textwrap.dedent(""" + import sys + import xml + import email + xml.alias = email + sys.modules['xml.alias'] = email + + lazy import xml.alias.message + assert xml.alias.message is sys.modules['xml.alias.message'] + assert xml.alias is email + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_available_partial_submodule(self): + code = textwrap.dedent(""" + import sys + from importlib.machinery import ModuleSpec + + sources = { + 'lazy_cycle': '', + 'lazy_cycle.a': 'value = 42\\nimport lazy_cycle.b\\n', + 'lazy_cycle.b': 'import lazy_cycle.a\\nassert lazy_cycle.a.value == 42\\n', + } + + class Loader: + def create_module(self, spec): + return None + + def exec_module(self, module): + exec(sources[module.__name__], module.__dict__) + + def find_spec(self, name, path=None, target=None): + if name in sources: + return ModuleSpec(name, self, is_package=name == 'lazy_cycle') + + sys.meta_path.insert(0, Loader()) + lazy import lazy_cycle.a + assert lazy_cycle.a.value == 42 + assert lazy_cycle.b is sys.modules['lazy_cycle.b'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_retries_after_self_import_during_initialization(self): + for reentry in ('__import__(module.__name__)', 'parent.broken'): + with self.subTest(reentry=reentry): + code = textwrap.dedent(f""" + import sys + import xml as parent + from importlib.machinery import ModuleSpec + + class Loader: + calls = 0 + + def find_spec(self, name, path=None, target=None): + if name == 'xml.broken': + return ModuleSpec(name, self) + + def create_module(self, spec): + return None + + def exec_module(self, module): + self.calls += 1 + {reentry} + raise RuntimeError('failed initialization') + + loader = Loader() + sys.meta_path.insert(0, loader) + lazy import xml.broken + for _ in range(2): + try: + xml.broken + except RuntimeError as exc: + assert str(exc) == 'failed initialization', exc + else: + raise AssertionError('returned failed partial module') + assert loader.calls == 2, loader.calls + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_retries_failed_parent_after_child_loaded(self): + for import_parent in (False, True): + with self.subTest(import_parent=import_parent): + code = textwrap.dedent(f""" + import importlib + import sys + from importlib.machinery import ModuleSpec + + sources = {{ + 'lazy_retry': '', + 'lazy_retry.a': 'import lazy_retry.a.b\\nvalue = 42\\n', + 'lazy_retry.a.b': '', + }} + + class Loader: + attempts = 0 + + def create_module(self, spec): + return None + + def exec_module(self, module): + exec(sources[module.__name__], module.__dict__) + if module.__name__ == 'lazy_retry.a': + self.attempts += 1 + if self.attempts == 1: + raise RuntimeError('first attempt') + + def find_spec(self, name, path=None, target=None): + if name in sources: + return ModuleSpec(name, self, is_package=name != 'lazy_retry.a.b') + + loader = Loader() + sys.meta_path.insert(0, loader) + lazy import lazy_retry.a.b + if {import_parent}: + lazy import lazy_retry.a + try: + importlib.import_module('lazy_retry.a') + except RuntimeError as exc: + assert str(exc) == 'first attempt', exc + else: + raise AssertionError('first import succeeded') + + assert 'lazy_retry.a' not in sys.modules + assert 'lazy_retry.a.b' in sys.modules + assert lazy_retry.a.b is sys.modules['lazy_retry.a.b'] + assert lazy_retry.a.value == 42 + assert loader.attempts == 2, loader.attempts + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_recursive_package_attribute(self): + code = textwrap.dedent(""" + import xml + xml.alias = xml + lazy import xml.alias.nonexistent + assert xml.alias is xml + """) + assert_python_ok("-c", code) + def test_lazy_import_pkg(self): """lazy import of package submodule should load the package.""" out = io.StringIO() @@ -624,6 +1419,22 @@ def test_dunder_lazy_import_builtins(self): self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) self.assertEqual(dunder_lazy_import_builtins.basic.basic2, 42) + @support.requires_subprocess() + def test_dunder_lazy_import_with_lazy_builtins(self): + code = textwrap.dedent(""" + placeholders = [__lazy_import__('sys')] + for _ in range(2): + placeholders.append(__lazy_import__( + 'sys', {'__builtins__': placeholders[-1]})) + try: + placeholders[-1].resolve() + except TypeError as exc: + assert 'not subscriptable' in str(exc), exc + else: + raise AssertionError('resolved with invalid builtins') + """) + assert_python_ok("-c", code) + class SysLazyImportsAPITests(LazyImportTestCase): """Tests for sys lazy imports API functions.""" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-15-00-00.gh-issue-158140.v5Aj2R.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-15-00-00.gh-issue-158140.v5Aj2R.rst new file mode 100644 index 00000000000000..3b2ad941d9031b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-15-00-00.gh-issue-158140.v5Aj2R.rst @@ -0,0 +1,4 @@ +Fix dotted lazy imports sharing a package name so that accessing one submodule +does not import the last declared sibling first. Resolving the package leaves +its pending submodules lazy, while nonexistent submodules still raise +:exc:`ModuleNotFoundError` when accessed. diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index 72624b746fe64c..b9223c8cd047bb 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -9,13 +9,16 @@ #include "pycore_lazyimportobject.h" #include "pycore_long.h" #include "pycore_moduleobject.h" +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pyerrors.h" #include "pycore_traceback.h" #include "pycore_tstate.h" typedef struct { PyObject_HEAD - PyObject *lz_builtins; // Roots own the mapping; projections retain the root. + // Roots retain the builtins mapping, or the original declaration for a + // published child. Projections leave this NULL and retain their source. + PyObject *lz_builtins; // A root stores its absolute name and original fromlist. A projection // stores its source placeholder and the attribute to import from it. PyObject *lz_from; @@ -23,6 +26,9 @@ typedef struct { // Declaration location. PyCodeObject *lz_code; int lz_instr_offset; + bool lz_submodule; + // Validate descendants only along this declaration's own path. + bool lz_validate; } PyLazyImportObject; #define PyLazyImportObject_CAST(op) ((PyLazyImportObject *)(op)) @@ -59,6 +65,8 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, m->lz_code = NULL; m->lz_instr_offset = -1; + m->lz_submodule = false; + m->lz_validate = true; if (frame != NULL) { m->lz_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); @@ -196,8 +204,9 @@ lazy_import_path(PyLazyImportObject *m) Py_DECREF(base); return res; } - if (m->lz_attr != NULL && - (!PyTuple_Check(m->lz_attr) || PyTuple_GET_SIZE(m->lz_attr) > 0)) { + if (m->lz_submodule || + (m->lz_attr != NULL && + (!PyTuple_Check(m->lz_attr) || PyTuple_GET_SIZE(m->lz_attr) > 0))) { return Py_NewRef(m->lz_from); } // __import__("a.b") returns the top-level package `a`. @@ -259,7 +268,29 @@ lazy_import_replay_from(PyThreadState *tstate, PyObject *mod, PyLazyImportObject *lz) { if (!PyLazyImport_CheckExact(lz->lz_from)) { - return Py_NewRef(mod); + PyObject *result = Py_NewRef(mod); + if (lz->lz_submodule) { + // A published child behaves like `import pkg.child as child`. + Py_ssize_t end = PyUnicode_GET_LENGTH(lz->lz_from); + Py_ssize_t dot = PyUnicode_FindChar(lz->lz_from, '.', 0, end, 1); + while (result != NULL && dot >= 0) { + Py_ssize_t start = dot + 1; + dot = PyUnicode_FindChar(lz->lz_from, '.', start, end, 1); + if (dot == -2) { + break; + } + PyObject *attr = PyUnicode_Substring( + lz->lz_from, start, dot < 0 ? end : dot); + PyObject *next = attr == NULL ? NULL : + _PyEval_ImportFrom(tstate, result, attr); + Py_XDECREF(attr); + Py_SETREF(result, lazy_import_resolve_result(tstate, next)); + } + if (dot == -2) { + Py_CLEAR(result); + } + } + return result; } PyObject *from = lazy_import_replay_from( tstate, mod, (PyLazyImportObject *)lz->lz_from); @@ -340,14 +371,181 @@ _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op) return active; } +// Published submodules must use import semantics, including validation of +// the full name, rather than a module's dynamic attribute fallback. +int +_PyLazyImport_IsSubmodule(PyObject *op) +{ + return ((PyLazyImportObject *)op)->lz_submodule; +} + +int +_PyLazyImport_IsStrictSubmodule(PyObject *op) +{ + PyLazyImportObject *lz = (PyLazyImportObject *)op; + return lz->lz_submodule && lz->lz_validate; +} + +static int +lazy_import_is_child(PyObject *name, PyObject *parent) +{ + Py_ssize_t len = PyUnicode_GET_LENGTH(parent); + if (PyUnicode_GET_LENGTH(name) <= len || + PyUnicode_READ_CHAR(name, len) != '.') { + return 0; + } + return PyUnicode_Tailmatch(name, parent, 0, len, -1); +} + +// Retain the declaration's hook and location without retaining other imports. +static PyObject * +lazy_import_submodule(PyLazyImportObject *source, PyObject *fullname, + bool validate) +{ + assert(!source->lz_submodule); + PyObject *result = _PyLazyImport_New(NULL, (PyObject *)source, fullname, NULL); + if (result != NULL) { + FT_ATOMIC_STORE_INT_RELAXED( + PyThreadState_GET()->interp->imports.has_lazy_submodules, 1); + PyLazyImportObject *child = (PyLazyImportObject *)result; + child->lz_submodule = true; + child->lz_validate = validate; + child->lz_code = (PyCodeObject *)Py_XNewRef(source->lz_code); + child->lz_instr_offset = source->lz_instr_offset; + } + return result; +} + +static int +lazy_import_bind_children(PyThreadState *tstate, PyLazyImportObject *origin, + PyObject *module, PyObject *name, bool validate) +{ + if (!PyModule_Check(module)) { + return 0; + } + PyObject *dict = _PyModule_GetDict(module); + int is_package = PyDict_Contains(dict, &_Py_ID(__path__)); + if (is_package <= 0) { + if (is_package < 0 || origin == NULL || !validate) { + return is_package; + } + // Validate only the declaration being resolved. An unused invalid + // declaration must not make unrelated imports of this module fail. + int matches = lazy_import_is_child(origin->lz_from, name); + if (matches <= 0) { + return matches; + } + PyObject *child = lazy_import_submodule(origin, origin->lz_from, true); + PyObject *value = child == NULL ? NULL : + _PyImport_LoadLazyImportTstate(tstate, child); + Py_XDECREF(child); + int res = value == NULL ? -1 : 0; + Py_XDECREF(value); + return res; + } + // Module subclasses retain their attribute and descriptor protocols. + if (!PyModule_CheckExact(module)) { + return 0; + } + if (_Py_EnterRecursiveCallTstate(tstate, " while binding lazy submodules")) { + return -1; + } + PyObject *pending = _PyImport_GetLazySubmodules(tstate, name); + int res = pending == NULL && PyErr_Occurred() ? -1 : 0; + if (pending != NULL) { + Py_ssize_t pos = 0; + PyObject *attr, *source; + while (PyDict_Next(pending, &pos, &attr, &source)) { + if (!PyLazyImport_CheckExact(source)) { + continue; + } + PyObject *fullname = PyUnicode_FromFormat("%U.%U", name, attr); + if (fullname == NULL) { + res = -1; + break; + } + PyLazyImportObject *declaration = (PyLazyImportObject *)source; + int matches = 0; + if (origin != NULL) { + matches = PyObject_RichCompareBool(origin->lz_from, fullname, Py_EQ); + if (matches == 0) { + matches = lazy_import_is_child(origin->lz_from, fullname); + } + if (matches < 0) { + Py_DECREF(fullname); + res = -1; + break; + } + if (matches) { + declaration = origin; + } + } + PyObject *existing; + res = PyDict_GetItemRef(dict, attr, &existing); + PyLazyImportObject *previous = + res > 0 && PyLazyImport_CheckExact(existing) ? + (PyLazyImportObject *)existing : NULL; + if (previous != NULL && !previous->lz_submodule) { + previous = NULL; + } + PyLazyImportObject *previous_source = previous == NULL ? NULL : + (PyLazyImportObject *)previous->lz_builtins; + if (matches && previous_source != NULL && + previous_source->lz_builtins != origin->lz_builtins) { + // A shared package attribute cannot keep two independently + // mutable import hooks. Complete this declaration with its + // own hook before either namespace loses its root binding. + PyObject *child = lazy_import_submodule(origin, origin->lz_from, true); + PyObject *value = child == NULL ? NULL : + _PyImport_LoadLazyImportTstate(tstate, child); + Py_XDECREF(child); + res = value == NULL ? -1 : 0; + Py_XDECREF(value); + } + else if (res >= 0 && + (existing == NULL || + (matches && previous != NULL && + (previous_source != origin || !previous->lz_validate)))) { + PyObject *child = lazy_import_submodule( + declaration, fullname, matches != 0); + if (child == NULL) { + res = -1; + } + else if (existing == NULL) { + res = PyDict_SetDefaultRef(dict, attr, child, NULL); + } + else { + res = _PyDict_ReplaceItemIf(dict, attr, existing, child); + } + Py_XDECREF(child); + } + else if (res > 0) { + // Existing values win. Already imported packages may still + // have pending children of their own. + res = lazy_import_bind_children( + tstate, origin, existing, fullname, validate); + } + Py_XDECREF(existing); + Py_DECREF(fullname); + if (res < 0) { + break; + } + } + Py_DECREF(pending); + } + _Py_LeaveRecursiveCallTstate(tstate); + return res < 0 ? -1 : 0; +} + static PyObject * lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, - PyObject **imported_module) + PyObject **imported_module, bool optional) { PyObject *obj = NULL; PyObject *fromlist = NULL; PyObject *import_func = NULL; PyObject *resolving = NULL; + bool default_import = false; assert(lazy_import != NULL); assert(PyLazyImport_CheckExact(lazy_import)); @@ -361,6 +559,15 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, } if (_PyLazyImport_IsResolving(tstate, lazy_import)) { + if (root->lz_submodule) { + // Eager imports in a cycle can access an already published, + // partially initialized module through its parent. + PyObject *partial = PyImport_GetModule(root->lz_from); + if (partial != Py_None && (partial != NULL || PyErr_Occurred())) { + return partial; + } + Py_XDECREF(partial); + } PyObject *name = lazy_import_name(lz); if (name == NULL) { return NULL; @@ -409,33 +616,128 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, globals = Py_None; } - if (PyMapping_GetOptionalItem(root->lz_builtins, &_Py_ID(__import__), - &import_func) < 0) { - goto done; + PyLazyImportObject *source = root; + if (source->lz_submodule) { + source = (PyLazyImportObject *)source->lz_builtins; } - if (import_func == NULL) { - PyErr_SetString(PyExc_ImportError, "__import__ not found"); - goto done; + assert(!source->lz_submodule); + bool optional_submodule = optional && root->lz_submodule && first == NULL; + if (!optional_submodule || root->lz_validate) { + if (PyMapping_GetOptionalItem(source->lz_builtins, &_Py_ID(__import__), + &import_func) < 0) { + goto done; + } + if (import_func == NULL) { + PyErr_SetString(PyExc_ImportError, "__import__ not found"); + goto done; + } + } + default_import = import_func == NULL || + _PyImport_IsDefaultImportFunc(tstate->interp, import_func); + optional_submodule = optional_submodule && default_import; + if (optional_submodule) { + // Unused siblings follow the pending-child importer. An active + // declaration still uses its current custom import hook. + obj = _PyImport_LoadLazySubmodule(tstate, root->lz_from); + if (obj == Py_None) { + Py_CLEAR(obj); + } + goto imported; + } + PyObject *name = Py_NewRef(root->lz_from); + if (first == NULL && !root->lz_submodule && + (fromlist == Py_None || + (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist) == 0)) && + default_import) { + Py_ssize_t dot = PyUnicode_FindChar( + name, '.', 0, PyUnicode_GET_LENGTH(name), 1); + if (dot == -2) { + Py_CLEAR(name); + } + else if (dot >= 0) { + PyObject *modules = _PyImport_GetModules(tstate->interp); + PyObject *module = NULL; + int cached = modules == NULL ? 0 : + PyMapping_GetOptionalItem(modules, name, &module); + if (cached > 0 && PyModule_Check(module)) { + PyObject *loaded = lazy_import_get_loaded_attr( + tstate, name, &_Py_ID(__name__)); + cached = loaded != NULL; + Py_XDECREF(loaded); + if (PyErr_Occurred()) { + cached = -1; + } + } + else if (module == Py_None) { + cached = 0; + } + Py_XDECREF(module); + if (cached < 0) { + Py_CLEAR(name); + } + else if (!cached) { + Py_SETREF(name, PyUnicode_Substring(name, 0, dot)); + } + } + if (name == NULL) { + goto done; + } } obj = _PyEval_ImportNameWithImport( tstate, import_func, globals, globals, - root->lz_from, fromlist, _PyLong_GetZero() + name, fromlist, _PyLong_GetZero() ); + if (obj != NULL && name != root->lz_from) { + int is_package = PyModule_CheckExact(obj) ? + PyDict_Contains(_PyModule_GetDict(obj), &_Py_ID(__path__)) : 0; + if (is_package < 0) { + Py_CLEAR(obj); + } + else if (!is_package) { + Py_DECREF(obj); + obj = _PyEval_ImportNameWithImport( + tstate, import_func, globals, globals, + root->lz_from, fromlist, _PyLong_GetZero()); + } + } + Py_DECREF(name); +imported: obj = lazy_import_resolve_result(tstate, obj); // The normal importer may publish this module on its parent. Custom // hooks retain control of their own assignments to the parent. if (imported_module != NULL && obj != NULL && PyModule_Check(obj) && - _PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { + default_import) { *imported_module = Py_NewRef(obj); } - if (obj != NULL && first != NULL) { + if (obj != NULL && + (first != NULL || (root->lz_submodule && !optional_submodule))) { // Keep the hook and root result alive until all attribute lookups finish. PyObject *from = obj; obj = lazy_import_replay_from(tstate, from, lz); Py_DECREF(from); } + if (obj != NULL && imported_module != NULL && + root->lz_submodule && + default_import) { + Py_XSETREF(*imported_module, Py_NewRef(obj)); + } + if (obj != NULL && PyModule_Check(obj)) { + PyObject *path = lazy_import_path(lz); + PyLazyImportObject *origin = + root->lz_validate && !optional_submodule ? source : NULL; + if (path == NULL || lazy_import_bind_children( + tstate, origin, obj, path, default_import) < 0) { + Py_CLEAR(obj); + } + Py_XDECREF(path); + } done: + if (obj != NULL && + default_import && + _PyImport_ClearLazySubmodules(tstate, root->lz_from) < 0) { + Py_CLEAR(obj); + } if (obj == NULL) { lazy_import_add_exception_cause(tstate, lz); } @@ -461,7 +763,7 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, PyObject * _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) { - return lazy_import_resolve_impl(tstate, lazy_import, NULL); + return lazy_import_resolve_impl(tstate, lazy_import, NULL, false); } // Loading pkg.child can replace a placeholder in pkg.child with the module @@ -510,17 +812,32 @@ lazy_import_replace_child(PyThreadState *tstate, PyObject *placeholder, return rc; } -PyObject * -_PyLazyImport_Reify(PyThreadState *tstate, PyObject *placeholder, - PyObject *name, PyObject *namespace) +static PyObject * +lazy_import_reify(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace, bool optional) { + int recursive = _PyLazyImport_IsResolving(tstate, placeholder); PyObject *imported_module = NULL; PyObject *value = lazy_import_resolve_impl( - tstate, placeholder, &imported_module); + tstate, placeholder, &imported_module, optional); if (value == NULL) { + if (imported_module != NULL && PyDict_CheckExact(namespace) && + _PyLazyImport_IsSubmodule(placeholder)) { + // Importing the parent can succeed before a required descendant + // fails. Retain the placeholder so the same lookup can retry. + PyObject *exc = _PyErr_GetRaisedException(tstate); + (void)_PyDict_ReplaceItemIf( + namespace, name, imported_module, placeholder); + _PyErr_SetRaisedException(tstate, exc); + } Py_XDECREF(imported_module); return NULL; } + // A cycle can inspect a partial module that may still fail to initialize. + if (recursive) { + Py_XDECREF(imported_module); + return value; + } int rc; if (PyDict_CheckExact(namespace)) { rc = _PyDict_ReplaceItemIf(namespace, name, placeholder, value); @@ -554,6 +871,20 @@ _PyLazyImport_Reify(PyThreadState *tstate, PyObject *placeholder, return value; } +PyObject * +_PyLazyImport_Reify(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace) +{ + return lazy_import_reify(tstate, placeholder, name, namespace, false); +} + +PyObject * +_PyLazyImport_ReifyOptional(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace) +{ + return lazy_import_reify(tstate, placeholder, name, namespace, true); +} + static PyObject * lazy_import_resolve(PyObject *self, PyObject *args) { diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 57df195b7f3201..55d8daabc4b29a 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1321,6 +1321,7 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { // When suppress=1, this function suppresses AttributeError. PyObject *attr, *mod_name, *getattr; + PyObject *missing_import = NULL; attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); if (attr == NULL) { if (suppress == 1) { @@ -1365,7 +1366,8 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) // the attribute before resolving a lazy import placeholder. // Resolving __getattr__ itself must not invoke the hook. getattr = NULL; - if (!_PyUnicode_EqualToASCIIString(name, "__getattr__") && + if (!_PyLazyImport_IsSubmodule(attr) && + !_PyUnicode_EqualToASCIIString(name, "__getattr__") && module_get_resolved_dict_item( m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { Py_DECREF(attr); @@ -1390,24 +1392,50 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) Py_DECREF(attr); return NULL; } - PyObject *new_value = _PyLazyImport_Reify( - tstate, attr, name, m->md_dict); + PyObject *new_value = + (suppress || !_PyLazyImport_IsStrictSubmodule(attr)) ? + _PyLazyImport_ReifyOptional(tstate, attr, name, m->md_dict) : + _PyLazyImport_Reify(tstate, attr, name, m->md_dict); + if (new_value == NULL && !PyErr_Occurred()) { + if (suppress) { + Py_DECREF(attr); + } + else { + missing_import = attr; + } + goto missing; + } Py_DECREF(attr); return new_value; } return attr; } +missing: if (module_get_resolved_dict_item( m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + Py_XDECREF(missing_import); return NULL; } if (getattr) { PyObject *result = PyObject_CallOneArg(getattr, name); - if (result == NULL && suppress == 1 && PyErr_ExceptionMatches(PyExc_AttributeError)) { - // suppress AttributeError + Py_DECREF(getattr); + if (result != NULL || !PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_XDECREF(missing_import); + return result; + } + if (suppress || missing_import != NULL) { PyErr_Clear(); } - Py_DECREF(getattr); + if (missing_import == NULL) { + return NULL; + } + } + if (missing_import != NULL) { + // A pending sibling permits dynamic attributes, but an actually + // missing module must still report its import name and declaration. + PyObject *result = _PyLazyImport_Reify( + PyThreadState_GET(), missing_import, name, m->md_dict); + Py_DECREF(missing_import); return result; } diff --git a/Python/import.c b/Python/import.c index 576b8d47639fa4..6dfa267b3fde6d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -297,6 +297,7 @@ get_importtime_name(PyObject *name) return encoded; } +// Return whether the module was still initializing when found in the cache. static int import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *name) { @@ -342,7 +343,7 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n #undef import_level } - return 0; + return rc; } static void remove_importlib_frames(PyThreadState *tstate); @@ -3972,6 +3973,43 @@ _PyImport_GetAbsName(PyThreadState *tstate, PyObject *name, PyObject *globals, } +// A cached eager import must also replace a published lazy binding. In +// particular, an earlier failed descendant must not keep hiding this module. +static int +import_bind_cached_submodule(PyThreadState *tstate, PyObject *name, PyObject *mod) +{ + PyObject *pending = LAZY_PENDING_SUBMODULES(tstate->interp); + if (pending == NULL || PyDict_GET_SIZE(pending) == 0) { + return 0; + } + Py_ssize_t end = PyUnicode_GET_LENGTH(name); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, -1); + if (dot < 0) { + return dot == -2 ? -1 : 0; + } + PyObject *parent_name = PyUnicode_Substring(name, 0, dot); + if (parent_name == NULL) { + return -1; + } + PyObject *parent = import_get_module(tstate, parent_name); + Py_DECREF(parent_name); + int res = 0; + if (parent != NULL && PyModule_CheckExact(parent)) { + PyObject *attr = PyUnicode_Substring(name, dot + 1, end); + PyObject *dict = _PyModule_GetDict(parent), *existing = NULL; + res = attr == NULL ? -1 : PyDict_GetItemRef(dict, attr, &existing); + if (res > 0 && PyLazyImport_CheckExact(existing) && + _PyLazyImport_IsSubmodule(existing)) { + res = _PyDict_ReplaceItemIf(dict, attr, existing, mod); + } + Py_XDECREF(existing); + Py_XDECREF(attr); + } + Py_XDECREF(parent); + return PyErr_Occurred() ? -1 : 0; +} + + PyObject * PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, @@ -4013,7 +4051,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } if (mod != NULL && mod != Py_None) { - if (import_ensure_initialized(tstate->interp, mod, abs_name) < 0) { + int initializing = import_ensure_initialized(tstate->interp, mod, abs_name); + if (initializing < 0) { goto error; } /* Verify the module is still in sys.modules. Another thread may have @@ -4036,6 +4075,11 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } else { Py_DECREF(mod_check); + if (!initializing && + FT_ATOMIC_LOAD_INT_RELAXED(interp->imports.has_lazy_submodules) && + import_bind_cached_submodule(tstate, abs_name, mod) < 0) { + goto error; + } } } else { @@ -4128,7 +4172,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } -// Ensure a set of pending submodule names exists for the parent. +// Ensure a dict of pending submodule names exists for the parent. // Returns a new reference. static PyObject * ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) @@ -4139,7 +4183,7 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) &lazy_submodules); if (err == 0) { // value isn't present - lazy_submodules = PySet_New(NULL); + lazy_submodules = PyDict_New(); if (lazy_submodules != NULL && _PyDict_SetItem_LockHeld(lazy_modules, parent, lazy_submodules) < 0) { @@ -4155,7 +4199,8 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) // is accessed, _Py_module_getattro_impl will check lazy_pending_submodules // and trigger the import. static int -register_lazy_on_parent(PyThreadState *tstate, PyObject *name) +register_lazy_on_parent(PyThreadState *tstate, PyObject *name, + PyObject *source, int required) { PyDictObject *pending = (PyDictObject *)LAZY_PENDING_SUBMODULES(tstate->interp); @@ -4178,19 +4223,24 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name) PyObject *submodules = ensure_lazy_pending_submodules(pending, parent); Py_DECREF(parent); - int err = submodules == NULL ? -1 : PySet_Add(submodules, child); + int err = -1; + if (submodules != NULL) { + err = required ? PyDict_SetItem(submodules, child, source) + : PyDict_SetDefaultRef(submodules, child, Py_False, NULL); + } Py_DECREF(child); Py_XDECREF(submodules); if (err < 0) { return -1; } end = dot; + required = 1; } } static int register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, - PyObject *from) + PyObject *from, PyObject *source) { PyObject *fromname = PyUnicode_FromFormat("%U.%U", abs_name, from); if (fromname == NULL) { @@ -4201,12 +4251,130 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, PyObject *lazy_modules = LAZY_MODULES(tstate->interp); int res = PySet_Add(lazy_modules, fromname); if (res == 0) { - res = register_lazy_on_parent(tstate, fromname); + res = register_lazy_on_parent(tstate, fromname, source, 0); } Py_DECREF(fromname); return res; } +// Return a private snapshot: publishing children can run finalizers. +// A missing registration returns NULL without an exception. +PyObject * +_PyImport_GetLazySubmodules(PyThreadState *tstate, PyObject *name) +{ + PyObject *pending = LAZY_PENDING_SUBMODULES(tstate->interp); + PyObject *children; + if (pending == NULL || PyDict_GetItemRef(pending, name, &children) <= 0) { + return NULL; + } + PyObject *result = PyDict_Copy(children); + Py_DECREF(children); + return result; +} + +// Release declarations on completed paths from the leaves upward. Keep the +// names for cached children whose parents can be replaced or fail to initialize. +int +_PyImport_ClearLazySubmodules(PyThreadState *tstate, PyObject *name) +{ + PyObject *pending = LAZY_PENDING_SUBMODULES(tstate->interp); + if (pending == NULL || PyDict_GET_SIZE(pending) == 0) { + return 0; + } + name = Py_NewRef(name); + int res = 0; + while (true) { + Py_ssize_t end = PyUnicode_GET_LENGTH(name); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, -1); + if (dot < 0) { + res = dot == -2 ? -1 : 0; + break; + } + PyObject *parent = NULL, *child = NULL, *siblings = NULL; + PyObject *source = NULL, *children = NULL, *spec = NULL; + PyObject *mod = NULL, *current = NULL; + res = 0; + parent = PyUnicode_Substring(name, 0, dot); + child = PyUnicode_Substring(name, dot + 1, end); + if (parent == NULL || child == NULL || + PyDict_GetItemRef(pending, parent, &siblings) <= 0 || + PyDict_GetItemRef(siblings, child, &source) <= 0 || + !PyLazyImport_CheckExact(source)) { + goto next; + } + mod = import_get_module(tstate, name); + if (mod == NULL || mod == Py_None) { + goto next; + } + int rc = PyObject_GetOptionalAttr(mod, &_Py_ID(__spec__), &spec); + if (rc < 0 || (rc > 0 && _PyModuleSpec_IsInitializing(spec) != 0)) { + // Cleanup must not turn successful imports into failures merely + // because a module supplies unusual spec metadata. + if (PyErr_ExceptionMatches(PyExc_Exception)) { + PyErr_Clear(); + } + goto next; + } + current = import_get_module(tstate, name); + if (current != mod) { + goto next; + } + // Registration writes descendants before ancestors. Capture the edge + // first, so a concurrent declaration either changes its identity or + // restores it after our conditional replacement. + children = _PyImport_GetLazySubmodules(tstate, name); + if (children != NULL) { + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(children, &pos, &key, &value)) { + if (PyLazyImport_CheckExact(value)) { + goto next; + } + } + } + if (!PyErr_Occurred()) { + res = _PyDict_ReplaceItemIf(siblings, child, source, Py_True); + if (res > 0) { + Py_SETREF(name, Py_NewRef(parent)); + } + } + next: + Py_XDECREF(current); + Py_XDECREF(mod); + Py_XDECREF(spec); + Py_XDECREF(parent); + Py_XDECREF(child); + Py_XDECREF(siblings); + Py_XDECREF(source); + Py_XDECREF(children); + if (PyErr_Occurred()) { + res = -1; + } + if (res <= 0) { + break; + } + } + Py_DECREF(name); + return res < 0 ? -1 : 0; +} + +// Return None when no module exists, without hiding exceptions from a loader. +PyObject * +_PyImport_LoadLazySubmodule(PyThreadState *tstate, PyObject *name) +{ + // Match eager from-imports: accept a partial module in an import cycle. + PyObject *mod = PyImport_GetModule(name); + if ((mod == NULL && !PyErr_Occurred()) || mod == Py_None) { + Py_XDECREF(mod); + mod = import_find_and_load_with_name( + tstate, name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); + } + if (mod == NULL) { + remove_importlib_frames(tstate); + } + return mod; +} + PyObject * _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, int *recheck_dict) @@ -4217,45 +4385,36 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, if (PyDict_GetItemRef(mod_dict, &_Py_ID(__name__), &mod_name) <= 0) { return NULL; } - PyObject *mod = NULL, *pending_set = NULL; + PyObject *mod = NULL, *pending_children = NULL; if (!PyUnicode_Check(mod_name)) { goto done; } PyThreadState *tstate = _PyThreadState_GET(); PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(tstate->interp); if (lazy_pending == NULL || - PyDict_GetItemRef(lazy_pending, mod_name, &pending_set) <= 0) { + PyDict_GetItemRef(lazy_pending, mod_name, &pending_children) <= 0) { goto done; } *recheck_dict = 1; - if (PySet_Contains(pending_set, attr_name) <= 0) { + if (PyDict_Contains(pending_children, attr_name) <= 0) { goto done; } PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); if (full_name == NULL) { goto done; } - // Match eager from-imports: accept a partial module in an import cycle. - mod = PyImport_GetModule(full_name); - if ((mod == NULL && !PyErr_Occurred()) || mod == Py_None) { - Py_XDECREF(mod); - mod = import_find_and_load_with_name( - tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); - } - Py_DECREF(full_name); - if (mod == NULL) { - remove_importlib_frames(tstate); - } + mod = _PyImport_LoadLazySubmodule(tstate, full_name); /* Keep missing children pending: a finder may provide them later. */ - else if (mod == Py_None || + if (mod != NULL && (mod == Py_None || PyDict_SetItem(mod_dict, attr_name, mod) < 0 || - PySet_Discard(pending_set, attr_name) < 0) { + _PyImport_ClearLazySubmodules(tstate, full_name) < 0)) { Py_CLEAR(mod); } + Py_DECREF(full_name); done: - Py_XDECREF(pending_set); + Py_XDECREF(pending_children); Py_DECREF(mod_name); return mod; } @@ -4364,13 +4523,13 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, if (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist)) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { if (register_from_lazy_on_parent(tstate, abs_name, - PyTuple_GET_ITEM(fromlist, i)) < 0) + PyTuple_GET_ITEM(fromlist, i), res) < 0) { goto error; } } } - else if (register_lazy_on_parent(tstate, abs_name) < 0) { + else if (register_lazy_on_parent(tstate, abs_name, res, 1) < 0) { goto error; } @@ -5398,6 +5557,9 @@ _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, if (PySet_Discard(LAZY_MODULES(interp), name) < 0) { return NULL; } + if (_PyImport_ClearLazySubmodules(_PyThreadState_GET(), name) < 0) { + return NULL; + } Py_RETURN_NONE; }