diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 7032b61d7654b8..81f1de3a5be865 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -23,6 +23,11 @@ PyAPI_FUNC(int) _PyDict_DelItemIf(PyObject *mp, PyObject *key, int (*predicate)(PyObject *value, void *arg), void *arg); +// Atomically replace an existing value if it is expected (by identity). +// Return 1 if replaced, 0 if absent or different, or -1 on error. +extern int _PyDict_ReplaceItemIf(PyObject *dict, PyObject *key, + PyObject *expected, PyObject *replacement); + // "KnownHash" variants // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, @@ -147,8 +152,8 @@ PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyOb */ PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); -PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); -PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); +// Return the borrowed source dictionary, or NULL if absent or on error. +PyAPI_FUNC(PyObject *) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); // Loads the __builtins__ object from the globals dict. Returns a new reference. extern PyObject *_PyDict_LoadBuiltinsFromGlobals(PyObject *globals); diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 669e328c266d00..8075ce66e2e335 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -32,20 +32,19 @@ extern int _PyImport_FixupBuiltin( PyObject *modules ); -extern PyObject * _PyImport_ResolveName( - PyThreadState *tstate, PyObject *name, PyObject *globals, int level); extern PyObject * _PyImport_GetAbsName( PyThreadState *tstate, PyObject *name, PyObject *globals, int level); // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); -typedef enum { - _Py_LAZY_SUBMODULE_ERROR = -1, - _Py_LAZY_SUBMODULE_NOT_FOUND = 0, - _Py_LAZY_SUBMODULE_LOADED = 1, -} _PyLazySubmoduleImportResult; -extern _PyLazySubmoduleImportResult _PyImport_TryLoadLazySubmodule( - PyObject *mod_name, PyObject *attr_name, PyObject **result); +// Bind a pending child on module before removing its registration. Return a +// new reference, NULL without an exception if absent, or NULL on error. +// Set recheck_dict if a concurrent lookup may have already bound the child. +extern PyObject *_PyImport_LoadLazySubmodule(PyThreadState *tstate, PyObject *name); +extern PyObject *_PyImport_GetLazySubmodules(PyThreadState *, PyObject *); +extern int _PyImport_ClearLazySubmodules(PyThreadState *, PyObject *); +extern PyObject * _PyImport_TryLoadLazySubmodule( + PyObject *module, PyObject *attr_name, int *recheck_dict); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 6c907e0cf79894..2692ac41cd6db4 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -352,16 +352,16 @@ struct _import_state { PyObject *lazy_import_func; int lazy_imports_mode; PyObject *lazy_imports_filter; - PyObject *lazy_importing_modules; // The set stored in sys.lazy_modules if values that have been // lazily imported. This value is only for debugging/introspection // purposes and is not used by the runtime. PyObject *lazy_modules; - // A dict mapping package names to a set of submodule names that - // have been imported lazily from packages which have been imported - // lazily. When the package is reified we need to add a - // LazyImportObject which refers to the submodule on the module. + // A dict mapping package names to dicts of pending child names. + // Values are declarations, True for completed imports, or False for + // optional from-import attributes. PyObject *lazy_pending_submodules; + // Cached imports may need to replace a published lazy binding. + int has_lazy_submodules; #ifdef Py_GIL_DISABLED PyMutex lazy_mutex; #endif diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index b81e4211b08ff3..8b2abf4bb5c1f9 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -14,20 +14,26 @@ extern "C" { PyAPI_DATA(PyTypeObject) PyLazyImport_Type; #define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type) -typedef struct { - PyObject_HEAD - PyObject *lz_builtins; - PyObject *lz_from; - PyObject *lz_attr; - // Frame information for the original import location. - PyCodeObject *lz_code; // Code object where the lazy import was created. - int lz_instr_offset; // Instruction offset where the lazy import was created. -} PyLazyImportObject; - - -PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import); PyAPI_FUNC(PyObject *) _PyLazyImport_New( - struct _PyInterpreterFrame *frame, PyObject *import_func, PyObject *from, PyObject *attr); + struct _PyInterpreterFrame *frame, PyObject *builtins, + PyObject *name, PyObject *fromlist); + +extern int _PyLazyImport_IsSubmodule(PyObject *op); +extern int _PyLazyImport_IsStrictSubmodule(PyObject *op); + +extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op); + +// Resolve a placeholder and replace its binding if it is unchanged or holds +// the child module published by the normal importer during resolution. +// namespace is the source captured during lookup, before resolution runs. +PyAPI_FUNC(PyObject *) _PyLazyImport_Reify( + PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *ns); + +// Optional module lookups preserve from-import and __getattr__ fallbacks. +extern PyObject *_PyLazyImport_ReifyOptional( + PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *ns); #ifdef __cplusplus } diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 38bccdd48c0f86..d3546119bc9ee0 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1253,7 +1253,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1476,6 +1476,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { _LOAD_FAST_CHECK, OPARG_SIMPLE, 0 } } }, [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _LOAD_FAST, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_DEREF, OPARG_SIMPLE, 0 } } }, + [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, OPARG_SIMPLE, 0 } } }, [LOAD_GLOBAL] = { .nuops = 2, .uops = { { _LOAD_GLOBAL, OPARG_SIMPLE, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 3 } } }, [LOAD_GLOBAL_BUILTIN] = { .nuops = 4, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_BUILTINS, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, OPERAND1_1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 4 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 4, .uops = { { _NOP, OPARG_SIMPLE, 1 }, { _LOAD_GLOBAL_MODULE, 1, 1 }, { _LOAD_GLOBAL_MODULE, OPERAND1_1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 4 } } }, diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index eb2b0c84acdc7c..58dd66b69efc9d 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -51,6 +51,9 @@ typedef struct _PyThreadStateImpl { PyObject *asyncio_running_loop; // Strong reference PyObject *asyncio_running_task; // Strong reference + // Reused for the thread's lifetime; owns placeholders while they resolve. + PyObject *lazy_imports; + // Distinguishes between yield and return from PyEval_EvalFrame(). // See gen_send_ex2() in Objects/genobject.c enum { diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f464effe23bc0b..325b811472f8f8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -190,6 +190,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG, + [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL_CONDITIONAL] = HAS_ARG_FLAG, @@ -1840,6 +1841,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_LOAD_FROM_DICT_OR_GLOBALS] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_FROM_DICT_OR_GLOBALS_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_LOAD_NAME] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -4306,6 +4316,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_LOCALS_r01] = _LOAD_LOCALS, [_LOAD_LOCALS_r12] = _LOAD_LOCALS, [_LOAD_LOCALS_r23] = _LOAD_LOCALS, + [_LOAD_FROM_DICT_OR_GLOBALS_r11] = _LOAD_FROM_DICT_OR_GLOBALS, [_LOAD_NAME_r01] = _LOAD_NAME, [_LOAD_GLOBAL_r00] = _LOAD_GLOBAL, [_PUSH_NULL_CONDITIONAL_r00] = _PUSH_NULL_CONDITIONAL, @@ -5810,6 +5821,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_FAST_CHECK_r23] = "_LOAD_FAST_CHECK_r23", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", [_LOAD_FROM_DICT_OR_DEREF_r11] = "_LOAD_FROM_DICT_OR_DEREF_r11", + [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", + [_LOAD_FROM_DICT_OR_GLOBALS_r11] = "_LOAD_FROM_DICT_OR_GLOBALS_r11", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", [_LOAD_GLOBAL_r00] = "_LOAD_GLOBAL_r00", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", @@ -6476,6 +6489,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_LOCALS: return 0; + case _LOAD_FROM_DICT_OR_GLOBALS: + return 1; case _LOAD_NAME: return 0; case _LOAD_GLOBAL: diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 5026c9670d81d2..a6c560bd105428 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -451,6 +451,801 @@ def test_lazy_with_from(self): class PackageTests(LazyImportTestCase): """Tests for lazy imports with packages.""" + @support.requires_subprocess() + def test_lazy_import_sibling_submodules(self): + for package, siblings in ( + ("test.test_lazy_import.data.pkg", ("b", "bar")), + ("test.test_lazy_import.data.metasyntactic.foo", ("ack", "bar")), + ): + for first, second in (siblings, siblings[::-1]): + with self.subTest(package=package, first=first): + code = textwrap.dedent(f""" + import sys + lazy import {package}.{first} + lazy import {package}.{second} + + def get_first(): + return {package}.{first} + + for _ in range(30): + assert get_first() is sys.modules['{package}.{first}'] + assert '{package}.{second}' not in sys.modules + assert {package}.{second} is sys.modules['{package}.{second}'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_siblings_through_module_attribute(self): + code = textwrap.dedent(""" + import sys + import types + namespace = types.ModuleType('lazy_siblings') + exec('lazy import xml.dom\\nlazy import xml.sax', namespace.__dict__) + assert namespace.xml.dom is sys.modules['xml.dom'] + assert 'xml.sax' not in sys.modules + assert namespace.xml.sax is sys.modules['xml.sax'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_blocked_sibling_stays_deferred(self): + code = textwrap.dedent(""" + import sys + sys.modules['xml.sax'] = None + lazy import xml.dom + lazy import xml.sax + + assert xml.dom is sys.modules['xml.dom'] + assert sys.modules['xml.sax'] is None + for _ in range(2): + try: + xml.sax + except ModuleNotFoundError as exc: + assert exc.name == 'xml.sax' + assert 'None in sys.modules' in str(exc), exc + else: + raise AssertionError('blocked sibling imported') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_parent_without_submodules(self): + code = textwrap.dedent(""" + import sys + lazy import test.test_lazy_import.data.pkg.b + lazy import test.test_lazy_import.data.pkg.bar + + assert test.__name__ == 'test' + assert 'test.test_lazy_import.data.pkg' not in sys.modules + assert test.test_lazy_import.data.pkg.__name__ == 'test.test_lazy_import.data.pkg' + assert 'test.test_lazy_import.data.pkg.b' not in sys.modules + assert 'test.test_lazy_import.data.pkg.bar' not in sys.modules + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_descendant_after_eager_parent(self): + code = textwrap.dedent(""" + import importlib + import sys + lazy import xml.dom.minidom + importlib.import_module('xml.dom') + + assert xml is sys.modules['xml'] + assert 'xml.dom.minidom' not in sys.modules + assert xml.dom.minidom is sys.modules['xml.dom.minidom'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_missing_nested_submodule(self): + package = 'test.test_lazy_import.data.metasyntactic.foo' + for preload in (False, True): + with self.subTest(preload=preload): + code = f"import {package}\n" if preload else "" + code += textwrap.dedent(f""" + lazy import {package}.nonexistent + + for _ in range(2): + try: + {package}.nonexistent + except ModuleNotFoundError as exc: + assert exc.name == '{package}.nonexistent' + assert exc.__cause__ is not None + else: + raise AssertionError('nonexistent module imported') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_sibling_error_declaration(self): + package = 'test.test_lazy_import.data.pkg' + for first, second in (('nonexistent', 'b'), ('b', 'nonexistent')): + with self.subTest(first=first): + declarations = ( + f'lazy import {package}.{first}\n' + f'lazy import {package}.{second}\n' + ) + code = textwrap.dedent(f""" + import traceback + exec(compile({declarations!r}, 'lazy_declarations.py', 'exec')) + try: + {package}.nonexistent + except ModuleNotFoundError as exc: + assert exc.name == '{package}.nonexistent' + assert exc.__cause__ is not None + origin = traceback.extract_tb(exc.__cause__.__traceback__)[-1] + assert origin.filename == 'lazy_declarations.py', origin + assert origin.lineno == {1 if first == 'nonexistent' else 2}, origin + else: + raise AssertionError('nonexistent module imported') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_preserves_assigned_package_attribute(self): + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg as package + package.b = 42 + lazy import test.test_lazy_import.data.pkg.b + + assert test.test_lazy_import.data.pkg.b == 42 + assert 'test.test_lazy_import.data.pkg.b' not in sys.modules + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_children_use_current_import_hook(self): + code = textwrap.dedent(""" + import builtins + import types + lazy import xml.dom.minidom + assert xml.__name__ == 'xml' + + child = types.ModuleType('xml.dom') + child.__path__ = [] + value = object() + calls = [] + original = builtins.__import__ + + def hook(name, *args): + calls.append(name) + if name == 'xml.dom': + xml.dom = child + elif name == 'xml.dom.minidom': + child.minidom = value + else: + raise AssertionError(name) + return xml + + builtins.__import__ = hook + try: + assert xml.dom.minidom is value + finally: + builtins.__import__ = original + assert calls == ['xml.dom', 'xml.dom.minidom'], calls + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_children_use_declaring_namespace_builtins(self): + code = textwrap.dedent(""" + import builtins + import types + import xml + + first = types.ModuleType('first') + second = types.ModuleType('second') + first.__dict__['__builtins__'] = builtins.__dict__.copy() + second.__dict__['__builtins__'] = builtins.__dict__.copy() + exec('lazy import xml.dom', first.__dict__) + exec('lazy import xml.sax', second.__dict__) + assert second.xml is xml + + def denied(*args): + raise AssertionError('used another namespace import hook') + + second.__dict__['__builtins__']['__import__'] = denied + assert first.xml.dom.__name__ == 'xml.dom' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_same_child_uses_declaring_namespace_builtins(self): + code = textwrap.dedent(""" + import builtins + import types + + first = types.ModuleType('first') + second = types.ModuleType('second') + first.__dict__['__builtins__'] = builtins.__dict__.copy() + second.__dict__['__builtins__'] = builtins.__dict__.copy() + exec('lazy import xml.dom', first.__dict__) + exec('lazy import xml.dom', second.__dict__) + + def denied(name, *args): + if name == 'xml.dom': + raise AssertionError('used another namespace import hook') + return builtins.__import__(name, *args) + + second.__dict__['__builtins__']['__import__'] = denied + assert first.xml.dom.__name__ == 'xml.dom' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_shared_parent_uses_declaring_namespace_builtins(self): + for resolve_first in (False, True): + with self.subTest(resolve_first=resolve_first): + code = textwrap.dedent(f""" + import builtins + import types + + first = types.ModuleType('first') + second = types.ModuleType('second') + first.__dict__['__builtins__'] = builtins.__dict__.copy() + second.__dict__['__builtins__'] = builtins.__dict__.copy() + exec('lazy import xml.dom.minidom', first.__dict__) + exec('lazy import xml.dom.pulldom', second.__dict__) + if {resolve_first}: + assert first.xml.__name__ == 'xml' + assert second.xml.__name__ == 'xml' + + def denied(*args): + raise AssertionError('used another namespace import hook') + + second.__dict__['__builtins__']['__import__'] = denied + assert first.xml.dom.minidom.__name__ == 'xml.dom.minidom' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_resolved_lazy_import_releases_namespace_builtins(self): + for mode in ('lazy', 'cached', 'eager'): + with self.subTest(mode=mode): + code = textwrap.dedent(f""" + import builtins + import gc + import importlib + import weakref + + class Payload: + pass + + payload = Payload() + payload_ref = weakref.ref(payload) + namespace_builtins = dict(vars(builtins), payload=payload) + namespace = {{'__builtins__': namespace_builtins}} + if {mode == 'cached'}: + importlib.import_module('xml.dom') + exec('lazy import xml.dom', namespace) + if {mode == 'eager'}: + importlib.import_module('xml.dom') + else: + exec('xml.dom', namespace) + + namespace.clear() + del namespace_builtins, payload + gc.collect() + assert payload_ref() is None + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_with_spec_initializing_error(self): + for eager in (False, True): + with self.subTest(eager=eager): + code = textwrap.dedent(f""" + import importlib + import sys + import warnings + from importlib.machinery import ModuleSpec + + class Spec: + @property + def _initializing(self): + raise RuntimeError('initializing lookup failed') + + class Loader: + def find_spec(self, name, path=None, target=None): + if name == 'xml.child': + return ModuleSpec(name, self) + + def create_module(self, spec): + return None + + def exec_module(self, module): + module.__spec__ = Spec() + + sys.meta_path.insert(0, Loader()) + warnings.simplefilter('error', ImportWarning) + lazy import xml.child + if {eager}: + child = importlib.import_module('xml.child') + else: + child = xml.child + assert child is sys.modules['xml.child'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_invalid_lazy_import_preserves_plain_import(self): + for separate_namespaces in (False, True): + with self.subTest(separate_namespaces=separate_namespaces): + if separate_namespaces: + code = textwrap.dedent(""" + import types + first = types.ModuleType('first') + second = types.ModuleType('second') + exec('lazy import math.pi', first.__dict__) + exec('lazy import math', second.__dict__) + assert second.math.sin(0) == 0 + """) + else: + code = textwrap.dedent(""" + lazy import math.pi + lazy import math + assert math.sin(0) == 0 + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_nested_nonpackage(self): + module = 'test.test_lazy_import.data.pkg.b' + for name in ('foo', 'nonexistent'): + with self.subTest(name=name): + code = textwrap.dedent(f""" + lazy import {module}.{name} + + for _ in range(2): + try: + {module}.{name} + except ModuleNotFoundError as exc: + assert exc.name == '{module}.{name}', exc.name + assert "'{module}' is not a package" in str(exc), exc + else: + raise AssertionError('imported from a nonpackage') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_nested_nonpackage_subclass(self): + code = textwrap.dedent(""" + import types + import xml.dom.minidom as leaf + + class Module(types.ModuleType): + pass + + leaf.__class__ = Module + lazy import xml.dom.minidom.Node + try: + xml.dom.minidom.Node + except ModuleNotFoundError as exc: + assert exc.name == 'xml.dom.minidom.Node', exc.name + else: + raise AssertionError('imported from a nonpackage') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_loaded_nonpackage_subclass(self): + code = textwrap.dedent(""" + import sys + import types + import xml + from importlib.machinery import ModuleSpec + + class Module(types.ModuleType): + pass + + class Loader: + def find_spec(self, name, path=None, target=None): + if name == 'xml.child': + return ModuleSpec(name, self) + + def create_module(self, spec): + return Module(spec.name) + + def exec_module(self, module): + module.value = 42 + + sys.meta_path.insert(0, Loader()) + lazy import xml.child.value + try: + xml.child.value + except ModuleNotFoundError as exc: + assert exc.name == 'xml.child.value', exc.name + else: + raise AssertionError('imported from a nonpackage') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_eager_import_after_invalid_lazy_import(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.pkg.b.foo + try: + test.test_lazy_import.data.pkg.b.foo + except ModuleNotFoundError: + pass + else: + raise AssertionError('imported from a nonpackage') + + import test.test_lazy_import.data.pkg.b + assert callable(test.test_lazy_import.data.pkg.b.foo) + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_eager_import_replaces_lazy_binding_to_cached_child(self): + code = textwrap.dedent(""" + import xml.dom.minidom as child + import xml.dom as parent + del parent.minidom + + lazy import xml.dom.minidom.Node + assert xml.__name__ == 'xml' + import xml.dom.minidom + assert xml.dom.minidom is child + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_invalid_lazy_import_preserves_sibling_from_import(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.pkg.b.foo + lazy import test.test_lazy_import.data.pkg.bar + test.test_lazy_import.data.pkg.bar + + from test.test_lazy_import.data.pkg import b + assert b.foo() == 'foo' + assert test.test_lazy_import.data.pkg.b.foo() == 'foo' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_submodule_ignores_package_getattr(self): + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg as package + original = package.__getattr__ + + def hook(name): + if name == 'b': + return 42 + return original(name) + + package.__getattr__ = hook + lazy import test.test_lazy_import.data.pkg.b + assert test.test_lazy_import.data.pkg.b is sys.modules['test.test_lazy_import.data.pkg.b'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_missing_submodule_ignores_package_getattr(self): + package = 'test.test_lazy_import.data.pkg' + for name in ('dynamic_attr', 'raising_attr'): + with self.subTest(name=name): + code = textwrap.dedent(f""" + lazy import {package}.{name} + + for _ in range(2): + try: + {package}.{name} + except ModuleNotFoundError as exc: + assert exc.name == '{package}.{name}', exc.name + else: + raise AssertionError('nonexistent module imported') + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_lazy_import_preserves_dynamic_from_import(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.pkg.dynamic_attr + from test.test_lazy_import.data.pkg import dynamic_attr + assert dynamic_attr == 'from_getattr' + + lazy from test.test_lazy_import.data.pkg import dynamic_attr as other + assert other == 'from_getattr' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_lazy_import_preserves_wrapped_import_hook(self): + code = textwrap.dedent(""" + import builtins + original = builtins.__import__ + builtins.__import__ = lambda *args, **kwargs: original(*args, **kwargs) + + lazy import test.test_lazy_import.data.pkg.dynamic_attr + lazy import test.test_lazy_import.data.pkg.bar + test.test_lazy_import.data.pkg.bar + from test.test_lazy_import.data.pkg import dynamic_attr + assert dynamic_attr == 'from_getattr' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_lazy_import_with_removed_import_hook(self): + code = textwrap.dedent(""" + import builtins + import types + namespace = types.ModuleType('namespace') + namespace.__dict__['__builtins__'] = builtins.__dict__.copy() + exec('lazy import test.test_lazy_import.data.pkg.dynamic_attr\\n' + 'lazy import test.test_lazy_import.data.pkg.bar', namespace.__dict__) + namespace.test.test_lazy_import.data.pkg.bar + del namespace.__dict__['__builtins__']['__import__'] + + from test.test_lazy_import.data.pkg import dynamic_attr + assert dynamic_attr == 'from_getattr' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_unused_lazy_import_preserves_dynamic_attribute(self): + package = 'test.test_lazy_import.data.pkg' + for name in ('dynamic_attr', 'b'): + with self.subTest(name=name): + code = textwrap.dedent(f""" + import sys + lazy import {package}.{name} + lazy import {package}.bar + {package}.bar + package = sys.modules['{package}'] + original = package.__getattr__ + + def hook(attr): + if attr == 'b': + return 42 + return original(attr) + + package.__getattr__ = hook + value = package.{name} + if {name == 'b'}: + assert value is sys.modules['{package}.b'] + else: + assert value == 'from_getattr', value + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_published_child_preserves_dynamic_from_import(self): + for prefix in ('', 'lazy '): + with self.subTest(prefix=prefix): + code = textwrap.dedent(f""" + import xml as package + package.__getattr__ = lambda name: 42 + lazy import xml.missing + lazy import xml + assert xml is package + + {prefix}from xml import missing + assert missing == 42 + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_published_child_preserves_loader_error(self): + for prefix in ('', 'lazy '): + with self.subTest(prefix=prefix): + code = textwrap.dedent(f""" + import sys + import xml as package + from importlib.machinery import ModuleSpec + + failure = ModuleNotFoundError('loader failed', name='xml.broken') + package.__getattr__ = lambda name: 42 + + class Loader: + calls = 0 + + def create_module(self, spec): + return None + + def exec_module(self, module): + self.calls += 1 + raise failure + + def find_spec(self, name, path=None, target=None): + if name == 'xml.broken': + return ModuleSpec(name, self) + + loader = Loader() + sys.meta_path.insert(0, loader) + lazy import xml.broken + lazy import xml + assert xml is package + + try: + exec('{prefix}from xml import broken') + broken + except ModuleNotFoundError as exc: + assert exc is failure, exc + else: + raise AssertionError('suppressed loader exception') + assert loader.calls == 1, loader.calls + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_aliased_package(self): + code = textwrap.dedent(""" + import sys + import xml + sys.modules['alias_xml'] = xml + lazy import alias_xml.dom + assert alias_xml is xml + assert alias_xml.dom is sys.modules['alias_xml.dom'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_aliased_package_siblings(self): + code = textwrap.dedent(""" + import sys + import xml + sys.modules['alias_xml'] = xml + lazy import alias_xml.dom + lazy import alias_xml.sax + + assert alias_xml is xml + assert 'alias_xml.dom' not in sys.modules + assert 'alias_xml.sax' not in sys.modules + assert alias_xml.dom is sys.modules['alias_xml.dom'] + assert 'alias_xml.sax' not in sys.modules + assert alias_xml.sax is sys.modules['alias_xml.sax'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_cached_intermediate_alias(self): + code = textwrap.dedent(""" + import sys + import xml + import email + xml.alias = email + sys.modules['xml.alias'] = email + + lazy import xml.alias.message + assert xml.alias.message is sys.modules['xml.alias.message'] + assert xml.alias is email + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_available_partial_submodule(self): + code = textwrap.dedent(""" + import sys + from importlib.machinery import ModuleSpec + + sources = { + 'lazy_cycle': '', + 'lazy_cycle.a': 'value = 42\\nimport lazy_cycle.b\\n', + 'lazy_cycle.b': 'import lazy_cycle.a\\nassert lazy_cycle.a.value == 42\\n', + } + + class Loader: + def create_module(self, spec): + return None + + def exec_module(self, module): + exec(sources[module.__name__], module.__dict__) + + def find_spec(self, name, path=None, target=None): + if name in sources: + return ModuleSpec(name, self, is_package=name == 'lazy_cycle') + + sys.meta_path.insert(0, Loader()) + lazy import lazy_cycle.a + assert lazy_cycle.a.value == 42 + assert lazy_cycle.b is sys.modules['lazy_cycle.b'] + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_retries_after_self_import_during_initialization(self): + for reentry in ('__import__(module.__name__)', 'parent.broken'): + with self.subTest(reentry=reentry): + code = textwrap.dedent(f""" + import sys + import xml as parent + from importlib.machinery import ModuleSpec + + class Loader: + calls = 0 + + def find_spec(self, name, path=None, target=None): + if name == 'xml.broken': + return ModuleSpec(name, self) + + def create_module(self, spec): + return None + + def exec_module(self, module): + self.calls += 1 + {reentry} + raise RuntimeError('failed initialization') + + loader = Loader() + sys.meta_path.insert(0, loader) + lazy import xml.broken + for _ in range(2): + try: + xml.broken + except RuntimeError as exc: + assert str(exc) == 'failed initialization', exc + else: + raise AssertionError('returned failed partial module') + assert loader.calls == 2, loader.calls + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_retries_failed_parent_after_child_loaded(self): + for import_parent in (False, True): + with self.subTest(import_parent=import_parent): + code = textwrap.dedent(f""" + import importlib + import sys + from importlib.machinery import ModuleSpec + + sources = {{ + 'lazy_retry': '', + 'lazy_retry.a': 'import lazy_retry.a.b\\nvalue = 42\\n', + 'lazy_retry.a.b': '', + }} + + class Loader: + attempts = 0 + + def create_module(self, spec): + return None + + def exec_module(self, module): + exec(sources[module.__name__], module.__dict__) + if module.__name__ == 'lazy_retry.a': + self.attempts += 1 + if self.attempts == 1: + raise RuntimeError('first attempt') + + def find_spec(self, name, path=None, target=None): + if name in sources: + return ModuleSpec(name, self, is_package=name != 'lazy_retry.a.b') + + loader = Loader() + sys.meta_path.insert(0, loader) + lazy import lazy_retry.a.b + if {import_parent}: + lazy import lazy_retry.a + try: + importlib.import_module('lazy_retry.a') + except RuntimeError as exc: + assert str(exc) == 'first attempt', exc + else: + raise AssertionError('first import succeeded') + + assert 'lazy_retry.a' not in sys.modules + assert 'lazy_retry.a.b' in sys.modules + assert lazy_retry.a.b is sys.modules['lazy_retry.a.b'] + assert lazy_retry.a.value == 42 + assert loader.attempts == 2, loader.attempts + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_lazy_import_recursive_package_attribute(self): + code = textwrap.dedent(""" + import xml + xml.alias = xml + lazy import xml.alias.nonexistent + assert xml.alias is xml + """) + assert_python_ok("-c", code) + def test_lazy_import_pkg(self): """lazy import of package submodule should load the package.""" out = io.StringIO() @@ -624,6 +1419,22 @@ def test_dunder_lazy_import_builtins(self): self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) self.assertEqual(dunder_lazy_import_builtins.basic.basic2, 42) + @support.requires_subprocess() + def test_dunder_lazy_import_with_lazy_builtins(self): + code = textwrap.dedent(""" + placeholders = [__lazy_import__('sys')] + for _ in range(2): + placeholders.append(__lazy_import__( + 'sys', {'__builtins__': placeholders[-1]})) + try: + placeholders[-1].resolve() + except TypeError as exc: + assert 'not subscriptable' in str(exc), exc + else: + raise AssertionError('resolved with invalid builtins') + """) + assert_python_ok("-c", code) + class SysLazyImportsAPITests(LazyImportTestCase): """Tests for sys lazy imports API functions.""" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-15-00-00.gh-issue-158140.v5Aj2R.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-15-00-00.gh-issue-158140.v5Aj2R.rst new file mode 100644 index 00000000000000..3b2ad941d9031b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-15-00-00.gh-issue-158140.v5Aj2R.rst @@ -0,0 +1,4 @@ +Fix dotted lazy imports sharing a package name so that accessing one submodule +does not import the last declared sibling first. Resolving the package leaves +its pending submodules lazy, while nonexistent submodules still raise +:exc:`ModuleNotFoundError` when accessed. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index a04770bd83b176..245b0793eea02b 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -9907,6 +9907,16 @@ JUMP_TO_LABEL(error); } } + else if (PyLazyImport_CheckExact(value_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + _PyFrame_StackPointerInvalidate(frame); + if (value_o == NULL) { + JUMP_TO_LABEL(error); + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9932,11 +9942,21 @@ _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); int err; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); _PyFrame_StackPointerInvalidate(frame); + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + err = -1; + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9947,81 +9967,17 @@ JUMP_TO_LABEL(error); } if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - } - JUMP_TO_LABEL(error); - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } - } - else { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(v)) { + JUMP_TO_LABEL(error); } } - v = PyStackRef_FromPyObjectSteal(v_o); + else { + v = PyStackRef_FromPyObjectSteal(v_o); + } stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10275,38 +10231,6 @@ if (v_o == NULL) { JUMP_TO_LABEL(error); } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - if (l_v == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(l_v); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 15377ac083c2b2..57874de6ee7497 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2009,6 +2009,31 @@ _PyDict_InsertSplitValue(PyDictObject *mp, PyObject *key, PyObject *value, Py_ss ASSERT_CONSISTENT(mp); } +// Replace a value at an existing entry. Steals the new value reference. +static void +replace_value(PyDictObject *mp, PyObject *key, Py_ssize_t ix, + PyObject *old_value, PyObject *value) +{ + if (old_value != value) { + _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); + assert(old_value != NULL); + if (DK_IS_UNICODE(mp->ma_keys)) { + if (_PyDict_HasSplitTable(mp)) { + STORE_SPLIT_VALUE(mp, ix, value); + } + else { + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); + } + } + else { + PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); + } + } + Py_DECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ +} + /* Internal routine to insert a new item into the table. Used both by the internal resize routine and by the public insert routine. @@ -2056,24 +2081,7 @@ insertdict(PyDictObject *mp, return 0; } - if (old_value != value) { - _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); - assert(old_value != NULL); - if (DK_IS_UNICODE(mp->ma_keys)) { - if (_PyDict_HasSplitTable(mp)) { - STORE_SPLIT_VALUE(mp, ix, value); - } - else { - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; - STORE_VALUE(ep, value); - } - } - else { - PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; - STORE_VALUE(ep, value); - } - } - Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ + replace_value(mp, key, ix, old_value, value); ASSERT_CONSISTENT(mp); Py_DECREF(key); return 0; @@ -2700,44 +2708,7 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) return rv; } -/* Fast version of global value lookup (LOAD_GLOBAL). - * Lookup in globals, then builtins. - * - * - * - * - * Raise an exception and return NULL if an error occurred (ex: computing the - * key hash failed, key comparison failed, ...). Return NULL if the key doesn't - * exist. Return the value if the key exists. - * - * Returns a new reference. - */ PyObject * -_PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) -{ - Py_ssize_t ix; - Py_hash_t hash; - PyObject *value; - - hash = _PyObject_HashDictKey(key); - if (hash == -1) { - return NULL; - } - - /* namespace 1: globals */ - ix = _Py_dict_lookup_threadsafe(globals, key, hash, &value); - if (ix == DKIX_ERROR) - return NULL; - if (ix != DKIX_EMPTY && value != NULL) - return value; - - /* namespace 2: builtins */ - ix = _Py_dict_lookup_threadsafe(builtins, key, hash, &value); - assert(ix >= 0 || value == NULL); - return value; -} - -void _PyDict_LoadGlobalStackRef(PyDictObject *globals, PyDictObject *builtins, PyObject *key, _PyStackRef *res) { Py_ssize_t ix; @@ -2746,21 +2717,22 @@ _PyDict_LoadGlobalStackRef(PyDictObject *globals, PyDictObject *builtins, PyObje hash = _PyObject_HashDictKey(key); if (hash == -1) { *res = PyStackRef_NULL; - return; + return NULL; } /* namespace 1: globals */ ix = _Py_dict_lookup_threadsafe_stackref(globals, key, hash, res); if (ix == DKIX_ERROR) { - return; + return NULL; } if (ix != DKIX_EMPTY && !PyStackRef_IsNull(*res)) { - return; + return (PyObject *)globals; } /* namespace 2: builtins */ ix = _Py_dict_lookup_threadsafe_stackref(builtins, key, hash, res); assert(ix >= 0 || PyStackRef_IsNull(*res)); + return PyStackRef_IsNull(*res) ? NULL : (PyObject *)builtins; } PyObject * @@ -3105,6 +3077,34 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, return res; } +int +_PyDict_ReplaceItemIf(PyObject *op, PyObject *key, + PyObject *expected, PyObject *replacement) +{ + assert(PyDict_Check(op)); + assert(expected != NULL && replacement != NULL); + Py_hash_t hash = PyObject_Hash(key); + if (hash == -1) { + return -1; + } + int result = 0; + Py_BEGIN_CRITICAL_SECTION(op); + PyDictObject *mp = (PyDictObject *)op; + PyObject *current; + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, ¤t); + if (ix == DKIX_ERROR) { + result = -1; + } + else if (current == expected) { + // Do not look up the key again: equality can execute Python code. + replace_value(mp, key, ix, current, Py_NewRef(replacement)); + ASSERT_CONSISTENT(mp); + result = 1; + } + Py_END_CRITICAL_SECTION(); + return result; +} + static void clear_embedded_values(PyDictValues *values, Py_ssize_t nentries) { diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index e008c076352ebb..b9223c8cd047bb 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -2,18 +2,43 @@ #include "Python.h" #include "pycore_ceval.h" -#include "pycore_frame.h" +#include "pycore_dict.h" +#include "pycore_gc.h" #include "pycore_import.h" #include "pycore_interpframe.h" #include "pycore_lazyimportobject.h" -#include "pycore_modsupport.h" +#include "pycore_long.h" +#include "pycore_moduleobject.h" +#include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_pyerrors.h" +#include "pycore_traceback.h" +#include "pycore_tstate.h" + +typedef struct { + PyObject_HEAD + // Roots retain the builtins mapping, or the original declaration for a + // published child. Projections leave this NULL and retain their source. + PyObject *lz_builtins; + // A root stores its absolute name and original fromlist. A projection + // stores its source placeholder and the attribute to import from it. + PyObject *lz_from; + PyObject *lz_attr; + // Declaration location. + PyCodeObject *lz_code; + int lz_instr_offset; + bool lz_submodule; + // Validate descendants only along this declaration's own path. + bool lz_validate; +} PyLazyImportObject; #define PyLazyImportObject_CAST(op) ((PyLazyImportObject *)(op)) +static PyObject *lazy_import_name(PyLazyImportObject *m); + PyObject * -_PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name, PyObject *fromlist) +_PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, + PyObject *name, PyObject *fromlist) { - PyLazyImportObject *m; if (!name || !(PyUnicode_Check(name) || PyLazyImport_CheckExact(name))) { PyErr_SetString(PyExc_TypeError, "expected str or lazy_import for name"); return NULL; @@ -26,7 +51,11 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name "lazy_import: fromlist must be None, a string, or a tuple"); return NULL; } - m = PyObject_GC_New(PyLazyImportObject, &PyLazyImport_Type); + assert(PyLazyImport_CheckExact(name) ? builtins == NULL : builtins != NULL); + assert(!PyLazyImport_CheckExact(name) || + (fromlist != NULL && PyUnicode_Check(fromlist))); + PyLazyImportObject *m = PyObject_GC_New( + PyLazyImportObject, &PyLazyImport_Type); if (m == NULL) { return NULL; } @@ -34,23 +63,84 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name m->lz_from = Py_NewRef(name); m->lz_attr = Py_XNewRef(fromlist); - // Capture frame information for the original import location. m->lz_code = NULL; m->lz_instr_offset = -1; + m->lz_submodule = false; + m->lz_validate = true; if (frame != NULL) { - PyCodeObject *code = _PyFrame_GetCode(frame); - if (code != NULL) { - m->lz_code = (PyCodeObject *)Py_NewRef(code); - // Calculate the instruction offset from the current frame. - m->lz_instr_offset = _PyInterpreterFrame_LASTI(frame); - } + m->lz_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + m->lz_instr_offset = _PyInterpreterFrame_LASTI(frame); } _PyObject_GC_TRACK(m); return (PyObject *)m; } +// Reuse concrete attributes of initialized modules without waiting for imports +// or resolving lazy attributes. Failed cache lookups are retried at resolution. +static PyObject * +lazy_import_get_loaded_attr(PyThreadState *tstate, PyObject *name, + PyObject *attr_name) +{ + PyObject *mod = NULL, *spec = NULL, *current = NULL, *attr = NULL; + PyObject *modules = Py_XNewRef(_PyImport_GetModules(tstate->interp)); + if (modules == NULL) { + return NULL; + } + int rc = PyMapping_GetOptionalItem(modules, name, &mod); + if (rc <= 0 || !PyModule_Check(mod)) { + goto done; + } + PyObject *dict = _PyModule_GetDict(mod); + if (PyObject_GetOptionalAttr(mod, &_Py_ID(__spec__), &spec) < 0 || + _PyModuleSpec_IsInitializing(spec) != 0) { + goto done; + } + // An initialization check can replace the module in sys.modules. + if (modules != _PyImport_GetModules(tstate->interp) || + PyMapping_GetOptionalItem(modules, name, ¤t) <= 0 || + current != mod) { + goto done; + } + if (PyDict_GetItemRef(dict, attr_name, &attr) < 0) { + goto done; + } + if (attr != NULL && PyLazyImport_CheckExact(attr)) { + Py_CLEAR(attr); + } + +done: + Py_XDECREF(current); + Py_XDECREF(spec); + Py_XDECREF(mod); + Py_DECREF(modules); + if (PyErr_ExceptionMatches(PyExc_Exception)) { + PyErr_Clear(); + } + return attr; +} + +PyObject * +_PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, + PyObject *v, PyObject *name) +{ + assert(PyLazyImport_CheckExact(v)); + assert(name); + assert(PyUnicode_Check(name)); + PyLazyImportObject *lz = PyLazyImportObject_CAST(v); + // Only `from a import b` can take b off an already imported a; + // `import a.b as c` has to import a.b first. + if (lz->lz_attr != NULL && PyTuple_Check(lz->lz_attr) && + PyTuple_GET_SIZE(lz->lz_attr) > 0) { + PyObject *attr = lazy_import_get_loaded_attr(tstate, lz->lz_from, name); + if (attr != NULL || PyErr_Occurred()) { + return attr; + } + } + return _PyLazyImport_New(frame, NULL, v, name); +} + static int lazy_import_traverse(PyObject *op, visitproc visit, void *arg) { @@ -85,23 +175,20 @@ lazy_import_dealloc(PyObject *op) static PyObject * lazy_import_getattro(PyObject *op, PyObject *name) { - PyObject *value = _PyObject_GenericGetAttrWithDict(op, name, NULL, /* suppress */1); - if (value == NULL) { - if (PyErr_Occurred()) { - // pass up non-AttributeError exception - return NULL; - } - PyObject *lz_name = _PyLazyImport_GetName(op); - if (lz_name == NULL) { - return NULL; - } - PyErr_Format(PyExc_AttributeError, - "cannot access attribute %R on unresolved lazy import %R", - name, lz_name); - Py_DECREF(lz_name); + PyObject *value = _PyObject_GenericGetAttrWithDict( + op, name, NULL, /* suppress */ 1); + if (value != NULL || PyErr_Occurred()) { + return value; + } + PyObject *lz_name = lazy_import_name(PyLazyImportObject_CAST(op)); + if (lz_name == NULL) { return NULL; } - return value; + PyErr_Format(PyExc_AttributeError, + "cannot access attribute %R on unresolved lazy import %R", + name, lz_name); + Py_DECREF(lz_name); + return NULL; } // The dotted name of the object that resolving the placeholder returns. @@ -117,8 +204,9 @@ lazy_import_path(PyLazyImportObject *m) Py_DECREF(base); return res; } - if (m->lz_attr != NULL && - (!PyTuple_Check(m->lz_attr) || PyTuple_GET_SIZE(m->lz_attr) > 0)) { + if (m->lz_submodule || + (m->lz_attr != NULL && + (!PyTuple_Check(m->lz_attr) || PyTuple_GET_SIZE(m->lz_attr) > 0))) { return Py_NewRef(m->lz_from); } // __import__("a.b") returns the top-level package `a`. @@ -160,12 +248,641 @@ lazy_import_repr(PyObject *op) return res; } -PyObject * -_PyLazyImport_GetName(PyObject *op) +// Consume a result, resolving a placeholder returned by an import hook or +// an attribute lookup under the same cycle and recursion checks. +static PyObject * +lazy_import_resolve_result(PyThreadState *tstate, PyObject *obj) +{ + if (obj == NULL || !PyLazyImport_CheckExact(obj)) { + return obj; + } + PyObject *result = _PyImport_LoadLazyImportTstate(tstate, obj); + Py_DECREF(obj); + return result; +} + +// Look up, in order, the attributes recorded from the root placeholder to lz +// on the module the root's import returned. +static PyObject * +lazy_import_replay_from(PyThreadState *tstate, PyObject *mod, + PyLazyImportObject *lz) +{ + if (!PyLazyImport_CheckExact(lz->lz_from)) { + PyObject *result = Py_NewRef(mod); + if (lz->lz_submodule) { + // A published child behaves like `import pkg.child as child`. + Py_ssize_t end = PyUnicode_GET_LENGTH(lz->lz_from); + Py_ssize_t dot = PyUnicode_FindChar(lz->lz_from, '.', 0, end, 1); + while (result != NULL && dot >= 0) { + Py_ssize_t start = dot + 1; + dot = PyUnicode_FindChar(lz->lz_from, '.', start, end, 1); + if (dot == -2) { + break; + } + PyObject *attr = PyUnicode_Substring( + lz->lz_from, start, dot < 0 ? end : dot); + PyObject *next = attr == NULL ? NULL : + _PyEval_ImportFrom(tstate, result, attr); + Py_XDECREF(attr); + Py_SETREF(result, lazy_import_resolve_result(tstate, next)); + } + if (dot == -2) { + Py_CLEAR(result); + } + } + return result; + } + PyObject *from = lazy_import_replay_from( + tstate, mod, (PyLazyImportObject *)lz->lz_from); + if (from == NULL) { + return NULL; + } + PyObject *obj = _PyEval_ImportFrom(tstate, from, lz->lz_attr); + Py_DECREF(from); + return lazy_import_resolve_result(tstate, obj); +} + +// Preserve the resolution error and attach the import's declaration location. +static void +lazy_import_add_exception_cause(PyThreadState *tstate, PyLazyImportObject *lz) +{ + if (!PyErr_Occurred() || lz->lz_code == NULL || lz->lz_instr_offset < 0) { + return; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyObject *name = lazy_import_name(lz); + if (name == NULL) { + goto done; + } + int lineno = PyCode_Addr2Line(lz->lz_code, lz->lz_instr_offset * 2); + const char *filename = PyUnicode_AsUTF8(lz->lz_code->co_filename); + if (filename == NULL) { + goto done; + } + const char *funcname = PyUnicode_AsUTF8(lz->lz_code->co_name); + if (funcname == NULL) { + goto done; + } + PyBaseExceptionObject *base_exc = (PyBaseExceptionObject *)exc; + if (base_exc->cause != NULL || base_exc->context != NULL || + base_exc->suppress_context) { + // Preserve the original chain, including `raise ... from None`. + PyObject *note = PyUnicode_FromFormat( + "lazy import of '%U' declared in %s at %s:%d", + name, funcname, filename, lineno); + if (note != NULL) { + PyObject *notes; + if (PyObject_GetOptionalAttr(exc, &_Py_ID(__notes__), ¬es) >= 0) { + if (notes == NULL || PySequence_Contains(notes, note) == 0) { + (void)_PyException_AddNote(exc, note); + } + Py_XDECREF(notes); + } + Py_DECREF(note); + } + goto done; + } + PyObject *msg = PyUnicode_FromFormat( + "lazy import of '%U' raised an exception during resolution", name); + if (msg == NULL) { + goto done; + } + PyObject *cause = PyObject_CallOneArg(PyExc_ImportError, msg); + Py_DECREF(msg); + if (cause == NULL) { + goto done; + } + _PyErr_SetRaisedException(tstate, cause); + _PyTraceback_Add(funcname, filename, lineno); + PyException_SetCause(exc, _PyErr_GetRaisedException(tstate)); + +done: + Py_XDECREF(name); + _PyErr_SetRaisedException(tstate, exc); +} + +int +_PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op) +{ + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(PyLazyImport_CheckExact(op)); + int active = ts->lazy_imports == NULL ? 0 : PySet_Contains(ts->lazy_imports, op); + assert(active >= 0); // Exact placeholders use identity hashing and equality. + return active; +} + +// Published submodules must use import semantics, including validation of +// the full name, rather than a module's dynamic attribute fallback. +int +_PyLazyImport_IsSubmodule(PyObject *op) +{ + return ((PyLazyImportObject *)op)->lz_submodule; +} + +int +_PyLazyImport_IsStrictSubmodule(PyObject *op) +{ + PyLazyImportObject *lz = (PyLazyImportObject *)op; + return lz->lz_submodule && lz->lz_validate; +} + +static int +lazy_import_is_child(PyObject *name, PyObject *parent) +{ + Py_ssize_t len = PyUnicode_GET_LENGTH(parent); + if (PyUnicode_GET_LENGTH(name) <= len || + PyUnicode_READ_CHAR(name, len) != '.') { + return 0; + } + return PyUnicode_Tailmatch(name, parent, 0, len, -1); +} + +// Retain the declaration's hook and location without retaining other imports. +static PyObject * +lazy_import_submodule(PyLazyImportObject *source, PyObject *fullname, + bool validate) +{ + assert(!source->lz_submodule); + PyObject *result = _PyLazyImport_New(NULL, (PyObject *)source, fullname, NULL); + if (result != NULL) { + FT_ATOMIC_STORE_INT_RELAXED( + PyThreadState_GET()->interp->imports.has_lazy_submodules, 1); + PyLazyImportObject *child = (PyLazyImportObject *)result; + child->lz_submodule = true; + child->lz_validate = validate; + child->lz_code = (PyCodeObject *)Py_XNewRef(source->lz_code); + child->lz_instr_offset = source->lz_instr_offset; + } + return result; +} + +static int +lazy_import_bind_children(PyThreadState *tstate, PyLazyImportObject *origin, + PyObject *module, PyObject *name, bool validate) +{ + if (!PyModule_Check(module)) { + return 0; + } + PyObject *dict = _PyModule_GetDict(module); + int is_package = PyDict_Contains(dict, &_Py_ID(__path__)); + if (is_package <= 0) { + if (is_package < 0 || origin == NULL || !validate) { + return is_package; + } + // Validate only the declaration being resolved. An unused invalid + // declaration must not make unrelated imports of this module fail. + int matches = lazy_import_is_child(origin->lz_from, name); + if (matches <= 0) { + return matches; + } + PyObject *child = lazy_import_submodule(origin, origin->lz_from, true); + PyObject *value = child == NULL ? NULL : + _PyImport_LoadLazyImportTstate(tstate, child); + Py_XDECREF(child); + int res = value == NULL ? -1 : 0; + Py_XDECREF(value); + return res; + } + // Module subclasses retain their attribute and descriptor protocols. + if (!PyModule_CheckExact(module)) { + return 0; + } + if (_Py_EnterRecursiveCallTstate(tstate, " while binding lazy submodules")) { + return -1; + } + PyObject *pending = _PyImport_GetLazySubmodules(tstate, name); + int res = pending == NULL && PyErr_Occurred() ? -1 : 0; + if (pending != NULL) { + Py_ssize_t pos = 0; + PyObject *attr, *source; + while (PyDict_Next(pending, &pos, &attr, &source)) { + if (!PyLazyImport_CheckExact(source)) { + continue; + } + PyObject *fullname = PyUnicode_FromFormat("%U.%U", name, attr); + if (fullname == NULL) { + res = -1; + break; + } + PyLazyImportObject *declaration = (PyLazyImportObject *)source; + int matches = 0; + if (origin != NULL) { + matches = PyObject_RichCompareBool(origin->lz_from, fullname, Py_EQ); + if (matches == 0) { + matches = lazy_import_is_child(origin->lz_from, fullname); + } + if (matches < 0) { + Py_DECREF(fullname); + res = -1; + break; + } + if (matches) { + declaration = origin; + } + } + PyObject *existing; + res = PyDict_GetItemRef(dict, attr, &existing); + PyLazyImportObject *previous = + res > 0 && PyLazyImport_CheckExact(existing) ? + (PyLazyImportObject *)existing : NULL; + if (previous != NULL && !previous->lz_submodule) { + previous = NULL; + } + PyLazyImportObject *previous_source = previous == NULL ? NULL : + (PyLazyImportObject *)previous->lz_builtins; + if (matches && previous_source != NULL && + previous_source->lz_builtins != origin->lz_builtins) { + // A shared package attribute cannot keep two independently + // mutable import hooks. Complete this declaration with its + // own hook before either namespace loses its root binding. + PyObject *child = lazy_import_submodule(origin, origin->lz_from, true); + PyObject *value = child == NULL ? NULL : + _PyImport_LoadLazyImportTstate(tstate, child); + Py_XDECREF(child); + res = value == NULL ? -1 : 0; + Py_XDECREF(value); + } + else if (res >= 0 && + (existing == NULL || + (matches && previous != NULL && + (previous_source != origin || !previous->lz_validate)))) { + PyObject *child = lazy_import_submodule( + declaration, fullname, matches != 0); + if (child == NULL) { + res = -1; + } + else if (existing == NULL) { + res = PyDict_SetDefaultRef(dict, attr, child, NULL); + } + else { + res = _PyDict_ReplaceItemIf(dict, attr, existing, child); + } + Py_XDECREF(child); + } + else if (res > 0) { + // Existing values win. Already imported packages may still + // have pending children of their own. + res = lazy_import_bind_children( + tstate, origin, existing, fullname, validate); + } + Py_XDECREF(existing); + Py_DECREF(fullname); + if (res < 0) { + break; + } + } + Py_DECREF(pending); + } + _Py_LeaveRecursiveCallTstate(tstate); + return res < 0 ? -1 : 0; +} + +static PyObject * +lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, + PyObject **imported_module, bool optional) { - PyLazyImportObject *lazy_import = PyLazyImportObject_CAST(op); + PyObject *obj = NULL; + PyObject *fromlist = NULL; + PyObject *import_func = NULL; + PyObject *resolving = NULL; + bool default_import = false; + assert(lazy_import != NULL); assert(PyLazyImport_CheckExact(lazy_import)); - return lazy_import_name(lazy_import); + + PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import; + + // Walk back to the placeholder IMPORT_NAME left, and the first lookup on it. + PyLazyImportObject *root = lz, *first = NULL; + while (PyLazyImport_CheckExact(root->lz_from)) { + first = root; + root = (PyLazyImportObject *)root->lz_from; + } + + if (_PyLazyImport_IsResolving(tstate, lazy_import)) { + if (root->lz_submodule) { + // Eager imports in a cycle can access an already published, + // partially initialized module through its parent. + PyObject *partial = PyImport_GetModule(root->lz_from); + if (partial != Py_None && (partial != NULL || PyErr_Occurred())) { + return partial; + } + Py_XDECREF(partial); + } + PyObject *name = lazy_import_name(lz); + if (name == NULL) { + return NULL; + } + PyObject *errmsg = PyUnicode_FromFormat( + "cannot import name %R (most likely due to a circular import)", + name); + Py_DECREF(name); + if (errmsg != NULL) { + PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg, + root->lz_from, NULL); + Py_DECREF(errmsg); + } + return NULL; + } + if (_Py_EnterRecursiveCallTstate(tstate, " while resolving a lazy import")) { + return NULL; + } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + if (ts->lazy_imports == NULL) { + ts->lazy_imports = PySet_New(NULL); + if (ts->lazy_imports == NULL) { + goto done; + } + } + resolving = ts->lazy_imports; + if (PySet_Add(resolving, lazy_import) < 0) { + goto done; + } + + // `from a import b, c`: import only the name being resolved. + // Keep an empty tuple intact for custom __import__ hooks. + if (first != NULL && root->lz_attr != NULL && + PyTuple_Check(root->lz_attr) && PyTuple_GET_SIZE(root->lz_attr) > 0) { + fromlist = PyTuple_Pack(1, first->lz_attr); + } + else { + fromlist = Py_NewRef(root->lz_attr != NULL ? root->lz_attr : Py_None); + } + if (fromlist == NULL) { + goto done; + } + + PyObject *globals = PyEval_GetGlobals(); + if (globals == NULL) { + globals = Py_None; + } + + PyLazyImportObject *source = root; + if (source->lz_submodule) { + source = (PyLazyImportObject *)source->lz_builtins; + } + assert(!source->lz_submodule); + bool optional_submodule = optional && root->lz_submodule && first == NULL; + if (!optional_submodule || root->lz_validate) { + if (PyMapping_GetOptionalItem(source->lz_builtins, &_Py_ID(__import__), + &import_func) < 0) { + goto done; + } + if (import_func == NULL) { + PyErr_SetString(PyExc_ImportError, "__import__ not found"); + goto done; + } + } + default_import = import_func == NULL || + _PyImport_IsDefaultImportFunc(tstate->interp, import_func); + optional_submodule = optional_submodule && default_import; + if (optional_submodule) { + // Unused siblings follow the pending-child importer. An active + // declaration still uses its current custom import hook. + obj = _PyImport_LoadLazySubmodule(tstate, root->lz_from); + if (obj == Py_None) { + Py_CLEAR(obj); + } + goto imported; + } + PyObject *name = Py_NewRef(root->lz_from); + if (first == NULL && !root->lz_submodule && + (fromlist == Py_None || + (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist) == 0)) && + default_import) { + Py_ssize_t dot = PyUnicode_FindChar( + name, '.', 0, PyUnicode_GET_LENGTH(name), 1); + if (dot == -2) { + Py_CLEAR(name); + } + else if (dot >= 0) { + PyObject *modules = _PyImport_GetModules(tstate->interp); + PyObject *module = NULL; + int cached = modules == NULL ? 0 : + PyMapping_GetOptionalItem(modules, name, &module); + if (cached > 0 && PyModule_Check(module)) { + PyObject *loaded = lazy_import_get_loaded_attr( + tstate, name, &_Py_ID(__name__)); + cached = loaded != NULL; + Py_XDECREF(loaded); + if (PyErr_Occurred()) { + cached = -1; + } + } + else if (module == Py_None) { + cached = 0; + } + Py_XDECREF(module); + if (cached < 0) { + Py_CLEAR(name); + } + else if (!cached) { + Py_SETREF(name, PyUnicode_Substring(name, 0, dot)); + } + } + if (name == NULL) { + goto done; + } + } + obj = _PyEval_ImportNameWithImport( + tstate, import_func, globals, globals, + name, fromlist, _PyLong_GetZero() + ); + if (obj != NULL && name != root->lz_from) { + int is_package = PyModule_CheckExact(obj) ? + PyDict_Contains(_PyModule_GetDict(obj), &_Py_ID(__path__)) : 0; + if (is_package < 0) { + Py_CLEAR(obj); + } + else if (!is_package) { + Py_DECREF(obj); + obj = _PyEval_ImportNameWithImport( + tstate, import_func, globals, globals, + root->lz_from, fromlist, _PyLong_GetZero()); + } + } + Py_DECREF(name); +imported: + obj = lazy_import_resolve_result(tstate, obj); + // The normal importer may publish this module on its parent. Custom + // hooks retain control of their own assignments to the parent. + if (imported_module != NULL && obj != NULL && PyModule_Check(obj) && + default_import) { + *imported_module = Py_NewRef(obj); + } + if (obj != NULL && + (first != NULL || (root->lz_submodule && !optional_submodule))) { + // Keep the hook and root result alive until all attribute lookups finish. + PyObject *from = obj; + obj = lazy_import_replay_from(tstate, from, lz); + Py_DECREF(from); + } + if (obj != NULL && imported_module != NULL && + root->lz_submodule && + default_import) { + Py_XSETREF(*imported_module, Py_NewRef(obj)); + } + if (obj != NULL && PyModule_Check(obj)) { + PyObject *path = lazy_import_path(lz); + PyLazyImportObject *origin = + root->lz_validate && !optional_submodule ? source : NULL; + if (path == NULL || lazy_import_bind_children( + tstate, origin, obj, path, default_import) < 0) { + Py_CLEAR(obj); + } + Py_XDECREF(path); + } + +done: + if (obj != NULL && + default_import && + _PyImport_ClearLazySubmodules(tstate, root->lz_from) < 0) { + Py_CLEAR(obj); + } + if (obj == NULL) { + lazy_import_add_exception_cause(tstate, lz); + } + assert(obj == NULL || !PyLazyImport_CheckExact(obj)); + if (resolving != NULL) { + // A failed set resize can leave the placeholder inserted. Removing by + // identity also permits greenlets to finish in a different order. + if (PySet_Discard(resolving, lazy_import) < 0) { + Py_CLEAR(obj); + } + if (PySet_GET_SIZE(resolving) == 0) { + // Keep the set, but release the capacity used by deep resolutions. + (void)PySet_Clear(resolving); + } + } + + Py_XDECREF(fromlist); + Py_XDECREF(import_func); + _Py_LeaveRecursiveCallTstate(tstate); + return obj; +} + +PyObject * +_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) +{ + return lazy_import_resolve_impl(tstate, lazy_import, NULL, false); +} + +// Loading pkg.child can replace a placeholder in pkg.child with the module +// before a from-import retrieves the value that belongs in that binding. +static int +lazy_import_replace_child(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace, + PyObject *child, PyObject *value) +{ + PyLazyImportObject *root = (PyLazyImportObject *)placeholder; + if (!PyLazyImport_CheckExact(root->lz_from)) { + return 0; + } + while (PyLazyImport_CheckExact(root->lz_from)) { + root = (PyLazyImportObject *)root->lz_from; + } + Py_ssize_t end = PyUnicode_GET_LENGTH(root->lz_from); + Py_ssize_t dot = PyUnicode_FindChar(root->lz_from, '.', 0, end, -1); + if (dot < 0) { + return dot == -1 ? 0 : -1; + } + if (end - dot - 1 != PyUnicode_GET_LENGTH(name)) { + return 0; + } + int matches = PyUnicode_Tailmatch(root->lz_from, name, dot + 1, end, 1); + if (matches <= 0) { + return matches; + } + PyObject *parent_name = PyUnicode_Substring(root->lz_from, 0, dot); + if (parent_name == NULL) { + return -1; + } + PyObject *modules = Py_XNewRef(_PyImport_GetModules(tstate->interp)); + PyObject *parent = NULL; + int rc = 0; + if (modules != NULL) { + rc = PyMapping_GetOptionalItem(modules, parent_name, &parent); + if (rc > 0 && PyModule_Check(parent) && + _PyModule_GetDict(parent) == namespace) { + rc = _PyDict_ReplaceItemIf(namespace, name, child, value); + } + } + Py_XDECREF(parent); + Py_XDECREF(modules); + Py_DECREF(parent_name); + return rc; +} + +static PyObject * +lazy_import_reify(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace, bool optional) +{ + int recursive = _PyLazyImport_IsResolving(tstate, placeholder); + PyObject *imported_module = NULL; + PyObject *value = lazy_import_resolve_impl( + tstate, placeholder, &imported_module, optional); + if (value == NULL) { + if (imported_module != NULL && PyDict_CheckExact(namespace) && + _PyLazyImport_IsSubmodule(placeholder)) { + // Importing the parent can succeed before a required descendant + // fails. Retain the placeholder so the same lookup can retry. + PyObject *exc = _PyErr_GetRaisedException(tstate); + (void)_PyDict_ReplaceItemIf( + namespace, name, imported_module, placeholder); + _PyErr_SetRaisedException(tstate, exc); + } + Py_XDECREF(imported_module); + return NULL; + } + // A cycle can inspect a partial module that may still fail to initialize. + if (recursive) { + Py_XDECREF(imported_module); + return value; + } + int rc; + if (PyDict_CheckExact(namespace)) { + rc = _PyDict_ReplaceItemIf(namespace, name, placeholder, value); + if (rc == 0 && imported_module != NULL) { + rc = lazy_import_replace_child( + tstate, placeholder, name, namespace, imported_module, value); + } + } + else if (Py_TYPE(namespace)->tp_as_mapping == NULL || + Py_TYPE(namespace)->tp_as_mapping->mp_ass_subscript == NULL) { + // Read-only namespaces can resolve a value without caching it. + Py_XDECREF(imported_module); + return value; + } + else { + // Custom namespaces retain their mapping protocol. Atomic replacement + // is only available for exact dictionaries. + PyObject *current; + rc = PyMapping_GetOptionalItem(namespace, name, ¤t); + if (rc > 0) { + if (current == placeholder) { + rc = PyObject_SetItem(namespace, name, value); + } + Py_DECREF(current); + } + } + if (rc < 0) { + Py_CLEAR(value); + } + Py_XDECREF(imported_module); + return value; +} + +PyObject * +_PyLazyImport_Reify(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace) +{ + return lazy_import_reify(tstate, placeholder, name, namespace, false); +} + +PyObject * +_PyLazyImport_ReifyOptional(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace) +{ + return lazy_import_reify(tstate, placeholder, name, namespace, true); } static PyObject * diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index b8cd6025c20ba5..55d8daabc4b29a 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -7,7 +7,7 @@ #include "pycore_fileutils.h" // _Py_wgetcwd #include "pycore_import.h" // _PyImport_GetNextModuleIndex() #include "pycore_interp.h" // PyInterpreterState.importlib -#include "pycore_lazyimportobject.h" // _PyLazyImportObject_Check() +#include "pycore_lazyimportobject.h" // PyLazyImport_CheckExact() #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() @@ -1299,32 +1299,21 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } -static PyObject * -try_load_lazy_submodule(PyModuleObject *m, PyObject *name) +static int +module_get_resolved_dict_item(PyObject *dict, PyObject *name, PyObject **result) { - PyObject *mod_name; - int rc = PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name); - if (rc <= 0) { - return NULL; - } - if (!PyUnicode_Check(mod_name)) { - Py_DECREF(mod_name); - return NULL; - } - PyObject *result = NULL; - _PyLazySubmoduleImportResult status = - _PyImport_TryLoadLazySubmodule(mod_name, name, &result); - Py_DECREF(mod_name); - if (status != _Py_LAZY_SUBMODULE_LOADED) { - assert(status == _Py_LAZY_SUBMODULE_ERROR || - status == _Py_LAZY_SUBMODULE_NOT_FOUND); - return NULL; + int rc = PyDict_GetItemRef(dict, name, result); + if (rc <= 0 || !PyLazyImport_CheckExact(*result)) { + return rc; } - if (PyDict_SetItem(m->md_dict, name, result) < 0) { - Py_DECREF(result); - return NULL; + PyThreadState *tstate = PyThreadState_GET(); + if (_PyLazyImport_IsResolving(tstate, *result)) { + Py_CLEAR(*result); + return 0; } - return result; + PyObject *value = _PyLazyImport_Reify(tstate, *result, name, dict); + Py_SETREF(*result, value); + return value == NULL ? -1 : 1; } PyObject* @@ -1332,12 +1321,55 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { // When suppress=1, this function suppresses AttributeError. PyObject *attr, *mod_name, *getattr; + PyObject *missing_import = NULL; attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); + if (attr == NULL) { + if (suppress == 1) { + if (PyErr_Occurred()) { + // pass up non-AttributeError exception + return NULL; + } + } + else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + // pass up non-AttributeError exception + return NULL; + } + PyErr_Clear(); + } + assert(m->md_dict != NULL); + int recheck_dict; + attr = _PyImport_TryLoadLazySubmodule((PyObject *)m, name, &recheck_dict); + if (attr != NULL || PyErr_Occurred()) { + return attr; + } + // A concurrent load may have bound the child and removed its pending + // registration after our initial dictionary lookup. + if (recheck_dict) { + if (Py_TYPE(m) != &PyModule_Type) { + PyObject *descr = _PyType_LookupRef(Py_TYPE(m), name); + if (descr != NULL) { + // Preserve the fallback after a descriptor raised. + recheck_dict = Py_TYPE(descr)->tp_descr_get == NULL; + Py_DECREF(descr); + } + } + if (recheck_dict && + PyDict_GetItemRef(m->md_dict, name, &attr) < 0) { + return NULL; + } + } + } if (attr) { if (PyLazyImport_CheckExact(attr)) { // gh-144957: Module __getattr__ should get a chance to provide // the attribute before resolving a lazy import placeholder. - if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + // Resolving __getattr__ itself must not invoke the hook. + getattr = NULL; + if (!_PyLazyImport_IsSubmodule(attr) && + !_PyUnicode_EqualToASCIIString(name, "__getattr__") && + module_get_resolved_dict_item( + m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { Py_DECREF(attr); return NULL; } @@ -1354,60 +1386,56 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) } PyErr_Clear(); } - PyObject *new_value = _PyImport_LoadLazyImportTstate( - PyThreadState_GET(), attr); - if (new_value == NULL) { - if (suppress && - PyErr_ExceptionMatches(PyExc_ImportCycleError)) { - // ImportCycleError is raised when a lazy object tries - // to import itself. In this case, the error should not - // propagate to the caller and instead treated as if the - // attribute doesn't exist. - PyErr_Clear(); - } + PyThreadState *tstate = PyThreadState_GET(); + if (suppress && _PyLazyImport_IsResolving(tstate, attr)) { + // Only direct reentry means this attribute is unavailable. Py_DECREF(attr); return NULL; } - - if (PyDict_SetItem(m->md_dict, name, new_value) < 0) { - Py_CLEAR(new_value); + PyObject *new_value = + (suppress || !_PyLazyImport_IsStrictSubmodule(attr)) ? + _PyLazyImport_ReifyOptional(tstate, attr, name, m->md_dict) : + _PyLazyImport_Reify(tstate, attr, name, m->md_dict); + if (new_value == NULL && !PyErr_Occurred()) { + if (suppress) { + Py_DECREF(attr); + } + else { + missing_import = attr; + } + goto missing; } Py_DECREF(attr); return new_value; } return attr; } - if (suppress == 1) { - if (PyErr_Occurred()) { - // pass up non-AttributeError exception - return NULL; - } - } - else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - // pass up non-AttributeError exception - return NULL; - } - PyErr_Clear(); - } - assert(m->md_dict != NULL); - attr = try_load_lazy_submodule(m, name); - if (attr != NULL) { - return attr; - } - if (PyErr_Occurred()) { - return NULL; - } - if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { +missing: + if (module_get_resolved_dict_item( + m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + Py_XDECREF(missing_import); return NULL; } if (getattr) { PyObject *result = PyObject_CallOneArg(getattr, name); - if (result == NULL && suppress == 1 && PyErr_ExceptionMatches(PyExc_AttributeError)) { - // suppress AttributeError + Py_DECREF(getattr); + if (result != NULL || !PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_XDECREF(missing_import); + return result; + } + if (suppress || missing_import != NULL) { PyErr_Clear(); } - Py_DECREF(getattr); + if (missing_import == NULL) { + return NULL; + } + } + if (missing_import != NULL) { + // A pending sibling permits dynamic attributes, but an actually + // missing module must still report its import name and declaration. + PyObject *result = _PyLazyImport_Reify( + PyThreadState_GET(), missing_import, name, m->md_dict); + Py_DECREF(missing_import); return result; } @@ -1589,11 +1617,14 @@ module_dir(PyObject *self, PyObject *args) if (dict != NULL) { if (PyDict_Check(dict)) { - PyObject *dirfunc = PyDict_GetItemWithError(dict, &_Py_ID(__dir__)); - if (dirfunc) { + PyObject *dirfunc; + int rc = module_get_resolved_dict_item( + dict, &_Py_ID(__dir__), &dirfunc); + if (rc > 0) { result = _PyObject_CallNoArgs(dirfunc); + Py_DECREF(dirfunc); } - else if (!PyErr_Occurred()) { + else if (rc == 0) { result = PyDict_Keys(dict); } } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fdfd5a0e609046..a425b572bb95be 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2193,80 +2193,30 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); int err; - PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); - + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); + if (v_o == NULL) { + err = -1; + } + } PyStackRef_CLOSE(mod_or_class_dict); ERROR_IF(err < 0); if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - ERROR_NO_POP(); - } - - if (PyLazyImport_CheckExact(v_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - Py_SETREF(v_o, l_v); - ERROR_IF(v_o == NULL); - } - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - ERROR_IF(err < 0); - if (v_o == NULL) { - /* namespace 2: builtins */ - v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - ERROR_IF(err < 0); - if (v_o == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - ERROR_IF(true); - } - } - if (PyLazyImport_CheckExact(v_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - Py_SETREF(v_o, l_v); - ERROR_IF(v_o == NULL); - } - } + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); + ERROR_IF(PyStackRef_IsNull(v)); + } + else { + v = PyStackRef_FromPyObjectSteal(v_o); } - v = PyStackRef_FromPyObjectSteal(v_o); } inst(LOAD_NAME, (-- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); ERROR_IF(v_o == NULL); - if (PyLazyImport_CheckExact(v_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - // cannot early-decref v_o as it may cause a side-effect on l_v - if (l_v == NULL) { - Py_DECREF(v_o); - ERROR_IF(true); - } - int err = PyDict_SetItem(GLOBALS(), name, l_v); - if (err < 0) { - Py_DECREF(v_o); - Py_DECREF(l_v); - ERROR_IF(true); - } - Py_SETREF(v_o, l_v); - } - v = PyStackRef_FromPyObjectSteal(v_o); } @@ -2428,6 +2378,13 @@ dummy_func( ERROR_NO_POP(); } } + else if (PyLazyImport_CheckExact(value_o)) { + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + if (value_o == NULL) { + ERROR_NO_POP(); + } + } PyStackRef_CLOSE(class_dict_st); value = PyStackRef_FromPyObjectSteal(value_o); } diff --git a/Python/ceval.c b/Python/ceval.c index c1fbcb0611384c..b90de789accd44 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3150,8 +3150,8 @@ _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, goto error; } - PyObject *args[6] = {name, globals, locals, fromlist, level, builtins}; - res = PyObject_Vectorcall(lazy_import_func, args, 6, NULL); + PyObject *args[5] = {name, globals, locals, fromlist, level}; + res = PyObject_Vectorcall(lazy_import_func, args, 5, NULL); error: Py_XDECREF(lazy_import_func); return res; @@ -3325,43 +3325,6 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) return NULL; } -PyObject * -_PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *v, PyObject *name) -{ - assert(PyLazyImport_CheckExact(v)); - assert(name); - assert(PyUnicode_Check(name)); - PyObject *ret; - PyLazyImportObject *d = (PyLazyImportObject *)v; - PyObject *mod = NULL; - // Only `from a import b` can take b off an already imported a; - // `import a.b as c` has to import a.b first. - if (d->lz_attr != NULL && PyTuple_Check(d->lz_attr) && - PyTuple_GET_SIZE(d->lz_attr) > 0) { - mod = PyImport_GetModule(d->lz_from); - } - if (mod != NULL) { - // Check if the module already has the attribute, if so, resolve it - // eagerly. - if (PyModule_Check(mod)) { - PyObject *mod_dict = PyModule_GetDict(mod); - if (mod_dict != NULL) { - if (PyDict_GetItemRef(mod_dict, name, &ret) < 0) { - Py_DECREF(mod); - return NULL; - } - if (ret != NULL) { - Py_DECREF(mod); - return ret; - } - } - } - Py_DECREF(mod); - } - - return _PyLazyImport_New(frame, d->lz_builtins, v, name); -} - #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ "BaseException is not allowed" @@ -3627,13 +3590,13 @@ _PyEval_GetANext(PyObject *aiter) void _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name, _PyStackRef *writeto) { + PyObject *namespace = globals; if (PyAnyDict_CheckExact(globals) && PyAnyDict_CheckExact(builtins)) { - _PyDict_LoadGlobalStackRef((PyDictObject *)globals, - (PyDictObject *)builtins, - name, writeto); + namespace = _PyDict_LoadGlobalStackRef((PyDictObject *)globals, + (PyDictObject *)builtins, + name, writeto); if (PyStackRef_IsNull(*writeto) && !PyErr_Occurred()) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ + // A missing key does not set an exception in the dictionary helper. _PyEval_FormatExcCheckArg(PyThreadState_GET(), PyExc_NameError, NAME_ERROR_MSG, name); } @@ -3648,6 +3611,7 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name } if (res == NULL) { /* namespace 2: builtins */ + namespace = builtins; if (PyMapping_GetOptionalItem(builtins, name, &res) < 0) { *writeto = PyStackRef_NULL; return; @@ -3665,20 +3629,10 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name PyObject *res_o = PyStackRef_AsPyObjectBorrow(*writeto); if (res_o != NULL && PyLazyImport_CheckExact(res_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(PyThreadState_GET(), res_o); + PyObject *l_v = _PyLazyImport_Reify( + PyThreadState_GET(), res_o, name, namespace); PyStackRef_CLOSE(writeto[0]); - if (l_v == NULL) { - assert(PyErr_Occurred()); - *writeto = PyStackRef_NULL; - return; - } - int err = PyDict_SetItem(globals, name, l_v); - if (err < 0) { - Py_DECREF(l_v); - *writeto = PyStackRef_NULL; - return; - } - *writeto = PyStackRef_FromPyObjectSteal(l_v); + *writeto = l_v == NULL ? PyStackRef_NULL : PyStackRef_FromPyObjectSteal(l_v); } } @@ -3709,32 +3663,39 @@ _PyEval_GetAwaitable(PyObject *iterable, int oparg) PyObject * _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name) { - PyObject *value; - if (frame->f_locals == NULL) { + PyObject *namespace = frame->f_locals; + if (namespace == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); return NULL; } - if (PyMapping_GetOptionalItem(frame->f_locals, name, &value) < 0) { + if (PyMapping_GetOptionalItem(namespace, name, &value) < 0) { return NULL; } if (value != NULL) { - return value; + goto found; } - if (PyDict_GetItemRef(frame->f_globals, name, &value) < 0) { + namespace = frame->f_globals; + if (PyDict_GetItemRef(namespace, name, &value) < 0) { return NULL; } if (value != NULL) { - return value; + goto found; } - if (PyMapping_GetOptionalItem(frame->f_builtins, name, &value) < 0) { + namespace = frame->f_builtins; + if (PyMapping_GetOptionalItem(namespace, name, &value) < 0) { return NULL; } if (value == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); + return NULL; + } +found: + if (PyLazyImport_CheckExact(value)) { + Py_SETREF(value, _PyLazyImport_Reify(tstate, value, name, namespace)); } return value; } diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 2e4e178b308406..749eaebca19dc5 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -627,7 +627,7 @@ PyDoc_STRVAR(_imp__set_lazy_attributes__doc__, "_set_lazy_attributes($module, modobj, name, /)\n" "--\n" "\n" -"Sets attributes to lazy submodules on the module, as side effects."); +"Remove the resolved module name from sys.lazy_modules."); #define _IMP__SET_LAZY_ATTRIBUTES_METHODDEF \ {"_set_lazy_attributes", _PyCFunction_CAST(_imp__set_lazy_attributes), METH_FASTCALL, _imp__set_lazy_attributes__doc__}, @@ -665,4 +665,4 @@ _imp__set_lazy_attributes(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=0974db098d601372 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ae9cf67e39955555 input=a9049054013a1b77]*/ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e3dc2e781a3beb..f34ec0431633e4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -10183,55 +10183,77 @@ break; } - /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - - case _LOAD_NAME_r01: { - CHECK_CURRENT_CACHED_VALUES(0); + case _LOAD_FROM_DICT_OR_GLOBALS_r11: { + CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef mod_or_class_dict; _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); + mod_or_class_dict = _stack_item_0; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); + int err; + stack_pointer[0] = mod_or_class_dict; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (PyLazyImport_CheckExact(v_o)) { + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); _PyFrame_StackPointerInvalidate(frame); - if (l_v == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + if (v_o == NULL) { + err = -1; } + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(mod_or_class_dict); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (v_o == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(l_v); - _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(v)) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); + } + else { + v = PyStackRef_FromPyObjectSteal(v_o); + } + _tos_cache0 = v; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _LOAD_NAME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); } v = PyStackRef_FromPyObjectSteal(v_o); _tos_cache0 = v; @@ -10603,6 +10625,17 @@ JUMP_TO_ERROR(); } } + else if (PyLazyImport_CheckExact(value_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + _PyFrame_StackPointerInvalidate(frame); + if (value_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 48f76ccf59f8bc..84b6a541f2c5bf 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9905,6 +9905,16 @@ JUMP_TO_LABEL(error); } } + else if (PyLazyImport_CheckExact(value_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + _PyFrame_StackPointerInvalidate(frame); + if (value_o == NULL) { + JUMP_TO_LABEL(error); + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9930,11 +9940,21 @@ _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); int err; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); _PyFrame_StackPointerInvalidate(frame); + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + err = -1; + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9945,81 +9965,17 @@ JUMP_TO_LABEL(error); } if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - } - JUMP_TO_LABEL(error); - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } - } - else { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(v)) { + JUMP_TO_LABEL(error); } } - v = PyStackRef_FromPyObjectSteal(v_o); + else { + v = PyStackRef_FromPyObjectSteal(v_o); + } stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10273,38 +10229,6 @@ if (v_o == NULL) { JUMP_TO_LABEL(error); } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - if (l_v == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(l_v); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; diff --git a/Python/import.c b/Python/import.c index 2fa63dd01833a6..6dfa267b3fde6d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -11,7 +11,6 @@ #include "pycore_interp.h" // struct _import_runtime_state #include "pycore_interpframe.h" #include "pycore_lazyimportobject.h" -#include "pycore_long.h" // _PyLong_GetZero #include "pycore_magic_number.h" // PYC_MAGIC_NUMBER_TOKEN #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_namespace.h" // _PyNamespace_Type @@ -25,7 +24,6 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_time.h" // _PyTime_AsMicroseconds() -#include "pycore_traceback.h" #include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() #include "pycore_weakref.h" // _PyWeakref_GET_REF() @@ -299,6 +297,7 @@ get_importtime_name(PyObject *name) return encoded; } +// Return whether the module was still initializing when found in the cache. static int import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *name) { @@ -344,7 +343,7 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n #undef import_level } - return 0; + return rc; } static void remove_importlib_frames(PyThreadState *tstate); @@ -3891,227 +3890,6 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level return NULL; } -PyObject * -_PyImport_ResolveName(PyThreadState *tstate, PyObject *name, - PyObject *globals, int level) -{ - return resolve_name(tstate, name, globals, level); -} - -// Look up, in order, the attributes recorded from the root placeholder to lz -// on the module the root's import returned. -static PyObject * -lazy_import_replay_from(PyThreadState *tstate, PyObject *mod, - PyLazyImportObject *lz) -{ - if (!PyLazyImport_CheckExact(lz->lz_from)) { - return Py_NewRef(mod); - } - PyObject *from = lazy_import_replay_from( - tstate, mod, (PyLazyImportObject *)lz->lz_from); - if (from == NULL) { - return NULL; - } - PyObject *obj = _PyEval_ImportFrom(tstate, from, lz->lz_attr); - Py_DECREF(from); - return obj; -} - -PyObject * -_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) -{ - PyObject *obj = NULL; - PyObject *fromlist = Py_None; - PyObject *import_func = NULL; - assert(lazy_import != NULL); - assert(PyLazyImport_CheckExact(lazy_import)); - - PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import; - PyInterpreterState *interp = tstate->interp; - - // Walk back to the placeholder IMPORT_NAME left, and the first lookup on it. - PyLazyImportObject *root = lz, *first = NULL; - while (PyLazyImport_CheckExact(root->lz_from)) { - first = root; - root = (PyLazyImportObject *)root->lz_from; - } - - // Acquire the global import lock to serialize reification - _PyImport_AcquireLock(interp); - - // Check if we are already importing this module, if so, then we want to - // return an error that indicates we've hit a cycle which will indicate - // the value isn't yet available. - PyObject *importing = interp->imports.lazy_importing_modules; - if (importing == NULL) { - importing = interp->imports.lazy_importing_modules = PySet_New(NULL); - if (importing == NULL) { - _PyImport_ReleaseLock(interp); - return NULL; - } - } - - assert(PyAnySet_CheckExact(importing)); - int is_loading = _PySet_Contains((PySetObject *)importing, lazy_import); - if (is_loading < 0) { - _PyImport_ReleaseLock(interp); - return NULL; - } - else if (is_loading == 1) { - PyObject *name = _PyLazyImport_GetName(lazy_import); - if (name == NULL) { - _PyImport_ReleaseLock(interp); - return NULL; - } - PyObject *errmsg = PyUnicode_FromFormat( - "cannot import name %R (most likely due to a circular import)", - name); - if (errmsg == NULL) { - Py_DECREF(name); - _PyImport_ReleaseLock(interp); - return NULL; - } - PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg, - root->lz_from, NULL); - Py_DECREF(errmsg); - Py_DECREF(name); - _PyImport_ReleaseLock(interp); - return NULL; - } - else if (PySet_Add(importing, lazy_import) < 0) { - goto error; - } - - if (root->lz_attr != NULL) { - // `from a import b, c`: import only the name being resolved. - // Keep an empty tuple intact for custom __import__ hooks. - fromlist = first && PyTuple_GET_SIZE(root->lz_attr) > 0 - ? PyTuple_Pack(1, first->lz_attr) - : Py_NewRef(root->lz_attr); - if (fromlist == NULL) { - goto error; - } - } - - PyObject *globals = PyEval_GetGlobals(); - - if (PyMapping_GetOptionalItem(root->lz_builtins, &_Py_ID(__import__), - &import_func) < 0) { - goto error; - } - if (import_func == NULL) { - PyErr_SetString(PyExc_ImportError, "__import__ not found"); - goto error; - } - obj = _PyEval_ImportNameWithImport( - tstate, import_func, globals, globals, - root->lz_from, fromlist, _PyLong_GetZero() - ); - if (obj == NULL) { - goto error; - } - - PyObject *from = obj; - obj = lazy_import_replay_from(tstate, from, lz); - Py_DECREF(from); - if (obj == NULL) { - goto error; - } - - assert(!PyLazyImport_CheckExact(obj)); - - goto ok; - -error: - Py_CLEAR(obj); - - // If an error occurred and we have frame information, add it to the - // exception. - if (PyErr_Occurred() && lz->lz_code != NULL && lz->lz_instr_offset >= 0) { - // Get the current exception - this already has the full traceback - // from the access point. - PyObject *exc = _PyErr_GetRaisedException(tstate); - - // Get import name - this can fail and set an exception. - PyObject *import_name = _PyLazyImport_GetName(lazy_import); - if (!import_name) { - // Failed to get import name, just restore original exception. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Resolve line number from instruction offset on demand. - int lineno = PyCode_Addr2Line((PyCodeObject *)lz->lz_code, - lz->lz_instr_offset*2); - - // Get strings - these can return NULL on encoding errors. - const char *filename_str = PyUnicode_AsUTF8(lz->lz_code->co_filename); - if (!filename_str) { - // Unicode conversion failed - clear error and restore original - // exception. - PyErr_Clear(); - Py_DECREF(import_name); - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - const char *funcname_str = PyUnicode_AsUTF8(lz->lz_code->co_name); - if (!funcname_str) { - // Unicode conversion failed - clear error and restore original - // exception. - PyErr_Clear(); - Py_DECREF(import_name); - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Create a cause exception showing where the lazy import was declared. - PyObject *msg = PyUnicode_FromFormat( - "lazy import of '%U' raised an exception during resolution", - import_name - ); - Py_DECREF(import_name); // Done with import_name. - - if (!msg) { - // Failed to create message - restore original exception. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - PyObject *cause_exc = PyObject_CallOneArg(PyExc_ImportError, msg); - Py_DECREF(msg); // Done with msg. - - if (!cause_exc) { - // Failed to create exception - restore original. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Add traceback entry for the lazy import declaration. - _PyErr_SetRaisedException(tstate, cause_exc); - _PyTraceback_Add(funcname_str, filename_str, lineno); - PyObject *cause_with_tb = _PyErr_GetRaisedException(tstate); - - // Set the cause on the original exception. - PyException_SetCause(exc, cause_with_tb); // Steals ref to cause_with_tb. - - // Restore the original exception with its full traceback. - _PyErr_SetRaisedException(tstate, exc); - } - -ok: - if (PySet_Discard(importing, lazy_import) < 0) { - Py_CLEAR(obj); - } - - // Release the global import lock. - _PyImport_ReleaseLock(interp); - - Py_XDECREF(fromlist); - Py_XDECREF(import_func); - return obj; -} - static PyObject * import_find_and_load_with_name(PyThreadState *tstate, PyObject *abs_name, PyObject *find_and_load, @@ -4180,9 +3958,9 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) tstate, abs_name, &_Py_ID(_find_and_load), NULL); } -static PyObject * -get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals, - int level) +PyObject * +_PyImport_GetAbsName(PyThreadState *tstate, PyObject *name, PyObject *globals, + int level) { if (level > 0) { return resolve_name(tstate, name, globals, level); @@ -4194,11 +3972,41 @@ get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals, return Py_NewRef(name); } -PyObject * -_PyImport_GetAbsName(PyThreadState *tstate, PyObject *name, - PyObject *globals, int level) + +// A cached eager import must also replace a published lazy binding. In +// particular, an earlier failed descendant must not keep hiding this module. +static int +import_bind_cached_submodule(PyThreadState *tstate, PyObject *name, PyObject *mod) { - return get_abs_name(tstate, name, globals, level); + PyObject *pending = LAZY_PENDING_SUBMODULES(tstate->interp); + if (pending == NULL || PyDict_GET_SIZE(pending) == 0) { + return 0; + } + Py_ssize_t end = PyUnicode_GET_LENGTH(name); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, -1); + if (dot < 0) { + return dot == -2 ? -1 : 0; + } + PyObject *parent_name = PyUnicode_Substring(name, 0, dot); + if (parent_name == NULL) { + return -1; + } + PyObject *parent = import_get_module(tstate, parent_name); + Py_DECREF(parent_name); + int res = 0; + if (parent != NULL && PyModule_CheckExact(parent)) { + PyObject *attr = PyUnicode_Substring(name, dot + 1, end); + PyObject *dict = _PyModule_GetDict(parent), *existing = NULL; + res = attr == NULL ? -1 : PyDict_GetItemRef(dict, attr, &existing); + if (res > 0 && PyLazyImport_CheckExact(existing) && + _PyLazyImport_IsSubmodule(existing)) { + res = _PyDict_ReplaceItemIf(dict, attr, existing, mod); + } + Py_XDECREF(existing); + Py_XDECREF(attr); + } + Py_XDECREF(parent); + return PyErr_Occurred() ? -1 : 0; } @@ -4232,7 +4040,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, goto error; } - abs_name = get_abs_name(tstate, name, globals, level); + abs_name = _PyImport_GetAbsName(tstate, name, globals, level); if (abs_name == NULL) { goto error; } @@ -4243,7 +4051,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } if (mod != NULL && mod != Py_None) { - if (import_ensure_initialized(tstate->interp, mod, abs_name) < 0) { + int initializing = import_ensure_initialized(tstate->interp, mod, abs_name); + if (initializing < 0) { goto error; } /* Verify the module is still in sys.modules. Another thread may have @@ -4266,6 +4075,11 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } else { Py_DECREF(mod_check); + if (!initializing && + FT_ATOMIC_LOAD_INT_RELAXED(interp->imports.has_lazy_submodules) && + import_bind_cached_submodule(tstate, abs_name, mod) < 0) { + goto error; + } } } else { @@ -4358,7 +4172,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } -// ensure we have the set for the parent module name in sys.lazy_modules. +// Ensure a dict of pending submodule names exists for the parent. // Returns a new reference. static PyObject * ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) @@ -4369,7 +4183,7 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) &lazy_submodules); if (err == 0) { // value isn't present - lazy_submodules = PySet_New(NULL); + lazy_submodules = PyDict_New(); if (lazy_submodules != NULL && _PyDict_SetItem_LockHeld(lazy_modules, parent, lazy_submodules) < 0) { @@ -4385,66 +4199,48 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) // is accessed, _Py_module_getattro_impl will check lazy_pending_submodules // and trigger the import. static int -register_lazy_on_parent(PyThreadState *tstate, PyObject *name) +register_lazy_on_parent(PyThreadState *tstate, PyObject *name, + PyObject *source, int required) { - int ret = -1; - PyObject *parent = NULL; - PyObject *child = NULL; - - PyInterpreterState *interp = tstate->interp; - PyObject *lazy_pending_submodules = LAZY_PENDING_SUBMODULES(interp); - assert(lazy_pending_submodules != NULL); - - Py_INCREF(name); + PyDictObject *pending = + (PyDictObject *)LAZY_PENDING_SUBMODULES(tstate->interp); + assert(pending != NULL); + Py_ssize_t end = PyUnicode_GET_LENGTH(name); while (true) { - Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, - PyUnicode_GET_LENGTH(name), -1); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, -1); if (dot < 0) { - PyObject *lazy_submodules = ensure_lazy_pending_submodules( - (PyDictObject *)lazy_pending_submodules, name); - if (lazy_submodules == NULL) { - goto done; - } - Py_DECREF(lazy_submodules); - ret = 0; - goto done; + return dot == -1 ? 0 : -1; } - parent = PyUnicode_Substring(name, 0, dot); + PyObject *parent = PyUnicode_Substring(name, 0, dot); if (parent == NULL) { - goto done; + return -1; } - Py_XDECREF(child); - child = PyUnicode_Substring(name, dot + 1, PyUnicode_GET_LENGTH(name)); + PyObject *child = PyUnicode_Substring(name, dot + 1, end); if (child == NULL) { - goto done; + Py_DECREF(parent); + return -1; } - PyObject *lazy_submodules = ensure_lazy_pending_submodules( - (PyDictObject *)lazy_pending_submodules, parent); - if (lazy_submodules == NULL) { - goto done; + PyObject *submodules = ensure_lazy_pending_submodules(pending, parent); + Py_DECREF(parent); + int err = -1; + if (submodules != NULL) { + err = required ? PyDict_SetItem(submodules, child, source) + : PyDict_SetDefaultRef(submodules, child, Py_False, NULL); } - - if (PySet_Add(lazy_submodules, child) < 0) { - Py_DECREF(lazy_submodules); - goto done; + Py_DECREF(child); + Py_XDECREF(submodules); + if (err < 0) { + return -1; } - Py_DECREF(lazy_submodules); - - Py_SETREF(name, parent); - parent = NULL; + end = dot; + required = 1; } - -done: - Py_XDECREF(child); - Py_XDECREF(parent); - Py_XDECREF(name); - return ret; } static int register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, - PyObject *from) + PyObject *from, PyObject *source) { PyObject *fromname = PyUnicode_FromFormat("%U.%U", abs_name, from); if (fromname == NULL) { @@ -4453,80 +4249,174 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, // Add the module name to sys.lazy_modules set (PEP 810). PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - if (PySet_Add(lazy_modules, fromname) < 0) { - Py_DECREF(fromname); - return -1; + int res = PySet_Add(lazy_modules, fromname); + if (res == 0) { + res = register_lazy_on_parent(tstate, fromname, source, 0); } - - int res = register_lazy_on_parent(tstate, fromname); Py_DECREF(fromname); return res; } -_PyLazySubmoduleImportResult -_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name, - PyObject **result) +// Return a private snapshot: publishing children can run finalizers. +// A missing registration returns NULL without an exception. +PyObject * +_PyImport_GetLazySubmodules(PyThreadState *tstate, PyObject *name) { - assert(result != NULL); - *result = NULL; - - PyThreadState *tstate = _PyThreadState_GET(); - PyInterpreterState *interp = tstate->interp; - PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); - if (lazy_pending == NULL) { - return _Py_LAZY_SUBMODULE_NOT_FOUND; + PyObject *pending = LAZY_PENDING_SUBMODULES(tstate->interp); + PyObject *children; + if (pending == NULL || PyDict_GetItemRef(pending, name, &children) <= 0) { + return NULL; } + PyObject *result = PyDict_Copy(children); + Py_DECREF(children); + return result; +} - PyObject *pending_set; - int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); - if (rc < 0) { - return _Py_LAZY_SUBMODULE_ERROR; +// Release declarations on completed paths from the leaves upward. Keep the +// names for cached children whose parents can be replaced or fail to initialize. +int +_PyImport_ClearLazySubmodules(PyThreadState *tstate, PyObject *name) +{ + PyObject *pending = LAZY_PENDING_SUBMODULES(tstate->interp); + if (pending == NULL || PyDict_GET_SIZE(pending) == 0) { + return 0; } - if (rc == 0) { - return _Py_LAZY_SUBMODULE_NOT_FOUND; + name = Py_NewRef(name); + int res = 0; + while (true) { + Py_ssize_t end = PyUnicode_GET_LENGTH(name); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, -1); + if (dot < 0) { + res = dot == -2 ? -1 : 0; + break; + } + PyObject *parent = NULL, *child = NULL, *siblings = NULL; + PyObject *source = NULL, *children = NULL, *spec = NULL; + PyObject *mod = NULL, *current = NULL; + res = 0; + parent = PyUnicode_Substring(name, 0, dot); + child = PyUnicode_Substring(name, dot + 1, end); + if (parent == NULL || child == NULL || + PyDict_GetItemRef(pending, parent, &siblings) <= 0 || + PyDict_GetItemRef(siblings, child, &source) <= 0 || + !PyLazyImport_CheckExact(source)) { + goto next; + } + mod = import_get_module(tstate, name); + if (mod == NULL || mod == Py_None) { + goto next; + } + int rc = PyObject_GetOptionalAttr(mod, &_Py_ID(__spec__), &spec); + if (rc < 0 || (rc > 0 && _PyModuleSpec_IsInitializing(spec) != 0)) { + // Cleanup must not turn successful imports into failures merely + // because a module supplies unusual spec metadata. + if (PyErr_ExceptionMatches(PyExc_Exception)) { + PyErr_Clear(); + } + goto next; + } + current = import_get_module(tstate, name); + if (current != mod) { + goto next; + } + // Registration writes descendants before ancestors. Capture the edge + // first, so a concurrent declaration either changes its identity or + // restores it after our conditional replacement. + children = _PyImport_GetLazySubmodules(tstate, name); + if (children != NULL) { + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(children, &pos, &key, &value)) { + if (PyLazyImport_CheckExact(value)) { + goto next; + } + } + } + if (!PyErr_Occurred()) { + res = _PyDict_ReplaceItemIf(siblings, child, source, Py_True); + if (res > 0) { + Py_SETREF(name, Py_NewRef(parent)); + } + } + next: + Py_XDECREF(current); + Py_XDECREF(mod); + Py_XDECREF(spec); + Py_XDECREF(parent); + Py_XDECREF(child); + Py_XDECREF(siblings); + Py_XDECREF(source); + Py_XDECREF(children); + if (PyErr_Occurred()) { + res = -1; + } + if (res <= 0) { + break; + } } + Py_DECREF(name); + return res < 0 ? -1 : 0; +} - int contains = PySet_Contains(pending_set, attr_name); - if (contains < 0) { - Py_DECREF(pending_set); - return _Py_LAZY_SUBMODULE_ERROR; +// Return None when no module exists, without hiding exceptions from a loader. +PyObject * +_PyImport_LoadLazySubmodule(PyThreadState *tstate, PyObject *name) +{ + // Match eager from-imports: accept a partial module in an import cycle. + PyObject *mod = PyImport_GetModule(name); + if ((mod == NULL && !PyErr_Occurred()) || mod == Py_None) { + Py_XDECREF(mod); + mod = import_find_and_load_with_name( + tstate, name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); } - if (contains == 0) { - Py_DECREF(pending_set); - return _Py_LAZY_SUBMODULE_NOT_FOUND; + if (mod == NULL) { + remove_importlib_frames(tstate); } + return mod; +} - PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); - if (full_name == NULL) { - Py_DECREF(pending_set); - return _Py_LAZY_SUBMODULE_ERROR; +PyObject * +_PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, + int *recheck_dict) +{ + *recheck_dict = 0; + PyObject *mod_dict = _PyModule_GetDict(module); + PyObject *mod_name; + if (PyDict_GetItemRef(mod_dict, &_Py_ID(__name__), &mod_name) <= 0) { + return NULL; } - - PyObject *mod = import_find_and_load_with_name( - tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); - if (mod == NULL) { - Py_DECREF(pending_set); - Py_DECREF(full_name); - remove_importlib_frames(tstate); - return _Py_LAZY_SUBMODULE_ERROR; + PyObject *mod = NULL, *pending_children = NULL; + if (!PyUnicode_Check(mod_name)) { + goto done; } - if (mod == Py_None) { - Py_DECREF(mod); - Py_DECREF(pending_set); - Py_DECREF(full_name); - return _Py_LAZY_SUBMODULE_NOT_FOUND; + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(tstate->interp); + if (lazy_pending == NULL || + PyDict_GetItemRef(lazy_pending, mod_name, &pending_children) <= 0) { + goto done; } - if (PySet_Discard(pending_set, attr_name) < 0) { - Py_DECREF(mod); - Py_DECREF(pending_set); - Py_DECREF(full_name); - return _Py_LAZY_SUBMODULE_ERROR; + *recheck_dict = 1; + if (PyDict_Contains(pending_children, attr_name) <= 0) { + goto done; + } + PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); + if (full_name == NULL) { + goto done; + } + mod = _PyImport_LoadLazySubmodule(tstate, full_name); + /* Keep missing children pending: a finder may provide them later. */ + if (mod != NULL && (mod == Py_None || + PyDict_SetItem(mod_dict, attr_name, mod) < 0 || + _PyImport_ClearLazySubmodules(tstate, full_name) < 0)) { + Py_CLEAR(mod); } - Py_DECREF(pending_set); Py_DECREF(full_name); - *result = mod; - return _Py_LAZY_SUBMODULE_LOADED; + +done: + Py_XDECREF(pending_children); + Py_DECREF(mod_name); + return mod; } PyObject * @@ -4546,12 +4436,11 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, return NULL; } - PyObject *abs_name = get_abs_name(tstate, name, globals, level); + PyObject *abs_name = _PyImport_GetAbsName(tstate, name, globals, level); if (abs_name == NULL) { return NULL; } - PyInterpreterState *interp = tstate->interp; _PyInterpreterFrame *frame = _PyEval_GetFrame(); if (frame == NULL || frame->f_globals != frame->f_locals) { Py_DECREF(abs_name); @@ -4560,71 +4449,69 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, return NULL; } + PyObject *res = NULL; + if (fromlist != NULL && PyUnicode_Check(fromlist)) { + fromlist = PyTuple_Pack(1, fromlist); + if (fromlist == NULL) { + goto done; + } + } + else { + fromlist = Py_NewRef(fromlist != NULL ? fromlist : Py_None); + } + // Check if the filter disables the lazy import. - // We must hold a reference to the filter while calling it to prevent - // use-after-free if another thread replaces it via - // PyImport_SetLazyImportsFilter. - LAZY_IMPORTS_LOCK(interp); - PyObject *filter = Py_XNewRef(LAZY_IMPORTS_FILTER(interp)); - LAZY_IMPORTS_UNLOCK(interp); + PyObject *filter = PyImport_GetLazyImportsFilter(); if (filter != NULL) { PyObject *modname; if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &modname) < 0) { Py_DECREF(filter); - Py_DECREF(abs_name); - return NULL; + goto done; } if (modname == NULL) { assert(!PyErr_Occurred()); modname = Py_NewRef(Py_None); } - if (fromlist == NULL) { - assert(!PyErr_Occurred()); - fromlist = Py_None; - } PyObject *args[] = {modname, abs_name, fromlist}; - PyObject *res = PyObject_Vectorcall(filter, args, 3, NULL); + PyObject *decision = PyObject_Vectorcall(filter, args, 3, NULL); Py_DECREF(modname); Py_DECREF(filter); - if (res == NULL) { - Py_DECREF(abs_name); - return NULL; + if (decision == NULL) { + goto done; } - int is_true = PyObject_IsTrue(res); - Py_DECREF(res); + int is_true = PyObject_IsTrue(decision); + Py_DECREF(decision); if (is_true < 0) { - Py_DECREF(abs_name); - return NULL; + goto done; } if (!is_true) { - Py_DECREF(abs_name); - return PyImport_ImportModuleLevelObject( + res = PyImport_ImportModuleLevelObject( name, globals, locals, fromlist, level ); + goto done; } } - // here, 'filter' is either NULL or is equivalent to a borrowed reference - if (fromlist && PyUnicode_Check(fromlist)) { - fromlist = PyTuple_Pack(1, fromlist); - if (fromlist == NULL) { - Py_DECREF(abs_name); - return NULL; + // Validate the entire fromlist before registering any pending imports. + if (PyTuple_Check(fromlist)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { + PyObject *item = PyTuple_GET_ITEM(fromlist, i); + if (!PyUnicode_Check(item)) { + _PyErr_Format(tstate, PyExc_TypeError, + "Item in ``from list'' must be str, not %.200s", + Py_TYPE(item)->tp_name); + goto done; + } } } - else { - Py_XINCREF(fromlist); - } - PyObject *res = _PyLazyImport_New(frame, builtins, abs_name, fromlist); + res = _PyLazyImport_New(frame, builtins, abs_name, fromlist); if (res == NULL) { - Py_XDECREF(fromlist); - Py_DECREF(abs_name); - return NULL; + goto done; } // Add the module name to sys.lazy_modules set (PEP 810). @@ -4633,27 +4520,26 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, goto error; } - if (fromlist && PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist)) { + if (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist)) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { if (register_from_lazy_on_parent(tstate, abs_name, - PyTuple_GET_ITEM(fromlist, i)) < 0) + PyTuple_GET_ITEM(fromlist, i), res) < 0) { goto error; } } } - else if (register_lazy_on_parent(tstate, abs_name) < 0) { + else if (register_lazy_on_parent(tstate, abs_name, res, 1) < 0) { goto error; } - Py_XDECREF(fromlist); - Py_DECREF(abs_name); - return res; + goto done; error: + Py_CLEAR(res); +done: Py_XDECREF(fromlist); Py_DECREF(abs_name); - Py_DECREF(res); - return NULL; + return res; } PyObject * @@ -4865,7 +4751,6 @@ _PyImport_ClearCore(PyInterpreterState *interp) Py_CLEAR(LAZY_IMPORT_FUNC(interp)); Py_CLEAR(interp->imports.lazy_pending_submodules); Py_CLEAR(interp->imports.lazy_modules); - Py_CLEAR(interp->imports.lazy_importing_modules); Py_CLEAR(interp->imports.lazy_imports_filter); } @@ -5660,18 +5545,21 @@ _imp._set_lazy_attributes modobj: object name: unicode / -Sets attributes to lazy submodules on the module, as side effects. +Remove the resolved module name from sys.lazy_modules. [clinic start generated code]*/ static PyObject * _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, PyObject *name) -/*[clinic end generated code: output=3369bb3242b1f043 input=38ea6f30956dd7d6]*/ +/*[clinic end generated code: output=3369bb3242b1f043 input=900339e013ab2b82]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); if (PySet_Discard(LAZY_MODULES(interp), name) < 0) { return NULL; } + if (_PyImport_ClearLazySubmodules(_PyThreadState_GET(), name) < 0) { + return NULL; + } Py_RETURN_NONE; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 21f275f27cafe0..89423d6c6900af 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2176,7 +2176,12 @@ break; } - /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ + case _LOAD_FROM_DICT_OR_GLOBALS: { + JitOptRef v; + v = sym_new_not_null(ctx); + stack_pointer[-1] = v; + break; + } case _LOAD_NAME: { JitOptRef v; diff --git a/Python/pystate.c b/Python/pystate.c index 737bf0f216bd34..1222c0632b7f7f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1630,6 +1630,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; + _tstate->lazy_imports = NULL; #ifdef _Py_TIER2 _tstate->jit_tracer_state = NULL; @@ -1873,6 +1874,9 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(tstate->context); + // Finalizers above may resolve imports and create this set. + Py_CLEAR(((_PyThreadStateImpl *)tstate)->lazy_imports); + #ifdef Py_GIL_DISABLED // Each thread should clear own freelists in free-threading builds. struct _Py_freelists *freelists = _Py_freelists_GET();