Skip to content

Commit 1f21498

Browse files
committed
Remove resolved names from sys.lazy_modules consistently
Names only left sys.lazy_modules through _imp._set_lazy_attributes(), which the import machinery calls from _find_and_load_unlocked(). Two cases never reached it, so their names were recorded and then kept forever: - A lazy import of a module already in sys.modules. _find_and_load() returns early, so nothing ever discards the name. Do not record it in the first place. - The "pkg.attr" entry for `lazy from pkg import attr`. The import machinery only discards module names, and attr is often not a module. Discard it when the lazy object is reified, where the name is already known and has been resolved either way. Submodules that are not yet loaded are still tracked: loading a package does not load its submodules, so those imports can still fire. Names whose reification failed also stay tracked, since the import can still happen.
1 parent bba171e commit 1f21498

3 files changed

Lines changed: 143 additions & 7 deletions

File tree

‎Lib/test/test_lazy_import/__init__.py‎

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,8 @@ def test_sys_lazy_modules(self):
5353
self.fail('lazy import failed')
5454

5555
self.assertFalse("test.test_lazy_import.data.basic2" in sys.modules)
56-
self.assertIn("test.test_lazy_import.data", sys.lazy_modules)
56+
# The package is already loaded, so it is not a pending import.
57+
self.assertNotIn("test.test_lazy_import.data", sys.lazy_modules)
5758
self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules)
5859
test.test_lazy_import.data.basic_from_unused.basic2
5960
self.assertNotIn("test.test_import.data", sys.lazy_modules)
@@ -1252,6 +1253,103 @@ def test_lazy_module_without_children_is_tracked(self):
12521253
""")
12531254
assert_python_ok("-c", code)
12541255

1256+
def test_already_loaded_module_is_not_tracked(self):
1257+
"""A lazy import of an already loaded module should not be tracked."""
1258+
code = textwrap.dedent("""
1259+
import sys
1260+
1261+
# Loaded by a regular import.
1262+
import json
1263+
lazy import json as lazy_json
1264+
assert "json" not in sys.lazy_modules, (
1265+
f"expected 'json' not in sys.lazy_modules, got {sys.lazy_modules}"
1266+
)
1267+
1268+
# Loaded by reifying an earlier lazy import.
1269+
lazy import base64
1270+
_ = base64.b64encode
1271+
lazy import base64 as lazy_base64
1272+
assert "base64" not in sys.lazy_modules, (
1273+
f"expected 'base64' not in sys.lazy_modules, got {sys.lazy_modules}"
1274+
)
1275+
""")
1276+
assert_python_ok("-c", code)
1277+
1278+
def test_already_loaded_submodule_is_not_tracked(self):
1279+
"""`lazy from` a loaded submodule should not be tracked either."""
1280+
code = textwrap.dedent("""
1281+
import sys
1282+
import test.test_lazy_import.data.pkg.b
1283+
lazy from test.test_lazy_import.data.pkg import b
1284+
assert "test.test_lazy_import.data.pkg.b" not in sys.lazy_modules, (
1285+
f"expected 'pkg.b' untracked, got {sys.lazy_modules}"
1286+
)
1287+
""")
1288+
assert_python_ok("-c", code)
1289+
1290+
def test_attribute_entry_removed_on_reification(self):
1291+
"""`lazy from x import attr` should untrack "x.attr" once resolved."""
1292+
code = textwrap.dedent("""
1293+
import sys
1294+
lazy from test.test_lazy_import.data.basic2 import x
1295+
assert "test.test_lazy_import.data.basic2.x" in sys.lazy_modules, (
1296+
f"expected 'basic2.x' tracked, got {sys.lazy_modules}"
1297+
)
1298+
_ = x
1299+
assert "test.test_lazy_import.data.basic2.x" not in sys.lazy_modules, (
1300+
f"expected 'basic2.x' untracked, got {sys.lazy_modules}"
1301+
)
1302+
""")
1303+
assert_python_ok("-c", code)
1304+
1305+
def test_failed_reification_stays_tracked(self):
1306+
"""A lazy import that fails to resolve must stay tracked."""
1307+
code = textwrap.dedent("""
1308+
import sys
1309+
lazy import test.test_lazy_import.data.broken_module
1310+
try:
1311+
_ = test.test_lazy_import.data.broken_module
1312+
except ValueError:
1313+
pass
1314+
else:
1315+
raise AssertionError("ValueError was not raised")
1316+
assert "test.test_lazy_import.data.broken_module" in sys.lazy_modules, (
1317+
f"failed reification must stay tracked, got {sys.lazy_modules}"
1318+
)
1319+
""")
1320+
assert_python_ok("-c", code)
1321+
1322+
def test_blocked_module_is_still_tracked(self):
1323+
"""A ``None`` entry in sys.modules must not count as loaded."""
1324+
code = textwrap.dedent("""
1325+
import sys
1326+
sys.modules['test.test_lazy_import.data.basic2'] = None
1327+
lazy import test.test_lazy_import.data.basic2
1328+
assert "test.test_lazy_import.data.basic2" in sys.lazy_modules, (
1329+
f"blocked module must stay tracked, got {sys.lazy_modules}"
1330+
)
1331+
""")
1332+
assert_python_ok("-c", code)
1333+
1334+
def test_pending_submodule_is_still_tracked(self):
1335+
"""`lazy from` a submodule that is not loaded must stay tracked."""
1336+
code = textwrap.dedent("""
1337+
import sys
1338+
lazy from test.test_lazy_import.data.pkg import b
1339+
assert "test.test_lazy_import.data.pkg.b" in sys.lazy_modules, (
1340+
f"expected 'pkg.b' tracked, got {sys.lazy_modules}"
1341+
)
1342+
import test.test_lazy_import.data.pkg
1343+
assert "test.test_lazy_import.data.pkg.b" not in sys.modules, (
1344+
"loading the package must not load the submodule"
1345+
)
1346+
assert "test.test_lazy_import.data.pkg.b" in sys.lazy_modules, (
1347+
f"loading the package must not untrack the submodule, "
1348+
f"got {sys.lazy_modules}"
1349+
)
1350+
""")
1351+
assert_python_ok("-c", code)
1352+
12551353

12561354
@support.requires_subprocess()
12571355
class CommandLineAndEnvVarTests(unittest.TestCase):
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Resolved names are now removed from :data:`sys.lazy_modules` more
2+
consistently: a lazy import of an already loaded module is no longer recorded,
3+
and reifying ``lazy from pkg import attr`` now discards the ``pkg.attr`` entry.

‎Python/import.c‎

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3898,6 +3898,20 @@ _PyImport_ResolveName(PyThreadState *tstate, PyObject *name,
38983898
return resolve_name(tstate, name, globals, level);
38993899
}
39003900

3901+
// The import machinery only discards module names, so this is what removes
3902+
// the "pkg.attr" entry left by `lazy from pkg import attr`, submodule or not.
3903+
static int
3904+
discard_reified_lazy_import(PyInterpreterState *interp, PyObject *lazy_import)
3905+
{
3906+
PyObject *name = _PyLazyImport_GetName(lazy_import);
3907+
if (name == NULL) {
3908+
return -1;
3909+
}
3910+
int res = PySet_Discard(LAZY_MODULES(interp), name);
3911+
Py_DECREF(name);
3912+
return res < 0 ? -1 : 0;
3913+
}
3914+
39013915
PyObject *
39023916
_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
39033917
{
@@ -4080,6 +4094,10 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
40804094
}
40814095

40824096
ok:
4097+
if (obj != NULL && discard_reified_lazy_import(interp, lazy_import) < 0) {
4098+
Py_CLEAR(obj);
4099+
}
4100+
40834101
if (PySet_Discard(importing, lazy_import) < 0) {
40844102
Py_CLEAR(obj);
40854103
}
@@ -4338,6 +4356,27 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals,
43384356
return final_mod;
43394357
}
43404358

4359+
// Check if a module is already loaded before adding it to sys.lazy_modules
4360+
static int
4361+
lazy_modules_add(PyThreadState *tstate, PyObject *name)
4362+
{
4363+
PyObject *modules = get_modules_dict(tstate, false);
4364+
if (modules == NULL) {
4365+
return -1;
4366+
}
4367+
PyObject *existing;
4368+
if (PyDict_GetItemRef(modules, name, &existing) < 0) {
4369+
return -1;
4370+
}
4371+
// A None entry blocks the import rather than satisfying it.
4372+
int loaded = (existing != NULL && existing != Py_None);
4373+
Py_XDECREF(existing);
4374+
if (loaded) {
4375+
return 0;
4376+
}
4377+
return PySet_Add(LAZY_MODULES(tstate->interp), name);
4378+
}
4379+
43414380
// ensure we have the set for the parent module name in sys.lazy_modules.
43424381
// Returns a new reference.
43434382
static PyObject *
@@ -4431,9 +4470,7 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name,
44314470
return -1;
44324471
}
44334472

4434-
// Add the module name to sys.lazy_modules set (PEP 810).
4435-
PyObject *lazy_modules = LAZY_MODULES(tstate->interp);
4436-
if (PySet_Add(lazy_modules, fromname) < 0) {
4473+
if (lazy_modules_add(tstate, fromname) < 0) {
44374474
Py_DECREF(fromname);
44384475
return -1;
44394476
}
@@ -4607,9 +4644,7 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate,
46074644
return NULL;
46084645
}
46094646

4610-
// Add the module name to sys.lazy_modules set (PEP 810).
4611-
PyObject *lazy_modules = LAZY_MODULES(tstate->interp);
4612-
if (PySet_Add(lazy_modules, abs_name) < 0) {
4647+
if (lazy_modules_add(tstate, abs_name) < 0) {
46134648
goto error;
46144649
}
46154650

0 commit comments

Comments
 (0)