Break reference cycles and purge stale caches in ctable layer

   - _RowIndexer: use weakref to parent CTable to break reference
     cycles, allowing CTable instances to be collected without gen-2 GC.

   - indexing.py: stop caching writable sidecar handles in the
     process-wide _SIDECAR_HANDLE_CACHE after construction — they kept
     NDArray objects alive indefinitely across tests on macOS/Python 3.14.

   - indexing.py: add _purge_stale_persistent_caches() to evict index
     cache entries whose backing paths no longer exist. Call it in
     _load_store, _open_query_cache_store, _open_sidecar_handle, and
     _gather_mmap_source.

   - ref.py: open urlpath operands in mode='r' instead of the default
     mode='a'; persistent recipe operands only need read access.

   - test_ctable_indexing.py: add tests verifying that sidecar handles
     are not cached after index creation, that CTables release without
     explicit GC, and that stale persistent caches are purged correctly.

Author: FrancescAlted

src/blosc2/ctable.py

@@ -16,6 +16,7 @@
 import os
 import pprint
 import shutil
+import weakref
 from collections.abc import Iterable
 from dataclasses import MISSING
 from textwrap import TextWrapper
@@ -448,10 +449,13 @@ def _find_physical_index(arr: blosc2.NDArray, logical_key: int) -> int:
 
 class _RowIndexer:
     def __init__(self, table):
-        self._table = table
+        self._table_ref = weakref.ref(table)
 
     def __getitem__(self, item):
-        return self._table._run_row_logic(item)
+        table = self._table_ref()
+        if table is None:
+            raise ReferenceError("owning CTable has been released")
+        return table._run_row_logic(item)
 
 
 class _Row:

src/blosc2/indexing.py

@@ -106,6 +106,48 @@ def _cleanup_in_memory_store(key: int) -> None:
     _hot_cache_clear(scope=("memory", key))
 
 
+def _persistent_cache_path_exists(path: str | int) -> bool:
+    if not isinstance(path, str):
+        return False
+    path_obj = Path(path)
+    return path_obj.exists() or path_obj.parent.exists()
+
+
+def _purge_stale_persistent_caches() -> None:
+    stale_scopes = {
+        key for key in _PERSISTENT_INDEXES if key[0] == "persistent" and not _persistent_cache_path_exists(key[1])
+    }
+    for key in stale_scopes:
+        _PERSISTENT_INDEXES.pop(key, None)
+
+    stale_data_keys = {
+        key for key in _DATA_CACHE if key[0][0] == "persistent" and not _persistent_cache_path_exists(key[0][1])
+    }
+    stale_scopes.update(key[0] for key in stale_data_keys)
+    for key in stale_data_keys:
+        _DATA_CACHE.pop(key, None)
+
+    stale_handle_keys = {
+        key
+        for key in _SIDECAR_HANDLE_CACHE
+        if key[0][0] == "persistent" and not _persistent_cache_path_exists(key[0][1])
+    }
+    stale_scopes.update(key[0] for key in stale_handle_keys)
+    for key in stale_handle_keys:
+        _SIDECAR_HANDLE_CACHE.pop(key, None)
+
+    stale_query_paths = [path for path in _QUERY_CACHE_STORE_HANDLES if not Path(path).exists()]
+    for path in stale_query_paths:
+        _QUERY_CACHE_STORE_HANDLES.pop(path, None)
+
+    stale_gather_paths = [path for path in _GATHER_MMAP_HANDLES if not Path(path).exists()]
+    for path in stale_gather_paths:
+        _GATHER_MMAP_HANDLES.pop(path, None)
+
+    for scope in stale_scopes:
+        _hot_cache_clear(scope=scope)
+
+
 @dataclass(slots=True)
 class IndexPlan:
     usable: bool
@@ -286,6 +328,7 @@ def _resolve_full_index_tmpdir(array: blosc2.NDArray, tmpdir: str | None) -> str
 
 
 def _load_store(array: blosc2.NDArray) -> dict:
+    _purge_stale_persistent_caches()
     if _is_persistent_array(array):
         key = _array_key(array)
         cached = _PERSISTENT_INDEXES.get(key)
@@ -405,6 +448,7 @@ def _open_query_cache_store(array: blosc2.NDArray, *, create: bool = False):
     Returns ``None`` if the array is not persistent.  When *create* is True the
     store is created if it does not yet exist.
     """
+    _purge_stale_persistent_caches()
     if not _is_persistent_array(array):
         return None
     path = _query_cache_payload_path(array)
@@ -920,6 +964,7 @@ def _invalidate_sidecar_cache_entries(array: blosc2.NDArray, token: str, categor
 
 
 def _open_sidecar_handle(array: blosc2.NDArray, token: str, category: str, name: str, path: str | None):
+    _purge_stale_persistent_caches()
     cache_key = _sidecar_handle_cache_key(array, token, category, name)
     cached = _SIDECAR_HANDLE_CACHE.get(cache_key)
     if cached is not None:
@@ -1108,8 +1153,11 @@ def _store_array_sidecar(
             kwargs["blocks"] = blocks
         if cparams is not None:
             kwargs["cparams"] = cparams
+        # Do not retain writable persistent handles in the process-wide cache.
+        # They keep native resources alive after index construction and can
+        # accumulate badly across tests on macOS/Python 3.14.
         handle = blosc2.asarray(data, **kwargs)
-        _SIDECAR_HANDLE_CACHE[handle_cache_key] = handle
+        del handle
         _DATA_CACHE.pop(cache_key, None)
     else:
         path = None
@@ -1152,10 +1200,9 @@ def _create_persistent_sidecar_handle(
         kwargs["cparams"] = cparams
     if length == 0:
         handle = blosc2.asarray(np.empty(0, dtype=dtype), **kwargs)
-        _SIDECAR_HANDLE_CACHE[_sidecar_handle_cache_key(array, token, category, name)] = handle
+        del handle
         return None, {"path": path, "dtype": dtype.descr if dtype.fields else dtype.str}
     handle = blosc2.empty((length,), dtype=dtype, **kwargs)
-    _SIDECAR_HANDLE_CACHE[_sidecar_handle_cache_key(array, token, category, name)] = handle
     return handle, {"path": path, "dtype": dtype.descr if dtype.fields else dtype.str}
 
 
@@ -4874,6 +4921,7 @@ def _gather_mmap_source(where_x):
     urlpath = getattr(where_x, "urlpath", None)
     if not _supports_block_reads(where_x) or urlpath is None:
         return where_x
+    _purge_stale_persistent_caches()
     urlpath = str(urlpath)
     handle = _GATHER_MMAP_HANDLES.get(urlpath)
     if handle is None:

src/blosc2/ref.py

@@ -118,7 +118,9 @@ def open(self):
         import blosc2
 
         if self.kind == "urlpath":
-            return blosc2.open(self.urlpath, mode="a")
+            # Structured refs are used to reopen operands for persisted recipes.
+            # Read-only access avoids allocating unnecessary writable state.
+            return blosc2.open(self.urlpath, mode="r")
         if self.kind == "dictstore_key":
             return blosc2.DictStore(self.urlpath, mode="r")[self.key]
         if self.kind == "c2array":

tests/ctable/test_ctable_indexing.py

@@ -10,6 +10,7 @@
 import dataclasses
 import shutil
 import tempfile
+import weakref
 from pathlib import Path
 
 import numpy as np
@@ -223,6 +224,33 @@ def test_create_index_persistent(tmpdir):
     assert sidecars, "No sidecar .b2nd files found"
 
 
+def test_create_index_persistent_does_not_cache_sidecar_handles(tmpdir):
+    import blosc2.indexing as indexing
+
+    path = str(tmpdir / "table.b2d")
+    t = _make_table(50, persistent_path=path)
+    t.create_index("id", kind=blosc2.IndexKind.FULL)
+
+    cached = [
+        key
+        for key in indexing._SIDECAR_HANDLE_CACHE
+        if key[0][0] == "persistent" and str(tmpdir) in key[0][1]
+    ]
+    assert cached == []
+
+
+def test_persistent_ctable_releases_immediately_without_gc(tmpdir):
+    path = str(tmpdir / "table.b2d")
+
+    def build_table():
+        t = _make_table(50, persistent_path=path)
+        t.create_index("id", kind=blosc2.IndexKind.FULL)
+        return weakref.ref(t)
+
+    table_ref = build_table()
+    assert table_ref() is None
+
+
 def test_catalog_survives_reopen(tmpdir):
     path = str(tmpdir / "table.b2d")
     t = _make_table(30, persistent_path=path)
@@ -250,6 +278,28 @@ def test_where_with_index_matches_scan_persistent(tmpdir):
     assert ids_idx == ids_scan
 
 
+def test_persistent_index_drop_releases_sidecars_without_gc(tmpdir):
+    import gc
+
+    def run_query_and_drop():
+        path = str(tmpdir / "table.b2d")
+        t = _make_table(200, persistent_path=path)
+        t.create_index("id")
+        result = t.where(t["id"] > 150)
+        ids = sorted(int(v) for v in result["id"].to_numpy())
+        assert ids == list(range(151, 200))
+        t.drop_index("id")
+
+    run_query_and_drop()
+
+    sidecars = [
+        obj
+        for obj in gc.get_objects()
+        if isinstance(obj, blosc2.NDArray) and obj.urlpath and str(tmpdir) in obj.urlpath and "__index__" in obj.urlpath
+    ]
+    assert sidecars == []
+
+
 def test_drop_index_persistent(tmpdir):
     path = str(tmpdir / "table.b2d")
     t = _make_table(30, persistent_path=path)
@@ -465,3 +515,23 @@ def test_indexed_ctable_b2z_double_open_append_no_corruption(tmp_path):
     assert t2.nrows == 50
     assert len(t2.indexes) == 1
     del t2
+
+
+def test_indexing_purges_stale_persistent_caches():
+    import blosc2.indexing as indexing
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        path = str(Path(tmpdir) / "table.b2d")
+        t = _make_table(50, persistent_path=path)
+        t.create_index("id")
+        _ = t.where(t["id"] > 10)
+        t.close()
+
+        assert any(tmpdir in key[1] for key in indexing._PERSISTENT_INDEXES if key[0] == "persistent")
+
+    indexing._purge_stale_persistent_caches()
+
+    assert all(tmpdir not in key[1] for key in indexing._PERSISTENT_INDEXES if key[0] == "persistent")
+    assert all(tmpdir not in key[0][1] for key in indexing._SIDECAR_HANDLE_CACHE if key[0][0] == "persistent")
+    assert all(tmpdir not in path for path in indexing._QUERY_CACHE_STORE_HANDLES)
+    assert all(tmpdir not in path for path in indexing._GATHER_MMAP_HANDLES)