Skip to content

Commit 7fbedef

Browse files
committed
Fix GC-induced thread hang on macOS with Python 3.14
Python 3.14 sets the gen-2 GC threshold to 0, so long-lived objects (like SChunk instances in pytest fixtures) never get auto-collected. Each SChunk carries 2×nthreads OS pthreads via its cctx/dctx. During pytest teardown, thousands of accumulated SChunks hit macOS's 6144 thread limit, causing an indefinite hang on gc.collect(). Three fixes applied: - Release the GIL during blosc2_schunk_free() in SChunk.__dealloc__ to prevent GIL deadlock when mass finalization triggers pthread_join - Add periodic gc.collect() every 50 tests in conftest.py to prevent thread accumulation past the macOS ceiling - Cap ThreadPoolExecutor in lazyexpr.py to os.cpu_count() workers (fixes #556) Closes #556
1 parent 2ca5eeb commit 7fbedef

3 files changed

Lines changed: 31 additions & 3 deletions

File tree

src/blosc2/blosc2_ext.pyx

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -476,7 +476,7 @@ cdef extern from "blosc2.h":
476476
int64_t blosc2_schunk_to_buffer(blosc2_schunk* schunk, uint8_t** cframe, c_bool* needs_free) nogil
477477
void blosc2_schunk_avoid_cframe_free(blosc2_schunk *schunk, c_bool avoid_cframe_free)
478478
int64_t blosc2_schunk_to_file(blosc2_schunk* schunk, const char* urlpath)
479-
int64_t blosc2_schunk_free(blosc2_schunk *schunk)
479+
int64_t blosc2_schunk_free(blosc2_schunk *schunk) nogil
480480
int64_t blosc2_schunk_append_chunk(blosc2_schunk *schunk, uint8_t *chunk, c_bool copy)
481481
int64_t blosc2_schunk_update_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t *chunk, c_bool copy)
482482
int64_t blosc2_schunk_insert_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t *chunk, c_bool copy)
@@ -2339,14 +2339,23 @@ cdef class SChunk:
23392339
self.schunk.cctx = NULL
23402340

23412341
def __dealloc__(self):
2342+
cdef blosc2_schunk *schunk_ptr
23422343
if self.schunk != NULL and not self._is_view:
23432344
# Free prefilters and postfilters params
23442345
if self.schunk.storage.cparams.prefilter != NULL:
23452346
self.remove_prefilter(func_name=None, _new_ctx=False)
23462347
if self.schunk.storage.dparams.postfilter != NULL:
23472348
self.remove_postfilter(func_name=None, _new_ctx=False)
23482349

2349-
blosc2_schunk_free(self.schunk)
2350+
# Release the GIL while freeing the C-Blosc2 super-chunk.
2351+
# blosc2_schunk_free -> blosc2_free_ctx -> release_threadpool
2352+
# joins worker pthreads; holding the GIL here can cause hangs
2353+
# when thousands of SChunks are finalized at once (e.g. during
2354+
# gc.collect() in Python 3.14+ where gen-2 threshold is 0).
2355+
schunk_ptr = self.schunk
2356+
self.schunk = NULL
2357+
with nogil:
2358+
blosc2_schunk_free(schunk_ptr)
23502359

23512360

23522361
# postfilter

src/blosc2/lazyexpr.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1182,7 +1182,7 @@ def get_chunk(arr, info, nchunk):
11821182
async def async_read_chunks(arrs, info, queue):
11831183
loop = asyncio.get_event_loop()
11841184
shape, chunks_ = arrs[0].shape, arrs[0].chunks
1185-
with concurrent.futures.ThreadPoolExecutor() as executor:
1185+
with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count() or 1) as executor:
11861186
my_chunk_iter = range(arrs[0].schunk.nchunks)
11871187
if len(info) == 5:
11881188
if info[-1] is not None:

tests/conftest.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
# SPDX-License-Identifier: BSD-3-Clause
66
#######################################################################
77

8+
import gc
89
import os
910
import sys
1011

@@ -14,6 +15,17 @@
1415
import blosc2
1516

1617

18+
# Each SChunk allocates C-level thread pools (pthreads) for its compression
19+
# and decompression contexts. Python 3.14 changed the GC gen-2 threshold
20+
# to 0, so long-lived objects are never collected automatically; they
21+
# accumulate until an explicit gc.collect() (e.g. pytest session cleanup).
22+
# Joining thousands of idle pthreads at once can hit the macOS thread-count
23+
# ceiling (6 144) and hang. Periodically forcing a full collection keeps
24+
# the thread count bounded.
25+
_GC_COLLECT_INTERVAL = 50 # collect every N tests
26+
_test_counter = 0
27+
28+
1729
def expected_nthreads(nthreads: int) -> int:
1830
return 1 if blosc2.IS_WASM else nthreads
1931

@@ -49,3 +61,10 @@ def pytest_runtest_call(item):
4961
item.runtest()
5062
except requests.exceptions.RequestException as exc:
5163
pytest.skip(f"Skipping network test due to request failure: {exc}")
64+
65+
66+
def pytest_runtest_teardown(item, nextitem):
67+
global _test_counter
68+
_test_counter += 1
69+
if _test_counter % _GC_COLLECT_INTERVAL == 0:
70+
gc.collect()

0 commit comments

Comments
 (0)