diff --git a/changelog.d/7889-block-pool-pressure.md b/changelog.d/7889-block-pool-pressure.md new file mode 100644 index 0000000000..717ed9eb7b --- /dev/null +++ b/changelog.d/7889-block-pool-pressure.md @@ -0,0 +1,3 @@ +Critical memory pressure now drains recycled arena blocks after the owed full +collection, and the recycled-block allowance is shared process-wide and scales +with constrained-device heap budgets. diff --git a/crates/perry-runtime/src/arena/block.rs b/crates/perry-runtime/src/arena/block.rs index 59ca0b6766..52b188a2c9 100644 --- a/crates/perry-runtime/src/arena/block.rs +++ b/crates/perry-runtime/src/arena/block.rs @@ -1,4 +1,5 @@ use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; /// Size of each arena block (1 MB — issue #179 tier 1 #1). /// @@ -79,7 +80,7 @@ fn block_size_for(min_size: usize) -> usize { // --------------------------------------------------------------------------- /// Owns the pooled blocks, so that a thread exiting with a non-empty pool -/// releases them instead of leaking up to [`BLOCK_POOL_CAP_BYTES`]. +/// releases them instead of leaking up to the process-wide pool cap. /// /// The ownership has to live *here* rather than in a drain called from /// `Arena::drop`: both are TLS destructors, their relative order is not @@ -91,11 +92,20 @@ fn block_size_for(min_size: usize) -> usize { /// arena and GC, so each exiting agent thread would otherwise strand its /// pooled blocks — unbounded growth across repeated spawns, in the one change /// whose purpose is lowering RSS. -struct BlockPool(Vec<(*mut u8, usize)>); +struct BlockPool { + blocks: Vec<(*mut u8, usize)>, + drain_requested: bool, +} impl Drop for BlockPool { fn drop(&mut self) { - for &(data, size) in &self.0 { + let bytes = self + .blocks + .iter() + .map(|&(_, size)| size) + .fold(0usize, usize::saturating_add); + block_pool_process_bytes_sub(bytes); + for &(data, size) in &self.blocks { if data.is_null() || size == 0 { continue; } @@ -113,18 +123,73 @@ impl Drop for BlockPool { } thread_local! { - static BLOCK_POOL: RefCell = const { RefCell::new(BlockPool(Vec::new())) }; + static BLOCK_POOL: RefCell = const { RefCell::new(BlockPool { + blocks: Vec::new(), + drain_requested: false, + }) }; static BLOCK_POOL_BYTES: Cell = const { Cell::new(0) }; } -/// Cap on pooled bytes: 64 MB, matching the young cap ceiling. Measured on +/// Process-wide cap on pooled bytes. It remains 64 MiB on unconstrained +/// desktop/server processes and scales to one eighth of a device/container +/// heap budget. A single global reservation closes the N-live-agents × 64 MiB +/// shape while retaining per-thread LIFO reuse. +/// +/// The original 64 MiB choice was measured on /// tree.ts (Mac mini M1, quiet): no pool -> 225 MB peak RSS; 64 MB pool -> /// 190 MB; 128 MB pool -> 210 MB. Bigger is NOT better — pooled pages are /// MADV_FREE'd but stay resident until the OS wants them, so an oversized /// pool trades fresh-segment growth for held free pages past the optimum. /// This is a cap, not a floor — the pool holds only blocks that were /// actually released, and the OS can take every pooled page under pressure. -const BLOCK_POOL_CAP_BYTES: usize = 64 * 1024 * 1024; +static BLOCK_POOL_PROCESS_BYTES: AtomicUsize = AtomicUsize::new(0); +#[cfg(test)] +static BLOCK_POOL_EXPLICIT_DRAINED_BYTES: AtomicUsize = AtomicUsize::new(0); + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct BlockPoolDrainStats { + pub(crate) blocks: usize, + pub(crate) bytes: usize, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ArenaBlockRelease { + Pooled, + Deallocated, +} + +fn block_pool_process_bytes_sub(bytes: usize) { + if bytes == 0 { + return; + } + BLOCK_POOL_PROCESS_BYTES + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + current.checked_sub(bytes) + }) + .unwrap_or_else(|current| { + panic!("process block-pool byte accounting underflow: {current} < {bytes}") + }); +} + +fn block_pool_process_try_reserve(size: usize) -> bool { + block_pool_counter_try_reserve(&BLOCK_POOL_PROCESS_BYTES, size, block_pool_cap_bytes()) +} + +pub(super) fn block_pool_counter_try_reserve( + counter: &AtomicUsize, + size: usize, + cap: usize, +) -> bool { + counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + current.checked_add(size).filter(|&next| next <= cap) + }) + .is_ok() +} + +fn block_pool_cap_bytes() -> usize { + crate::gc::gc_block_pool_cap_bytes() +} /// Offer a released block to the pool. Returns false (caller deallocs) when /// the pool is full or the block is null. @@ -132,14 +197,17 @@ pub(crate) fn block_pool_put(data: *mut u8, size: usize) -> bool { if data.is_null() || size == 0 { return false; } - if BLOCK_POOL_BYTES.with(Cell::get).saturating_add(size) > BLOCK_POOL_CAP_BYTES { + let cap = block_pool_cap_bytes(); + if BLOCK_POOL_BYTES.with(Cell::get).saturating_add(size) > cap + || !block_pool_process_try_reserve(size) + { return false; } #[cfg(unix)] unsafe { libc::madvise(data as *mut libc::c_void, size, libc::MADV_FREE); } - BLOCK_POOL.with(|p| p.borrow_mut().0.push((data, size))); + BLOCK_POOL.with(|p| p.borrow_mut().blocks.push((data, size))); BLOCK_POOL_BYTES.with(|c| c.set(c.get().saturating_add(size))); true } @@ -147,18 +215,95 @@ pub(crate) fn block_pool_put(data: *mut u8, size: usize) -> bool { fn block_pool_take(size: usize) -> Option<*mut u8> { let taken = BLOCK_POOL.with(|p| { let mut pool = p.borrow_mut(); - let idx = pool.0.iter().rposition(|&(_, s)| s == size)?; - Some(pool.0.swap_remove(idx).0) + let idx = pool.blocks.iter().rposition(|&(_, s)| s == size)?; + Some(pool.blocks.swap_remove(idx).0) })?; BLOCK_POOL_BYTES.with(|c| c.set(c.get().saturating_sub(size))); + block_pool_process_bytes_sub(size); Some(taken) } +/// Release an arena block through the one pool-or-deallocate funnel. The +/// disposition is returned so GC telemetry can distinguish idle mappings kept +/// for reuse from bytes actually handed to the allocator. +pub(crate) fn release_arena_block(data: *mut u8, size: usize) -> ArenaBlockRelease { + if block_pool_put(data, size) { + return ArenaBlockRelease::Pooled; + } + if !data.is_null() && size != 0 { + let layout = Layout::from_size_align(size, 16).unwrap(); + unsafe { + // #4665: test builds retain otherwise-freed mappings so stale raw + // GC pointers remain readable. The production disposition is still + // Deallocated; focused pool tests observe the explicit drain census. + if !cfg!(test) { + std::alloc::dealloc(data, layout); + } + } + } + ArenaBlockRelease::Deallocated +} + +/// Drain the current thread's retained blocks through real allocator +/// deallocation in production. Logical removal is still performed under +/// `cfg(test)`; #4665 suppresses the final `dealloc` there. +pub(crate) fn drain_block_pool() -> BlockPoolDrainStats { + let entries = BLOCK_POOL.with(|pool| std::mem::take(&mut pool.borrow_mut().blocks)); + let bytes = entries + .iter() + .map(|&(_, size)| size) + .fold(0usize, usize::saturating_add); + let tracked = BLOCK_POOL_BYTES.with(|cell| cell.replace(0)); + debug_assert_eq!(tracked, bytes, "thread block-pool byte accounting drifted"); + block_pool_process_bytes_sub(bytes); + + for &(data, size) in &entries { + if data.is_null() || size == 0 { + continue; + } + let layout = Layout::from_size_align(size, 16).unwrap(); + unsafe { + if !cfg!(test) { + std::alloc::dealloc(data, layout); + } + } + } + #[cfg(test)] + BLOCK_POOL_EXPLICIT_DRAINED_BYTES.fetch_add(bytes, Ordering::Relaxed); + BlockPoolDrainStats { + blocks: entries.len(), + bytes, + } +} + +pub(crate) fn request_block_pool_drain() { + BLOCK_POOL.with(|pool| pool.borrow_mut().drain_requested = true); +} + +/// Called only from full-cycle publication. A critical-pressure request stays +/// sticky through unsafe/deferred periods and is retired after the owed full +/// collection has completed all arena reclamation. +pub(crate) fn drain_block_pool_if_requested() -> BlockPoolDrainStats { + let requested = BLOCK_POOL.with(|pool| { + let mut pool = pool.borrow_mut(); + std::mem::replace(&mut pool.drain_requested, false) + }); + if !requested { + return BlockPoolDrainStats::default(); + } + drain_block_pool() +} + #[cfg(test)] pub(crate) fn block_pool_bytes_for_test() -> usize { BLOCK_POOL_BYTES.with(Cell::get) } +#[cfg(test)] +pub(crate) fn block_pool_explicit_drained_bytes_for_test() -> usize { + BLOCK_POOL_EXPLICIT_DRAINED_BYTES.load(Ordering::Relaxed) +} + fn try_alloc_block(min_size: usize, injectable: bool) -> Option { let size = block_size_for(min_size); let layout = Layout::from_size_align(size, 16).unwrap(); @@ -189,7 +334,8 @@ fn try_alloc_block(min_size: usize, injectable: bool) -> Option { } /// Reserve a block, running one emergency full collection if the OS refuses -/// memory — idle-block dealloc and the malloc sweep can return real pages. +/// memory — idle-block release, pool draining, and the malloc sweep can return +/// real pages. /// /// **NO `&mut Arena` BORROW MAY BE LIVE ACROSS THIS CALL (#7022).** The /// emergency collection allocates into the arenas exactly like the @@ -332,7 +478,7 @@ impl Drop for Arena { fn drop(&mut self) { for block in &self.blocks { // Skip tombstoned slots (gen-GC Phase C4b-δ): C4b-δ - // deallocates fully-idle nursery blocks back to the OS + // releases fully-idle nursery blocks through the pool/allocator // and leaves a `data = null, size = 0` tombstone in the // Vec to keep block-index semantics stable across GC // cycles. `dealloc(null, …)` is UB. @@ -369,7 +515,7 @@ impl Arena { /// (`data = null, size = 0`) instead of an eagerly-mapped 1 MB /// block, so JS-touching threads that never allocate in this /// region (spawn workers, tokio callers) don't pay the block. - /// The tombstone shape is exactly the one C4b-δ dealloc leaves + /// The tombstone shape is exactly the one C4b-δ block release leaves /// behind, so every walker/reset/alloc path already handles it: /// the first `alloc` misses the tombstone, and the slow path's /// `install_fresh_block` replaces the tombstone slot in place. @@ -520,9 +666,9 @@ impl Arena { return ptr; } // Still no room anywhere — need a fresh block. C4b-δ: - // prefer reusing a tombstoned slot (a block deallocated by + // prefer reusing a tombstoned slot (a block released by // `arena_reset_empty_blocks` after staying idle past the - // dealloc threshold) over growing the Vec, so block_idx + // release threshold) over growing the Vec, so block_idx // semantics stay bounded even on workloads that churn // through nursery blocks. self.alloc_fresh_block(size, align) @@ -762,9 +908,9 @@ thread_local! { /// `gc_check_trigger()` calls it on every `gc_malloc`, so for an /// 80-block working set the per-allocation overhead was ~250 ns /// just to recompute a total that almost never changes (only on - /// fresh-block alloc and tombstone dealloc). Maintained via deltas + /// fresh-block alloc and tombstone release). Maintained via deltas /// at the four mutation sites (Arena::new initial block, fresh - /// alloc into a tombstone slot or the end, and dealloc inside + /// alloc into a tombstone slot or the end, and release inside /// `arena_reset_empty_blocks`). pub(crate) static ARENA_TOTAL_BYTES: std::cell::Cell = const { std::cell::Cell::new(0) }; @@ -785,8 +931,8 @@ thread_local! { /// `old_arena_reclaim_selected_dead_blocks`, /// `OldArenaReclaimDeadBlocksState::process_block`) which zero /// old block offsets on sweep/defrag. - /// Block install/dealloc paths don't touch it: fresh blocks start - /// at offset 0 and blocks are only deallocated after their offset + /// Block install/release paths don't touch it: fresh blocks start + /// at offset 0 and blocks are only released after their offset /// was already zeroed. `old_gen_in_use_bytes()` (stats.rs) /// debug-asserts this cache against the O(blocks) recompute so a /// missed mutation site fails tests instead of silently skewing @@ -838,7 +984,7 @@ thread_local! { /// (same lifetime contract as longlived blocks from the nursery /// reset path), and never feed the inline bump allocator. Full /// mark-sweep can reclaim completely dead old blocks through the - /// dedicated old-arena reset/deallocation path. + /// dedicated old-arena reset/release path. pub(crate) static OLD_ARENA: UnsafeCell = UnsafeCell::new(Arena::new_lazy(HeapGeneration::Old, HeapSpace::Old)); diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 66c7f4eee0..12a5855c9d 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -32,9 +32,11 @@ pub(crate) use allocators::{ inactive_survivor_index, with_survivor_arena, with_survivor_arena_mut, }; pub(crate) use block::{ - arena_cell_alloc, block_pool_put, old_gen_in_use_bytes_sub, Arena, ArenaBlock, ACTIVE_SURVIVOR, - ARENA, ARENA_TOTAL_BYTES, BLOCK_SIZE, FRESH_GENERAL_BLOCK_MIN_USED_BYTES, INLINE_STATE, - LONGLIVED_ARENA, OLD_ARENA, OLD_GEN_IN_USE_BYTES, SURVIVOR_ARENA_0, SURVIVOR_ARENA_1, + arena_cell_alloc, drain_block_pool_if_requested, old_gen_in_use_bytes_sub, release_arena_block, + request_block_pool_drain, Arena, ArenaBlock, ArenaBlockRelease, BlockPoolDrainStats, + ACTIVE_SURVIVOR, ARENA, ARENA_TOTAL_BYTES, BLOCK_SIZE, FRESH_GENERAL_BLOCK_MIN_USED_BYTES, + INLINE_STATE, LONGLIVED_ARENA, OLD_ARENA, OLD_GEN_IN_USE_BYTES, SURVIVOR_ARENA_0, + SURVIVOR_ARENA_1, }; /// #7469 hot-TLS plumbing — see `crate::tls_hot`. The `*_hot_addr` half is /// consumed by `tls_hot::fill`; the `hot_*` half is the cached accessor the @@ -42,13 +44,14 @@ pub(crate) use block::{ pub(crate) use block::{arena_hot_addr, hot_arena, hot_inline_state, inline_state_hot_addr}; #[cfg(test)] pub(crate) use block::{ - block_pool_bytes_for_test, force_next_block_alloc_failure, gc_trigger_arena_borrow_depth, - gc_trigger_arena_calls, reset_gc_trigger_arena_probe, + block_pool_bytes_for_test, block_pool_explicit_drained_bytes_for_test, block_pool_put, + force_next_block_alloc_failure, gc_trigger_arena_borrow_depth, gc_trigger_arena_calls, + reset_gc_trigger_arena_probe, }; pub(crate) use page_meta::{ address_span_overlaps_pages, defer_old_object_page_registration, register_block_space, register_old_object_pages, unregister_block_generation, unregister_old_block_pages, - OLD_GEN_RECLAIM_RETURNED_BYTES, OLD_GEN_RECLAIM_REUSABLE_BYTES, + OLD_GEN_RECLAIM_POOLED_BYTES, OLD_GEN_RECLAIM_RETURNED_BYTES, OLD_GEN_RECLAIM_REUSABLE_BYTES, }; pub(crate) use page_meta::{page_generation_cache_hot_addr, page_generations_hot_addr}; diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index 95645f35f7..b751a2e1dc 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -294,6 +294,7 @@ pub(crate) struct OldPageSummary { pub(crate) live_bytes: usize, pub(crate) dead_bytes: usize, pub(crate) reusable_bytes: usize, + pub(crate) pooled_bytes: usize, pub(crate) returned_bytes: usize, pub(crate) pinned_bytes: usize, pub(crate) object_count: usize, @@ -326,6 +327,7 @@ thread_local! { RefCell::new(crate::fast_hash::new_ptr_hash_map()); pub(crate) static OLD_GEN_RECLAIM_REUSABLE_BYTES: Cell = const { Cell::new(0) }; + pub(crate) static OLD_GEN_RECLAIM_POOLED_BYTES: Cell = const { Cell::new(0) }; pub(crate) static OLD_GEN_RECLAIM_RETURNED_BYTES: Cell = const { Cell::new(0) }; /// Monotonic per-cycle epoch for old-page `dirty_slots` (#6181). Bumped @@ -1011,6 +1013,7 @@ pub(crate) fn old_pages_begin_gc_cycle() { // it on first touch this cycle (`old_page_account_dirty_slot`). OLD_GEN_PAGE_DIRTY_EPOCH.with(|epoch| epoch.set(epoch.get().wrapping_add(1))); OLD_GEN_RECLAIM_REUSABLE_BYTES.with(|bytes| bytes.set(0)); + OLD_GEN_RECLAIM_POOLED_BYTES.with(|bytes| bytes.set(0)); OLD_GEN_RECLAIM_RETURNED_BYTES.with(|bytes| bytes.set(0)); } @@ -1168,6 +1171,7 @@ pub(crate) fn old_page_summary() -> OldPageSummary { } } summary.reusable_bytes = OLD_GEN_RECLAIM_REUSABLE_BYTES.with(|bytes| bytes.get()); + summary.pooled_bytes = OLD_GEN_RECLAIM_POOLED_BYTES.with(|bytes| bytes.get()); summary.returned_bytes = OLD_GEN_RECLAIM_RETURNED_BYTES.with(|bytes| bytes.get()); summary }) diff --git a/crates/perry-runtime/src/arena/quarantine.rs b/crates/perry-runtime/src/arena/quarantine.rs index 2603e711a6..f051308d91 100644 --- a/crates/perry-runtime/src/arena/quarantine.rs +++ b/crates/perry-runtime/src/arena/quarantine.rs @@ -586,8 +586,7 @@ pub(crate) fn copying_quarantine_from_spaces_and_flip() -> ArenaResetStats { ArenaResetStats { reset_blocks, reusable_bytes, - deallocated_blocks: 0, - deallocated_bytes: 0, + ..ArenaResetStats::default() } } diff --git a/crates/perry-runtime/src/arena/reset.rs b/crates/perry-runtime/src/arena/reset.rs index 4d4250d411..8045decf27 100644 --- a/crates/perry-runtime/src/arena/reset.rs +++ b/crates/perry-runtime/src/arena/reset.rs @@ -142,8 +142,7 @@ pub(crate) fn copying_reset_from_spaces_and_flip() -> ArenaResetStats { ArenaResetStats { reset_blocks, reusable_bytes, - deallocated_blocks: 0, - deallocated_bytes: 0, + ..ArenaResetStats::default() } } @@ -258,8 +257,8 @@ pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats { }); } - // Gen-GC Phase C4b-δ: deallocate fully-idle blocks back to - // the OS. A block becomes a dealloc candidate when: + // Gen-GC Phase C4b-δ: release fully-idle blocks from arena + // accounting. A block becomes a release candidate when: // - it's not the current allocator target // - it's outside the `keep_low..=current` register-miss // window (already excluded from reset above for the @@ -269,9 +268,9 @@ pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats { // reset this cycle or never used since the prior reset), // - it's not already a tombstone. // Each candidate's `dead_cycles` increments per cycle; once - // it reaches `DEALLOC_DEAD_CYCLES`, we hand the underlying - // allocation back to glibc/jemalloc/whatever via `dealloc` - // and leave a `data = null, size = 0` tombstone in the Vec + // it reaches `DEALLOC_DEAD_CYCLES`, we offer the underlying + // allocation to the bounded recycled pool (or call `dealloc` + // when the pool refuses it) and leave a null/zero tombstone in the Vec // so block-index semantics stay stable for the rest of the // GC cycle. Future allocations preferentially reuse // tombstoned slots (`Arena::alloc`'s slow path) before @@ -281,14 +280,15 @@ pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats { // Threshold tuning: 2 cycles. A block resets on cycle N // (`dead_cycles=1` after this loop), and on cycle N+1 either // gets reused (offset > 0, dead_cycles back to 0) or stays - // idle (`dead_cycles=2` ⇒ dealloc). Two cycles is the + // idle (`dead_cycles=2` ⇒ release). Two cycles is the // minimum that gives the bump allocator one cycle to reuse // a freshly-reset block before declaring it truly idle — // catches the `bench_json_roundtrip` case (only 2-3 GCs // per run) while still letting tight allocation loops keep // hot blocks alive across consecutive resets. const DEALLOC_DEAD_CYCLES: u32 = 2; - let mut deallocated_ranges: Vec<(usize, usize)> = Vec::new(); + let mut removed_ranges: Vec<(usize, usize)> = Vec::new(); + let mut release_stats = ArenaResetStats::default(); for (i, block) in arena.blocks.iter_mut().enumerate() { if block.data.is_null() { continue; @@ -309,15 +309,10 @@ pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats { if block.dead_cycles >= DEALLOC_DEAD_CYCLES { let base = block.data as usize; let size = block.size; - let layout = Layout::from_size_align(block.size, 16).unwrap(); unregister_block_generation(base, size); - deallocated_ranges.push((base, size)); - // #4665: in test builds keep freed blocks mapped (no munmap) so - // unit tests holding raw GC pointers across a collection read stale - // bytes instead of SIGSEGV-ing on an unmapped page. - if !block_pool_put(block.data, block.size) && !cfg!(test) { - std::alloc::dealloc(block.data, layout); - } + removed_ranges.push((base, size)); + let release = release_arena_block(block.data, block.size); + release_stats.record_block_release(size, release); ARENA_TOTAL_BYTES.with(|t| t.set(t.get().saturating_sub(block.size))); block.data = std::ptr::null_mut(); block.size = 0; @@ -326,33 +321,29 @@ pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats { } } let reset_blocks = reset_block_ranges.len(); - let deallocated_blocks = deallocated_ranges.len(); - let deallocated_bytes: usize = deallocated_ranges.iter().map(|&(_, s)| s).sum(); let reusable_bytes: usize = reset_block_ranges .iter() .filter(|&&(base, _, _)| { - !deallocated_ranges + !removed_ranges .iter() - .any(|&(deallocated_base, _)| deallocated_base == base) + .any(|&(removed_base, _)| removed_base == base) }) .map(|&(_, _, used)| used) .sum(); let stats = ArenaResetStats { reset_blocks, reusable_bytes, - deallocated_blocks, - deallocated_bytes, + ..release_stats }; - if !deallocated_ranges.is_empty() { - // Drop free-list entries pointing into deallocated - // blocks — same reasoning as the reset path, but the - // memory is now gone, not just reusable. + if !removed_ranges.is_empty() { + // Drop free-list entries pointing into removed blocks — whether + // pooled or unmapped, they no longer belong to this arena. crate::gc::ARENA_FREE_LIST.with(|fl| { let mut fl = fl.borrow_mut(); fl.retain(|&(ptr, _)| { let p = ptr as usize; - !deallocated_ranges + !removed_ranges .iter() .any(|&(base, size)| p >= base && p < base + size) }); @@ -362,19 +353,21 @@ pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats { }); if std::env::var_os("PERRY_GC_DIAG").is_some() { eprintln!( - "[gc-dealloc] freed {} blocks ({} bytes) back to OS", - deallocated_ranges.len(), - deallocated_bytes + "[gc-block-release] removed {} blocks ({} bytes): pooled={} bytes, deallocated={} bytes", + stats.removed_blocks, + stats.removed_bytes, + stats.pooled_bytes, + stats.deallocated_bytes ); } } - if reset_block_ranges.is_empty() && deallocated_ranges.is_empty() { + if reset_block_ranges.is_empty() && removed_ranges.is_empty() { stats } else { // Walk back the `current` index to the first reset block — // i.e., one with `offset == 0`. Skip tombstones (data.is_null()) - // — the inline allocator can't bump from a deallocated slot. + // — the inline allocator can't bump from a released slot. // If we just picked the first block with any free space we'd // land on the live block that still has 80 bytes left at the // end (not enough for a 96-byte class instance), and the next @@ -388,7 +381,7 @@ pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats { } } // If `new_current` ended up pointing at a tombstone (the only - // remaining offset==0 entries are deallocated slots), keep + // remaining offset==0 entries are released slots), keep // `arena.current` where it was — the next `Arena::alloc` slow // path will tombstone-reuse a slot and update `current` then. if !arena.blocks[new_current].data.is_null() { @@ -446,7 +439,7 @@ pub(crate) struct ArenaResetEmptyBlocksState { cursor: usize, changed: bool, reset_ranges: Vec<(usize, usize, usize)>, - deallocated_ranges: Vec<(usize, usize)>, + removed_ranges: Vec<(usize, usize)>, stats: ArenaResetStats, } @@ -459,7 +452,7 @@ impl ArenaResetEmptyBlocksState { cursor: 0, changed: false, reset_ranges: Vec::new(), - deallocated_ranges: Vec::new(), + removed_ranges: Vec::new(), stats: ArenaResetStats::default(), } } @@ -491,8 +484,9 @@ impl ArenaResetEmptyBlocksState { self.subphase = GeneralResetSubphase::Finish; continue; } - if let Some((base, size)) = self.process_dealloc_block(self.cursor) { - self.deallocated_ranges.push((base, size)); + if let Some((base, size, release)) = self.process_dealloc_block(self.cursor) { + self.removed_ranges.push((base, size)); + self.stats.record_block_release(size, release); free_list_ranges.push((base, size)); } self.cursor += 1; @@ -549,7 +543,10 @@ impl ArenaResetEmptyBlocksState { }) } - fn process_dealloc_block(&mut self, block_idx: usize) -> Option<(usize, usize)> { + fn process_dealloc_block( + &mut self, + block_idx: usize, + ) -> Option<(usize, usize, ArenaBlockRelease)> { let snapshot = self.snapshots.get(block_idx).copied().unwrap_or_default(); if snapshot.data == 0 { return None; @@ -586,35 +583,27 @@ impl ArenaResetEmptyBlocksState { let base = block.data as usize; let size = block.size; - let layout = Layout::from_size_align(block.size, 16).unwrap(); unregister_block_generation(base, size); - // #4665: in test builds keep freed blocks mapped (no munmap) so - // unit tests holding raw GC pointers across a collection read stale - // bytes instead of SIGSEGV-ing on an unmapped page. - if !block_pool_put(block.data, block.size) && !cfg!(test) { - std::alloc::dealloc(block.data, layout); - } + let release = release_arena_block(block.data, block.size); ARENA_TOTAL_BYTES.with(|total| total.set(total.get().saturating_sub(size))); block.data = std::ptr::null_mut(); block.size = 0; block.offset = 0; block.dead_cycles = 0; self.changed = true; - Some((base, size)) + Some((base, size, release)) }) } fn finish(&mut self) { - let deallocated_blocks = self.deallocated_ranges.len(); - let deallocated_bytes: usize = self.deallocated_ranges.iter().map(|&(_, size)| size).sum(); let reusable_bytes: usize = self .reset_ranges .iter() .filter(|&&(base, _, _)| { !self - .deallocated_ranges + .removed_ranges .iter() - .any(|&(deallocated_base, _)| deallocated_base == base) + .any(|&(removed_base, _)| removed_base == base) }) .map(|&(_, _, used)| used) .sum(); @@ -622,8 +611,7 @@ impl ArenaResetEmptyBlocksState { self.stats = ArenaResetStats { reset_blocks: self.reset_ranges.len(), reusable_bytes, - deallocated_blocks, - deallocated_bytes, + ..self.stats }; if !self.changed { @@ -733,7 +721,7 @@ impl SurvivorArenaReclaimState { return; } - with_survivor_arena_mut(self.arena_idx, |arena| unsafe { + with_survivor_arena_mut(self.arena_idx, |arena| { let keep_idx = arena .blocks .get(arena.current) @@ -782,21 +770,14 @@ impl SurvivorArenaReclaimState { let base = block.data as usize; let size = block.size; - let layout = Layout::from_size_align(size, 16).unwrap(); unregister_block_generation(base, size); - // #4665: in test builds keep freed blocks mapped (no munmap) so - // unit tests holding raw GC pointers across a collection read stale - // bytes instead of SIGSEGV-ing on an unmapped page. - if !block_pool_put(block.data, block.size) && !cfg!(test) { - std::alloc::dealloc(block.data, layout); - } + let release = release_arena_block(block.data, block.size); ARENA_TOTAL_BYTES.with(|total| total.set(total.get().saturating_sub(size))); block.data = std::ptr::null_mut(); block.size = 0; block.offset = 0; block.dead_cycles = 0; - self.stats.deallocated_blocks = self.stats.deallocated_blocks.saturating_add(1); - self.stats.deallocated_bytes = self.stats.deallocated_bytes.saturating_add(size); + self.stats.record_block_release(size, release); }); } @@ -862,16 +843,18 @@ impl SurvivorArenaReclaimDeadBlocksState { match self.active { 0 => { let before = self.state0.stats; - if self.state0.step(budget) { - self.stats = self.add_delta(self.stats, before, self.state0.stats); + let finished = self.state0.step(budget); + self.stats = self.add_delta(self.stats, before, self.state0.stats); + if finished { self.active = 1; } false } 1 => { let before = self.state1.stats; - if self.state1.step(budget) { - self.stats = self.add_delta(self.stats, before, self.state1.stats); + let finished = self.state1.step(budget); + self.stats = self.add_delta(self.stats, before, self.state1.stats); + if finished { self.active = 2; return true; } @@ -907,6 +890,18 @@ impl SurvivorArenaReclaimDeadBlocksState { .deallocated_bytes .saturating_sub(before.deallocated_bytes), ); + total.removed_blocks = total + .removed_blocks + .saturating_add(after.removed_blocks.saturating_sub(before.removed_blocks)); + total.removed_bytes = total + .removed_bytes + .saturating_add(after.removed_bytes.saturating_sub(before.removed_bytes)); + total.pooled_blocks = total + .pooled_blocks + .saturating_add(after.pooled_blocks.saturating_sub(before.pooled_blocks)); + total.pooled_bytes = total + .pooled_bytes + .saturating_add(after.pooled_bytes.saturating_sub(before.pooled_bytes)); total } } @@ -969,6 +964,7 @@ impl OldArenaReclaimDeadBlocksState { self.finish(); OLD_GEN_RECLAIM_REUSABLE_BYTES .with(|bytes| bytes.set(self.stats.reusable_bytes)); + OLD_GEN_RECLAIM_POOLED_BYTES.with(|bytes| bytes.set(self.stats.pooled_bytes)); OLD_GEN_RECLAIM_RETURNED_BYTES .with(|bytes| bytes.set(self.stats.deallocated_bytes)); self.subphase = RegionReclaimSubphase::Done; @@ -1042,21 +1038,14 @@ impl OldArenaReclaimDeadBlocksState { return; } - let layout = Layout::from_size_align(size, 16).unwrap(); unregister_block_generation(base, size); - // #4665: in test builds keep freed blocks mapped (no munmap) so - // unit tests holding raw GC pointers across a collection read stale - // bytes instead of SIGSEGV-ing on an unmapped page. - if !block_pool_put(block.data, block.size) && !cfg!(test) { - std::alloc::dealloc(block.data, layout); - } + let release = release_arena_block(block.data, block.size); ARENA_TOTAL_BYTES.with(|total| total.set(total.get().saturating_sub(size))); block.data = std::ptr::null_mut(); block.size = 0; block.offset = 0; block.dead_cycles = 0; - self.stats.deallocated_blocks = self.stats.deallocated_blocks.saturating_add(1); - self.stats.deallocated_bytes = self.stats.deallocated_bytes.saturating_add(size); + self.stats.record_block_release(size, release); }); } @@ -1137,21 +1126,14 @@ pub(crate) fn old_arena_reclaim_dead_blocks(block_has_live: &[bool]) -> ArenaRes continue; } - let layout = Layout::from_size_align(size, 16).unwrap(); unregister_block_generation(base, size); - // #4665: in test builds keep freed blocks mapped (no munmap) so - // unit tests holding raw GC pointers across a collection read stale - // bytes instead of SIGSEGV-ing on an unmapped page. - if !block_pool_put(block.data, block.size) && !cfg!(test) { - std::alloc::dealloc(block.data, layout); - } + let release = release_arena_block(block.data, block.size); ARENA_TOTAL_BYTES.with(|total| total.set(total.get().saturating_sub(size))); block.data = std::ptr::null_mut(); block.size = 0; block.offset = 0; block.dead_cycles = 0; - stats.deallocated_blocks = stats.deallocated_blocks.saturating_add(1); - stats.deallocated_bytes = stats.deallocated_bytes.saturating_add(size); + stats.record_block_release(size, release); } if changed { @@ -1183,6 +1165,7 @@ pub(crate) fn old_arena_reclaim_dead_blocks(block_has_live: &[bool]) -> ArenaRes }); OLD_GEN_RECLAIM_REUSABLE_BYTES.with(|bytes| bytes.set(stats.reusable_bytes)); + OLD_GEN_RECLAIM_POOLED_BYTES.with(|bytes| bytes.set(stats.pooled_bytes)); OLD_GEN_RECLAIM_RETURNED_BYTES.with(|bytes| bytes.set(stats.deallocated_bytes)); stats } @@ -1240,21 +1223,14 @@ pub(crate) fn old_arena_reclaim_selected_dead_blocks( continue; } - let layout = Layout::from_size_align(size, 16).unwrap(); unregister_block_generation(base, size); - // #4665: in test builds keep freed blocks mapped (no munmap) so - // unit tests holding raw GC pointers across a collection read stale - // bytes instead of SIGSEGV-ing on an unmapped page. - if !block_pool_put(block.data, block.size) && !cfg!(test) { - std::alloc::dealloc(block.data, layout); - } + let release = release_arena_block(block.data, block.size); ARENA_TOTAL_BYTES.with(|total| total.set(total.get().saturating_sub(size))); block.data = std::ptr::null_mut(); block.size = 0; block.offset = 0; block.dead_cycles = 0; - stats.deallocated_blocks = stats.deallocated_blocks.saturating_add(1); - stats.deallocated_bytes = stats.deallocated_bytes.saturating_add(size); + stats.record_block_release(size, release); } if changed { @@ -1286,6 +1262,7 @@ pub(crate) fn old_arena_reclaim_selected_dead_blocks( }); OLD_GEN_RECLAIM_REUSABLE_BYTES.with(|bytes| bytes.set(stats.reusable_bytes)); + OLD_GEN_RECLAIM_POOLED_BYTES.with(|bytes| bytes.set(stats.pooled_bytes)); OLD_GEN_RECLAIM_RETURNED_BYTES.with(|bytes| bytes.set(stats.deallocated_bytes)); stats } @@ -1295,7 +1272,7 @@ fn reclaim_dead_survivor_arena_blocks( block_start: usize, block_has_live: &[bool], ) -> ArenaResetStats { - with_survivor_arena_mut(arena_idx, |arena| unsafe { + with_survivor_arena_mut(arena_idx, |arena| { let keep_idx = arena .blocks .get(arena.current) @@ -1340,21 +1317,14 @@ fn reclaim_dead_survivor_arena_blocks( let base = block.data as usize; let size = block.size; - let layout = Layout::from_size_align(size, 16).unwrap(); unregister_block_generation(base, size); - // #4665: in test builds keep freed blocks mapped (no munmap) so - // unit tests holding raw GC pointers across a collection read stale - // bytes instead of SIGSEGV-ing on an unmapped page. - if !block_pool_put(block.data, block.size) && !cfg!(test) { - std::alloc::dealloc(block.data, layout); - } + let release = release_arena_block(block.data, block.size); ARENA_TOTAL_BYTES.with(|total| total.set(total.get().saturating_sub(size))); block.data = std::ptr::null_mut(); block.size = 0; block.offset = 0; block.dead_cycles = 0; - stats.deallocated_blocks = stats.deallocated_blocks.saturating_add(1); - stats.deallocated_bytes = stats.deallocated_bytes.saturating_add(size); + stats.record_block_release(size, release); } if changed { @@ -1394,6 +1364,10 @@ pub(crate) fn survivor_arena_reclaim_dead_blocks(block_has_live: &[bool]) -> Are ArenaResetStats { reset_blocks: stats0.reset_blocks.saturating_add(stats1.reset_blocks), reusable_bytes: stats0.reusable_bytes.saturating_add(stats1.reusable_bytes), + removed_blocks: stats0.removed_blocks.saturating_add(stats1.removed_blocks), + removed_bytes: stats0.removed_bytes.saturating_add(stats1.removed_bytes), + pooled_blocks: stats0.pooled_blocks.saturating_add(stats1.pooled_blocks), + pooled_bytes: stats0.pooled_bytes.saturating_add(stats1.pooled_bytes), deallocated_blocks: stats0 .deallocated_blocks .saturating_add(stats1.deallocated_blocks), diff --git a/crates/perry-runtime/src/arena/tests.rs b/crates/perry-runtime/src/arena/tests.rs index fb54316ed6..feea248e26 100644 --- a/crates/perry-runtime/src/arena/tests.rs +++ b/crates/perry-runtime/src/arena/tests.rs @@ -130,7 +130,7 @@ fn survivor_reclaim_resets_dead_blocks() { assert_eq!(survivor_after, 0); assert!(stats.reset_blocks > 0); - assert!(stats.reusable_bytes > 0 || stats.deallocated_bytes > 0); + assert!(stats.reusable_bytes > 0 || stats.removed_bytes > 0); assert!( after_reclaim.total_reserved_bytes <= after_alloc.total_reserved_bytes, "dead survivor blocks should become reusable or be returned" @@ -138,6 +138,35 @@ fn survivor_reclaim_resets_dead_blocks() { }); } +#[test] +fn budgeted_survivor_reclaim_accumulates_release_stats_across_slices() { + run_with_fresh_arenas(|| { + for _ in 0..3 { + let ptr = arena_alloc_gc_survivor(BLOCK_SIZE, 8, GC_TYPE_STRING); + assert!(!ptr.is_null()); + } + + let snapshots = arena_block_snapshots(); + let block_has_live = vec![false; snapshots.len()]; + let mut reclaim = SurvivorArenaReclaimDeadBlocksState::new(&block_has_live, &snapshots); + let mut slices = 0; + while !reclaim.step(1) { + slices += 1; + assert!(slices < 32, "one-unit survivor reclaim must converge"); + } + + let stats = reclaim.stats(); + assert!(slices > 3, "the test must span multiple reclamation slices"); + assert_eq!(stats.reset_blocks, 3); + assert_eq!(stats.removed_blocks, 2); + assert!(stats.removed_bytes >= 2 * BLOCK_SIZE); + assert_eq!(stats.pooled_blocks, 2); + assert_eq!(stats.pooled_bytes, stats.removed_bytes); + assert_eq!(stats.deallocated_blocks, 0); + assert_eq!(stats.deallocated_bytes, 0); + }); +} + fn page_range_for(base: usize, size: usize) -> std::ops::RangeInclusive { generation_page_for_addr(base)..=generation_page_for_addr(base + size - 1) } @@ -554,6 +583,8 @@ fn generation_metadata_arena_reset_stats_reports_reusable_bytes_for_retained_res assert_eq!(stats.reusable_bytes, before_offset); assert_eq!(stats.deallocated_blocks, 0); assert_eq!(stats.deallocated_bytes, 0); + assert_eq!(stats.pooled_blocks, 0); + assert_eq!(stats.pooled_bytes, 0); ARENA.with(|a| unsafe { let arena = &*a.get(); assert!(!arena.blocks[idx].data.is_null()); @@ -567,8 +598,8 @@ fn generation_metadata_removed_on_nursery_block_deallocation() { run_with_fresh_arenas(|| { let (idx, base, _size, stats) = reset_old_nursery_block(1); assert!( - stats.deallocated_blocks >= 1, - "test setup should deallocate at least one nursery block" + stats.removed_blocks >= 1, + "test setup should remove at least one nursery block" ); ARENA.with(|a| unsafe { let arena = &*a.get(); @@ -581,13 +612,17 @@ fn generation_metadata_removed_on_nursery_block_deallocation() { } #[test] -fn generation_metadata_arena_reset_stats_reports_deallocated_blocks_as_returned_not_reusable() { +fn generation_metadata_arena_reset_stats_distinguishes_pooled_from_deallocated_blocks() { run_with_fresh_arenas(|| { let (idx, base, size, _before_offset, stats) = reset_single_reclaimable_nursery_block(1); assert_eq!(stats.reset_blocks, 1); assert_eq!(stats.reusable_bytes, 0); - assert_eq!(stats.deallocated_blocks, 1); - assert_eq!(stats.deallocated_bytes, size); + assert_eq!(stats.removed_blocks, 1); + assert_eq!(stats.removed_bytes, size); + assert_eq!(stats.pooled_blocks, 1); + assert_eq!(stats.pooled_bytes, size); + assert_eq!(stats.deallocated_blocks, 0); + assert_eq!(stats.deallocated_bytes, 0); ARENA.with(|a| unsafe { let arena = &*a.get(); assert!(arena.blocks[idx].data.is_null()); @@ -602,7 +637,7 @@ fn generation_metadata_registered_on_tombstone_reuse() { run_with_fresh_arenas(|| { let (idx, _base, _size, stats) = reset_old_nursery_block(1); assert!( - stats.deallocated_blocks >= 1, + stats.removed_blocks >= 1, "test setup should create a nursery tombstone" ); @@ -1169,6 +1204,76 @@ fn block_pool_is_per_thread_and_drops_with_its_thread() { assert_eq!(block_pool_bytes_for_test(), before); } +/// #7875: per-thread LIFO ownership must not multiply the allowance by the +/// number of simultaneously-live `perry/thread` agents. Four threads race to +/// reserve 8 MiB against the same process counter under a 2 MiB cap; the +/// census reaches the cap, never four copies of it, and returns to zero after +/// the simulated owners release their shares. The production wrapper passes +/// `BLOCK_POOL_PROCESS_BYTES` to this exact reservation primitive. +#[test] +fn block_pool_cap_is_process_wide_across_live_threads() { + let counter = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ready = std::sync::Arc::new(std::sync::Barrier::new(5)); + let release = std::sync::Arc::new(std::sync::Barrier::new(5)); + let mut threads = Vec::new(); + + for _ in 0..4 { + let counter = counter.clone(); + let ready = ready.clone(); + let release = release.clone(); + threads.push(std::thread::spawn(move || { + let mut reserved = 0; + for _ in 0..2 { + if super::block::block_pool_counter_try_reserve( + &counter, + BLOCK_SIZE, + 2 * BLOCK_SIZE, + ) { + reserved += BLOCK_SIZE; + } + } + ready.wait(); + release.wait(); + counter.fetch_sub(reserved, std::sync::atomic::Ordering::Relaxed); + })); + } + + ready.wait(); + assert_eq!( + counter.load(std::sync::atomic::Ordering::Relaxed), + 2 * BLOCK_SIZE, + "all live threads together must share one process-wide cap" + ); + release.wait(); + for thread in threads { + thread.join().expect("pool worker must exit cleanly"); + } + assert_eq!( + counter.load(std::sync::atomic::Ordering::Relaxed), + 0, + "each owner must release its share of the process census" + ); +} + +#[test] +fn allocation_failure_recovery_drains_mismatched_pooled_blocks() { + let layout = std::alloc::Layout::from_size_align(BLOCK_SIZE, 16).unwrap(); + let raw = unsafe { std::alloc::alloc(layout) }; + assert!(!raw.is_null()); + assert!(block_pool_put(raw, BLOCK_SIZE)); + + force_next_block_alloc_failure(); + let block = crate::arena::block::reserve_arena_block(BLOCK_SIZE + 1); + assert_eq!( + block_pool_bytes_for_test(), + 0, + "emergency full collection must drain blocks unusable for the failed size" + ); + + let returned_layout = std::alloc::Layout::from_size_align(block.size, 16).unwrap(); + unsafe { std::alloc::dealloc(block.data, returned_layout) }; +} + // --------------------------------------------------------------------------- // #7624: deferred old-object page registration. // diff --git a/crates/perry-runtime/src/arena/walk.rs b/crates/perry-runtime/src/arena/walk.rs index f9c4749f8b..2eb35cc0eb 100644 --- a/crates/perry-runtime/src/arena/walk.rs +++ b/crates/perry-runtime/src/arena/walk.rs @@ -389,10 +389,35 @@ pub(crate) struct ArenaTelemetrySnapshot { pub struct ArenaResetStats { pub reset_blocks: usize, pub reusable_bytes: usize, + /// Blocks removed from arena reservation accounting, whether retained in + /// the recycled pool or actually returned to the allocator. + pub removed_blocks: usize, + pub removed_bytes: usize, + /// Removed blocks retained as discarded, reusable mappings. + pub pooled_blocks: usize, + pub pooled_bytes: usize, + /// Removed blocks actually handed to `dealloc` in production. pub deallocated_blocks: usize, pub deallocated_bytes: usize, } +impl ArenaResetStats { + pub(crate) fn record_block_release(&mut self, size: usize, release: ArenaBlockRelease) { + self.removed_blocks = self.removed_blocks.saturating_add(1); + self.removed_bytes = self.removed_bytes.saturating_add(size); + match release { + ArenaBlockRelease::Pooled => { + self.pooled_blocks = self.pooled_blocks.saturating_add(1); + self.pooled_bytes = self.pooled_bytes.saturating_add(size); + } + ArenaBlockRelease::Deallocated => { + self.deallocated_blocks = self.deallocated_blocks.saturating_add(1); + self.deallocated_bytes = self.deallocated_bytes.saturating_add(size); + } + } + } +} + fn arena_region_telemetry(arena: &Arena) -> ArenaRegionTelemetry { ArenaRegionTelemetry { in_use_bytes: arena.blocks.iter().map(|b| b.offset).sum(), diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index ba25a56838..b46f476708 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -1649,8 +1649,7 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( crate::arena::ArenaResetStats { reset_blocks: 0, reusable_bytes: 0, - deallocated_blocks: 0, - deallocated_bytes: 0, + ..crate::arena::ArenaResetStats::default() }, promotion_stats, ) @@ -1747,6 +1746,12 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( reusable_bytes: reset.reusable_bytes, returned_bytes: reset.deallocated_bytes, reset_blocks: reset.reset_blocks, + removed_blocks: reset.removed_blocks, + removed_bytes: reset.removed_bytes, + pooled_blocks: reset.pooled_blocks, + pooled_bytes: reset.pooled_bytes, + pool_drained_blocks: 0, + pool_drained_bytes: 0, deallocated_blocks: reset.deallocated_blocks, deallocated_bytes: reset.deallocated_bytes, retained_forwarded_stub_objects: 0, diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 19154c3190..6384d6ba41 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -1840,6 +1840,26 @@ impl GcCycleState { } ReclaimSubphase::MallocTrim => { let reclaim_start = trace_phase_start(&self.trace); + // #7875: a critical-pressure / allocation-failure request + // drains only after a FULL sweep has released all idle + // arena blocks, and before allocator pressure relief so + // the newly-deallocated mappings participate in the trim. + if self.minor.is_none() { + let drained: crate::arena::BlockPoolDrainStats = + crate::arena::drain_block_pool_if_requested(); + if let Some(trace) = self.trace.as_mut() { + trace.sweep.pool_drained_blocks = drained.blocks; + trace.sweep.pool_drained_bytes = drained.bytes; + trace.sweep.returned_bytes = + trace.sweep.returned_bytes.saturating_add(drained.bytes); + trace.sweep.deallocated_blocks = trace + .sweep + .deallocated_blocks + .saturating_add(drained.blocks); + trace.sweep.deallocated_bytes = + trace.sweep.deallocated_bytes.saturating_add(drained.bytes); + } + } let trim = run_malloc_trim(self.progress_kind); if let Some(trace) = self.trace.as_mut() { if trim.status == AllocatorMaintenanceStatus::Executed { diff --git a/crates/perry-runtime/src/gc/heap_budget.rs b/crates/perry-runtime/src/gc/heap_budget.rs index 3279266a3c..e4d69dd74a 100644 --- a/crates/perry-runtime/src/gc/heap_budget.rs +++ b/crates/perry-runtime/src/gc/heap_budget.rs @@ -130,6 +130,19 @@ budget_scaled_accessor!( 32, 1024 * 1024 ); + +/// Process-wide recycled arena-block allowance. The historical 64 MiB remains +/// unchanged on unconstrained desktop/server processes, while constrained +/// processes spend at most one eighth of their heap budget on idle mappings. +/// One 1 MiB block is the minimum useful reserve. +pub(crate) fn gc_block_pool_cap_bytes() -> usize { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| gc_block_pool_cap_with_budget(gc_heap_budget_bytes())) +} + +pub(super) fn gc_block_pool_cap_with_budget(budget: Option) -> usize { + budget_scaled_with(budget, 64 * 1024 * 1024, 1, 8, 1024 * 1024) +} budget_scaled_accessor!( gc_old_gen_reclaim_threshold_dyn_bytes, GC_OLD_GEN_RECLAIM_THRESHOLD_BYTES, diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 6116adfd97..6e9d14a1a4 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -620,6 +620,9 @@ pub(crate) fn gc_try_emergency_reclaim() -> bool { return false; } IN_EMERGENCY.with(|c| c.set(true)); + // A failed reservation can coexist with differently-sized pooled blocks. + // Make the emergency full return those mappings before the one retry. + crate::arena::request_block_pool_drain(); let _scan = roots::ManualGcScanGuard::force_full_scan(ConservativeScanSite::EmergencyReclaim); let _ = gc_collect_emergency_full(); IN_EMERGENCY.with(|c| c.set(false)); diff --git a/crates/perry-runtime/src/gc/oldgen.rs b/crates/perry-runtime/src/gc/oldgen.rs index ffa8e230bf..f3b35aff6a 100644 --- a/crates/perry-runtime/src/gc/oldgen.rs +++ b/crates/perry-runtime/src/gc/oldgen.rs @@ -114,6 +114,14 @@ pub(super) struct SweepTraceStats { pub(super) reusable_bytes: usize, pub(super) returned_bytes: usize, pub(super) reset_blocks: usize, + pub(super) removed_blocks: usize, + pub(super) removed_bytes: usize, + pub(super) pooled_blocks: usize, + pub(super) pooled_bytes: usize, + /// Pooled mappings explicitly deallocated after a critical-pressure or + /// allocation-failure full cycle. Included in returned/deallocated totals. + pub(super) pool_drained_blocks: usize, + pub(super) pool_drained_bytes: usize, pub(super) deallocated_blocks: usize, // Compatibility alias for returned_bytes. pub(super) deallocated_bytes: usize, @@ -1054,6 +1062,22 @@ fn legacy_sweep_with_age_bump_and_old_reclaim_targets( .reusable_bytes .saturating_add(survivor_reset.reusable_bytes) .saturating_add(old_reset.reusable_bytes), + removed_blocks: nursery_reset + .removed_blocks + .saturating_add(survivor_reset.removed_blocks) + .saturating_add(old_reset.removed_blocks), + removed_bytes: nursery_reset + .removed_bytes + .saturating_add(survivor_reset.removed_bytes) + .saturating_add(old_reset.removed_bytes), + pooled_blocks: nursery_reset + .pooled_blocks + .saturating_add(survivor_reset.pooled_blocks) + .saturating_add(old_reset.pooled_blocks), + pooled_bytes: nursery_reset + .pooled_bytes + .saturating_add(survivor_reset.pooled_bytes) + .saturating_add(old_reset.pooled_bytes), deallocated_blocks: nursery_reset .deallocated_blocks .saturating_add(survivor_reset.deallocated_blocks) @@ -1070,6 +1094,12 @@ fn legacy_sweep_with_age_bump_and_old_reclaim_targets( reusable_bytes: reset.reusable_bytes, returned_bytes: reset.deallocated_bytes, reset_blocks: reset.reset_blocks, + removed_blocks: reset.removed_blocks, + removed_bytes: reset.removed_bytes, + pooled_blocks: reset.pooled_blocks, + pooled_bytes: reset.pooled_bytes, + pool_drained_blocks: 0, + pool_drained_bytes: 0, deallocated_blocks: reset.deallocated_blocks, deallocated_bytes: reset.deallocated_bytes, retained_forwarded_stub_objects, @@ -1244,6 +1274,12 @@ impl IncrementalSweepState { reusable_bytes: reset.reusable_bytes, returned_bytes: reset.deallocated_bytes, reset_blocks: reset.reset_blocks, + removed_blocks: reset.removed_blocks, + removed_bytes: reset.removed_bytes, + pooled_blocks: reset.pooled_blocks, + pooled_bytes: reset.pooled_bytes, + pool_drained_blocks: 0, + pool_drained_bytes: 0, deallocated_blocks: reset.deallocated_blocks, deallocated_bytes: reset.deallocated_bytes, retained_forwarded_stub_objects: self.arena.retained_forwarded_stub_objects, @@ -1655,6 +1691,10 @@ fn add_reset_stats( crate::arena::ArenaResetStats { reset_blocks: lhs.reset_blocks.saturating_add(rhs.reset_blocks), reusable_bytes: lhs.reusable_bytes.saturating_add(rhs.reusable_bytes), + removed_blocks: lhs.removed_blocks.saturating_add(rhs.removed_blocks), + removed_bytes: lhs.removed_bytes.saturating_add(rhs.removed_bytes), + pooled_blocks: lhs.pooled_blocks.saturating_add(rhs.pooled_blocks), + pooled_bytes: lhs.pooled_bytes.saturating_add(rhs.pooled_bytes), deallocated_blocks: lhs .deallocated_blocks .saturating_add(rhs.deallocated_blocks), diff --git a/crates/perry-runtime/src/gc/pressure.rs b/crates/perry-runtime/src/gc/pressure.rs index 64573c41b2..ab907dbd5c 100644 --- a/crates/perry-runtime/src/gc/pressure.rs +++ b/crates/perry-runtime/src/gc/pressure.rs @@ -73,6 +73,14 @@ pub extern "C" fn js_gc_memory_pressure(level: u32) -> u32 { } }); + // Critical pressure owes both a full collection and a post-reclaim drain. + // Arm these before every deferral guard so an unsafe callback cannot return + // `deferred` while remembering only the nursery-sized trigger clamp. + if level >= 2 { + GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(true)); + crate::arena::request_block_pool_drain(); + } + let blocked = GC_FLAGS.with(|f| f.get()) & (GC_FLAG_IN_ALLOC | GC_FLAG_SUPPRESSED) != 0 || gc_blocked_by_unsafe_zone() || GC_ROOT_LOCK_DEPTH.with(|depth| depth.get() != 0) @@ -89,9 +97,6 @@ pub extern "C" fn js_gc_memory_pressure(level: u32) -> u32 { // the sticky "a full old-gen reclaim is owed" flag `gc_budgeted_due_trigger` // reads, so the safepoint drain runs a full mark-sweep rather than a minor. if roots::shadow_stack_has_active_frame() { - if level >= 2 { - GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(true)); - } if !GC_SAFEPOINT_PENDING.with(std::cell::Cell::get) { GC_SAFEPOINT_DEFER_ARENA_BASE.with(|base| base.set(total)); // Through the helper, never the `Cell`: it also arms the global diff --git a/crates/perry-runtime/src/gc/telemetry.rs b/crates/perry-runtime/src/gc/telemetry.rs index 0b357261bb..0e60316ee9 100644 --- a/crates/perry-runtime/src/gc/telemetry.rs +++ b/crates/perry-runtime/src/gc/telemetry.rs @@ -982,6 +982,7 @@ impl GcCycleTrace { "live_bytes": self.old_pages.live_bytes, "dead_bytes": self.old_pages.dead_bytes, "reusable_bytes": self.old_pages.reusable_bytes, + "pooled_bytes": self.old_pages.pooled_bytes, "returned_bytes": self.old_pages.returned_bytes, "pinned_bytes": self.old_pages.pinned_bytes, "object_count": self.old_pages.object_count, @@ -1110,6 +1111,12 @@ impl GcCycleTrace { "reusable_bytes": self.sweep.reusable_bytes, "returned_bytes": self.sweep.returned_bytes, "reset_blocks": self.sweep.reset_blocks, + "removed_blocks": self.sweep.removed_blocks, + "removed_bytes": self.sweep.removed_bytes, + "pooled_blocks": self.sweep.pooled_blocks, + "pooled_bytes": self.sweep.pooled_bytes, + "pool_drained_blocks": self.sweep.pool_drained_blocks, + "pool_drained_bytes": self.sweep.pool_drained_bytes, "deallocated_blocks": self.sweep.deallocated_blocks, "deallocated_bytes": self.sweep.deallocated_bytes, "retained_forwarded_stub_objects": self.sweep.retained_forwarded_stub_objects, diff --git a/crates/perry-runtime/src/gc/tests/block_pool_pressure.rs b/crates/perry-runtime/src/gc/tests/block_pool_pressure.rs new file mode 100644 index 0000000000..621ebcc21e --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/block_pool_pressure.rs @@ -0,0 +1,107 @@ +use super::super::*; +use super::support::*; + +fn seed_one_pooled_block() { + let layout = std::alloc::Layout::from_size_align(crate::arena::BLOCK_SIZE, 16).unwrap(); + let raw = unsafe { std::alloc::alloc(layout) }; + assert!(!raw.is_null(), "test block allocation must succeed"); + assert!( + crate::arena::block_pool_put(raw, crate::arena::BLOCK_SIZE), + "a fresh current-thread pool must accept one block" + ); +} + +#[test] +fn critical_pressure_drains_the_current_thread_block_pool() { + let _isolation = GcTestIsolationGuard::new(); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + + seed_one_pooled_block(); + let drained_before = crate::arena::block_pool_explicit_drained_bytes_for_test(); + assert!( + crate::arena::block_pool_bytes_for_test() >= crate::arena::BLOCK_SIZE, + "LIVE SUBJECT: the pool contains retained capacity before pressure" + ); + + assert_eq!( + js_gc_memory_pressure(2), + 2, + "safe critical pressure collects" + ); + assert_eq!( + crate::arena::block_pool_bytes_for_test(), + 0, + "critical pressure must return every current-thread pooled block" + ); + assert!( + crate::arena::block_pool_explicit_drained_bytes_for_test() + >= drained_before.saturating_add(crate::arena::BLOCK_SIZE), + "drain telemetry must count the retained mapping actually released" + ); +} + +#[test] +fn deferred_critical_pressure_drains_after_the_owed_full_cycle() { + let _isolation = GcTestIsolationGuard::new(); + let _pacing = crate::gc::policy::force_moving_gc_pacing(); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + reset_shadow_stack(); + + seed_one_pooled_block(); + let drained_before = crate::arena::block_pool_explicit_drained_bytes_for_test(); + let frame = js_shadow_frame_push(1); + assert_eq!( + js_gc_memory_pressure(2), + 1, + "a live generated frame defers critical pressure" + ); + assert!( + crate::arena::block_pool_bytes_for_test() >= crate::arena::BLOCK_SIZE, + "the pool must remain owned until the full cycle actually completes" + ); + js_shadow_frame_pop(frame); + + js_gc_loop_safepoint(); + assert_eq!( + crate::arena::block_pool_bytes_for_test(), + 0, + "the deferred full-cycle publication must consume the sticky drain" + ); + assert!( + crate::arena::block_pool_explicit_drained_bytes_for_test() + >= drained_before.saturating_add(crate::arena::BLOCK_SIZE), + "honest drain telemetry must include the seeded retained block" + ); +} + +#[test] +fn blocked_critical_pressure_keeps_the_full_cycle_and_drain_sticky() { + let _isolation = GcTestIsolationGuard::new(); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + seed_one_pooled_block(); + + let previous_flags = GC_FLAGS.with(|flags| { + let previous = flags.get(); + flags.set(previous | GC_FLAG_SUPPRESSED); + previous + }); + let result = js_gc_memory_pressure(2); + GC_FLAGS.with(|flags| flags.set(previous_flags)); + + assert_eq!( + result, 1, + "suppressed allocation bookkeeping defers pressure" + ); + assert!( + GC_OLD_RECLAIM_PENDING.with(std::cell::Cell::get), + "critical pressure must retain the full-cycle debt across every guard" + ); + assert!(crate::arena::block_pool_bytes_for_test() >= crate::arena::BLOCK_SIZE); + + gc_check_trigger(); + assert_eq!( + crate::arena::block_pool_bytes_for_test(), + 0, + "the allocation-point full-cycle backstop must consume the drain" + ); +} diff --git a/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs b/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs index a057d57349..be9e54cd7e 100644 --- a/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs +++ b/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs @@ -326,7 +326,11 @@ fn old_generation_targeted_and_full_reclaim_are_bounded_and_publish_telemetry() "targeted old reclaim should not complete in one work unit" ); let targeted_stats = complete_incremental_sweep(&mut targeted_sweep); - assert!(targeted_stats.returned_bytes > 0 || targeted_stats.reusable_bytes > 0); + assert!( + targeted_stats.returned_bytes > 0 + || targeted_stats.pooled_bytes > 0 + || targeted_stats.reusable_bytes > 0 + ); let targeted_summary = crate::arena::old_page_summary(); assert_eq!( targeted_summary.returned_bytes, @@ -336,6 +340,7 @@ fn old_generation_targeted_and_full_reclaim_are_bounded_and_publish_telemetry() targeted_summary.reusable_bytes, targeted_stats.reusable_bytes ); + assert_eq!(targeted_summary.pooled_bytes, targeted_stats.pooled_bytes); let _full_dead_a = crate::arena::arena_alloc_gc_old(900 * 1024, 8, GC_TYPE_STRING) as usize; let _full_dead_b = crate::arena::arena_alloc_gc_old(900 * 1024, 8, GC_TYPE_STRING) as usize; @@ -346,10 +351,15 @@ fn old_generation_targeted_and_full_reclaim_are_bounded_and_publish_telemetry() "full old reclaim should not complete in one work unit" ); let full_stats = complete_incremental_sweep(&mut full_sweep); - assert!(full_stats.returned_bytes > 0 || full_stats.reusable_bytes > 0); + assert!( + full_stats.returned_bytes > 0 + || full_stats.pooled_bytes > 0 + || full_stats.reusable_bytes > 0 + ); let full_summary = crate::arena::old_page_summary(); assert_eq!(full_summary.returned_bytes, full_stats.returned_bytes); assert_eq!(full_summary.reusable_bytes, full_stats.reusable_bytes); + assert_eq!(full_summary.pooled_bytes, full_stats.pooled_bytes); } #[test] diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 5fecce3fbd..48b61cce99 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -2,6 +2,7 @@ mod alloc; mod barrier; mod barrier_arming; mod barrier_decoded_parent; +mod block_pool_pressure; mod budgeted_step_api; mod buffer_bound_method_name; mod buffer_side_tables; diff --git a/crates/perry-runtime/src/gc/tests/oldgen.rs b/crates/perry-runtime/src/gc/tests/oldgen.rs index 51c95fa676..1e34d5b173 100644 --- a/crates/perry-runtime/src/gc/tests/oldgen.rs +++ b/crates/perry-runtime/src/gc/tests/oldgen.rs @@ -955,10 +955,18 @@ fn test_old_page_defrag_trace_json_distinguishes_moved_from_reclaimable() { trace.evacuation.old_page_moved_bytes = 64; trace.evacuation.released_original_objects = 1; trace.evacuation.released_original_bytes = 64; + trace.old_pages.pooled_bytes = 48; trace.sweep.dead_bytes = 192; trace.sweep.freed_bytes = 192; trace.sweep.reusable_bytes = 128; trace.sweep.returned_bytes = 32; + trace.sweep.removed_blocks = 2; + trace.sweep.removed_bytes = 96; + trace.sweep.pooled_blocks = 1; + trace.sweep.pooled_bytes = 64; + trace.sweep.pool_drained_blocks = 1; + trace.sweep.pool_drained_bytes = 16; + trace.sweep.deallocated_blocks = 1; trace.sweep.deallocated_bytes = 32; let event = trace.into_json(GcStepSnapshot::current()); @@ -987,10 +995,18 @@ fn test_old_page_defrag_trace_json_distinguishes_moved_from_reclaimable() { event["evacuation"]["released_original_returned_bytes"].as_u64(), Some(0) ); + assert_eq!(event["old_pages"]["pooled_bytes"].as_u64(), Some(48)); assert_eq!(event["sweep"]["dead_bytes"].as_u64(), Some(192)); assert_eq!(event["sweep"]["freed_bytes"].as_u64(), Some(192)); assert_eq!(event["sweep"]["reusable_bytes"].as_u64(), Some(128)); assert_eq!(event["sweep"]["returned_bytes"].as_u64(), Some(32)); + assert_eq!(event["sweep"]["removed_blocks"].as_u64(), Some(2)); + assert_eq!(event["sweep"]["removed_bytes"].as_u64(), Some(96)); + assert_eq!(event["sweep"]["pooled_blocks"].as_u64(), Some(1)); + assert_eq!(event["sweep"]["pooled_bytes"].as_u64(), Some(64)); + assert_eq!(event["sweep"]["pool_drained_blocks"].as_u64(), Some(1)); + assert_eq!(event["sweep"]["pool_drained_bytes"].as_u64(), Some(16)); + assert_eq!(event["sweep"]["deallocated_blocks"].as_u64(), Some(1)); assert_eq!(event["sweep"]["deallocated_bytes"].as_u64(), Some(32)); } diff --git a/crates/perry-runtime/src/gc/tests/triggers.rs b/crates/perry-runtime/src/gc/tests/triggers.rs index d4444075ca..3bb82357de 100644 --- a/crates/perry-runtime/src/gc/tests/triggers.rs +++ b/crates/perry-runtime/src/gc/tests/triggers.rs @@ -370,6 +370,21 @@ fn test_budget_scaled_clamps_only_under_budget() { assert_eq!(budget_scaled_with(Some(MB), 128 * MB, 1, 4, 2 * MB), 2 * MB); } +#[test] +fn test_block_pool_allowance_scales_below_small_heap_budgets() { + use super::super::heap_budget::gc_block_pool_cap_with_budget; + const MB: usize = 1024 * 1024; + + assert_eq!(gc_block_pool_cap_with_budget(None), 64 * MB); + assert_eq!(gc_block_pool_cap_with_budget(Some(64 * MB)), 8 * MB); + assert_eq!(gc_block_pool_cap_with_budget(Some(32 * MB)), 4 * MB); + assert_eq!(gc_block_pool_cap_with_budget(Some(8 * MB)), MB); + assert!( + gc_block_pool_cap_with_budget(Some(32 * MB)) < 32 * MB, + "a small PERRY_GC_HEAP_LIMIT cannot coexist with the fixed 64 MiB reserve" + ); +} + // ─────────────────────────────────────────────────────────────────────────── // #7024: the alloc-point deferral must be REACHABLE at the moment a nursery // trigger becomes due, at every heap budget.