diff --git a/changelog.d/7892-slice-weak-processing.md b/changelog.d/7892-slice-weak-processing.md new file mode 100644 index 0000000000..a852b332b6 --- /dev/null +++ b/changelog.d/7892-slice-weak-processing.md @@ -0,0 +1,3 @@ +Full and fallback garbage collections now process weak-reference holders from +their registry in budgeted slices instead of scanning the entire live heap in +one atomic pause. diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index b46f476708..b952ab4344 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -842,8 +842,8 @@ impl CopyingNurseryCollector { // never tombstoned and FinalizationRegistry never fired while // copied-minor was the operative cycle. Repair an already-moved // target's address now and queue the slot so `repair_weak_slots` - // fixes targets evacuated after this visit; the after-mark pass - // (`process_weak_targets_after_mark`) then tombstones dead ones. + // fixes targets evacuated after this visit; the registry pass then + // tombstones dead ones. // No remembered-set entry either — the write barrier skips weak // slots the same way. if !parent_header.is_null() @@ -898,8 +898,8 @@ impl CopyingNurseryCollector { /// Second pass over the weak target slots collected during the scan: /// a weak target evacuated via a strong edge AFTER its slot was /// visited still points at the from-space original — rewrite it to - /// the forwarding address so `process_weak_targets_after_mark` (and - /// the mutator) read the live copy. Targets never forwarded are + /// the forwarding address so weak processing (and the mutator) read the + /// live copy. Targets never forwarded are /// either old-gen/pinned live (no rewrite needed) or dead (left for /// the after-mark tombstone pass). pub(super) unsafe fn repair_weak_slots(&mut self) { @@ -1589,8 +1589,8 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( // allocated. `process_weak_targets_from_registry` instead walks only the // registered holders and classifies targets with the O(1) page-metadata // classifier the copy already built (`collector.ptrs`) — no BTreeSet, no - // arena walk. The full-cycle path (cycle.rs `WeakProcessing`) is - // untouched and still uses the valid-pointer set it built for its trace. + // arena walk. The full-cycle path (cycle.rs `WeakProcessing`) now uses the + // same registry, with its existing valid-pointer set for liveness. unsafe { collector.repair_weak_slots(); } diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 6384d6ba41..b248e44310 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -872,6 +872,7 @@ struct AtomicFinalizeCycleState { subphase: AtomicFinalizeSubphase, barrier_drain: Option, remembered_rebuild: Option, + weak_processing: Option, /// Budgeted cycles insert FinalRootRemark after BarrierSeedDrain; /// synchronous cycles have no mutator windows and skip it. remark: bool, @@ -890,6 +891,7 @@ impl AtomicFinalizeCycleState { subphase: AtomicFinalizeSubphase::BarrierSeedDrain, barrier_drain: None, remembered_rebuild: None, + weak_processing: None, remark, } } @@ -1035,6 +1037,20 @@ impl GcCycleState { self.phase } + #[cfg(test)] + pub(super) fn atomic_finalize_subphase_for_tests(&self) -> Option<&'static str> { + let subphase = self.atomic_finalize.as_ref()?.subphase; + Some(match subphase { + AtomicFinalizeSubphase::WeakProcessing => "weak_processing", + AtomicFinalizeSubphase::MinorPrelude => "minor_prelude", + AtomicFinalizeSubphase::BarrierSeedDrain => "barrier_seed_drain", + AtomicFinalizeSubphase::FinalRootRemark => "final_root_remark", + AtomicFinalizeSubphase::RememberedSetRebuild => "remembered_set_rebuild", + AtomicFinalizeSubphase::DisableBarrier => "disable_barrier", + AtomicFinalizeSubphase::Done => "done", + }) + } + pub(super) fn collection_kind(&self) -> GcCollectionKind { self.collection_kind } @@ -1305,15 +1321,16 @@ impl GcCycleState { .as_ref() .expect("atomic finalize state exists") .subphase; - // SLICED subphases (seed drain, full-cycle RS rebuild) honor the - // caller's budget and may return to the mutator; the ATOMIC TAIL - // (remark → weak → barrier-off → Sweep) runs to the phase - // transition in this single pause so no mutator window can - // invalidate the near-final mark set. + // SLICED subphases honor the caller's budget and may return to the + // mutator. WeakProcessing is safe to slice while the barrier and + // allocate-black births stay active: a target the mutator can + // still name was marked at remark or was born black, and holders + // born after the registry snapshot wait for the next cycle. let sliced = matches!( subphase, AtomicFinalizeSubphase::BarrierSeedDrain | AtomicFinalizeSubphase::RememberedSetRebuild + | AtomicFinalizeSubphase::WeakProcessing ); let sub_budget = if sliced { budget.work_units @@ -1399,18 +1416,30 @@ impl GcCycleState { // record is guaranteed by the record's pending-flag reset; // delivery happens at the explicit-`gc()` tail or the next // microtask-pump drain (`drain_pending_finalization_jobs`). - crate::weakref::process_weak_targets_after_mark( - valid_ptrs, minor_only, /* enqueue_callbacks = */ true, - ); - let next = if minor_only { - AtomicFinalizeSubphase::MinorPrelude - } else { - AtomicFinalizeSubphase::DisableBarrier + let done = { + let state = self + .atomic_finalize + .as_mut() + .expect("atomic finalize state exists"); + let weak = state + .weak_processing + .get_or_insert_with(crate::weakref::FullWeakProcessingState::new); + weak.step( + valid_ptrs, minor_only, /* enqueue_callbacks = */ true, budget, + ) }; - self.atomic_finalize - .as_mut() - .expect("atomic finalize state exists") - .subphase = next; + if done { + let state = self + .atomic_finalize + .as_mut() + .expect("atomic finalize state exists"); + state.weak_processing = None; + state.subphase = if minor_only { + AtomicFinalizeSubphase::MinorPrelude + } else { + AtomicFinalizeSubphase::DisableBarrier + }; + } } AtomicFinalizeSubphase::MinorPrelude => { if budget == 0 { diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 26f52d2b30..d70201e59c 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -3275,10 +3275,10 @@ fn gc_budgeted_step_work_units_inner_with_progress( /// the incremental sweep-parking hole (#6180): a pure compute loop that never /// reaches the event pump still finishes the cycle (and disables the mark /// barrier / reclaims memory) purely from the allocations it keeps making, so -/// RSS stays bounded. `AtomicFinalizeSubphase::WeakProcessing` is the one -/// phase step that is not yet internally sliced, so the assist that lands on it -/// runs it whole — a single O(live-weak-holders) spike per cycle; slicing it is -/// a tracked follow-up (pause-quality, not correctness). +/// RSS stays bounded. `AtomicFinalizeSubphase::WeakProcessing` snapshots the +/// live-holder registry and consumes at most the supplied number of holders per +/// assist, so unrelated heap size cannot turn one assist into a whole-arena +/// pause. fn gc_mutator_assist_step_work_units_inner_with_progress( work_units: usize, start_progress_kind: GcProgressKind, diff --git a/crates/perry-runtime/src/gc/tests/copying/weak_holder_registry.rs b/crates/perry-runtime/src/gc/tests/copying/weak_holder_registry.rs index c4b6a54246..581e22d237 100644 --- a/crates/perry-runtime/src/gc/tests/copying/weak_holder_registry.rs +++ b/crates/perry-runtime/src/gc/tests/copying/weak_holder_registry.rs @@ -204,6 +204,35 @@ fn test_weak_holder_latch_clears_after_transient_weakmap_dies() { ); } +/// Full/fallback weak processing must scale with registered weak holders, not +/// with unrelated arena population. One live WeakRef is held constant while +/// the second collection sees a much larger heap. +#[test] +fn test_full_weak_processing_work_is_independent_of_unrelated_heap_size() { + let _guard = CopyingNurseryTestGuard::new(1_001); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + crate::weakref::test_support::clear_weak_holders(); + + let target = crate::object::js_object_alloc(0, 0); + let weak_ref = crate::weakref::js_weakref_new(f64::from_bits(obj_bits(target))); + js_shadow_slot_set(0, obj_bits(weak_ref)); + + gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct)); + let small_heap_work = crate::weakref::test_support::full_weak_processing_work_units(); + assert_eq!(small_heap_work, 1, "exactly one holder was registered"); + + for slot in 1..=1_000 { + let unrelated = crate::object::js_object_alloc(0, 0); + js_shadow_slot_set(slot, obj_bits(unrelated)); + } + gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct)); + let large_heap_work = crate::weakref::test_support::full_weak_processing_work_units(); + assert_eq!( + large_heap_work, small_heap_work, + "one-holder weak work grew with unrelated heap: small={small_heap_work}, large={large_heap_work}" + ); +} + /// (5) Cross-cycle registry currency: a WeakMap entry survives three /// consecutive moving minors with its holder evacuated (address changing) each /// time; the registry tracks the moved holder so a key that dies on cycle 3 is @@ -277,6 +306,139 @@ extern "C" fn finreg_registry_test_callback( f64::from_bits(crate::value::TAG_UNDEFINED) } +/// Build one holder of every kind, move all four through a copied minor, then +/// drop their weak targets and age the graph out of block persistence. Also +/// adds one dead holder and one stale registry address so the following +/// full/fallback pass must classify both before dereferencing. +fn prepare_moved_full_path_weak_holders() { + crate::weakref::test_support::clear_weak_holders(); + assert_eq!(crate::weakref::pending_finalization_jobs_count(), 0); + + let weak_target = crate::object::js_object_alloc(0, 0); + let weak_ref = crate::weakref::js_weakref_new(f64::from_bits(obj_bits(weak_target))); + js_shadow_slot_set(0, obj_bits(weak_ref)); + js_shadow_slot_set(4, obj_bits(weak_target)); + + let map = crate::weakref::js_weakmap_new(); + let map_key = crate::object::js_object_alloc(0, 0); + js_shadow_slot_set(1, obj_bits(map)); + js_shadow_slot_set(5, obj_bits(map_key)); + crate::weakref::js_weakmap_set( + f64::from_bits(js_shadow_slot_get(1)), + f64::from_bits(js_shadow_slot_get(5)), + f64::from_bits(crate::value::TAG_TRUE), + ); + + let set = crate::weakref::js_weakset_new(); + let set_key = crate::object::js_object_alloc(0, 0); + js_shadow_slot_set(2, obj_bits(set)); + js_shadow_slot_set(6, obj_bits(set_key)); + crate::weakref::js_weakset_add( + f64::from_bits(js_shadow_slot_get(2)), + f64::from_bits(js_shadow_slot_get(6)), + ); + + let callback = crate::closure::js_closure_alloc(finreg_registry_test_callback as *const u8, 0); + let registry = crate::weakref::js_finreg_new(f64::from_bits(ptr_bits(callback as usize))); + let finreg_target = crate::object::js_object_alloc(0, 0); + js_shadow_slot_set(3, obj_bits(registry)); + js_shadow_slot_set(7, obj_bits(finreg_target)); + crate::weakref::js_finreg_register( + f64::from_bits(js_shadow_slot_get(3)), + f64::from_bits(js_shadow_slot_get(7)), + f64::from_bits(crate::value::TAG_TRUE), + f64::from_bits(crate::value::TAG_UNDEFINED), + ); + + let before_move = crate::weakref::test_support::weak_holder_addresses(); + assert_eq!( + before_move.len(), + 4, + "one holder of every kind is registered" + ); + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert_copied_minor_trace(&trace, true, CopiedMinorFallbackReason::None, false); + let after_move = crate::weakref::test_support::weak_holder_addresses(); + assert_eq!(after_move.len(), 4); + assert!( + before_move.iter().all(|addr| !after_move.contains(addr)), + "every registered holder must be rekeyed after evacuation" + ); + + for slot in 4..8 { + js_shadow_slot_set(slot, 0); + } + + // A dead holder must be pruned without dispatch. The fabricated stale + // address must be rejected by ValidPointerSet before any header read. + let dead_target = crate::object::js_object_alloc(0, 0); + let _dead_holder = crate::weakref::js_weakref_new(f64::from_bits(obj_bits(dead_target))); + crate::weakref::test_support::register_weak_holder_address(0x1234_5678); + assert_eq!( + crate::weakref::test_support::weak_holder_addresses().len(), + 6 + ); + + let aged_from = crate::arena::general_block_count(); + while crate::arena::general_block_count().saturating_sub(aged_from) < 7 { + for _ in 0..64 { + let _ = crate::arena::arena_alloc_gc(4096, 8, GC_TYPE_STRING); + } + } +} + +fn assert_full_path_weak_results() { + assert_eq!( + crate::weakref::js_weakref_deref(f64::from_bits(js_shadow_slot_get(0))).to_bits(), + crate::value::TAG_UNDEFINED, + "WeakRef target must be tombstoned" + ); + for slot in [1, 2] { + let collection = (js_shadow_slot_get(slot) & POINTER_MASK) as *const crate::ObjectHeader; + assert!( + crate::weakref::weak_collection_entries(collection).is_empty(), + "WeakMap/WeakSet dead-key entry must be tombstoned" + ); + } + assert_eq!( + crate::weakref::pending_finalization_jobs_count(), + 1, + "FinalizationRegistry target must enqueue exactly one cleanup job" + ); + assert_eq!( + crate::weakref::test_support::weak_holder_addresses().len(), + 4, + "the dead holder and stale address must be pruned; live holders remain" + ); +} + +#[test] +fn test_full_registry_path_handles_all_weak_kinds_moved_dead_and_stale_holders() { + let _guard = CopyingNurseryTestGuard::new(8); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + prepare_moved_full_path_weak_holders(); + + gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct)); + assert_full_path_weak_results(); +} + +#[test] +fn test_fallback_registry_path_handles_all_weak_kinds_moved_dead_and_stale_holders() { + let _guard = CopyingNurseryTestGuard::new(8); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + prepare_moved_full_path_weak_holders(); + + let _barrier_guard = GeneratedWriteBarrierTestGuard::inactive(); + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert_copied_minor_trace( + &trace, + false, + CopiedMinorFallbackReason::BarriersInactive, + false, + ); + assert_full_path_weak_results(); +} + /// (6) FinalizationRegistry: a registered target that dies across a moving minor /// enqueues its cleanup job through the registry-based pass (the #6192 /// automatic-cycle delivery must be preserved — the registry holder is the diff --git a/crates/perry-runtime/src/gc/tests/copying/weak_semantics.rs b/crates/perry-runtime/src/gc/tests/copying/weak_semantics.rs index 910e0d39f0..21dc222b33 100644 --- a/crates/perry-runtime/src/gc/tests/copying/weak_semantics.rs +++ b/crates/perry-runtime/src/gc/tests/copying/weak_semantics.rs @@ -5,8 +5,8 @@ //! entries never cleared and FinalizationRegistry never fired while //! copied-minor was the operative cycle. The scan now records weak slots //! without evacuating, `repair_weak_slots` fixes addresses of targets moved -//! via strong edges, and `process_weak_targets_after_mark` runs on the fast -//! path (gated on the weak-holder latch). +//! via strong edges, and the registry-scoped weak pass runs on the fast path +//! (gated on the weak-holder latch). use super::*; diff --git a/crates/perry-runtime/src/gc/tests/cycle_state.rs b/crates/perry-runtime/src/gc/tests/cycle_state.rs index 00a16ed620..25453db861 100644 --- a/crates/perry-runtime/src/gc/tests/cycle_state.rs +++ b/crates/perry-runtime/src/gc/tests/cycle_state.rs @@ -1084,6 +1084,71 @@ fn full_atomic_finalize_slices_barrier_seed_drain_with_tiny_budget() { } } +#[test] +fn full_atomic_finalize_slices_weak_holders_with_tiny_budget() { + const HOLDERS: u32 = 8; + let _guard = CopyingNurseryTestGuard::new(HOLDERS); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + crate::weakref::test_support::clear_weak_holders(); + + for slot in 0..HOLDERS { + let target = crate::object::js_object_alloc(0, 0); + let weak_ref = crate::weakref::js_weakref_new(f64::from_bits(ptr_bits(target as usize))); + js_shadow_slot_set(slot, ptr_bits(weak_ref as usize)); + } + + // Recent blocks are conservatively persisted for register-held values. + // Move the holders/targets outside that window so weak-only targets are + // genuinely white at finalization. + let aged_from = crate::arena::general_block_count(); + while crate::arena::general_block_count().saturating_sub(aged_from) < 7 { + for _ in 0..64 { + let _ = crate::arena::arena_alloc_gc(4096, 8, GC_TYPE_STRING); + } + } + + let mut state = GcCycleState::new_full(trace_snapshot(GcTriggerKind::Direct)); + run_cycle_until_phase(&mut state, GcCyclePhase::AtomicFinalize); + let mut setup_steps = 0usize; + while state.atomic_finalize_subphase_for_tests() != Some("weak_processing") { + state.step(GcWorkBudget::bounded(1)); + setup_steps += 1; + assert!(setup_steps < 100_000, "weak processing was never reached"); + } + + let after_first_slice = crate::weakref::test_support::full_weak_processing_work_units(); + assert_eq!( + after_first_slice, 1, + "the step that enters weak processing must consume one holder" + ); + state.step(GcWorkBudget::bounded(1)); + assert_eq!( + crate::weakref::test_support::full_weak_processing_work_units(), + 2, + "a one-unit step must consume exactly one additional holder" + ); + assert_eq!( + state.atomic_finalize_subphase_for_tests(), + Some("weak_processing"), + "multiple holders must keep weak processing parked across steps" + ); + + run_cycle_in_single_unit_steps(&mut state); + let _ = state.take_outcome().expect("cycle should complete"); + assert_eq!( + crate::weakref::test_support::full_weak_processing_work_units(), + HOLDERS as usize + ); + for slot in 0..HOLDERS { + let weak_ref = f64::from_bits(js_shadow_slot_get(slot)); + assert_eq!( + crate::weakref::js_weakref_deref(weak_ref).to_bits(), + crate::value::TAG_UNDEFINED, + "weak-only target must be tombstoned" + ); + } +} + #[test] fn bounded_full_cycle_preserves_roots_and_reclaims_unreachable_objects() { let _guard = CopyingNurseryTestGuard::new(1); diff --git a/crates/perry-runtime/src/weakref.rs b/crates/perry-runtime/src/weakref.rs index d0b2c695f4..7d1d20c2e9 100644 --- a/crates/perry-runtime/src/weakref.rs +++ b/crates/perry-runtime/src/weakref.rs @@ -21,6 +21,9 @@ use crate::value::{ }; use std::cell::RefCell; +#[cfg(test)] +pub(crate) mod test_support; + const TAG_UNDEFINED: u64 = 0x7FFC_0000_0000_0001; const TAG_TRUE: u64 = 0x7FFC_0000_0000_0004; const TAG_FALSE: u64 = 0x7FFC_0000_0000_0003; @@ -667,9 +670,9 @@ pub(crate) fn scan_pending_finalization_jobs_roots_mut( // shared between two passes that decide "was this weak target collected?" // differently: // -// * The FULL / fallback cycle (`process_weak_targets_after_mark`, driven from -// cycle.rs `WeakProcessing`) probes the `ValidPointerSet` it already built -// for its main trace. UNCHANGED behavior — see `weak_target_should_clear`. +// * The FULL / fallback cycle (`FullWeakProcessingState`, driven from cycle.rs +// `WeakProcessing`) probes the `ValidPointerSet` it already built for its +// main trace. See `weak_target_should_clear`. // * The copied-minor fast path (`process_weak_targets_from_registry`) probes // the copy's O(1) page-metadata classifier (`CopyingPointerSet`), avoiding // both the full-heap BTreeSet build and the whole-arena walk. See @@ -816,15 +819,12 @@ unsafe fn classify_gc_type_child( (unsafe { (*cp.header).obj_type } == obj_type).then_some(ptr) } -/// What the copied-minor pass should do with a registered holder address. +/// What a registry-scoped weak pass should do with a holder address. enum HolderDisposition { - /// Live holder scanned this cycle (its weak slots are repaired): rekey the - /// registry to this current address and process it. + /// Live holder at its current address: process its weak slots. Process(usize), /// Cannot be proven dead in a minor (an unmarked OLD/longlived holder — a - /// minor doesn't mark old-gen) AND its weak slots may be stale/unrepaired: - /// leave it registered untouched and let a full GC resolve it. Mirrors the - /// original arena walk, which only ever processed MARKED objects. + /// minor doesn't mark old-gen): leave it registered for a full GC. Keep, /// Provably dead (unmarked nursery holder) or unclassifiable (stale / /// recycled address): remove it from the registry. @@ -871,35 +871,102 @@ unsafe fn resolve_weak_holder_copied( } } -/// Full / fallback-cycle weak processing. Walks EVERY live object in the arena -/// to find the three weak-holder class_ids and tombstones dead weak targets -/// using the `ValidPointerSet` the caller built for its main trace. UNCHANGED -/// by #6182 (the registry optimization is copied-minor-only). -pub(crate) fn process_weak_targets_after_mark( +/// Resumable full/fallback weak processing. The holder registry is snapshotted +/// once, then each call consumes at most `budget` holders. This makes the work +/// O(registered weak holders), rather than O(all arena objects), and lets a +/// budgeted GC return to the mutator between holders. +/// +/// Snapshotting is intentional: budgeted cycles are non-moving, while +/// synchronous moving cycles pass an unlimited budget and cannot expose a +/// mutator window. Holders allocated after the snapshot are allocate-black and +/// therefore cannot lose a target in the current cycle; the next collection +/// processes them. +pub(crate) struct FullWeakProcessingState { + holders: Vec, + cursor: usize, +} + +impl FullWeakProcessingState { + pub(crate) fn new() -> Self { + let holders = WEAK_HOLDERS.with(|holders| holders.borrow().iter().copied().collect()); + #[cfg(test)] + test_support::reset_full_weak_processing_work_units(); + Self { holders, cursor: 0 } + } + + /// Process up to `budget` registered holders. A FinalizationRegistry is + /// one holder/work unit; its record array stays atomic so unregistering + /// cannot interleave with and reorder an in-progress registry scan. + pub(crate) fn step( + &mut self, + valid_ptrs: &crate::gc::ValidPointerSet, + minor_only: bool, + enqueue_callbacks: bool, + budget: usize, + ) -> bool { + if budget == 0 { + return self.cursor == self.holders.len(); + } + let stop = self.holders.len().min(self.cursor.saturating_add(budget)); + let liveness = FullCycleLiveness { + valid_ptrs, + minor_only, + }; + while self.cursor < stop { + let addr = self.holders[self.cursor]; + self.cursor += 1; + #[cfg(test)] + test_support::note_full_weak_processing_work_unit(); + match unsafe { resolve_weak_holder_full(valid_ptrs, addr, minor_only) } { + HolderDisposition::Drop => { + WEAK_HOLDERS.with(|holders| { + holders.borrow_mut().remove(&addr); + }); + } + HolderDisposition::Keep => {} + HolderDisposition::Process(current) => unsafe { + dispatch_weak_holder( + current as *mut ObjectHeader, + &liveness, + enqueue_callbacks, + ); + }, + } + } + self.cursor == self.holders.len() + } +} + +/// Validate a registry entry before dereferencing it. Full cycles can prove +/// every unmarked holder dead. Fallback minors may only prove that for nursery +/// holders; unmarked old holders stay registered for the next full cycle. +unsafe fn resolve_weak_holder_full( valid_ptrs: &crate::gc::ValidPointerSet, + addr: usize, minor_only: bool, - enqueue_callbacks: bool, -) { - // #6180 pause floor: the whole-heap walk below exists only to FIND weak - // holders (WeakRef / FinalizationRegistry / WeakMap-entry objects). The - // #6182 registry tracks every live holder — if none exist (the common - // case), the entire O(heap) pass is a no-op. This is the single largest - // atomic-finalize cost for weakref-free programs. - if !weak_target_holders_allocated() { - return; +) -> HolderDisposition { + if !valid_ptrs.contains(&addr) { + return HolderDisposition::Drop; + } + let header = header_from_user_addr(addr); + if (*header).obj_type != crate::gc::GC_TYPE_OBJECT { + return HolderDisposition::Drop; + } + let obj = addr as *mut ObjectHeader; + if !matches!( + (*obj).class_id, + CLASS_ID_WEAKREF | CLASS_ID_FINALIZATION_REGISTRY | CLASS_ID_WEAK_ENTRY + ) { + return HolderDisposition::Drop; + } + if header_is_live(header) { + return HolderDisposition::Process(addr); + } + if minor_only && !crate::arena::pointer_in_nursery(addr) { + HolderDisposition::Keep + } else { + HolderDisposition::Drop } - let liveness = FullCycleLiveness { - valid_ptrs, - minor_only, - }; - crate::arena::arena_walk_objects(|header_ptr| unsafe { - let header = header_ptr as *mut crate::gc::GcHeader; - if (*header).obj_type != crate::gc::GC_TYPE_OBJECT || !header_is_live(header) { - return; - } - let obj = header_ptr.add(crate::gc::GC_HEADER_SIZE) as *mut ObjectHeader; - dispatch_weak_holder(obj, &liveness, enqueue_callbacks); - }); } /// Copied-minor weak processing (#6182). Iterates ONLY the registered holders diff --git a/crates/perry-runtime/src/weakref/test_support.rs b/crates/perry-runtime/src/weakref/test_support.rs new file mode 100644 index 0000000000..9bfc37102c --- /dev/null +++ b/crates/perry-runtime/src/weakref/test_support.rs @@ -0,0 +1,35 @@ +use super::WEAK_HOLDERS; + +thread_local! { + static FULL_WEAK_PROCESSING_WORK_UNITS: std::cell::Cell = + const { std::cell::Cell::new(0) }; +} + +pub(crate) fn full_weak_processing_work_units() -> usize { + FULL_WEAK_PROCESSING_WORK_UNITS.with(std::cell::Cell::get) +} + +pub(crate) fn reset_full_weak_processing_work_units() { + FULL_WEAK_PROCESSING_WORK_UNITS.with(|units| units.set(0)); +} + +pub(crate) fn note_full_weak_processing_work_unit() { + FULL_WEAK_PROCESSING_WORK_UNITS.with(|units| units.set(units.get().saturating_add(1))); +} + +pub(crate) fn clear_weak_holders() { + WEAK_HOLDERS.with(|holders| holders.borrow_mut().clear()); +} + +pub(crate) fn weak_holder_addresses() -> Vec { + let mut addresses = + WEAK_HOLDERS.with(|holders| holders.borrow().iter().copied().collect::>()); + addresses.sort_unstable(); + addresses +} + +pub(crate) fn register_weak_holder_address(addr: usize) { + WEAK_HOLDERS.with(|holders| { + holders.borrow_mut().insert(addr); + }); +}