perf(gc): slice full weak processing by holder

changelog.d/7892-slice-weak-processing.md

@@ -0,0 +1,3 @@
+Full and fallback garbage collections now process weak-reference holders from
+their registry in budgeted slices instead of scanning the entire live heap in
+one atomic pause.

crates/perry-runtime/src/gc/copying.rs

@@ -842,8 +842,8 @@ impl CopyingNurseryCollector {
         // never tombstoned and FinalizationRegistry never fired while
         // copied-minor was the operative cycle. Repair an already-moved
         // target's address now and queue the slot so `repair_weak_slots`
-        // fixes targets evacuated after this visit; the after-mark pass
-        // (`process_weak_targets_after_mark`) then tombstones dead ones.
+        // fixes targets evacuated after this visit; the registry pass then
+        // tombstones dead ones.
         // No remembered-set entry either — the write barrier skips weak
         // slots the same way.
         if !parent_header.is_null()
@@ -898,8 +898,8 @@ impl CopyingNurseryCollector {
     /// Second pass over the weak target slots collected during the scan:
     /// a weak target evacuated via a strong edge AFTER its slot was
     /// visited still points at the from-space original — rewrite it to
-    /// the forwarding address so `process_weak_targets_after_mark` (and
-    /// the mutator) read the live copy. Targets never forwarded are
+    /// the forwarding address so weak processing (and the mutator) read the
+    /// live copy. Targets never forwarded are
     /// either old-gen/pinned live (no rewrite needed) or dead (left for
     /// the after-mark tombstone pass).
     pub(super) unsafe fn repair_weak_slots(&mut self) {
@@ -1589,8 +1589,8 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility(
     // allocated. `process_weak_targets_from_registry` instead walks only the
     // registered holders and classifies targets with the O(1) page-metadata
     // classifier the copy already built (`collector.ptrs`) — no BTreeSet, no
-    // arena walk. The full-cycle path (cycle.rs `WeakProcessing`) is
-    // untouched and still uses the valid-pointer set it built for its trace.
+    // arena walk. The full-cycle path (cycle.rs `WeakProcessing`) now uses the
+    // same registry, with its existing valid-pointer set for liveness.
     unsafe {
         collector.repair_weak_slots();
     }

crates/perry-runtime/src/gc/cycle.rs

@@ -872,6 +872,7 @@ struct AtomicFinalizeCycleState {
     subphase: AtomicFinalizeSubphase,
     barrier_drain: Option<TraceWorklistCycleState>,
     remembered_rebuild: Option<OldToYoungRememberedRebuildState>,
+    weak_processing: Option<crate::weakref::FullWeakProcessingState>,
     /// Budgeted cycles insert FinalRootRemark after BarrierSeedDrain;
     /// synchronous cycles have no mutator windows and skip it.
     remark: bool,
@@ -890,6 +891,7 @@ impl AtomicFinalizeCycleState {
             subphase: AtomicFinalizeSubphase::BarrierSeedDrain,
             barrier_drain: None,
             remembered_rebuild: None,
+            weak_processing: None,
             remark,
         }
     }
@@ -1035,6 +1037,20 @@ impl GcCycleState {
         self.phase
     }
 
+    #[cfg(test)]
+    pub(super) fn atomic_finalize_subphase_for_tests(&self) -> Option<&'static str> {
+        let subphase = self.atomic_finalize.as_ref()?.subphase;
+        Some(match subphase {
+            AtomicFinalizeSubphase::WeakProcessing => "weak_processing",
+            AtomicFinalizeSubphase::MinorPrelude => "minor_prelude",
+            AtomicFinalizeSubphase::BarrierSeedDrain => "barrier_seed_drain",
+            AtomicFinalizeSubphase::FinalRootRemark => "final_root_remark",
+            AtomicFinalizeSubphase::RememberedSetRebuild => "remembered_set_rebuild",
+            AtomicFinalizeSubphase::DisableBarrier => "disable_barrier",
+            AtomicFinalizeSubphase::Done => "done",
+        })
+    }
+
     pub(super) fn collection_kind(&self) -> GcCollectionKind {
         self.collection_kind
     }
@@ -1305,15 +1321,16 @@ impl GcCycleState {
                 .as_ref()
                 .expect("atomic finalize state exists")
                 .subphase;
-            // SLICED subphases (seed drain, full-cycle RS rebuild) honor the
-            // caller's budget and may return to the mutator; the ATOMIC TAIL
-            // (remark → weak → barrier-off → Sweep) runs to the phase
-            // transition in this single pause so no mutator window can
-            // invalidate the near-final mark set.
+            // SLICED subphases honor the caller's budget and may return to the
+            // mutator. WeakProcessing is safe to slice while the barrier and
+            // allocate-black births stay active: a target the mutator can
+            // still name was marked at remark or was born black, and holders
+            // born after the registry snapshot wait for the next cycle.
             let sliced = matches!(
                 subphase,
                 AtomicFinalizeSubphase::BarrierSeedDrain
                     | AtomicFinalizeSubphase::RememberedSetRebuild
+                    | AtomicFinalizeSubphase::WeakProcessing
             );
             let sub_budget = if sliced {
                 budget.work_units
@@ -1399,18 +1416,30 @@ impl GcCycleState {
                 // record is guaranteed by the record's pending-flag reset;
                 // delivery happens at the explicit-`gc()` tail or the next
                 // microtask-pump drain (`drain_pending_finalization_jobs`).
-                crate::weakref::process_weak_targets_after_mark(
-                    valid_ptrs, minor_only, /* enqueue_callbacks = */ true,
-                );
-                let next = if minor_only {
-                    AtomicFinalizeSubphase::MinorPrelude
-                } else {
-                    AtomicFinalizeSubphase::DisableBarrier
+                let done = {
+                    let state = self
+                        .atomic_finalize
+                        .as_mut()
+                        .expect("atomic finalize state exists");
+                    let weak = state
+                        .weak_processing
+                        .get_or_insert_with(crate::weakref::FullWeakProcessingState::new);
+                    weak.step(
+                        valid_ptrs, minor_only, /* enqueue_callbacks = */ true, budget,
+                    )
                 };
-                self.atomic_finalize
-                    .as_mut()
-                    .expect("atomic finalize state exists")
-                    .subphase = next;
+                if done {
+                    let state = self
+                        .atomic_finalize
+                        .as_mut()
+                        .expect("atomic finalize state exists");
+                    state.weak_processing = None;
+                    state.subphase = if minor_only {
+                        AtomicFinalizeSubphase::MinorPrelude
+                    } else {
+                        AtomicFinalizeSubphase::DisableBarrier
+                    };
+                }
             }
             AtomicFinalizeSubphase::MinorPrelude => {
                 if budget == 0 {

crates/perry-runtime/src/gc/policy.rs

@@ -3275,10 +3275,10 @@ fn gc_budgeted_step_work_units_inner_with_progress(
 /// the incremental sweep-parking hole (#6180): a pure compute loop that never
 /// reaches the event pump still finishes the cycle (and disables the mark
 /// barrier / reclaims memory) purely from the allocations it keeps making, so
-/// RSS stays bounded. `AtomicFinalizeSubphase::WeakProcessing` is the one
-/// phase step that is not yet internally sliced, so the assist that lands on it
-/// runs it whole — a single O(live-weak-holders) spike per cycle; slicing it is
-/// a tracked follow-up (pause-quality, not correctness).
+/// RSS stays bounded. `AtomicFinalizeSubphase::WeakProcessing` snapshots the
+/// live-holder registry and consumes at most the supplied number of holders per
+/// assist, so unrelated heap size cannot turn one assist into a whole-arena
+/// pause.
 fn gc_mutator_assist_step_work_units_inner_with_progress(
     work_units: usize,
     start_progress_kind: GcProgressKind,

crates/perry-runtime/src/gc/tests/copying/weak_holder_registry.rs

@@ -204,6 +204,35 @@ fn test_weak_holder_latch_clears_after_transient_weakmap_dies() {
     );
 }
 
+/// Full/fallback weak processing must scale with registered weak holders, not
+/// with unrelated arena population. One live WeakRef is held constant while
+/// the second collection sees a much larger heap.
+#[test]
+fn test_full_weak_processing_work_is_independent_of_unrelated_heap_size() {
+    let _guard = CopyingNurseryTestGuard::new(1_001);
+    let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    crate::weakref::test_support::clear_weak_holders();
+
+    let target = crate::object::js_object_alloc(0, 0);
+    let weak_ref = crate::weakref::js_weakref_new(f64::from_bits(obj_bits(target)));
+    js_shadow_slot_set(0, obj_bits(weak_ref));
+
+    gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct));
+    let small_heap_work = crate::weakref::test_support::full_weak_processing_work_units();
+    assert_eq!(small_heap_work, 1, "exactly one holder was registered");
+
+    for slot in 1..=1_000 {
+        let unrelated = crate::object::js_object_alloc(0, 0);
+        js_shadow_slot_set(slot, obj_bits(unrelated));
+    }
+    gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct));
+    let large_heap_work = crate::weakref::test_support::full_weak_processing_work_units();
+    assert_eq!(
+        large_heap_work, small_heap_work,
+        "one-holder weak work grew with unrelated heap: small={small_heap_work}, large={large_heap_work}"
+    );
+}
+
 /// (5) Cross-cycle registry currency: a WeakMap entry survives three
 /// consecutive moving minors with its holder evacuated (address changing) each
 /// time; the registry tracks the moved holder so a key that dies on cycle 3 is
@@ -277,6 +306,139 @@ extern "C" fn finreg_registry_test_callback(
     f64::from_bits(crate::value::TAG_UNDEFINED)
 }
 
+/// Build one holder of every kind, move all four through a copied minor, then
+/// drop their weak targets and age the graph out of block persistence. Also
+/// adds one dead holder and one stale registry address so the following
+/// full/fallback pass must classify both before dereferencing.
+fn prepare_moved_full_path_weak_holders() {
+    crate::weakref::test_support::clear_weak_holders();
+    assert_eq!(crate::weakref::pending_finalization_jobs_count(), 0);
+
+    let weak_target = crate::object::js_object_alloc(0, 0);
+    let weak_ref = crate::weakref::js_weakref_new(f64::from_bits(obj_bits(weak_target)));
+    js_shadow_slot_set(0, obj_bits(weak_ref));
+    js_shadow_slot_set(4, obj_bits(weak_target));
+
+    let map = crate::weakref::js_weakmap_new();
+    let map_key = crate::object::js_object_alloc(0, 0);
+    js_shadow_slot_set(1, obj_bits(map));
+    js_shadow_slot_set(5, obj_bits(map_key));
+    crate::weakref::js_weakmap_set(
+        f64::from_bits(js_shadow_slot_get(1)),
+        f64::from_bits(js_shadow_slot_get(5)),
+        f64::from_bits(crate::value::TAG_TRUE),
+    );
+
+    let set = crate::weakref::js_weakset_new();
+    let set_key = crate::object::js_object_alloc(0, 0);
+    js_shadow_slot_set(2, obj_bits(set));
+    js_shadow_slot_set(6, obj_bits(set_key));
+    crate::weakref::js_weakset_add(
+        f64::from_bits(js_shadow_slot_get(2)),
+        f64::from_bits(js_shadow_slot_get(6)),
+    );
+
+    let callback = crate::closure::js_closure_alloc(finreg_registry_test_callback as *const u8, 0);
+    let registry = crate::weakref::js_finreg_new(f64::from_bits(ptr_bits(callback as usize)));
+    let finreg_target = crate::object::js_object_alloc(0, 0);
+    js_shadow_slot_set(3, obj_bits(registry));
+    js_shadow_slot_set(7, obj_bits(finreg_target));
+    crate::weakref::js_finreg_register(
+        f64::from_bits(js_shadow_slot_get(3)),
+        f64::from_bits(js_shadow_slot_get(7)),
+        f64::from_bits(crate::value::TAG_TRUE),
+        f64::from_bits(crate::value::TAG_UNDEFINED),
+    );
+
+    let before_move = crate::weakref::test_support::weak_holder_addresses();
+    assert_eq!(
+        before_move.len(),
+        4,
+        "one holder of every kind is registered"
+    );
+    let trace = collect_minor_trace(GcTriggerKind::Direct);
+    assert_copied_minor_trace(&trace, true, CopiedMinorFallbackReason::None, false);
+    let after_move = crate::weakref::test_support::weak_holder_addresses();
+    assert_eq!(after_move.len(), 4);
+    assert!(
+        before_move.iter().all(|addr| !after_move.contains(addr)),
+        "every registered holder must be rekeyed after evacuation"
+    );
+
+    for slot in 4..8 {
+        js_shadow_slot_set(slot, 0);
+    }
+
+    // A dead holder must be pruned without dispatch. The fabricated stale
+    // address must be rejected by ValidPointerSet before any header read.
+    let dead_target = crate::object::js_object_alloc(0, 0);
+    let _dead_holder = crate::weakref::js_weakref_new(f64::from_bits(obj_bits(dead_target)));
+    crate::weakref::test_support::register_weak_holder_address(0x1234_5678);
+    assert_eq!(
+        crate::weakref::test_support::weak_holder_addresses().len(),
+        6
+    );
+
+    let aged_from = crate::arena::general_block_count();
+    while crate::arena::general_block_count().saturating_sub(aged_from) < 7 {
+        for _ in 0..64 {
+            let _ = crate::arena::arena_alloc_gc(4096, 8, GC_TYPE_STRING);
+        }
+    }
+}
+
+fn assert_full_path_weak_results() {
+    assert_eq!(
+        crate::weakref::js_weakref_deref(f64::from_bits(js_shadow_slot_get(0))).to_bits(),
+        crate::value::TAG_UNDEFINED,
+        "WeakRef target must be tombstoned"
+    );
+    for slot in [1, 2] {
+        let collection = (js_shadow_slot_get(slot) & POINTER_MASK) as *const crate::ObjectHeader;
+        assert!(
+            crate::weakref::weak_collection_entries(collection).is_empty(),
+            "WeakMap/WeakSet dead-key entry must be tombstoned"
+        );
+    }
+    assert_eq!(
+        crate::weakref::pending_finalization_jobs_count(),
+        1,
+        "FinalizationRegistry target must enqueue exactly one cleanup job"
+    );
+    assert_eq!(
+        crate::weakref::test_support::weak_holder_addresses().len(),
+        4,
+        "the dead holder and stale address must be pruned; live holders remain"
+    );
+}
+
+#[test]
+fn test_full_registry_path_handles_all_weak_kinds_moved_dead_and_stale_holders() {
+    let _guard = CopyingNurseryTestGuard::new(8);
+    let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    prepare_moved_full_path_weak_holders();
+
+    gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct));
+    assert_full_path_weak_results();
+}
+
+#[test]
+fn test_fallback_registry_path_handles_all_weak_kinds_moved_dead_and_stale_holders() {
+    let _guard = CopyingNurseryTestGuard::new(8);
+    let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    prepare_moved_full_path_weak_holders();
+
+    let _barrier_guard = GeneratedWriteBarrierTestGuard::inactive();
+    let trace = collect_minor_trace(GcTriggerKind::Direct);
+    assert_copied_minor_trace(
+        &trace,
+        false,
+        CopiedMinorFallbackReason::BarriersInactive,
+        false,
+    );
+    assert_full_path_weak_results();
+}
+
 /// (6) FinalizationRegistry: a registered target that dies across a moving minor
 /// enqueues its cleanup job through the registry-based pass (the #6192
 /// automatic-cycle delivery must be preserved — the registry holder is the

crates/perry-runtime/src/gc/tests/copying/weak_semantics.rs

@@ -5,8 +5,8 @@
 //! entries never cleared and FinalizationRegistry never fired while
 //! copied-minor was the operative cycle. The scan now records weak slots
 //! without evacuating, `repair_weak_slots` fixes addresses of targets moved
-//! via strong edges, and `process_weak_targets_after_mark` runs on the fast
-//! path (gated on the weak-holder latch).
+//! via strong edges, and the registry-scoped weak pass runs on the fast path
+//! (gated on the weak-holder latch).
 
 use super::*;