From 0f5a049a2d26259605edea4391c998971ad3af1d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 13 Oct 2026 18:58:39 +0900 Subject: [PATCH] Directly call functions in C binding This commit removes RubyUpcalls and instead directly calls functions in the C binding from Rust. This also allows LTO to inline C calls from Rust. --- gc/mmtk/mmtk.c | 92 +++++++---------- gc/mmtk/mmtk.h | 88 ++++++++++------- gc/mmtk/src/abi.rs | 57 +++-------- gc/mmtk/src/active_plan.rs | 6 +- gc/mmtk/src/api.rs | 8 +- gc/mmtk/src/binding.rs | 13 +-- gc/mmtk/src/collection.rs | 18 ++-- gc/mmtk/src/lib.rs | 17 +--- gc/mmtk/src/object_model.rs | 4 +- gc/mmtk/src/pinning_registry.rs | 2 +- gc/mmtk/src/scanning.rs | 12 +-- gc/mmtk/src/upcalls.rs | 170 ++++++++++++++++++++++++++++++++ gc/mmtk/src/weak_proc.rs | 10 +- 13 files changed, 304 insertions(+), 193 deletions(-) create mode 100644 gc/mmtk/src/upcalls.rs diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index 3c4133d..cf80d98 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -114,21 +114,31 @@ RB_THREAD_LOCAL_SPECIFIER VALUE marking_parent_object; #define MMTK_ALLOCATION_SEMANTICS_DEFAULT 0 #define MMTK_ALLOCATION_SEMANTICS_LOS 2 +/* Functions called from the Rust binding (upcalls). They are not static so + * that the linker can resolve them from libmmtk_ruby.a, but they are hidden + * so that calls do not go through the PLT and so that LTO can inline them + * into their Rust callers. */ +#ifdef __GNUC__ +# define RB_MMTK_UPCALL __attribute__((visibility("hidden"))) +#else +# define RB_MMTK_UPCALL +#endif + static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin); -static void +RB_MMTK_UPCALL void rb_mmtk_init_gc_worker_thread(MMTk_VMWorkerThread gc_thread_tls) { rb_mmtk_gc_thread_tls = gc_thread_tls; } -static bool +RB_MMTK_UPCALL bool rb_mmtk_is_mutator(void) { return ruby_native_thread_p(); } -static void +RB_MMTK_UPCALL void rb_mmtk_stop_the_world(void) { struct objspace *objspace = rb_gc_get_objspace(); @@ -147,7 +157,7 @@ rb_mmtk_stop_the_world(void) } } -static void +RB_MMTK_UPCALL void rb_mmtk_resume_mutators(bool current_gc_may_move) { struct objspace *objspace = rb_gc_get_objspace(); @@ -169,7 +179,7 @@ rb_mmtk_resume_mutators(bool current_gc_may_move) static void mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache); -static void +RB_MMTK_UPCALL void rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator) { struct objspace *objspace = rb_gc_get_objspace(); @@ -233,26 +243,26 @@ rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator) RB_GC_VM_UNLOCK(lock_lev); } -static void +RB_MMTK_UPCALL void rb_mmtk_before_updating_jit_code(void) { rb_gc_before_updating_jit_code(); } -static void +RB_MMTK_UPCALL void rb_mmtk_after_updating_jit_code(void) { rb_gc_after_updating_jit_code(); } -static size_t +RB_MMTK_UPCALL size_t rb_mmtk_number_of_mutators(void) { struct objspace *objspace = rb_gc_get_objspace(); return objspace->live_ractor_cache_count; } -static void +RB_MMTK_UPCALL void rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), void *data) { struct objspace *objspace = rb_gc_get_objspace(); @@ -263,7 +273,7 @@ rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), v } } -static void +RB_MMTK_UPCALL void rb_mmtk_scan_gc_roots(void) { struct objspace *objspace = rb_gc_get_objspace(); @@ -279,7 +289,7 @@ pin_value(st_data_t key, st_data_t value, st_data_t data) return ST_CONTINUE; } -static void +RB_MMTK_UPCALL void rb_mmtk_scan_objspace(void) { struct objspace *objspace = rb_gc_get_objspace(); @@ -304,7 +314,7 @@ rb_mmtk_scan_objspace(void) } } -static void +RB_MMTK_UPCALL void rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference to) { rb_gc_move_obj_during_marking((VALUE)from, (VALUE)to); @@ -320,13 +330,13 @@ rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference } } -static size_t +RB_MMTK_UPCALL size_t rb_mmtk_obj_optimal_size(MMTk_ObjectReference obj) { return rb_gc_obj_optimal_size((VALUE)obj); } -static void +RB_MMTK_UPCALL void rb_mmtk_update_object_references(MMTk_ObjectReference mmtk_object) { VALUE object = (VALUE)mmtk_object; @@ -338,7 +348,7 @@ rb_mmtk_update_object_references(MMTk_ObjectReference mmtk_object) } } -static void +RB_MMTK_UPCALL void rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object) { marking_parent_object = (VALUE)object; @@ -346,7 +356,7 @@ rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object) marking_parent_object = 0; } -static void +RB_MMTK_UPCALL void rb_mmtk_handle_weak_references(MMTk_ObjectReference mmtk_object, bool moving) { VALUE object = (VALUE)mmtk_object; @@ -362,7 +372,7 @@ rb_mmtk_handle_weak_references(MMTk_ObjectReference mmtk_object, bool moving) marking_parent_object = 0; } -static void +RB_MMTK_UPCALL void rb_mmtk_call_obj_free(MMTk_ObjectReference object) { VALUE obj = (VALUE)object; @@ -383,7 +393,7 @@ rb_mmtk_call_obj_free(MMTk_ObjectReference object) #endif } -static size_t +RB_MMTK_UPCALL size_t rb_mmtk_vm_live_bytes(void) { return 0; @@ -438,7 +448,7 @@ rb_mmtk_update_finalizer_table_replace_i(st_data_t *key, st_data_t *value, st_da return ST_CONTINUE; } -static void +RB_MMTK_UPCALL void rb_mmtk_update_finalizer_table(void) { struct objspace *objspace = rb_gc_get_objspace(); @@ -451,7 +461,7 @@ rb_mmtk_update_finalizer_table(void) ); } -static int +RB_MMTK_UPCALL int rb_mmtk_global_tables_count(void) { return RB_GC_VM_WEAK_TABLE_COUNT; @@ -483,7 +493,7 @@ rb_mmtk_update_global_tables_replace_i(VALUE *ptr, void *data) return ST_CONTINUE; } -static void +RB_MMTK_UPCALL void rb_mmtk_update_global_tables(int table, bool moving) { MMTK_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT); @@ -497,14 +507,6 @@ rb_mmtk_update_global_tables(int table, bool moving) ); } -static bool -rb_mmtk_special_const_p(MMTk_ObjectReference object) -{ - VALUE obj = (VALUE)object; - - return RB_SPECIAL_CONST_P(obj); -} - RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 1, 2) RBIMPL_ATTR_NORETURN() static void @@ -532,46 +534,20 @@ rb_mmtk_gc_thread_bug(const char *msg, ...) } RBIMPL_ATTR_NORETURN() -static void +RB_MMTK_UPCALL void rb_mmtk_gc_thread_panic_handler(void) { rb_mmtk_gc_thread_bug("MMTk GC thread panicked"); } RBIMPL_ATTR_NORETURN() -static void +RB_MMTK_UPCALL void rb_mmtk_mutator_thread_panic_handler(void) { rb_bug("Ruby mutator thread panicked"); } // Bootup -MMTk_RubyUpcalls ruby_upcalls = { - rb_mmtk_init_gc_worker_thread, - rb_mmtk_is_mutator, - rb_mmtk_stop_the_world, - rb_mmtk_resume_mutators, - rb_mmtk_block_for_gc, - rb_mmtk_before_updating_jit_code, - rb_mmtk_after_updating_jit_code, - rb_mmtk_number_of_mutators, - rb_mmtk_get_mutators, - rb_mmtk_scan_gc_roots, - rb_mmtk_scan_objspace, - rb_mmtk_move_obj_during_marking, - rb_mmtk_obj_optimal_size, - rb_mmtk_update_object_references, - rb_mmtk_call_gc_mark_children, - rb_mmtk_handle_weak_references, - rb_mmtk_call_obj_free, - rb_mmtk_vm_live_bytes, - rb_mmtk_update_global_tables, - rb_mmtk_global_tables_count, - rb_mmtk_update_finalizer_table, - rb_mmtk_special_const_p, - rb_mmtk_mutator_thread_panic_handler, - rb_mmtk_gc_thread_panic_handler, -}; // Use max 80% of the available memory by default for MMTk #define RB_MMTK_HEAP_LIMIT_PERC 80 @@ -603,7 +579,7 @@ rb_gc_impl_objspace_alloc(void) MMTk_RubyBindingOptions binding_options = { .suffix_size = 0, }; - mmtk_init_binding(builder, &binding_options, &ruby_upcalls); + mmtk_init_binding(builder, &binding_options); the_objspace = calloc(1, sizeof(struct objspace)); } diff --git a/gc/mmtk/mmtk.h b/gc/mmtk/mmtk.h index cd0c8c1..cdcae3f 100644 --- a/gc/mmtk/mmtk.h +++ b/gc/mmtk/mmtk.h @@ -36,6 +36,12 @@ typedef struct MMTk_RubyBindingOptions { size_t suffix_size; } MMTk_RubyBindingOptions; +typedef struct MMTk_RawVecOfObjRef { + MMTk_ObjectReference *ptr; + size_t len; + size_t capa; +} MMTk_RawVecOfObjRef; + typedef MMTk_ObjectReference (*MMTk_ObjectClosureFunction)(void*, void*, MMTk_ObjectReference, bool); typedef struct MMTk_ObjectClosure { @@ -55,39 +61,6 @@ typedef struct MMTk_GCThreadTLS { struct MMTk_ObjectClosure object_closure; } MMTk_GCThreadTLS; -typedef struct MMTk_RubyUpcalls { - void (*init_gc_worker_thread)(struct MMTk_GCThreadTLS *gc_worker_tls); - bool (*is_mutator)(void); - void (*stop_the_world)(void); - void (*resume_mutators)(bool gc_may_move); - void (*block_for_gc)(MMTk_VMMutatorThread tls); - void (*before_updating_jit_code)(void); - void (*after_updating_jit_code)(void); - size_t (*number_of_mutators)(void); - void (*get_mutators)(void (*visit_mutator)(MMTk_Mutator*, void*), void *data); - void (*scan_gc_roots)(void); - void (*scan_objspace)(void); - void (*move_obj_during_marking)(MMTk_ObjectReference from, MMTk_ObjectReference to); - size_t (*obj_optimal_size)(MMTk_ObjectReference object); - void (*update_object_references)(MMTk_ObjectReference object); - void (*call_gc_mark_children)(MMTk_ObjectReference object); - void (*handle_weak_references)(MMTk_ObjectReference object, bool moving); - void (*call_obj_free)(MMTk_ObjectReference object); - size_t (*vm_live_bytes)(void); - void (*update_global_tables)(int tbl_idx, bool moving); - int (*global_tables_count)(void); - void (*update_finalizer_table)(void); - bool (*special_const_p)(MMTk_ObjectReference object); - void (*mutator_thread_panic_handler)(void); - void (*gc_thread_panic_handler)(void); -} MMTk_RubyUpcalls; - -typedef struct MMTk_RawVecOfObjRef { - MMTk_ObjectReference *ptr; - size_t len; - size_t capa; -} MMTk_RawVecOfObjRef; - bool mmtk_is_live_object(MMTk_ObjectReference object); bool mmtk_is_reachable(MMTk_ObjectReference object); @@ -95,8 +68,7 @@ bool mmtk_is_reachable(MMTk_ObjectReference object); MMTk_Builder *mmtk_builder_default(void); void mmtk_init_binding(MMTk_Builder *builder, - const struct MMTk_RubyBindingOptions *binding_options, - const struct MMTk_RubyUpcalls *upcalls); + const struct MMTk_RubyBindingOptions *binding_options); size_t mmtk_get_vo_bit_log_region_size(void); @@ -181,4 +153,50 @@ size_t mmtk_heap_max(void); bool mmtk_is_mmtk_object(MMTk_Address addr); +extern void rb_mmtk_init_gc_worker_thread(struct MMTk_GCThreadTLS *gc_worker_tls); + +extern bool rb_mmtk_is_mutator(void); + +extern void rb_mmtk_stop_the_world(void); + +extern void rb_mmtk_resume_mutators(bool gc_may_move); + +extern void rb_mmtk_block_for_gc(MMTk_VMMutatorThread tls); + +extern void rb_mmtk_before_updating_jit_code(void); + +extern void rb_mmtk_after_updating_jit_code(void); + +extern size_t rb_mmtk_number_of_mutators(void); + +extern void rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator*, void*), void *data); + +extern void rb_mmtk_scan_gc_roots(void); + +extern void rb_mmtk_scan_objspace(void); + +extern void rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference to); + +extern size_t rb_mmtk_obj_optimal_size(MMTk_ObjectReference object); + +extern void rb_mmtk_update_object_references(MMTk_ObjectReference object); + +extern void rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object); + +extern void rb_mmtk_handle_weak_references(MMTk_ObjectReference object, bool moving); + +extern void rb_mmtk_call_obj_free(MMTk_ObjectReference object); + +extern size_t rb_mmtk_vm_live_bytes(void); + +extern void rb_mmtk_update_global_tables(int tbl_idx, bool moving); + +extern int rb_mmtk_global_tables_count(void); + +extern void rb_mmtk_update_finalizer_table(void); + +extern void rb_mmtk_mutator_thread_panic_handler(void); + +extern void rb_mmtk_gc_thread_panic_handler(void); + #endif /* MMTK_H */ diff --git a/gc/mmtk/src/abi.rs b/gc/mmtk/src/abi.rs index 171b75e..b9bb41a 100644 --- a/gc/mmtk/src/abi.rs +++ b/gc/mmtk/src/abi.rs @@ -1,11 +1,8 @@ use crate::Ruby; -use crate::api::RubyMutator; use crate::extra_assert; -use libc::c_int; use mmtk::scheduler::GCWorker; use mmtk::util::Address; use mmtk::util::ObjectReference; -use mmtk::util::VMMutatorThread; use mmtk::util::VMWorkerThread; // For the C binding @@ -47,7 +44,6 @@ impl HiddenHeader { } /// Provide convenient methods for accessing Ruby objects. -/// TODO: Wrap C functions in `RubyUpcalls` as Rust-friendly methods. pub struct RubyObjectAccess { objref: ObjectReference, } @@ -161,10 +157,23 @@ impl ObjectClosure { std::ptr::fn_addr_eq(self.c_function, THE_UNREGISTERED_CLOSURE_FUNC), "set_temporarily_and_run_code is recursively called." ); - self.c_function = Self::c_function_registered::; - self.rust_closure = &mut visit_object as *mut F1 as *mut libc::c_void; + // The C code reads these fields through a thread-local pointer that is + // invisible to the Rust compiler, and this function's callers are C + // functions whose bodies the optimizer cannot see either. Plain + // stores could therefore be eliminated as dead (the reset store below + // overwrites them without any Rust-visible read in between). Use + // volatile writes, which the optimizer must not eliminate or reorder. + unsafe { + std::ptr::write_volatile(&mut self.c_function, Self::c_function_registered::); + std::ptr::write_volatile( + &mut self.rust_closure, + &mut visit_object as *mut F1 as *mut libc::c_void, + ); + } let result = f(); - *self = Default::default(); + unsafe { + std::ptr::write_volatile(&mut *self, ObjectClosure::default()); + } result } @@ -294,40 +303,6 @@ pub struct RubyBindingOptions { pub suffix_size: usize, } -#[repr(C)] -#[derive(Clone)] -pub struct RubyUpcalls { - pub init_gc_worker_thread: extern "C" fn(gc_worker_tls: *mut GCThreadTLS), - pub is_mutator: extern "C" fn() -> bool, - pub stop_the_world: extern "C" fn(), - pub resume_mutators: extern "C" fn(gc_may_move: bool), - pub block_for_gc: extern "C" fn(tls: VMMutatorThread), - pub before_updating_jit_code: extern "C" fn(), - pub after_updating_jit_code: extern "C" fn(), - pub number_of_mutators: extern "C" fn() -> usize, - pub get_mutators: extern "C" fn( - visit_mutator: extern "C" fn(*mut RubyMutator, *mut libc::c_void), - data: *mut libc::c_void, - ), - pub scan_gc_roots: extern "C" fn(), - pub scan_objspace: extern "C" fn(), - pub move_obj_during_marking: extern "C" fn(from: ObjectReference, to: ObjectReference), - pub obj_optimal_size: extern "C" fn(object: ObjectReference) -> usize, - pub update_object_references: extern "C" fn(object: ObjectReference), - pub call_gc_mark_children: extern "C" fn(object: ObjectReference), - pub handle_weak_references: extern "C" fn(object: ObjectReference, moving: bool), - pub call_obj_free: extern "C" fn(object: ObjectReference), - pub vm_live_bytes: extern "C" fn() -> usize, - pub update_global_tables: extern "C" fn(tbl_idx: c_int, moving: bool), - pub global_tables_count: extern "C" fn() -> c_int, - pub update_finalizer_table: extern "C" fn(), - pub special_const_p: extern "C" fn(object: ObjectReference) -> bool, - pub mutator_thread_panic_handler: extern "C" fn(), - pub gc_thread_panic_handler: extern "C" fn(), -} - -unsafe impl Sync for RubyUpcalls {} - #[repr(C)] #[derive(Clone)] pub struct HeapBounds { diff --git a/gc/mmtk/src/active_plan.rs b/gc/mmtk/src/active_plan.rs index c859733..d789073 100644 --- a/gc/mmtk/src/active_plan.rs +++ b/gc/mmtk/src/active_plan.rs @@ -12,11 +12,11 @@ pub struct VMActivePlan {} impl ActivePlan for VMActivePlan { fn number_of_mutators() -> usize { - (upcalls().number_of_mutators)() + upcalls::number_of_mutators() } fn is_mutator(_tls: VMThread) -> bool { - (upcalls().is_mutator)() + upcalls::is_mutator() } fn mutator(_tls: VMMutatorThread) -> &'static mut Mutator { @@ -25,7 +25,7 @@ impl ActivePlan for VMActivePlan { fn mutators<'a>() -> Box> + 'a> { let mut mutators = VecDeque::new(); - (upcalls().get_mutators)( + upcalls::get_mutators( add_mutator_to_vec, &mut mutators as *mut VecDeque<&mut Mutator> as _, ); diff --git a/gc/mmtk/src/api.rs b/gc/mmtk/src/api.rs index a592280..ef0d7bf 100644 --- a/gc/mmtk/src/api.rs +++ b/gc/mmtk/src/api.rs @@ -12,7 +12,6 @@ use crate::Ruby; use crate::RubySlot; use crate::abi::RawVecOfObjRef; use crate::abi::RubyBindingOptions; -use crate::abi::RubyUpcalls; use crate::binding; use crate::binding::RubyBinding; use crate::heap::CPU_HEAP_TRIGGER_CONFIG; @@ -220,12 +219,7 @@ pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder { pub unsafe extern "C" fn mmtk_init_binding( builder: *mut MMTKBuilder, binding_options: *const RubyBindingOptions, - upcalls: *const RubyUpcalls, ) { - crate::MUTATOR_THREAD_PANIC_HANDLER - .set((unsafe { (*upcalls).clone() }).mutator_thread_panic_handler) - .unwrap_or_else(|_| panic!("MUTATOR_THREAD_PANIC_HANDLER is already initialized")); - crate::set_panic_hook(); let builder: Box = unsafe { Box::from_raw(builder) }; @@ -233,7 +227,7 @@ pub unsafe extern "C" fn mmtk_init_binding( let mmtk_boxed = mmtk_init(&builder); let mmtk_static = Box::leak(Box::new(mmtk_boxed)); - let mut binding = RubyBinding::new(mmtk_static, &binding_options, upcalls); + let mut binding = RubyBinding::new(mmtk_static, &binding_options); binding .weak_proc .init_parallel_obj_free_candidates(memory_manager::num_of_workers(binding.mmtk)); diff --git a/gc/mmtk/src/binding.rs b/gc/mmtk/src/binding.rs index 5610480..be2fa07 100644 --- a/gc/mmtk/src/binding.rs +++ b/gc/mmtk/src/binding.rs @@ -7,7 +7,6 @@ use mmtk::MMTK; use mmtk::util::ObjectReference; use crate::Ruby; -use crate::abi; use crate::abi::RubyBindingOptions; use crate::pinning_registry::PinningRegistry; use crate::weak_proc::WeakProcessor; @@ -31,7 +30,6 @@ impl RubyBindingFast { pub struct RubyBinding { pub mmtk: &'static MMTK, pub options: RubyBindingOptions, - pub upcalls: *const abi::RubyUpcalls, pub plan_name: Mutex>, pub weak_proc: WeakProcessor, pub pinning_registry: PinningRegistry, @@ -43,11 +41,7 @@ unsafe impl Sync for RubyBinding {} unsafe impl Send for RubyBinding {} impl RubyBinding { - pub fn new( - mmtk: &'static MMTK, - binding_options: &RubyBindingOptions, - upcalls: *const abi::RubyUpcalls, - ) -> Self { + pub fn new(mmtk: &'static MMTK, binding_options: &RubyBindingOptions) -> Self { unsafe { crate::BINDING_FAST.suffix_size = binding_options.suffix_size; } @@ -55,7 +49,6 @@ impl RubyBinding { Self { mmtk, options: binding_options.clone(), - upcalls, plan_name: Mutex::new(None), weak_proc: WeakProcessor::new(), pinning_registry: PinningRegistry::new(), @@ -64,10 +57,6 @@ impl RubyBinding { } } - pub fn upcalls(&self) -> &'static abi::RubyUpcalls { - unsafe { &*self.upcalls as &'static abi::RubyUpcalls } - } - pub fn get_plan_name_c(&self) -> *const libc::c_char { let mut plan_name = self.plan_name.lock().unwrap(); if plan_name.is_none() { diff --git a/gc/mmtk/src/collection.rs b/gc/mmtk/src/collection.rs index d7e44a3..89d587f 100644 --- a/gc/mmtk/src/collection.rs +++ b/gc/mmtk/src/collection.rs @@ -29,18 +29,18 @@ impl Collection for VMCollection { where F: FnMut(&'static mut mmtk::Mutator), { - (upcalls().stop_the_world)(); + upcalls::stop_the_world(); if crate::mmtk().get_plan().current_gc_may_move_object() { CURRENT_GC_MAY_MOVE.store(true, Ordering::Relaxed); - (upcalls().before_updating_jit_code)(); + upcalls::before_updating_jit_code(); } else { CURRENT_GC_MAY_MOVE.store(false, Ordering::Relaxed); } crate::binding().pinning_registry.pin_children(tls); - (upcalls().get_mutators)( + upcalls::get_mutators( Self::notify_mutator_ready::, &mut mutator_visitor as *mut F as *mut _, ); @@ -50,14 +50,14 @@ impl Collection for VMCollection { let current_gc_may_move = CURRENT_GC_MAY_MOVE.load(Ordering::Relaxed); if current_gc_may_move { - (upcalls().after_updating_jit_code)(); + upcalls::after_updating_jit_code(); } - (upcalls().resume_mutators)(current_gc_may_move); + upcalls::resume_mutators(current_gc_may_move); } fn block_for_gc(tls: VMMutatorThread) { - (upcalls().block_for_gc)(tls); + upcalls::block_for_gc(tls); } fn out_of_memory(_tls: VMThread, err_kind: AllocationError) { @@ -68,7 +68,7 @@ impl Collection for VMCollection { // The OS refused an mmap. This is unrecoverable, so abort the // process via the same panic handler used for GC-thread panics. AllocationError::MmapOutOfMemory => { - (upcalls().mutator_thread_panic_handler)(); + upcalls::mutator_thread_panic_handler(); } } } @@ -84,7 +84,7 @@ impl Collection for VMCollection { let ptr_worker = &mut *worker as *mut GCWorker; let gc_thread_tls = Box::into_raw(Box::new(GCThreadTLS::for_worker(ptr_worker))); - (upcalls().init_gc_worker_thread)(gc_thread_tls); + upcalls::init_gc_worker_thread(gc_thread_tls); memory_manager::start_worker( mmtk(), GCThreadTLS::to_vwt(gc_thread_tls), @@ -103,7 +103,7 @@ impl Collection for VMCollection { } fn vm_live_bytes() -> usize { - (upcalls().vm_live_bytes)() + upcalls::vm_live_bytes() } fn create_gc_trigger() -> Box> { diff --git a/gc/mmtk/src/lib.rs b/gc/mmtk/src/lib.rs index b9deace..e84d7af 100644 --- a/gc/mmtk/src/lib.rs +++ b/gc/mmtk/src/lib.rs @@ -13,7 +13,6 @@ use std::panic::PanicHookInfo; use std::sync::Mutex; use std::thread::ThreadId; -use abi::RubyUpcalls; use binding::RubyBinding; use binding::RubyBindingFast; use mmtk::MMTK; @@ -32,6 +31,7 @@ pub mod object_model; pub mod pinning_registry; pub mod reference_glue; pub mod scanning; +pub mod upcalls; pub mod utils; pub mod weak_proc; @@ -59,11 +59,6 @@ impl VMBinding for Ruby { type VMMemorySlice = RubyMemorySlice; } -/// The callback for mutator thread panic handler (which calls rb_bug to output -/// debugging information such as the Ruby backtrace and memory maps). -/// This is set before BINDING is set because mmtk_init could panic. -pub static MUTATOR_THREAD_PANIC_HANDLER: OnceCell = OnceCell::new(); - /// The singleton object for the Ruby binding itself. pub static BINDING: OnceCell = OnceCell::new(); @@ -81,10 +76,6 @@ pub fn mmtk() -> &'static MMTK { binding().mmtk } -pub fn upcalls() -> &'static RubyUpcalls { - binding().upcalls() -} - pub static GC_THREADS: OnceCell>> = OnceCell::new(); pub(crate) fn register_gc_thread(thread_id: ThreadId) { @@ -135,12 +126,10 @@ pub(crate) fn set_panic_hook() { if is_gc_thread(std::thread::current().id()) { handle_gc_thread_panic(panic_info); - (crate::binding().upcalls().gc_thread_panic_handler)(); + upcalls::gc_thread_panic_handler(); } else { old_hook(panic_info); - (crate::MUTATOR_THREAD_PANIC_HANDLER - .get() - .expect("MUTATOR_THREAD_PANIC_HANDLER is not set"))(); + upcalls::mutator_thread_panic_handler(); } })); } diff --git a/gc/mmtk/src/object_model.rs b/gc/mmtk/src/object_model.rs index 8b9cecf..b615ade 100644 --- a/gc/mmtk/src/object_model.rs +++ b/gc/mmtk/src/object_model.rs @@ -53,7 +53,7 @@ impl ObjectModel for VMObjectModel { let from_start = from_acc.obj_start(); let object_size = from_acc.object_size(); - let optimal_payload = (crate::binding().upcalls().obj_optimal_size)(from); + let optimal_payload = crate::upcalls::obj_optimal_size(from); let new_size = if optimal_payload != 0 { let size = (RubyObjectAccess::prefix_size() + optimal_payload @@ -93,7 +93,7 @@ impl ObjectModel for VMObjectModel { } } - (crate::binding().upcalls().move_obj_during_marking)(from, to_obj); + crate::upcalls::move_obj_during_marking(from, to_obj); copy_context.post_copy(to_obj, new_size, semantics); trace!("Copied object from {} to {}", from, to_obj); diff --git a/gc/mmtk/src/pinning_registry.rs b/gc/mmtk/src/pinning_registry.rs index 3698fce..7b295d4 100644 --- a/gc/mmtk/src/pinning_registry.rs +++ b/gc/mmtk/src/pinning_registry.rs @@ -125,7 +125,7 @@ impl GCWork for PinPinningChildren { .set_temporarily_and_run_code(visit_object, || { for obj in self.pinning_objs.iter().cloned() { log::trace!(" Pinning: {}", obj); - (upcalls().call_gc_mark_children)(obj); + upcalls::call_gc_mark_children(obj); } }); diff --git a/gc/mmtk/src/scanning.rs b/gc/mmtk/src/scanning.rs index ee7c993..7b59483 100644 --- a/gc/mmtk/src/scanning.rs +++ b/gc/mmtk/src/scanning.rs @@ -76,10 +76,10 @@ impl Scanning for VMScanning { gc_tls .object_closure .set_temporarily_and_run_code(visit_object, || { - (upcalls().call_gc_mark_children)(object); + upcalls::call_gc_mark_children(object); if crate::mmtk().get_plan().current_gc_may_move_object() { - (upcalls().update_object_references)(object); + upcalls::update_object_references(object); } }); } @@ -256,11 +256,11 @@ macro_rules! define_global_root_scanner { } define_global_root_scanner!(ScanGCRoots, { - (crate::upcalls().scan_gc_roots)(); + crate::upcalls::scan_gc_roots(); }); define_global_root_scanner!(ScanObjspace, { - (crate::upcalls().scan_objspace)(); + crate::upcalls::scan_objspace(); }); struct ScanWbUnprotectedRoots> { @@ -275,10 +275,10 @@ impl> GCWork for ScanWbUnprotectedRoots { for object in self.objects.iter().copied() { if object.is_reachable() { debug!("[wb_unprot_roots] Visiting WB-unprotected object (parent): {object}"); - (upcalls().call_gc_mark_children)(object); + upcalls::call_gc_mark_children(object); if crate::mmtk().get_plan().current_gc_may_move_object() { - (upcalls().update_object_references)(object); + upcalls::update_object_references(object); } } else { debug!( diff --git a/gc/mmtk/src/upcalls.rs b/gc/mmtk/src/upcalls.rs new file mode 100644 index 0000000..c0c9cf2 --- /dev/null +++ b/gc/mmtk/src/upcalls.rs @@ -0,0 +1,170 @@ +//! Upcalls from MMTk to the Ruby VM. +//! +//! The upcalls are implemented in C (`gc/mmtk/mmtk.c`) and declared here as +//! `extern "C"` functions. Since `libmmtk_ruby.a` is statically linked into +//! `librubygc` together with the C code, the linker resolves these symbols +//! directly. There is no function pointer indirection, and calls can be +//! inlined when building with cross-language LTO. + +use libc::c_int; +use libc::c_void; + +use mmtk::util::ObjectReference; +use mmtk::util::VMMutatorThread; + +use crate::abi::GCThreadTLS; +use crate::api::RubyMutator; + +// The C functions must have external linkage (i.e. not `static`) so that the +// linker can resolve these symbols. +unsafe extern "C" { + fn rb_mmtk_init_gc_worker_thread(gc_worker_tls: *mut GCThreadTLS); + fn rb_mmtk_is_mutator() -> bool; + fn rb_mmtk_stop_the_world(); + fn rb_mmtk_resume_mutators(gc_may_move: bool); + fn rb_mmtk_block_for_gc(tls: VMMutatorThread); + fn rb_mmtk_before_updating_jit_code(); + fn rb_mmtk_after_updating_jit_code(); + fn rb_mmtk_number_of_mutators() -> usize; + fn rb_mmtk_get_mutators( + visit_mutator: extern "C" fn(*mut RubyMutator, *mut c_void), + data: *mut c_void, + ); + fn rb_mmtk_scan_gc_roots(); + fn rb_mmtk_scan_objspace(); + fn rb_mmtk_move_obj_during_marking(from: ObjectReference, to: ObjectReference); + fn rb_mmtk_obj_optimal_size(object: ObjectReference) -> usize; + fn rb_mmtk_update_object_references(object: ObjectReference); + fn rb_mmtk_call_gc_mark_children(object: ObjectReference); + fn rb_mmtk_handle_weak_references(object: ObjectReference, moving: bool); + fn rb_mmtk_call_obj_free(object: ObjectReference); + fn rb_mmtk_vm_live_bytes() -> usize; + fn rb_mmtk_update_global_tables(tbl_idx: c_int, moving: bool); + fn rb_mmtk_global_tables_count() -> c_int; + fn rb_mmtk_update_finalizer_table(); + fn rb_mmtk_mutator_thread_panic_handler() -> !; + fn rb_mmtk_gc_thread_panic_handler() -> !; +} + +// `gc_worker_tls` comes from a Box::into_raw'd GCThreadTLS and is valid. +#[allow(clippy::not_unsafe_ptr_arg_deref)] +#[inline] +pub fn init_gc_worker_thread(gc_worker_tls: *mut GCThreadTLS) { + unsafe { rb_mmtk_init_gc_worker_thread(gc_worker_tls) } +} + +#[inline] +pub fn is_mutator() -> bool { + unsafe { rb_mmtk_is_mutator() } +} + +#[inline] +pub fn stop_the_world() { + unsafe { rb_mmtk_stop_the_world() } +} + +#[inline] +pub fn resume_mutators(gc_may_move: bool) { + unsafe { rb_mmtk_resume_mutators(gc_may_move) } +} + +#[inline] +pub fn block_for_gc(tls: VMMutatorThread) { + unsafe { rb_mmtk_block_for_gc(tls) } +} + +#[inline] +pub fn before_updating_jit_code() { + unsafe { rb_mmtk_before_updating_jit_code() } +} + +#[inline] +pub fn after_updating_jit_code() { + unsafe { rb_mmtk_after_updating_jit_code() } +} + +#[inline] +pub fn number_of_mutators() -> usize { + unsafe { rb_mmtk_number_of_mutators() } +} + +// `data` is an opaque pointer passed through to `visit_mutator`; the caller +// guarantees it is valid for the duration of the call. +#[allow(clippy::not_unsafe_ptr_arg_deref)] +#[inline] +pub fn get_mutators( + visit_mutator: extern "C" fn(*mut RubyMutator, *mut c_void), + data: *mut c_void, +) { + unsafe { rb_mmtk_get_mutators(visit_mutator, data) } +} + +#[inline] +pub fn scan_gc_roots() { + unsafe { rb_mmtk_scan_gc_roots() } +} + +#[inline] +pub fn scan_objspace() { + unsafe { rb_mmtk_scan_objspace() } +} + +#[inline] +pub fn move_obj_during_marking(from: ObjectReference, to: ObjectReference) { + unsafe { rb_mmtk_move_obj_during_marking(from, to) } +} + +#[inline] +pub fn obj_optimal_size(object: ObjectReference) -> usize { + unsafe { rb_mmtk_obj_optimal_size(object) } +} + +#[inline] +pub fn update_object_references(object: ObjectReference) { + unsafe { rb_mmtk_update_object_references(object) } +} + +#[inline] +pub fn call_gc_mark_children(object: ObjectReference) { + unsafe { rb_mmtk_call_gc_mark_children(object) } +} + +#[inline] +pub fn handle_weak_references(object: ObjectReference, moving: bool) { + unsafe { rb_mmtk_handle_weak_references(object, moving) } +} + +#[inline] +pub fn call_obj_free(object: ObjectReference) { + unsafe { rb_mmtk_call_obj_free(object) } +} + +#[inline] +pub fn vm_live_bytes() -> usize { + unsafe { rb_mmtk_vm_live_bytes() } +} + +#[inline] +pub fn update_global_tables(tbl_idx: c_int, moving: bool) { + unsafe { rb_mmtk_update_global_tables(tbl_idx, moving) } +} + +#[inline] +pub fn global_tables_count() -> c_int { + unsafe { rb_mmtk_global_tables_count() } +} + +#[inline] +pub fn update_finalizer_table() { + unsafe { rb_mmtk_update_finalizer_table() } +} + +#[inline] +pub fn mutator_thread_panic_handler() -> ! { + unsafe { rb_mmtk_mutator_thread_panic_handler() } +} + +#[inline] +pub fn gc_thread_panic_handler() -> ! { + unsafe { rb_mmtk_gc_thread_panic_handler() } +} diff --git a/gc/mmtk/src/weak_proc.rs b/gc/mmtk/src/weak_proc.rs index a3b0b4c..2592490 100644 --- a/gc/mmtk/src/weak_proc.rs +++ b/gc/mmtk/src/weak_proc.rs @@ -120,7 +120,7 @@ impl WeakProcessor { worker.add_work(WorkBucketStage::Prepare, UpdateFinalizerTable); - let global_tables_count = (crate::upcalls().global_tables_count)(); + let global_tables_count = crate::upcalls::global_tables_count(); let work_packets = (0..global_tables_count) .map(|i| Box::new(UpdateGlobalTables { idx: i }) as _) .collect(); @@ -143,7 +143,7 @@ fn process_obj_free_candidates(obj_free_candidates: &mut Vec) { trace!("Forwarding obj_free candidate: {object} -> {new_object}"); new_candidates.push(new_object); } else { - (upcalls().call_obj_free)(object); + upcalls::call_obj_free(object); } } @@ -224,7 +224,7 @@ impl ProcessWeakReferences { } if object.is_reachable() { - (upcalls().handle_weak_references)(object, moving_gc); + upcalls::handle_weak_references(object, moving_gc); true } else { @@ -263,7 +263,7 @@ trait GlobalTableProcessingWork { struct UpdateFinalizerTable; impl GlobalTableProcessingWork for UpdateFinalizerTable { fn process_table(&mut self) { - (crate::upcalls().update_finalizer_table)(); + crate::upcalls::update_finalizer_table(); } } impl GCWork for UpdateFinalizerTable { @@ -277,7 +277,7 @@ struct UpdateGlobalTables { } impl GlobalTableProcessingWork for UpdateGlobalTables { fn process_table(&mut self) { - (crate::upcalls().update_global_tables)( + crate::upcalls::update_global_tables( self.idx, crate::mmtk().get_plan().current_gc_may_move_object(), )