diff --git a/changelog.d/6811-object-write-fast-paths.md b/changelog.d/6811-object-write-fast-paths.md new file mode 100644 index 0000000000..bb1bc4a4e8 --- /dev/null +++ b/changelog.d/6811-object-write-fast-paths.md @@ -0,0 +1 @@ +perf(runtime, codegen): accelerate ordinary existing-field writes with header-first receiver classification and static-key write PICs, and version dense same-shape two-field numeric object loops into a once-validated call-free clone. The #6759 write micro now measures 5 ms in Perry versus 7 ms in Node.js, while descriptors, holes, mixed layouts, proxies, and diagnostic modes retain the full semantic fallback. diff --git a/crates/perry-codegen/src/block.rs b/crates/perry-codegen/src/block.rs index 97a42f2c48..50d0d35d11 100644 --- a/crates/perry-codegen/src/block.rs +++ b/crates/perry-codegen/src/block.rs @@ -453,6 +453,17 @@ impl LlBlock { r } + /// Sequentially-consistent atomic load for globals shared with runtime + /// atomics. The explicit alignment is required by LLVM atomic loads. + pub fn load_atomic_seq_cst(&mut self, ty: LlvmType, ptr: &str, alignment: u32) -> String { + let r = self.reg(); + self.emit(format!( + "{} = load atomic {}, ptr {} seq_cst, align {}", + r, ty, ptr, alignment + )); + r + } + /// (Issue #52) Load tagged with `!invariant.load !0`. LLVM's GVN + /// LICM are allowed to hoist these loads out of any enclosing loop — /// the contract is that the loaded memory does not change between diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index 60b6f31115..aad31ed271 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -834,6 +834,7 @@ pub(super) fn compile_closure( integer_locals: native_facts.integer_locals(), unsigned_i32_locals: native_facts.unsigned_i32_locals(), shadow_slot_map, + persistent_shadow_slots: std::collections::HashSet::new(), shadow_slot_clears_after_stmt, arena_state_slot: None, class_keys_slots: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 6d74b68974..0d8c725a99 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -747,6 +747,7 @@ pub(super) fn compile_module_entry( integer_locals: main_native_facts.integer_locals(), unsigned_i32_locals: main_native_facts.unsigned_i32_locals(), shadow_slot_map: main_shadow_slot_map, + persistent_shadow_slots: std::collections::HashSet::new(), shadow_slot_clears_after_stmt: main_shadow_slot_clears_after_stmt, arena_state_slot: None, class_keys_slots: HashMap::new(), @@ -1347,6 +1348,7 @@ pub(super) fn compile_module_entry( integer_locals: init_native_facts.integer_locals(), unsigned_i32_locals: init_native_facts.unsigned_i32_locals(), shadow_slot_map: init_shadow_slot_map, + persistent_shadow_slots: std::collections::HashSet::new(), shadow_slot_clears_after_stmt: init_shadow_slot_clears_after_stmt, arena_state_slot: None, class_keys_slots: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 28c3368f15..0caf218499 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -542,6 +542,7 @@ pub(super) fn compile_function( integer_locals: native_facts.integer_locals(), unsigned_i32_locals: native_facts.unsigned_i32_locals(), shadow_slot_map, + persistent_shadow_slots: std::collections::HashSet::new(), shadow_slot_clears_after_stmt, arena_state_slot: None, class_keys_slots: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index d88c0e1f4c..5de56e0385 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -460,6 +460,7 @@ pub(super) fn compile_method( integer_locals: native_facts.integer_locals(), unsigned_i32_locals: native_facts.unsigned_i32_locals(), shadow_slot_map, + persistent_shadow_slots: std::collections::HashSet::new(), shadow_slot_clears_after_stmt, arena_state_slot: None, class_keys_slots: HashMap::new(), @@ -1458,6 +1459,7 @@ pub(super) fn compile_static_method( integer_locals: native_facts.integer_locals(), unsigned_i32_locals: native_facts.unsigned_i32_locals(), shadow_slot_map, + persistent_shadow_slots: std::collections::HashSet::new(), shadow_slot_clears_after_stmt, arena_state_slot: None, class_keys_slots: HashMap::new(), diff --git a/crates/perry-codegen/src/expr/index_get.rs b/crates/perry-codegen/src/expr/index_get.rs index e59cc1193e..f185322980 100644 --- a/crates/perry-codegen/src/expr/index_get.rs +++ b/crates/perry-codegen/src/expr/index_get.rs @@ -19,8 +19,9 @@ use super::{ array_kind_fact, buffer_access_materialization_reason, emit_typed_feedback_register_site, expr_has_numeric_pointer_free_array_layout, int_range_expr, lower_buffer_load, lower_expr, lower_expr_as_i32, lower_typed_array_load, materialize_js_value, raw_f64_layout_fact, - try_lower_flat_const_index_get, unbox_str_handle, unbox_to_i64, BufferAccessSpec, FnCtx, - PackedF64LoopFact, TypedFeedbackContract, TypedFeedbackKind, + try_lower_flat_const_index_get, typed_feedback_emission_enabled, unbox_str_handle, + unbox_to_i64, BufferAccessSpec, FnCtx, PackedF64LoopFact, TypedFeedbackContract, + TypedFeedbackKind, }; fn is_width_tracked_typed_array_receiver(ctx: &FnCtx<'_>, object: &Expr) -> bool { @@ -345,26 +346,94 @@ fn lower_guarded_array_index_get( let fallback_label = ctx.block_label(fallback_idx); let merge_label = ctx.block_label(merge_idx); - let guard_ok = { - let blk = ctx.block(); - let guard_fn = if require_numeric_layout { - "js_typed_feedback_numeric_array_index_get_guard" - } else { - "js_typed_feedback_plain_array_index_get_guard" + if !require_numeric_layout && !typed_feedback_emission_enabled() { + // Normal builds do not collect feedback. Inline the plain-array + // structural guard instead of paying an out-of-line call merely to + // rediscover the same header facts before the direct slot load below. + // Prototype-chain invalidators are summarized by one sticky runtime + // byte; per-array descriptors and forwarding remain receiver-local. + let deref_idx = ctx.new_block(&format!("{}.guard.deref", block_prefix)); + let deref_label = ctx.block_label(deref_idx); + { + let blk = ctx.block(); + let arr_bits = blk.bitcast_double_to_i64(arr_box); + let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); + let tag = blk.lshr(I64, &arr_bits, "48"); + let is_pointer = blk.icmp_eq(I64, &tag, "32765"); // POINTER_TAG + let above_handle_band = blk.icmp_ugt(I64, &arr_handle, "1048575"); + let heap_candidate = blk.and(I1, &is_pointer, &above_handle_band); + blk.cond_br(&heap_candidate, &deref_label, &fallback_label); + } + + ctx.current_block = deref_idx; + { + let blk = ctx.block(); + let arr_bits = blk.bitcast_double_to_i64(arr_box); + let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); + + let gc_type_addr = blk.sub(I64, &arr_handle, "8"); + let gc_type_ptr = blk.inttoptr(I64, &gc_type_addr); + let gc_type = blk.load(I8, &gc_type_ptr); + let is_array = blk.icmp_eq(I8, &gc_type, "1"); // GC_TYPE_ARRAY + + let gc_flags_addr = blk.sub(I64, &arr_handle, "7"); + let gc_flags_ptr = blk.inttoptr(I64, &gc_flags_addr); + let gc_flags = blk.load(I8, &gc_flags_ptr); + let forwarded_bits = blk.and(I8, &gc_flags, "128"); + let not_forwarded = blk.icmp_eq(I8, &forwarded_bits, "0"); + + let reserved_addr = blk.sub(I64, &arr_handle, "6"); + let reserved_ptr = blk.inttoptr(I64, &reserved_addr); + let reserved = blk.load(I16, &reserved_ptr); + let descriptor_bits = blk.and(I16, &reserved, "1024"); + let no_descriptors = blk.icmp_eq(I16, &descriptor_bits, "0"); + + let invalidated = blk.load_volatile(I8, "@PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED"); + let default_prototype_chain = blk.icmp_eq(I8, &invalidated, "0"); + + let arr_ptr = blk.inttoptr(I64, &arr_handle); + let length = blk.load(I32, &arr_ptr); + let capacity_ptr = blk.gep(I8, &arr_ptr, &[(I64, "4")]); + let capacity = blk.load(I32, &capacity_ptr); + let index_nonnegative = blk.icmp_slt(I32, idx_i32, "0"); + let index_nonnegative = blk.icmp_eq(I1, &index_nonnegative, "false"); + let index_in_bounds = blk.icmp_ult(I32, idx_i32, &length); + let length_sane = blk.icmp_ule(I32, &length, "16000000"); + let capacity_sane = blk.icmp_ule(I32, &capacity, "16000000"); + let length_within_capacity = blk.icmp_ule(I32, &length, &capacity); + + let mut guard_ok = blk.and(I1, &is_array, ¬_forwarded); + guard_ok = blk.and(I1, &guard_ok, &no_descriptors); + guard_ok = blk.and(I1, &guard_ok, &default_prototype_chain); + guard_ok = blk.and(I1, &guard_ok, &index_nonnegative); + guard_ok = blk.and(I1, &guard_ok, &index_in_bounds); + guard_ok = blk.and(I1, &guard_ok, &length_sane); + guard_ok = blk.and(I1, &guard_ok, &capacity_sane); + guard_ok = blk.and(I1, &guard_ok, &length_within_capacity); + blk.cond_br(&guard_ok, &fast_label, &fallback_label); + } + } else { + let guard_ok = { + let blk = ctx.block(); + let guard_fn = if require_numeric_layout { + "js_typed_feedback_numeric_array_index_get_guard" + } else { + "js_typed_feedback_plain_array_index_get_guard" + }; + let guard_i32 = blk.call( + I32, + guard_fn, + &[ + (I64, &feedback_site_id), + (DOUBLE, arr_box), + (I32, idx_i32), + (I32, "1"), + ], + ); + blk.icmp_ne(I32, &guard_i32, "0") }; - let guard_i32 = blk.call( - I32, - guard_fn, - &[ - (I64, &feedback_site_id), - (DOUBLE, arr_box), - (I32, idx_i32), - (I32, "1"), - ], - ); - blk.icmp_ne(I32, &guard_i32, "0") - }; - ctx.block().cond_br(&guard_ok, &fast_label, &fallback_label); + ctx.block().cond_br(&guard_ok, &fast_label, &fallback_label); + } ctx.current_block = fallback_idx; // Materialize the f64 index only here (cold path) so the int→fp conversion diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 152dd75073..3175129c66 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -101,7 +101,8 @@ pub(crate) use range_facts::{ }; pub(crate) use strings::emit_string_literal_global; pub(crate) use typed_feedback::{ - emit_typed_feedback_register_site, native_region_slug, TypedFeedbackContract, TypedFeedbackKind, + emit_typed_feedback_register_site, native_region_slug, typed_feedback_emission_enabled, + TypedFeedbackContract, TypedFeedbackKind, }; pub(crate) use url_helpers::lower_url_string_getter; pub(crate) use v8_interop::{ @@ -128,7 +129,7 @@ mod shadow_slot; pub(crate) use dispatch::{lower_expr, lower_math_operand}; pub(crate) use shadow_slot::{ emit_shadow_slot_bind_for_local, emit_shadow_slot_clear, emit_shadow_slot_update_for_expr, - expr_is_known_non_pointer_shadow_value, + enable_persistent_shadow_slot_for_array_alias, expr_is_known_non_pointer_shadow_value, }; /// One in-flight inline-constructor return target. See @@ -598,6 +599,12 @@ pub(crate) struct FnCtx<'a> { /// the frame reflects the live pointer state at the following /// safepoint. Today — just tracked, not consumed. pub shadow_slot_map: std::collections::HashMap, + /// Shadow slots bound once in the function-entry setup and deliberately + /// kept active until return. This is used for immutable loop aliases read + /// from an already-rooted array: the local alloca is stable, and retaining + /// its current value for the function lifetime avoids per-iteration TLS + /// bind/clear traffic without weakening GC reachability. + pub persistent_shadow_slots: std::collections::HashSet, /// Top-level statement index → shadow-frame slot indices that can be /// cleared after lowering that statement. Built once per user function /// from HIR local-reference last-use information. diff --git a/crates/perry-codegen/src/expr/proxy_reflect.rs b/crates/perry-codegen/src/expr/proxy_reflect.rs index 1fe9667455..b6de695c53 100644 --- a/crates/perry-codegen/src/expr/proxy_reflect.rs +++ b/crates/perry-codegen/src/expr/proxy_reflect.rs @@ -9,12 +9,13 @@ use perry_hir::Expr; use crate::nanbox::{double_literal, POINTER_MASK_I64}; use crate::native_value::MaterializationReason; -use crate::type_analysis::{is_array_expr, is_string_expr, receiver_class_name}; -use crate::types::{DOUBLE, I32, I64, PTR}; +use crate::type_analysis::{is_array_expr, is_numeric_expr, is_string_expr, receiver_class_name}; +use crate::types::{DOUBLE, I1, I16, I32, I64, I8, PTR}; use super::{ - downgrade_buffer_aliases_in_expr, lower_expr, nanbox_pointer_inline, proxy_build_args_array, - unbox_str_handle, unbox_to_i64, FnCtx, + downgrade_buffer_aliases_in_expr, emit_jsvalue_slot_store_scalar_aware_on_block, + expr_produces_non_pointer_bits_by_construction, lower_expr, nanbox_pointer_inline, + proxy_build_args_array, unbox_str_handle, unbox_to_i64, FnCtx, }; fn downgrade_unknown_call_expr(ctx: &mut FnCtx<'_>, expr: &Expr) { @@ -219,6 +220,237 @@ fn put_value_static_property_fast_path( } } +/// Monomorphic inline cache for a static-name `PutValue` whose target and +/// receiver are the same expression. +/// +/// Sloppy script writes cannot reuse `PropertySet` because its fallback throws +/// on rejected writes. This diamond keeps the strict-aware runtime on every +/// miss, then turns a settled existing-own-data store into a keys-token compare +/// plus a direct slot write. Mutable semantic state (freeze/descriptor flags) +/// is rechecked on every hit. +fn lower_put_value_static_write_ic( + ctx: &mut FnCtx<'_>, + target: &Expr, + key: &Expr, + value: &Expr, + receiver: &Expr, + strict: bool, +) -> Result> { + let Expr::String(_) = key else { + return Ok(None); + }; + if !same_put_value_receiver_expr(target, receiver) || crate::codegen::full_outline_ic_enabled() + { + return Ok(None); + } + // The assignment reference (target + static key) is evaluated before the + // RHS. Until PutValue reference temporaries have dedicated GC roots, an + // allocating/calling RHS could move the already-evaluated target while its + // SSA value remains stale. Keep the inline PIC to call-free expressions; + // the existing runtime lowering handles every other RHS. + if !put_value_rhs_is_safepoint_free(ctx, value) { + return Ok(None); + } + + downgrade_unknown_call_expr(ctx, target); + downgrade_unknown_call_expr(ctx, key); + downgrade_unknown_call_expr(ctx, value); + downgrade_unknown_call_expr(ctx, receiver); + let target_value = lower_expr(ctx, target)?; + let key_value = lower_expr(ctx, key)?; + let stored_value = lower_expr(ctx, value)?; + + let target_bits = ctx.block().bitcast_double_to_i64(&target_value); + let key_bits = ctx.block().bitcast_double_to_i64(&key_value); + let key_handle = ctx.block().and(I64, &key_bits, POINTER_MASK_I64); + let target_handle = ctx.block().and(I64, &target_bits, POINTER_MASK_I64); + + let site_id = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let cache_name = format!("perry_ic_{}", site_id); + ctx.pending_declares + .push((format!("__ic_decl_{}", site_id), DOUBLE, vec![])); + ctx.ic_globals.push(cache_name.clone()); + let cache_ref = format!("@{}", cache_name); + + // Branch before the first header load so primitives, forged non-pointer + // bit patterns, and native handle ids can never be dereferenced by the + // inline checks. + let target_tag = ctx.block().lshr(I64, &target_bits, "48"); + let pointer_tag = ctx.block().icmp_eq(I64, &target_tag, "32765"); // 0x7FFD + let above_handles = ctx.block().icmp_ugt(I64, &target_handle, "1048575"); // 0x100000 + let heap_candidate = ctx.block().and(I1, &pointer_tag, &above_handles); + let guard_idx = ctx.new_block("put.pic.guard"); + let hit_idx = ctx.new_block("put.pic.hit"); + let miss_idx = ctx.new_block("put.pic.miss"); + let merge_idx = ctx.new_block("put.pic.merge"); + let guard_label = ctx.block_label(guard_idx); + let hit_label = ctx.block_label(hit_idx); + let miss_label = ctx.block_label(miss_idx); + let merge_label = ctx.block_label(merge_idx); + ctx.block() + .cond_br(&heap_candidate, &guard_label, &miss_label); + + ctx.current_block = guard_idx; + let safe_target = target_handle.clone(); + + let gc_type_addr = ctx.block().sub(I64, &safe_target, "8"); + let gc_type_ptr = ctx.block().inttoptr(I64, &gc_type_addr); + let gc_type = ctx.block().load(I8, &gc_type_ptr); + let gc_object = ctx.block().icmp_eq(I8, &gc_type, "2"); + let gc_flags_addr = ctx.block().sub(I64, &safe_target, "7"); + let gc_flags_ptr = ctx.block().inttoptr(I64, &gc_flags_addr); + let gc_flags = ctx.block().load(I8, &gc_flags_ptr); + let forwarded = ctx.block().and(I8, &gc_flags, "128"); + let not_forwarded = ctx.block().icmp_eq(I8, &forwarded, "0"); + + // Existing-own overwrite guards. Bit 12 is the per-object typed-layout + // intact bit: the runtime miss downgrades it before priming this cache, so + // same-shape siblings take one miss each before direct stores are allowed. + const BLOCKING_FLAGS: u16 = 0x1907; // frozen/sealed/noextend/TA-proto/descriptors/typed-intact + let reserved_addr = ctx.block().sub(I64, &safe_target, "6"); + let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); + let reserved = ctx.block().load(I16, &reserved_ptr); + let blocked = ctx.block().and(I16, &reserved, &BLOCKING_FLAGS.to_string()); + let flags_clear = ctx.block().icmp_eq(I16, &blocked, "0"); + + let object_type_ptr = ctx.block().inttoptr(I64, &safe_target); + let object_type = ctx.block().load(I32, &object_type_ptr); + let regular = ctx.block().icmp_eq(I32, &object_type, "1"); + let class_addr = ctx.block().add(I64, &safe_target, "4"); + let class_ptr = ctx.block().inttoptr(I64, &class_addr); + let class_id = ctx.block().load(I32, &class_ptr); + let class_nonzero = ctx.block().icmp_ne(I32, &class_id, "0"); + let not_native_module = ctx.block().icmp_ne(I32, &class_id, "-2"); + + let keys_addr = ctx.block().add(I64, &safe_target, "16"); + let keys_ptr = ctx.block().inttoptr(I64, &keys_addr); + let keys = ctx.block().load(I64, &keys_ptr); + + // Mirror the read PIC's #6804 discriminated shape token. Plain objects + // carrying a never-reused runtime ShapeId compare by that stable id, + // lifted above the pointer range with bit 62. Class instances and + // unstamped receivers compare by their shared keys pointer. The runtime + // miss publishes the same token representation. + let parent_class_addr = ctx.block().add(I64, &safe_target, "8"); + let parent_class_ptr = ctx.block().inttoptr(I64, &parent_class_addr); + let parent_class_id = ctx.block().load(I32, &parent_class_ptr); + let shape_id_rel = ctx.block().add(I32, &parent_class_id, "-2147483648"); + let has_shape_id = ctx.block().icmp_ult(I32, &shape_id_rel, "1073741824"); + let shape_id64 = ctx.block().zext(I32, &parent_class_id, I64); + let shape_id_token = ctx.block().or(I64, &shape_id64, "4611686018427387904"); + let shape_token = ctx + .block() + .select(I1, &has_shape_id, I64, &shape_id_token, &keys); + let cached_token_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); + let cached_token = ctx.block().load(I64, &cached_token_ptr); + let token_match = ctx.block().icmp_eq(I64, &shape_token, &cached_token); + let token_nonzero = ctx.block().icmp_ne(I64, &shape_token, "0"); + + let cached_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]); + let slot = ctx.block().load(I64, &cached_slot_ptr); + let field_count_addr = ctx.block().add(I64, &safe_target, "12"); + let field_count_ptr = ctx.block().inttoptr(I64, &field_count_addr); + let field_count = ctx.block().load(I32, &field_count_ptr); + let field_count64 = ctx.block().zext(I32, &field_count, I64); + let below_floor = ctx.block().icmp_ult(I64, &field_count64, "4"); + let inline_limit = ctx + .block() + .select(I1, &below_floor, I64, "4", &field_count64); + let slot_in_bounds = ctx.block().icmp_ult(I64, &slot, &inline_limit); + + let mut hit = ctx.block().and(I1, &heap_candidate, &gc_object); + hit = ctx.block().and(I1, &hit, ¬_forwarded); + hit = ctx.block().and(I1, &hit, &flags_clear); + hit = ctx.block().and(I1, &hit, ®ular); + hit = ctx.block().and(I1, &hit, &class_nonzero); + hit = ctx.block().and(I1, &hit, ¬_native_module); + hit = ctx.block().and(I1, &hit, &token_match); + hit = ctx.block().and(I1, &hit, &token_nonzero); + hit = ctx.block().and(I1, &hit, &slot_in_bounds); + + ctx.block().cond_br(&hit, &hit_label, &miss_label); + + ctx.current_block = hit_idx; + let pointer_possible = !(is_numeric_expr(ctx, value) + || expr_produces_non_pointer_bits_by_construction(ctx, value)); + { + let header_size = + crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string(); + let blk = ctx.block(); + let slot_offset = blk.shl(I64, &slot, "3"); + let fields_base = blk.add(I64, &target_handle, &header_size); + let field_addr = blk.add(I64, &fields_base, &slot_offset); + let field_ptr = blk.inttoptr(I64, &field_addr); + if pointer_possible { + let slot_i32 = blk.trunc(I64, &slot, I32); + emit_jsvalue_slot_store_scalar_aware_on_block( + blk, + &field_ptr, + &stored_value, + &target_handle, + &slot_i32, + true, + &target_bits, + &field_addr, + true, + ); + } else { + // A non-pointer overwrite cannot create a young edge or make a GC + // pointer layout less conservative. The per-object typed layout + // bit was already cleared on the miss that primed this cache. + // GC_STORE_AUDIT(POINTER_FREE): this branch only stores a value + // proven unable to contain GC pointer bits. + blk.store(DOUBLE, &stored_value, &field_ptr); + } + blk.br(&merge_label); + } + let hit_end_label = ctx.block().label.clone(); + + ctx.current_block = miss_idx; + let strict_i32 = if strict { "1" } else { "0" }; + let miss_value = ctx.block().call( + DOUBLE, + "js_put_value_set_ic_miss", + &[ + (DOUBLE, &target_value), + (I64, &key_handle), + (DOUBLE, &stored_value), + (I32, strict_i32), + (PTR, &cache_ref), + ], + ); + let miss_end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + let result = ctx.block().phi( + DOUBLE, + &[ + (&stored_value, &hit_end_label), + (&miss_value, &miss_end_label), + ], + ); + Ok(Some(result)) +} + +fn put_value_rhs_is_safepoint_free(ctx: &FnCtx<'_>, expr: &Expr) -> bool { + match expr { + Expr::LocalGet(_) + | Expr::Number(_) + | Expr::Integer(_) + | Expr::Bool(_) + | Expr::Null + | Expr::Undefined + | Expr::String(_) => true, + Expr::Binary { left, right, .. } if is_numeric_expr(ctx, expr) => { + put_value_rhs_is_safepoint_free(ctx, left) + && put_value_rhs_is_safepoint_free(ctx, right) + } + _ => false, + } +} + fn same_side_effect_free_receiver(target: &Expr, receiver: &Expr) -> bool { match (target, receiver) { (Expr::LocalGet(id), Expr::LocalGet(receiver_id)) => id == receiver_id, @@ -632,6 +864,11 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { }, ); } + if let Some(result) = + lower_put_value_static_write_ic(ctx, target, key, value, receiver, *strict)? + { + return Ok(result); + } downgrade_unknown_call_expr(ctx, target); downgrade_unknown_call_expr(ctx, key); downgrade_unknown_call_expr(ctx, value); diff --git a/crates/perry-codegen/src/expr/shadow_slot.rs b/crates/perry-codegen/src/expr/shadow_slot.rs index 6045041616..03e45a9e8d 100644 --- a/crates/perry-codegen/src/expr/shadow_slot.rs +++ b/crates/perry-codegen/src/expr/shadow_slot.rs @@ -57,16 +57,58 @@ pub(crate) fn expr_is_known_non_pointer_shadow_value(ctx: &FnCtx<'_>, expr: &Exp } pub(crate) fn emit_shadow_slot_clear(ctx: &mut FnCtx<'_>, slot_idx: u32) { + if ctx.persistent_shadow_slots.contains(&slot_idx) { + return; + } ctx.block().call_void( "js_shadow_slot_set", &[(I32, &slot_idx.to_string()), (I64, "0")], ); } +/// Bind an immutable `const item = rootedArray[index]` local once in the +/// function-entry setup and retain its current value until return. +/// +/// The alloca is entry-hoisted and initialized to `undefined`, so the early +/// bind is valid even when the declaration itself sits in a loop or branch. +/// Every later iteration writes the same alloca, which the GC scanner follows +/// through `slot_ptrs`. Pointer-capable updates still emit the root shading +/// barrier required when an incremental collection has already scanned roots; +/// only the repeated TLS slot rebinding and lexical-death clear are removed. +pub(crate) fn enable_persistent_shadow_slot_for_array_alias( + ctx: &mut FnCtx<'_>, + local_id: u32, + init: &Expr, +) { + let Expr::IndexGet { object, .. } = init else { + return; + }; + if !matches!(object.as_ref(), Expr::LocalGet(_)) { + return; + } + let Some(slot_idx) = ctx.shadow_slot_map.get(&local_id).copied() else { + return; + }; + let Some(local_slot) = ctx.locals.get(&local_id).cloned() else { + return; + }; + if !ctx.persistent_shadow_slots.insert(slot_idx) { + return; + } + let slot_idx_string = slot_idx.to_string(); + ctx.func.entry_setup_call_void( + "js_shadow_slot_bind", + &[(I32, &slot_idx_string), (PTR, &local_slot)], + ); +} + pub(crate) fn emit_shadow_slot_bind_for_local(ctx: &mut FnCtx<'_>, local_id: u32) { let Some(slot_idx) = ctx.shadow_slot_map.get(&local_id).copied() else { return; }; + if ctx.persistent_shadow_slots.contains(&slot_idx) { + return; + } let Some(local_slot) = ctx.locals.get(&local_id).cloned() else { return; }; @@ -76,6 +118,25 @@ pub(crate) fn emit_shadow_slot_bind_for_local(ctx: &mut FnCtx<'_>, local_id: u32 ); } +fn emit_persistent_shadow_root_barrier(ctx: &mut FnCtx<'_>, value_bits: &str) { + let active = + ctx.block() + .load_atomic_seq_cst(I32, "@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT", 4); + let barrier_needed = ctx.block().icmp_ne(I32, &active, "0"); + let barrier_idx = ctx.new_block("shadow.root.barrier"); + let done_idx = ctx.new_block("shadow.root.barrier.done"); + let barrier_label = ctx.block_label(barrier_idx); + let done_label = ctx.block_label(done_idx); + ctx.block() + .cond_br(&barrier_needed, &barrier_label, &done_label); + + ctx.current_block = barrier_idx; + ctx.block() + .call_void("js_write_barrier_root_nanbox", &[(I64, value_bits)]); + ctx.block().br(&done_label); + ctx.current_block = done_idx; +} + pub(crate) fn emit_shadow_slot_update_for_expr( ctx: &mut FnCtx<'_>, local_id: u32, @@ -93,14 +154,20 @@ pub(crate) fn emit_shadow_slot_update_for_expr( let Some(slot_idx) = ctx.shadow_slot_map.get(&local_id).copied() else { return; }; + if ctx.persistent_shadow_slots.contains(&slot_idx) { + if !expr_is_known_non_pointer_shadow_value(ctx, rhs) { + let value_bits = ctx.block().bitcast_double_to_i64(value_reg); + emit_persistent_shadow_root_barrier(ctx, &value_bits); + } + return; + } if expr_is_known_non_pointer_shadow_value(ctx, rhs) { emit_shadow_slot_clear(ctx, slot_idx); } else { + // Every caller has already stored the new value in the local alloca. + // `js_shadow_slot_bind` copies that slot into the shadow frame, marks + // it active, and runs the root barrier, so a following slot-set call + // only repeated the same TLS lookup, copy, and barrier. emit_shadow_slot_bind_for_local(ctx, local_id); - let v_i64 = ctx.block().bitcast_double_to_i64(value_reg); - ctx.block().call_void( - "js_shadow_slot_set", - &[(I32, &slot_idx.to_string()), (I64, &v_i64)], - ); } } diff --git a/crates/perry-codegen/src/expr/typed_feedback.rs b/crates/perry-codegen/src/expr/typed_feedback.rs index 146c6bf517..bb7a78d5e3 100644 --- a/crates/perry-codegen/src/expr/typed_feedback.rs +++ b/crates/perry-codegen/src/expr/typed_feedback.rs @@ -246,7 +246,7 @@ fn emit_typed_feedback_bytes_global( /// build (`PERRY_TYPED_FEEDBACK=1 perry app.ts -o app && ./app`, env inherited /// by the run) emits and uses it. The site-id is still allocated and returned /// so the shape *guard* call is unchanged — guards stay correct either way. -fn typed_feedback_emission_enabled() -> bool { +pub(crate) fn typed_feedback_emission_enabled() -> bool { // Read fresh (not cached) so tests that toggle the env per-case observe the // change. At compile time this is a cheap getenv per property-access site. std::env::var_os("PERRY_TYPED_FEEDBACK").is_some() diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index 179190e53c..2d1a50cf65 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -29,6 +29,14 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // when it is non-zero (descriptors / typed-feedback in use). Defined in // perry-runtime as `PERRY_CLASS_FIELD_INLINE_GUARD_DISABLED`. module.add_external_global("PERRY_CLASS_FIELD_INLINE_GUARD_DISABLED", I8); + // Sticky summary of indexed Array/Object prototype pollution and custom + // Array [[Prototype]] installation. Normal compiled programs read this + // byte directly in the inline plain-array index guard. + module.add_external_global("PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED", I8); + // Process-wide count of threads with an active incremental marking + // barrier. Persistent shadow-slot updates use zero as an authoritative + // fast skip before calling the TLS-backed root barrier. + module.add_external_global("PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT", I32); // #5525 follow-up: the process-global typed-array kind cache + the // "any exotic views live" guard, exported from perry-runtime so the codegen // can emit a guarded *inline* typed-array element load at the access site @@ -445,6 +453,16 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { DOUBLE, &[DOUBLE, DOUBLE, DOUBLE, DOUBLE, I32], ); + module.declare_function( + "js_put_value_set_ic_miss", + DOUBLE, + &[DOUBLE, I64, DOUBLE, I32, PTR], + ); + module.declare_function( + "js_object_array_numeric_write2_guard", + I64, + &[DOUBLE, DOUBLE, DOUBLE, I32], + ); module.declare_function( "js_super_put_value_set", DOUBLE, diff --git a/crates/perry-codegen/src/stmt/let_stmt.rs b/crates/perry-codegen/src/stmt/let_stmt.rs index 6eebc91026..0df164096c 100644 --- a/crates/perry-codegen/src/stmt/let_stmt.rs +++ b/crates/perry-codegen/src/stmt/let_stmt.rs @@ -1123,6 +1123,11 @@ pub(crate) fn lower_let( } ctx.locals.insert(id, slot.clone()); ctx.local_types.insert(id, refined_ty.clone()); + if !mutable { + if let Some(init_expr) = init { + crate::expr::enable_persistent_shadow_slot_for_array_alias(ctx, id, init_expr); + } + } // Int32 specialization (issue #48): if this local qualifies as // integer-valued (all writes are `| 0` / `>>> 0` / bitwise / int // literal / ++/--), allocate a parallel i32 slot. Update/LocalSet diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index 9ea26b3f77..e94dde3be3 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -1842,6 +1842,435 @@ const CLASS_FIELD_LOOP_CLASS_DENYLIST: &[&str] = &[ "Function", ]; +#[derive(Clone)] +enum ObjectArrayWriteNumber { + OuterCounter, + InnerCounter, + Constant(f64), + Add(Box, Box), + Sub(Box, Box), +} + +struct ObjectArrayWrite2Loop { + outer_counter_id: u32, + outer_start: i32, + outer_bound: i32, + inner_counter_id: u32, + inner_bound: i32, + array_id: u32, + alias_id: u32, + properties: [String; 2], + values: [ObjectArrayWriteNumber; 2], +} + +fn match_nonnegative_constant_i32(expr: &perry_hir::Expr) -> Option { + match expr { + perry_hir::Expr::Integer(n) => i32::try_from(*n).ok().filter(|n| *n >= 0), + perry_hir::Expr::Number(n) + if n.is_finite() && n.fract() == 0.0 && *n >= 0.0 && *n <= i32::MAX as f64 => + { + Some(*n as i32) + } + _ => None, + } +} + +fn match_object_array_write_number( + expr: &perry_hir::Expr, + outer_counter_id: u32, + inner_counter_id: u32, +) -> Option { + use perry_hir::{BinaryOp, Expr}; + match expr { + Expr::LocalGet(id) if *id == outer_counter_id => Some(ObjectArrayWriteNumber::OuterCounter), + Expr::LocalGet(id) if *id == inner_counter_id => Some(ObjectArrayWriteNumber::InnerCounter), + Expr::Integer(n) if (-i64::from(i32::MAX)..=i64::from(i32::MAX)).contains(n) => { + Some(ObjectArrayWriteNumber::Constant(*n as f64)) + } + Expr::Number(n) if n.is_finite() => Some(ObjectArrayWriteNumber::Constant(*n)), + Expr::Binary { op, left, right } if matches!(op, BinaryOp::Add | BinaryOp::Sub) => { + let left = match_object_array_write_number(left, outer_counter_id, inner_counter_id)?; + let right = match_object_array_write_number(right, outer_counter_id, inner_counter_id)?; + Some(if matches!(op, BinaryOp::Add) { + ObjectArrayWriteNumber::Add(Box::new(left), Box::new(right)) + } else { + ObjectArrayWriteNumber::Sub(Box::new(left), Box::new(right)) + }) + } + _ => None, + } +} + +fn match_constant_counted_for( + init: Option<&Stmt>, + condition: Option<&perry_hir::Expr>, + update: Option<&perry_hir::Expr>, +) -> Option<(u32, i32, i32)> { + use perry_hir::{CompareOp, Expr, UpdateOp}; + let (counter_id, start) = match init? { + Stmt::Let { + id, + init: Some(start), + .. + } => (*id, match_nonnegative_constant_i32(start)?), + _ => return None, + }; + let bound = match condition? { + Expr::Compare { + op: CompareOp::Lt, + left, + right, + } if matches!(left.as_ref(), Expr::LocalGet(id) if *id == counter_id) => { + match_nonnegative_constant_i32(right)? + } + _ => return None, + }; + if !matches!( + update?, + Expr::Update { + id, + op: UpdateOp::Increment, + .. + } if *id == counter_id + ) || start >= bound + { + return None; + } + Some((counter_id, start, bound)) +} + +/// Match the #6809 object-write micro shape. This is deliberately a separate, +/// much narrower proof than generic loop purity: the fast clone has no side +/// exits after its one runtime scan, so it may commit multiple stores per +/// iteration without a replay protocol. +fn match_object_array_write2_loop( + ctx: &FnCtx<'_>, + init: Option<&Stmt>, + condition: Option<&perry_hir::Expr>, + update: Option<&perry_hir::Expr>, + body: &[Stmt], +) -> Option { + use perry_hir::Expr; + + if !ctx.pending_labels.is_empty() { + return None; + } + let (outer_counter_id, outer_start, outer_bound) = + match_constant_counted_for(init, condition, update)?; + let [Stmt::For { + init: inner_init, + condition: inner_condition, + update: inner_update, + body: inner_body, + }] = body + else { + return None; + }; + let (inner_counter_id, inner_start, inner_bound) = match_constant_counted_for( + inner_init.as_deref(), + inner_condition.as_ref(), + inner_update.as_ref(), + )?; + // Starting at zero lets the runtime preflight prove one contiguous dense + // prefix and keeps the raw element address calculation minimal. + if inner_start != 0 + || inner_bound > 16_000_000 + || outer_counter_id == inner_counter_id + || ctx.boxed_vars.contains(&outer_counter_id) + || ctx.boxed_vars.contains(&inner_counter_id) + { + return None; + } + + let [Stmt::Let { + id: alias_id, + mutable: false, + init: Some(Expr::IndexGet { object, index }), + .. + }, Stmt::Expr(first), Stmt::Expr(second)] = inner_body.as_slice() + else { + return None; + }; + let (Expr::LocalGet(array_id), Expr::LocalGet(index_id)) = (object.as_ref(), index.as_ref()) + else { + return None; + }; + if *index_id != inner_counter_id + || *array_id == outer_counter_id + || *array_id == inner_counter_id + || *array_id == *alias_id + || ctx.boxed_vars.contains(array_id) + || ctx.boxed_vars.contains(alias_id) + || ctx.module_globals.contains_key(array_id) + || !ctx.locals.contains_key(array_id) + || ctx.scalar_replaced.contains_key(array_id) + || ctx.pod_records.contains_key(array_id) + { + return None; + } + + let match_store = |effect: &Expr| -> Option<(String, ObjectArrayWriteNumber)> { + let Expr::PutValueSet { + target, + key, + value, + receiver, + .. + } = effect + else { + return None; + }; + if !matches!( + (target.as_ref(), receiver.as_ref()), + (Expr::LocalGet(target_id), Expr::LocalGet(receiver_id)) + if target_id == alias_id && receiver_id == alias_id + ) { + return None; + } + let Expr::String(property) = key.as_ref() else { + return None; + }; + let value = match_object_array_write_number(value, outer_counter_id, inner_counter_id)?; + Some((property.clone(), value)) + }; + let (property_1, value_1) = match_store(first)?; + let (property_2, value_2) = match_store(second)?; + + Some(ObjectArrayWrite2Loop { + outer_counter_id, + outer_start, + outer_bound, + inner_counter_id, + inner_bound, + array_id: *array_id, + alias_id: *alias_id, + properties: [property_1, property_2], + values: [value_1, value_2], + }) +} + +fn emit_object_array_write_number( + ctx: &mut FnCtx<'_>, + expr: &ObjectArrayWriteNumber, + outer: &str, + inner: &str, +) -> String { + match expr { + ObjectArrayWriteNumber::OuterCounter => outer.to_string(), + ObjectArrayWriteNumber::InnerCounter => inner.to_string(), + ObjectArrayWriteNumber::Constant(n) => crate::nanbox::double_literal(*n), + ObjectArrayWriteNumber::Add(left, right) => { + let left = emit_object_array_write_number(ctx, left, outer, inner); + let right = emit_object_array_write_number(ctx, right, outer, inner); + ctx.block().fadd(&left, &right) + } + ObjectArrayWriteNumber::Sub(left, right) => { + let left = emit_object_array_write_number(ctx, left, outer, inner); + let right = emit_object_array_write_number(ctx, right, outer, inner); + ctx.block().fsub(&left, &right) + } + } +} + +/// Whole-nest versioning for a dense array of same-shape objects. +/// +/// The runtime helper validates every receiver and resolves both slots before +/// the first store. The successful clone contains no calls, allocations, +/// barriers, or side exits, so all raw pointers remain valid for the complete +/// outer × inner nest. A failed proof enters the untouched generic clone. +fn lower_object_array_write2_versioned_for( + ctx: &mut FnCtx<'_>, + init: Option<&Stmt>, + condition: Option<&perry_hir::Expr>, + update: Option<&perry_hir::Expr>, + body: &[Stmt], +) -> Result { + let Some(matched) = match_object_array_write2_loop(ctx, init, condition, update, body) else { + return Ok(false); + }; + + let slow_pre_idx = ctx.new_block("object_array_write2.loop.slow.preheader"); + let merge_idx = ctx.new_block("object_array_write2.loop.merge"); + let slow_pre_label = ctx.block_label(slow_pre_idx); + let merge_label = ctx.block_label(merge_idx); + + let key_1_idx = ctx.strings.intern(&matched.properties[0]); + let key_2_idx = ctx.strings.intern(&matched.properties[1]); + let key_1_global = format!("@{}", ctx.strings.entry(key_1_idx).handle_global); + let key_2_global = format!("@{}", ctx.strings.entry(key_2_idx).handle_global); + let array_box = lower_expr(ctx, &perry_hir::Expr::LocalGet(matched.array_id))?; + let (key_1_box, key_2_box) = { + let blk = ctx.block(); + ( + blk.load(DOUBLE, &key_1_global), + blk.load(DOUBLE, &key_2_global), + ) + }; + let packed_slots = ctx.block().call( + I64, + "js_object_array_numeric_write2_guard", + &[ + (DOUBLE, &array_box), + (DOUBLE, &key_1_box), + (DOUBLE, &key_2_box), + (I32, &matched.inner_bound.to_string()), + ], + ); + let preheader_idx = ctx.current_block; + let preheader_label = ctx.block().label.clone(); + + // Emit the fallback first. Besides preserving the original semantics, this + // creates the ordinary local slots for the nested counter, allowing the + // fast completion block to synchronize loop variables before the merge. + ctx.current_block = slow_pre_idx; + lower_for_after_init( + ctx, + init, + condition, + update, + body, + "for.object_array_write2_slow", + )?; + if !ctx.block().is_terminated() { + ctx.block().br(&merge_label); + } + + let fast_outer_cond_idx = ctx.new_block("object_array_write2.loop.fast.outer.cond"); + let fast_inner_pre_idx = ctx.new_block("object_array_write2.loop.fast.inner.preheader"); + let fast_inner_cond_idx = ctx.new_block("object_array_write2.loop.fast.inner.cond"); + let fast_inner_body_idx = ctx.new_block("object_array_write2.loop.fast.inner.body"); + let fast_inner_exit_idx = ctx.new_block("object_array_write2.loop.fast.inner.exit"); + let fast_done_idx = ctx.new_block("object_array_write2.loop.fast.done"); + let fast_outer_cond_label = ctx.block_label(fast_outer_cond_idx); + let fast_inner_pre_label = ctx.block_label(fast_inner_pre_idx); + let fast_inner_cond_label = ctx.block_label(fast_inner_cond_idx); + let fast_inner_body_label = ctx.block_label(fast_inner_body_idx); + let fast_inner_exit_label = ctx.block_label(fast_inner_exit_idx); + let fast_done_label = ctx.block_label(fast_done_idx); + + let (slot_1, slot_2, array_ptr) = { + let blk = ctx + .func + .block_mut(preheader_idx) + .expect("object-array preheader block must exist"); + let encoded_1 = blk.and(I64, &packed_slots, "4294967295"); + let encoded_2 = blk.lshr(I64, &packed_slots, "32"); + let slot_1 = blk.sub(I64, &encoded_1, "1"); + let slot_2 = blk.sub(I64, &encoded_2, "1"); + let array_bits = blk.bitcast_double_to_i64(&array_box); + let array_handle = blk.and(I64, &array_bits, crate::nanbox::POINTER_MASK_I64); + let array_ptr = blk.inttoptr(I64, &array_handle); + (slot_1, slot_2, array_ptr) + }; + + let fast_scan_start = fast_outer_cond_idx; + let (outer_next, inner_next) = { + let blk = ctx + .func + .block_mut(preheader_idx) + .expect("object-array preheader block must exist"); + (blk.fresh_reg(), blk.fresh_reg()) + }; + ctx.current_block = fast_outer_cond_idx; + let outer = ctx.block().phi( + I32, + &[ + (&matched.outer_start.to_string(), &preheader_label), + (&outer_next, &fast_inner_exit_label), + ], + ); + let outer_double = ctx.block().sitofp(I32, &outer, DOUBLE); + let outer_more = ctx + .block() + .icmp_slt(I32, &outer, &matched.outer_bound.to_string()); + ctx.block() + .cond_br(&outer_more, &fast_inner_pre_label, &fast_done_label); + + ctx.current_block = fast_inner_pre_idx; + ctx.block().br(&fast_inner_cond_label); + + ctx.current_block = fast_inner_cond_idx; + let inner = ctx.block().phi( + I32, + &[ + ("0", &fast_inner_pre_label), + (&inner_next, &fast_inner_body_label), + ], + ); + let inner_more = ctx + .block() + .icmp_slt(I32, &inner, &matched.inner_bound.to_string()); + ctx.block() + .cond_br(&inner_more, &fast_inner_body_label, &fast_inner_exit_label); + + ctx.current_block = fast_inner_body_idx; + let inner_double = ctx.block().sitofp(I32, &inner, DOUBLE); + let object_ptr = { + let blk = ctx.block(); + let inner_i64 = blk.sext(I32, &inner, I64); + let element_word = blk.add(I64, &inner_i64, "1"); + let element_ptr = blk.gep_inbounds(I64, &array_ptr, &[(I64, &element_word)]); + let object_box = blk.load(DOUBLE, &element_ptr); + let object_bits = blk.bitcast_double_to_i64(&object_box); + let object_handle = blk.and(I64, &object_bits, crate::nanbox::POINTER_MASK_I64); + blk.inttoptr(I64, &object_handle) + }; + let header_words = + (crate::target_layout::object_header_size_bytes(ctx.target_triple) / 8).to_string(); + for (slot, value) in [(&slot_1, &matched.values[0]), (&slot_2, &matched.values[1])] { + let value = emit_object_array_write_number(ctx, value, &outer_double, &inner_double); + let field_ptr = { + let blk = ctx.block(); + let field_word = blk.add(I64, slot, &header_words); + blk.gep_inbounds(I64, &object_ptr, &[(I64, &field_word)]) + }; + // GC_STORE_AUDIT(POINTER_FREE): the versioned loop emits only numeric + // values into fields proven numeric by the entry guard. + ctx.block().store(DOUBLE, &value, &field_ptr); + } + ctx.block() + .emit_raw(format!("{} = add i32 {}, 1", inner_next, inner)); + ctx.block().br(&fast_inner_cond_label); + + ctx.current_block = fast_inner_exit_idx; + ctx.block() + .emit_raw(format!("{} = add i32 {}, 1", outer_next, outer)); + ctx.block().br(&fast_outer_cond_label); + + // Keep the ordinary counter slots coherent on the fast edge. The values + // are normally block-scoped, but this also preserves transformed `var` + // cases and future HIR consumers without adding work inside either loop. + ctx.current_block = fast_done_idx; + for (id, final_value) in [ + (matched.outer_counter_id, matched.outer_bound), + (matched.inner_counter_id, matched.inner_bound), + ] { + if let Some(slot) = ctx.locals.get(&id).cloned() { + let value = crate::nanbox::double_literal(final_value as f64); + ctx.block().store(DOUBLE, &value, &slot); + } + if let Some(slot) = ctx.i32_counter_slots.get(&id).cloned() { + ctx.block().store(I32, &final_value.to_string(), &slot); + } + } + ctx.block().br(&merge_label); + + let fast_call_free = (fast_scan_start..ctx.func.num_blocks()) + .all(|idx| !ctx.func.blocks()[idx].contains_gc_unsafe_call()); + ctx.current_block = preheader_idx; + let guard_ok = ctx.block().icmp_ne(I64, &packed_slots, "0"); + if fast_call_free { + ctx.block() + .cond_br(&guard_ok, &fast_outer_cond_label, &slow_pre_label); + } else { + ctx.block().br(&slow_pre_label); + } + + ctx.current_block = merge_idx; + let _ = matched.alias_id; + Ok(true) +} + #[derive(Clone, Copy)] enum ClassFieldLoopBound { /// `i < `. @@ -3199,6 +3628,13 @@ pub(crate) fn lower_for( lower_stmt(ctx, init_stmt)?; } + // #6809: validate a dense, same-shape object array once and run the + // complete nested two-field numeric write loop without receiver/shape + // guards or runtime calls in either hot loop. + if lower_object_array_write2_versioned_for(ctx, init, condition, update, body)? { + return Ok(()); + } + if let Some(matched) = match_numeric_bulk_fill_loop(ctx, init, condition, update, body) { if lower_numeric_bulk_fill_loop(ctx, matched)? { return Ok(()); diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index f666e211f4..a38d027712 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -13441,6 +13441,193 @@ fn put_value_set_index_keeps_the_numeric_array_fast_path() { ); } +#[test] +fn static_put_value_uses_write_pic_for_call_free_rhs() { + let object = 1u32; + let left = 2u32; + let right = 3u32; + let module = module_with_classes_and_params( + "static_put_value_write_pic", + Vec::new(), + vec![ + param(object, "object", Type::Any), + param(left, "left", Type::Number), + param(right, "right", Type::Number), + ], + Type::Any, + vec![Stmt::Return(Some(Expr::PutValueSet { + target: Box::new(Expr::LocalGet(object)), + key: Box::new(Expr::String("x".to_string())), + value: Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(left)), + right: Box::new(Expr::LocalGet(right)), + }), + receiver: Box::new(Expr::LocalGet(object)), + strict: false, + }))], + ); + + let ir = compile_ir_for_module_with_opts(module, empty_opts()).unwrap(); + assert!( + ir.contains("call double @js_put_value_set_ic_miss"), + "a static existing-field write with a call-free numeric RHS should emit the guarded PIC:\n{ir}" + ); + assert!( + ir.contains("put.pic.guard") && ir.contains("put.pic.hit") && ir.contains("put.pic.miss"), + "the PIC must branch before header dereferences and retain a semantic miss path" + ); + assert!( + ir.contains("4611686018427387904") && ir.contains("1073741824"), + "the write PIC must mirror the read PIC's discriminated, never-reused ShapeId token" + ); +} + +#[test] +fn static_put_value_rejects_write_pic_when_rhs_can_allocate() { + let object = 1u32; + let module = module_with_classes_and_params( + "allocating_rhs_put_value_write_pic", + Vec::new(), + vec![param(object, "object", Type::Any)], + Type::Any, + vec![Stmt::Return(Some(Expr::PutValueSet { + target: Box::new(Expr::LocalGet(object)), + key: Box::new(Expr::String("x".to_string())), + value: Box::new(Expr::Object(vec![("value".to_string(), Expr::Integer(1))])), + receiver: Box::new(Expr::LocalGet(object)), + strict: false, + }))], + ); + + let ir = compile_ir_for_module_with_opts(module, empty_opts()).unwrap(); + assert!( + !ir.contains("call double @js_put_value_set_ic_miss"), + "an allocating RHS must stay on the rooted generic PutValue path" + ); + assert!( + ir.contains("call double @js_put_value_set("), + "the rejected PIC case must retain the complete strict/sloppy runtime semantics:\n{ir}" + ); +} + +#[test] +fn nested_same_shape_object_writes_version_the_whole_loop() { + let objects = 1u32; + let outer = 2u32; + let inner = 3u32; + let object = 4u32; + let store = |property: &str, op: BinaryOp| { + Stmt::Expr(Expr::PutValueSet { + target: Box::new(Expr::LocalGet(object)), + key: Box::new(Expr::String(property.to_string())), + value: Box::new(Expr::Binary { + op, + left: Box::new(Expr::LocalGet(outer)), + right: Box::new(Expr::LocalGet(inner)), + }), + receiver: Box::new(Expr::LocalGet(object)), + strict: false, + }) + }; + let body = vec![ + Stmt::Let { + id: objects, + name: "objects".to_string(), + ty: Type::Array(Box::new(Type::Any)), + mutable: false, + init: Some(Expr::Array(vec![])), + }, + Stmt::For { + init: Some(Box::new(Stmt::Let { + id: outer, + name: "outer".to_string(), + ty: Type::Number, + mutable: true, + init: Some(Expr::Integer(0)), + })), + condition: Some(Expr::Compare { + op: CompareOp::Lt, + left: Box::new(Expr::LocalGet(outer)), + right: Box::new(Expr::Integer(20)), + }), + update: Some(Expr::Update { + id: outer, + op: UpdateOp::Increment, + prefix: false, + }), + body: vec![Stmt::For { + init: Some(Box::new(Stmt::Let { + id: inner, + name: "inner".to_string(), + ty: Type::Number, + mutable: true, + init: Some(Expr::Integer(0)), + })), + condition: Some(Expr::Compare { + op: CompareOp::Lt, + left: Box::new(Expr::LocalGet(inner)), + right: Box::new(Expr::Integer(10)), + }), + update: Some(Expr::Update { + id: inner, + op: UpdateOp::Increment, + prefix: false, + }), + body: vec![ + Stmt::Let { + id: object, + name: "object".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::IndexGet { + object: Box::new(Expr::LocalGet(objects)), + index: Box::new(Expr::LocalGet(inner)), + }), + }, + store("c", BinaryOp::Add), + store("d", BinaryOp::Sub), + ], + }], + }, + ]; + + let ir = compile_ir("nested_object_write2_loop", body); + assert_eq!( + ir.matches("call i64 @js_object_array_numeric_write2_guard") + .count(), + 1, + "the complete receiver/shape/slot scan must run once before the loop nest:\n{ir}" + ); + assert!( + ir.contains("object_array_write2.loop.fast.outer.cond") + && ir.contains("object_array_write2.loop.fast.inner.body") + && ir.contains("object_array_write2.loop.slow.preheader"), + "the proof must retain distinct call-free and semantic fallback clones:\n{ir}" + ); + let fast_body = ir + .split("\nobject_array_write2.loop.fast.inner.body") + .nth(1) + .and_then(|tail| { + tail.split("\nobject_array_write2.loop.fast.inner.exit") + .next() + }) + .expect("fast inner-loop block"); + assert!( + !fast_body.contains("call "), + "the successful raw-pointer clone must stay call/GC-free:\n{fast_body}" + ); + assert!( + fast_body.matches("store double").count() >= 2 + && fast_body.contains("getelementptr inbounds i64"), + "the fast clone should be two direct numeric field stores:\n{fast_body}" + ); + assert!( + ir.contains("call double @js_put_value_set_ic_miss"), + "a failed whole-array proof must retain the original PutValue semantics:\n{ir}" + ); +} + #[path = "native_proof_regressions/invalidation.rs"] mod invalidation; diff --git a/crates/perry-codegen/tests/shadow_slot_hygiene.rs b/crates/perry-codegen/tests/shadow_slot_hygiene.rs index 69ae8ade80..0602a8082b 100644 --- a/crates/perry-codegen/tests/shadow_slot_hygiene.rs +++ b/crates/perry-codegen/tests/shadow_slot_hygiene.rs @@ -213,6 +213,38 @@ fn top_level_loop_shadow_module() -> Module { module } +fn persistent_index_alias_shadow_module() -> Module { + let mut module = top_level_shadow_module("entry_persistent_index_alias_shadow.ts"); + module.init = vec![ + Stmt::Let { + id: 21, + name: "items".to_string(), + ty: Type::Array(Box::new(Type::Any)), + mutable: false, + init: Some(Expr::Array(vec![Expr::MapNew])), + }, + Stmt::For { + init: None, + condition: Some(Expr::Bool(false)), + update: None, + body: vec![ + Stmt::Let { + id: 22, + name: "item".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::IndexGet { + object: Box::new(Expr::LocalGet(21)), + index: Box::new(Expr::Integer(0)), + }), + }, + Stmt::Expr(Expr::LocalGet(22)), + ], + }, + ]; + module +} + fn flat_const_row_alias_shadow_module() -> Module { Module { name: "entry_flat_const_shadow.ts".to_string(), @@ -545,8 +577,8 @@ fn function_shadow_slots_clear_dead_values_and_skip_numeric_roots() { ); let dead_write = ir - .find("call void @js_shadow_slot_set(i32 0, i64 %") - .expect("dead array let should write its pointer to shadow slot 0"); + .find("call void @js_shadow_slot_bind(i32 0, ptr %") + .expect("dead array let should bind its pointer local to shadow slot 0"); let dead_clear = ir[dead_write..] .find("call void @js_shadow_slot_set(i32 0, i64 0)") .map(|offset| dead_write + offset) @@ -612,8 +644,8 @@ fn entry_module_top_level_shadow_slots_update_and_clear() { ); let dead_write = main_ir - .find("call void @js_shadow_slot_set(i32 0, i64 %") - .expect("top-level pointer let should write its pointer to shadow slot 0"); + .find("call void @js_shadow_slot_bind(i32 0, ptr %") + .expect("top-level pointer let should bind its pointer local to shadow slot 0"); let dead_clear = main_ir[dead_write..] .find("call void @js_shadow_slot_set(i32 0, i64 0)") .map(|offset| dead_write + offset) @@ -657,8 +689,8 @@ fn non_entry_module_init_body_gets_post_init_shadow_frame() { assert!(strings_init < frame_push); assert!(frame_push < user_alloc); assert!( - init_ir.contains("call void @js_shadow_slot_set(i32 0, i64 %"), - "non-entry top-level pointer local should update its shadow slot" + init_ir.contains("call void @js_shadow_slot_bind(i32 0, ptr %"), + "non-entry top-level pointer local should bind its shadow slot" ); assert!( init_ir.contains("call void @js_shadow_frame_pop"), @@ -674,8 +706,8 @@ fn top_level_loop_body_shadow_slots_clear_each_iteration() { let main_ir = function_slice(&ir, "main"); let body_write = main_ir - .find("call void @js_shadow_slot_set(i32 0, i64 %") - .expect("loop-body pointer local should write its shadow slot"); + .find("call void @js_shadow_slot_bind(i32 0, ptr %") + .expect("loop-body pointer local should bind its shadow slot"); let body_clear = main_ir[body_write..] .find("call void @js_shadow_slot_set(i32 0, i64 0)") .map(|offset| body_write + offset) @@ -687,6 +719,41 @@ fn top_level_loop_body_shadow_slots_clear_each_iteration() { assert!(body_write < body_clear); assert!(body_clear < loop_backedge); + assert!( + !main_ir[body_write..body_clear].contains("call void @js_shadow_slot_set(i32 0, i64 %"), + "binding the initialized local already copies and barriers its value" + ); +} + +#[test] +fn immutable_index_alias_binds_once_but_keeps_incremental_root_barrier() { + let ir = String::from_utf8( + compile_module(&persistent_index_alias_shadow_module(), entry_opts()).unwrap(), + ) + .expect("LLVM IR should be UTF-8"); + let main_ir = function_slice(&ir, "main"); + + assert_eq!( + main_ir + .matches("call void @js_shadow_slot_bind(i32 1, ptr %") + .count(), + 1, + "loop-local index alias should bind its entry-hoisted alloca once" + ); + assert!( + !main_ir.contains("call void @js_shadow_slot_set(i32 1, i64 0)"), + "persistent index alias must not be cleared on each backedge" + ); + assert!( + main_ir.contains("call void @js_write_barrier_root_nanbox(i64 %"), + "pointer-capable alias updates must still shade a newly installed root" + ); + assert!( + main_ir.contains( + "load atomic i32, ptr @PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT seq_cst, align 4" + ) && main_ir.contains("shadow.root.barrier"), + "an inactive incremental collector should skip the TLS-backed root barrier call" + ); } #[test] @@ -722,9 +789,9 @@ fn reassigned_any_from_number_to_pointer_reserves_and_updates_shadow_slot() { .find("call i64 @js_array_alloc") .expect("pointer reassignment should allocate an array"); let slot_update = fn_ir[array_alloc..] - .find("call void @js_shadow_slot_set(i32 0, i64 %") + .find("call void @js_shadow_slot_bind(i32 0, ptr %") .map(|offset| array_alloc + offset) - .expect("pointer reassignment should update the reserved shadow slot"); + .expect("pointer reassignment should bind the reserved shadow slot"); assert!(array_alloc < slot_update); } @@ -744,8 +811,10 @@ fn mixed_any_writes_keep_alias_shadow_slots_precise() { ); for slot_idx in 0..3 { assert!( - fn_ir.contains(&format!("call void @js_shadow_slot_set(i32 {slot_idx}")), - "expected writes or clears for shadow slot {slot_idx}:\n{fn_ir}" + fn_ir.contains(&format!( + "call void @js_shadow_slot_bind(i32 {slot_idx}, ptr %" + )) || fn_ir.contains(&format!("call void @js_shadow_slot_set(i32 {slot_idx}")), + "expected binds or clears for shadow slot {slot_idx}:\n{fn_ir}" ); } } diff --git a/crates/perry-runtime/src/array/indexing.rs b/crates/perry-runtime/src/array/indexing.rs index 5c02712204..6f2887bdb1 100644 --- a/crates/perry-runtime/src/array/indexing.rs +++ b/crates/perry-runtime/src/array/indexing.rs @@ -2,7 +2,7 @@ use super::header::{array_numeric_layout, NumericArrayLayout}; use super::*; use std::ptr; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering}; const MAX_DENSE_ARRAY_GROW_LENGTH: u32 = 1_000_000; @@ -60,6 +60,18 @@ static ARRAY_PROTO_HAS_INDEX: AtomicBool = AtomicBool::new(false); static OBJECT_PROTO_ADDR: AtomicUsize = AtomicUsize::new(usize::MAX); static OBJECT_PROTO_HAS_INDEX: AtomicBool = AtomicBool::new(false); +/// Sticky summary of the process-wide conditions that invalidate codegen's +/// inline plain-array index guard. The generated guard loads this byte +/// directly; keeping the three rare prototype conditions behind one exported +/// byte avoids an out-of-line runtime call on every array read. +#[no_mangle] +pub static PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED: AtomicU8 = AtomicU8::new(0); + +#[inline] +pub(crate) fn invalidate_array_index_fast_path() { + PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED.store(1, Ordering::Relaxed); +} + pub(crate) fn object_prototype_addr() -> usize { let cached = OBJECT_PROTO_ADDR.load(Ordering::Relaxed); if cached != usize::MAX { @@ -95,6 +107,7 @@ pub(crate) fn note_object_prototype_index_write(obj: usize) { if !OBJECT_PROTO_HAS_INDEX.load(Ordering::Relaxed) && obj != 0 && obj == object_prototype_addr() { OBJECT_PROTO_HAS_INDEX.store(true, Ordering::Relaxed); + invalidate_array_index_fast_path(); } } @@ -169,6 +182,7 @@ pub(crate) fn array_prototype_addr() -> usize { pub(crate) fn note_array_index_write(arr: usize) { if !ARRAY_PROTO_HAS_INDEX.load(Ordering::Relaxed) && arr != 0 && arr == array_prototype_addr() { ARRAY_PROTO_HAS_INDEX.store(true, Ordering::Relaxed); + invalidate_array_index_fast_path(); } } diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index dfd573103b..705f61e95b 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -81,9 +81,10 @@ pub use self::immutable::{ pub(crate) use self::indexing::{ array_has_own_index, array_iteration_is_exotic, array_proto_iterator_modified, array_prototype_addr, array_prototype_has_index_flag, array_spec_get, array_spec_has_index, - keys_array_len_capped_to_capacity, note_array_proto_iterator_write, - note_object_prototype_index_write, object_prototype_addr, object_prototype_addr_matches, - object_prototype_has_index_flag, + invalidate_array_index_fast_path, keys_array_len_capped_to_capacity, + note_array_proto_iterator_write, note_object_prototype_index_write, object_prototype_addr, + object_prototype_addr_matches, object_prototype_has_index_flag, + PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED, }; pub use self::indexing::{ js_array_get_element, js_array_get_element_f64, js_array_get_f64, js_array_get_f64_unchecked, diff --git a/crates/perry-runtime/src/gc/barrier.rs b/crates/perry-runtime/src/gc/barrier.rs index 03d6744fe7..f8ecd6ea8b 100644 --- a/crates/perry-runtime/src/gc/barrier.rs +++ b/crates/perry-runtime/src/gc/barrier.rs @@ -766,17 +766,45 @@ thread_local! { pub(super) static GENERATED_WRITE_BARRIERS_EMITTED: AtomicUsize = AtomicUsize::new(0); +/// Number of threads whose incremental mark barrier is currently active. +/// +/// Generated code reads this before calling a root barrier from a persistent +/// shadow-slot update. Zero is authoritative: the current thread cannot have +/// an active incremental cycle, so a root store needs no shading and can skip +/// the Rust/TLS call entirely. A non-zero value is deliberately conservative: +/// another thread's cycle may be active while this thread's is not, in which +/// case the ordinary barrier call observes its null thread-local pointer and +/// returns. The count (rather than a bool) prevents one worker disabling its +/// cycle from hiding another worker's still-active cycle. +#[no_mangle] +pub static PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT: std::sync::atomic::AtomicU32 = + std::sync::atomic::AtomicU32::new(0); + pub(super) fn incremental_mark_barrier_enable(valid_ptrs: &ValidPointerSet, minor_only: bool) { INCREMENTAL_MARK_BARRIER_MINOR_ONLY.with(|cell| cell.set(minor_only)); - INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| { + let newly_active = INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| { + let newly_active = cell.get().is_null(); cell.set(valid_ptrs as *const ValidPointerSet); + newly_active }); + if newly_active { + PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT.fetch_add(1, Ordering::SeqCst); + } } pub(super) fn incremental_mark_barrier_disable() { - INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| { + let was_active = INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| { + let was_active = !cell.get().is_null(); cell.set(std::ptr::null()); + was_active }); + if was_active { + let _ = PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT.fetch_update( + Ordering::SeqCst, + Ordering::SeqCst, + |count| count.checked_sub(1), + ); + } INCREMENTAL_MARK_BARRIER_MINOR_ONLY.with(|cell| cell.set(false)); // Keep allocate-black aligned with the barrier (see enable). Sweep-phase // births need no mark either: both the arena cursor and the malloc sweep diff --git a/crates/perry-runtime/src/gc/tests/barrier.rs b/crates/perry-runtime/src/gc/tests/barrier.rs index 83daee0808..9e6facb649 100644 --- a/crates/perry-runtime/src/gc/tests/barrier.rs +++ b/crates/perry-runtime/src/gc/tests/barrier.rs @@ -63,6 +63,32 @@ fn remembered_maintenance_entry_count() -> usize { dirty_old + external_dirty + fallback } +#[test] +fn incremental_mark_barrier_active_count_tracks_thread_activation() { + let _guard = GcTestIsolationGuard::new(); + incremental_mark_barrier_disable(); + let before = PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT.load(Ordering::SeqCst); + let valid_ptrs = ValidPointerSet::new(); + + let active = IncrementalMarkBarrierTestGuard::new(&valid_ptrs); + assert_eq!( + PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT.load(Ordering::SeqCst), + before + 1 + ); + // Replacing the active cycle state on the same thread must not count the + // thread twice. + incremental_mark_barrier_enable(&valid_ptrs, true); + assert_eq!( + PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT.load(Ordering::SeqCst), + before + 1 + ); + drop(active); + assert_eq!( + PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT.load(Ordering::SeqCst), + before + ); +} + #[test] fn test_write_barrier_old_to_young_records() { let _guard = GcTestIsolationGuard::new(); diff --git a/crates/perry-runtime/src/gc/tests/dead_owner_side_tables.rs b/crates/perry-runtime/src/gc/tests/dead_owner_side_tables.rs index 24fcd58b46..e8f53d0a1a 100644 --- a/crates/perry-runtime/src/gc/tests/dead_owner_side_tables.rs +++ b/crates/perry-runtime/src/gc/tests/dead_owner_side_tables.rs @@ -637,6 +637,10 @@ fn test_object_meta_prototype_survives_copied_minor_move() { Some(ptr_bits(old_proto)), "test premise: the meta-resident prototype reads back before the GC" ); + assert!( + crate::object::prototype_chain::object_has_prototype_override(old_owner), + "test premise: the per-instance override bit lives in the meta record" + ); js_shadow_slot_set(0, ptr_bits(old_owner)); js_shadow_slot_set(1, ptr_bits(old_proto)); @@ -653,6 +657,10 @@ fn test_object_meta_prototype_survives_copied_minor_move() { new_proto, "the meta record's prototype slot must be rewritten to the moved proto" ); + assert!( + crate::object::prototype_chain::object_has_prototype_override(new_owner), + "the non-pointer meta flags must travel with the copied record" + ); js_shadow_slot_set(0, 0); js_shadow_slot_set(1, 0); diff --git a/crates/perry-runtime/src/gc/types.rs b/crates/perry-runtime/src/gc/types.rs index 50b94ea261..f48fd2d837 100644 --- a/crates/perry-runtime/src/gc/types.rs +++ b/crates/perry-runtime/src/gc/types.rs @@ -942,13 +942,6 @@ pub const OBJ_FLAG_ARRAY_DESCRIPTORS: u16 = 0x400; // `GC_TYPE_OBJECT`. Set-only (clearing a descriptor leaves it set; the slow // path is always correct). pub const OBJ_FLAG_HAS_DESCRIPTORS: u16 = 0x800; -// This specific object's [[Prototype]] was overridden per-instance -// (`Object.setPrototypeOf` / `__proto__` recording via -// `object_set_static_prototype`). Class-keyed interception caches -// (`object::prop_plan`) must not apply a class-chain verdict to an object -// whose own chain diverges. Bit 12; only meaningful for `GC_TYPE_OBJECT`. -// Set-only, travels with the object across evacuation. -pub const OBJ_FLAG_PROTO_OVERRIDE: u16 = 0x1000; // #2145: this object is a per-kind `.prototype` whose // `[[Prototype]]` is the shared `%TypedArray%.prototype` intrinsic. // `Object.getPrototypeOf(Int8Array.prototype)` returns the cached diff --git a/crates/perry-runtime/src/object/class_registry/prototype_objects.rs b/crates/perry-runtime/src/object/class_registry/prototype_objects.rs index 14460033f3..6f77c345e3 100644 --- a/crates/perry-runtime/src/object/class_registry/prototype_objects.rs +++ b/crates/perry-runtime/src/object/class_registry/prototype_objects.rs @@ -314,7 +314,7 @@ unsafe fn inherited_proto_accessor_value( key: *const crate::StringHeader, receiver: f64, ) -> Option { - if key.is_null() || !crate::state::state().descriptors.accessors_in_use.get() { + if key.is_null() || !crate::object::object_has_descriptors(proto_obj as usize) { return None; } let key_ptr = (key as *const u8).add(std::mem::size_of::()); diff --git a/crates/perry-runtime/src/object/collection_proto_thunks.rs b/crates/perry-runtime/src/object/collection_proto_thunks.rs index 62a3eff3ff..7e1761f715 100644 --- a/crates/perry-runtime/src/object/collection_proto_thunks.rs +++ b/crates/perry-runtime/src/object/collection_proto_thunks.rs @@ -234,17 +234,20 @@ fn install_collection_size_getter(proto_obj: *mut ObjectHeader, name: &str, func super::native_module::set_builtin_closure_length(closure as usize, 0); let key = crate::string::js_string_from_bytes(name.as_ptr(), name.len() as u32); super::object_ops::ensure_key_in_keys_array(proto_obj, key); - super::set_accessor_descriptor( + // #6809: gate-neutral BUILTIN install. `.size` reads on live Map/Set + // receivers are gc-type-routed inside the generic getter (they never + // dispatch through the gated accessor tables), so this entry exists + // for reflection only — and this installer runs inside + // `populate_global_this_builtins`, where the user-install variant + // flipped the process-wide descriptor gates at startup for every + // program touching a builtin global. + super::set_builtin_accessor_descriptor( proto_obj as usize, name.to_string(), super::AccessorDescriptor { get: crate::value::js_nanbox_pointer(closure as i64).to_bits(), set: 0, }, - ); - super::set_property_attrs( - proto_obj as usize, - name.to_string(), super::PropertyAttrs::new(true, false, true), ); super::set_builtin_property_attrs( diff --git a/crates/perry-runtime/src/object/descriptor_state.rs b/crates/perry-runtime/src/object/descriptor_state.rs index 2a3d512d12..0ea529e559 100644 --- a/crates/perry-runtime/src/object/descriptor_state.rs +++ b/crates/perry-runtime/src/object/descriptor_state.rs @@ -865,8 +865,9 @@ pub(crate) fn clear_accessor_descriptor(obj: usize, key: &str) { /// /// `Object.getOwnPropertyDescriptor` reads `ACCESSOR_DESCRIPTORS` and /// `PROPERTY_DESCRIPTORS` *unconditionally*, so the descriptor is fully -/// reflectable — but the hot object get/set paths (which only consult the -/// side tables once a gate has flipped) keep skipping the HashMap lookup. +/// reflectable. The owning object's `OBJ_FLAG_HAS_DESCRIPTORS` bit lets direct +/// reads/writes consult the side tables without flipping a process-wide gate; +/// unrelated objects keep skipping the HashMap lookup. /// This matters because built-in prototype accessors such as /// `%TypedArray%.prototype.length` are installed lazily at globalThis /// init for *every* program that merely touches a builtin global; flipping @@ -880,6 +881,7 @@ pub(crate) fn set_builtin_accessor_descriptor( attrs: PropertyAttrs, ) { super::prop_plan::prop_plan_epoch_bump(); + note_descriptor_target(obj); note_accessor_descriptor_key(&key); // #6759 Phase C2: the meta summary must over-approximate the tables // even for gate-neutral builtin installs — the (unconditionally diff --git a/crates/perry-runtime/src/object/disposable_proto_thunks.rs b/crates/perry-runtime/src/object/disposable_proto_thunks.rs index c83b771b1c..14b6869976 100644 --- a/crates/perry-runtime/src/object/disposable_proto_thunks.rs +++ b/crates/perry-runtime/src/object/disposable_proto_thunks.rs @@ -155,21 +155,11 @@ fn install_disposed_getter(proto_obj: *mut ObjectHeader, func_ptr: *const u8) { super::native_module::set_bound_native_closure_name(closure, "get disposed"); super::native_module::set_builtin_closure_length(closure as usize, 0); let getter_bits = crate::value::js_nanbox_pointer(closure as i64).to_bits(); - let key = crate::string::js_string_from_bytes(b"disposed".as_ptr(), 8); - super::object_ops::ensure_key_in_keys_array(proto_obj, key); - super::set_accessor_descriptor( - proto_obj as usize, - "disposed".to_string(), - super::AccessorDescriptor { - get: getter_bits, - set: 0, - }, - ); - super::set_property_attrs( - proto_obj as usize, - "disposed".to_string(), - super::PropertyAttrs::new(true, false, true), - ); + // #6809: reads have a dedicated native-instance route, while direct + // prototype reads use the per-owner descriptor flag. Keep startup + // gate-neutral so this builtin accessor does not poison every dynamic + // object write in the process. + super::object_ops::install_builtin_getter(proto_obj, "disposed", getter_bits); } } diff --git a/crates/perry-runtime/src/object/field_get_set/accessors.rs b/crates/perry-runtime/src/object/field_get_set/accessors.rs index df5d159f5f..3e78eabce5 100644 --- a/crates/perry-runtime/src/object/field_get_set/accessors.rs +++ b/crates/perry-runtime/src/object/field_get_set/accessors.rs @@ -279,79 +279,20 @@ pub(crate) unsafe fn invoke_accessor_setter(set_bits: u64, receiver: f64, value: super::super::js_implicit_this_set(prev); } -/// #4140: builtin *reflection-only* accessors — most prominently the four -/// `%TypedArray%.prototype` getters (`length`/`byteLength`/`byteOffset`/ -/// `buffer`) — are installed via [`super::super::set_builtin_accessor_descriptor`], -/// which deliberately does NOT flip the `ACCESSORS_IN_USE` hot-path gate (these -/// getters are never written and exist purely so reflection sees them, see -/// #2060). The downside: a plain *value* read that resolves to the hosting -/// prototype object (e.g. `Uint8Array.prototype.buffer`, where the per-kind -/// proto inherits from the shared `%TypedArray%.prototype`) skips the gated -/// accessor short-circuit and returns the empty backing slot — `undefined` -/// instead of Node's `TypeError`. -/// -/// Invoke the real getter here for the one builtin object that hosts these -/// getters, guarded by a cheap pointer compare so ordinary reads pay nothing. -/// The receiver is the intrinsic prototype itself, which is never a concrete -/// typed array (real `TypedArray` instances short-circuit far earlier in -/// `js_object_get_field_by_name`), so the getter always throws the spec -/// `TypeError` — matching `Uint8Array.prototype.buffer` in Node. When the gate -/// IS on, the inline short-circuit below already handles this, so bail. +/// Invoke an accessor owned by a descriptor-marked object before its empty +/// backing slot is read. Gate-neutral builtin installs deliberately leave the +/// process-wide `ACCESSORS_IN_USE` flag clear, but stamp their owner with +/// `OBJ_FLAG_HAS_DESCRIPTORS`; the caller checks that bit before entering this +/// helper, so ordinary object reads pay only the already-loaded header-bit +/// test. This also makes direct reads of builtin prototype accessors preserve +/// their real behavior (`Set.prototype.size` throws, `RegExp.prototype.source` +/// returns `"(?:)"`, and so on) once startup is descriptor-gate-free. pub(crate) unsafe fn builtin_reflection_accessor_read( obj: *const ObjectHeader, key_bytes: &[u8], ) -> Option { - // Only the four `%TypedArray%.prototype` accessor names — the cheap key - // filter keeps this off every other property read entirely. - if !matches!( - key_bytes, - b"buffer" | b"byteLength" | b"byteOffset" | b"length" - ) { - return None; - } - // This helper runs before the heavy object validation further down, so a - // caller that passes a NaN-boxed number / raw `f64` as `obj` (e.g. the - // dynamic `arr.length = …` set path threading a numeric value through the - // generic getter) must not be dereferenced. A genuine heap pointer has its - // top 16 bits clear; reject anything else and confirm it points at a real - // GC object before reading its header below. - if (obj as u64) >> 48 != 0 || !super::super::is_valid_obj_ptr(obj as *const u8) { - return None; - } - let intrinsic_proto = - super::super::TYPED_ARRAY_INTRINSIC_PROTO_PTR.load(std::sync::atomic::Ordering::Relaxed); - if intrinsic_proto == 0 { - return None; - } - // Fire for the shared `%TypedArray%.prototype` intrinsic itself and for - // every per-kind prototype (`Uint8Array.prototype`, …). The per-kind protos - // carry `OBJ_FLAG_TYPED_ARRAY_PROTO` and resolve their `[[Prototype]]` to - // the intrinsic only through `Object.getPrototypeOf`'s flag check — they - // have `class_id == 0` and no recorded static-prototype link, so the normal - // chain walk in this function never reaches the intrinsic where these - // accessors live, and the read silently returned the empty slot - // (`undefined`) instead of Node's `TypeError`. None of these objects is a - // concrete typed array (real instances short-circuit far earlier via the - // `TYPED_ARRAY_REGISTRY` arm), so invoking the getter with the proto as the - // receiver always throws — matching `Uint8Array.prototype.buffer` in Node. - // #4140. - let is_intrinsic = obj as i64 == intrinsic_proto; - // `OBJ_FLAG_TYPED_ARRAY_PROTO` lives in the shared `_reserved` word, whose - // bits mean different things for `GC_TYPE_ARRAY` (raw-f64 layout, arguments, - // survival age, …). The per-kind typed-array prototypes are always plain - // `GC_TYPE_OBJECT`s, so gate the flag read on the object type — otherwise a - // regular array whose `_reserved` happens to have bit 0x100 set would be - // misread as a typed-array prototype and its `.length` get would crash. - let gc = (obj as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; - let is_perkind_proto = (*gc).obj_type == crate::gc::GC_TYPE_OBJECT - && ((*gc)._reserved & crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO) != 0; - if !is_intrinsic && !is_perkind_proto { - return None; - } - // The accessor descriptors live on the intrinsic prototype, not the per-kind - // protos, so always resolve the getter off the intrinsic. let name = std::str::from_utf8(key_bytes).ok()?; - let acc = get_accessor_descriptor(intrinsic_proto as usize, name)?; + let acc = get_accessor_descriptor(obj as usize, name)?; if acc.get == 0 { return Some(JSValue::undefined()); } @@ -365,7 +306,7 @@ pub(crate) unsafe fn builtin_reflection_accessor_read( /// typed arrays, so a method invoked directly on them (e.g. /// `Int8Array.prototype.entries()`) must fail `ValidateTypedArray` and throw a /// `TypeError`. Mirrors the per-kind/intrinsic detection in -/// `builtin_reflection_accessor_read`. +/// the builtin-accessor read path. pub(crate) unsafe fn is_typed_array_prototype(addr: usize) -> bool { if addr == 0 || (addr as u64) >> 48 != 0 || !super::super::is_valid_obj_ptr(addr as *const u8) { return false; diff --git a/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs b/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs index 27940a979f..ba450e5113 100644 --- a/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs +++ b/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs @@ -1197,6 +1197,26 @@ pub(crate) fn get_field_by_name_object_tail( } } + // `DisposableStack` and `AsyncDisposableStack` are reserved native + // class ids rather than registered JS classes, so their instance + // getter cannot be found through the class-prototype registry. Route + // the one native data accessor directly. An own property installed + // with `defineProperty` still shadows the inherited getter. + if !key.is_null() + && ((*obj).class_id == crate::disposable::CLASS_ID_DISPOSABLE_STACK + || (*obj).class_id == crate::disposable::CLASS_ID_ASYNC_DISPOSABLE_STACK) + { + let key_ptr = (key as *const u8).add(std::mem::size_of::()); + let key_len = (*key).byte_len as usize; + let key_bytes = std::slice::from_raw_parts(key_ptr, key_len); + if key_bytes == b"disposed" && !own_key_present(obj as *mut ObjectHeader, key) { + return JSValue::from_bits( + crate::disposable::js_disposable_stack_disposed(obj as *mut ObjectHeader) + .to_bits(), + ); + } + } + // #1387: `PerformanceEntry#toJSON` is a synthesized (non-enumerable) // method — entry objects are plain shaped objects with no stored // `toJSON` field, so a `entry.toJSON` read (e.g. `typeof entry.toJSON`) @@ -1508,13 +1528,13 @@ pub(crate) fn get_field_by_name_object_tail( (key as *const u8).add(std::mem::size_of::()), (*key).byte_len as usize, ); - // #4140: builtin reflection-only accessors (e.g. the - // `%TypedArray%.prototype` getters) don't flip `ACCESSORS_IN_USE`, so the - // gated short-circuits below skip them on a plain value read. Handle the - // hosting prototype object here — a cheap pointer compare for everything - // else — before the slot scan returns the empty backing field. - if let Some(v) = builtin_reflection_accessor_read(obj, key_bytes) { - return v; + // Gate-neutral builtin accessors mark only their owning object. Consult + // the descriptor table before an accessor's empty backing slot is read; + // unrelated objects pay only this already-loaded header-bit test. + if (*gc_header)._reserved & crate::gc::OBJ_FLAG_HAS_DESCRIPTORS != 0 { + if let Some(v) = builtin_reflection_accessor_read(obj, key_bytes) { + return v; + } } let key_hash = { let mut h: u32 = 0x811c9dc5; diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index ac5bb135dc..dec68ccd0c 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -411,8 +411,13 @@ pub extern "C" fn js_object_get_field_ic_miss( (obj as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; (*gc_header).obj_type == crate::gc::GC_TYPE_OBJECT }; + let has_own_descriptors = is_object && super::super::object_has_descriptors(obj as usize); let is_regular = is_object && (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR; - if can_cache && is_regular { + // Gate-neutral builtin accessors deliberately leave the process-wide + // accessor latch clear. Their owner bit must still block this PIC: + // its generated hit path is a raw slot load and would otherwise turn + // `Set.prototype.size` into `undefined` instead of invoking the getter. + if can_cache && is_regular && !has_own_descriptors { let keys = (*obj).keys_array; if keys.is_null() || (keys as usize) <= 0x10000 { let value = js_object_get_field_by_name(obj, key); diff --git a/crates/perry-runtime/src/object/field_set_by_name.rs b/crates/perry-runtime/src/object/field_set_by_name.rs index 3ffc5cfc98..060712498d 100644 --- a/crates/perry-runtime/src/object/field_set_by_name.rs +++ b/crates/perry-runtime/src/object/field_set_by_name.rs @@ -6,6 +6,113 @@ use super::*; +/// Non-allocating-in-the-GC-heap overwrite for an existing own data field. +/// +/// This is the common assignment case for ordinary objects. It is deliberately +/// conservative: anything with per-object semantics (descriptors, URL backing +/// state, a changed prototype, frozen-family flags, or a special object class) +/// falls through to the complete `[[Set]]` implementation. +/// +/// The key must already be the canonical interned heap string emitted by +/// codegen. No arena allocation occurs here, so callers may use this before +/// opening a `RuntimeHandleScope`. +#[inline] +pub(crate) unsafe fn try_existing_own_data_overwrite( + obj: *mut ObjectHeader, + key: *const crate::StringHeader, + value: f64, +) -> bool { + let obj_addr = obj as usize; + let key_addr = key as usize; + if obj.is_null() || key.is_null() { + return false; + } + + let Some(obj_gc) = crate::value::addr_class::try_read_gc_header(obj_addr) else { + return false; + }; + const BLOCKING_FLAGS: u16 = crate::gc::OBJ_FLAG_FROZEN + | crate::gc::OBJ_FLAG_SEALED + | crate::gc::OBJ_FLAG_NO_EXTEND + | crate::gc::OBJ_FLAG_HAS_DESCRIPTORS + | crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO; + if obj_gc.obj_type != crate::gc::GC_TYPE_OBJECT + || obj_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || obj_gc._reserved & BLOCKING_FLAGS != 0 + || (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR + || (*obj).class_id == NATIVE_MODULE_CLASS_ID + || crate::array::object_prototype_addr_matches(obj_addr) + // URL's visible fields are live views over one backing URL. An own + // slot exists for e.g. `pathname`, but its setter must also rebuild + // `href`/`origin`; do not mistake that slot for ordinary data. + || ((*obj).class_id == 0 && crate::url::is_url_object_shape(obj)) + { + return false; + } + + let Some(key_gc) = crate::value::addr_class::try_read_gc_header(key_addr) else { + return false; + }; + if key_gc.obj_type != crate::gc::GC_TYPE_STRING + || key_gc.gc_flags & (crate::gc::GC_FLAG_FORWARDED | crate::gc::GC_FLAG_INTERNED) + != crate::gc::GC_FLAG_INTERNED + { + return false; + } + + let keys = (*obj).keys_array; + let keys_addr = keys as usize; + if keys.is_null() || (keys_addr as u64) >> 48 != 0 { + return false; + } + let Some(keys_gc) = crate::value::addr_class::try_read_gc_header(keys_addr) else { + return false; + }; + if keys_gc.obj_type != crate::gc::GC_TYPE_ARRAY + || keys_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + { + return false; + } + + let mut own_idx = super::prop_plan::read_plan_lookup(keys_addr, key_addr); + if own_idx.is_none() { + let key_count = crate::array::keys_array_len_capped_to_capacity(keys); + if key_count > 4096 { + return false; + } + for i in 0..key_count { + let kv = crate::array::js_array_get(keys, i as u32); + if crate::string::js_string_key_matches(kv, key) { + super::prop_plan::read_plan_record(keys_addr, key_addr, i as u32); + own_idx = Some(i as u32); + break; + } + } + } + let Some(idx) = own_idx else { + return false; + }; + + let vbits = value.to_bits(); + let vbits = if (vbits >> 48) == 0x7FFD && (vbits & 0x0000_FFFF_FFFF_FFFF) == 0 { + crate::value::TAG_UNDEFINED + } else { + vbits + }; + super::mark_object_dynamic_shape_unknown(obj); + let alloc_limit = + std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize; + if (idx as usize) < alloc_limit { + store_object_field_slot(obj, idx as usize, vbits); + if idx >= (*obj).field_count { + (*obj).field_count = idx + 1; + } + } else { + overflow_set(obj_addr, idx as usize, vbits); + } + true +} + /// Fast transition-cache-backed dynamic property write. /// /// This is intentionally narrower than `js_object_set_field_by_name`: it only @@ -52,6 +159,10 @@ pub extern "C" fn js_object_set_field_by_name_transition_fast( return 0; } + if unsafe { try_existing_own_data_overwrite(obj, key, value) } { + return 1; + } + let scope = crate::gc::RuntimeHandleScope::new(); let obj_handle = scope.root_raw_mut_ptr(obj); let key_handle = scope.root_string_ptr(key); @@ -80,12 +191,26 @@ pub extern "C" fn js_object_set_field_by_name_transition_fast( | crate::gc::OBJ_FLAG_NO_EXTEND // #6084 item 6: an own descriptor on THIS object (accessor or // non-writable) must route through the full setter semantics. - | crate::gc::OBJ_FLAG_HAS_DESCRIPTORS) + | crate::gc::OBJ_FLAG_HAS_DESCRIPTORS + | crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO) != 0 { return 0; } - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR || (*obj).class_id != 0 { + if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR + || (*obj).class_id == NATIVE_MODULE_CLASS_ID + { + return 0; + } + + let key_gc = + (key as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + + // The append-transition half below is intentionally restricted to + // class-id-zero plain objects. Existing own-data overwrites were + // already handled by `try_existing_own_data_overwrite` before the + // rooting scope. + if (*obj).class_id != 0 { return 0; } @@ -101,8 +226,6 @@ pub extern "C" fn js_object_set_field_by_name_transition_fast( return 0; } - let key_gc = - (key as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; if (*key_gc).obj_type != crate::gc::GC_TYPE_STRING { return 0; } @@ -299,9 +422,6 @@ pub extern "C" fn js_object_set_field_by_name( key: *const crate::StringHeader, value: f64, ) { - // #6759 A: one state fetch for the whole write path — the descriptor - // gates below all reuse it. - let st = crate::state::state(); // #5135: the receiver may be a Proxy id arriving with its NaN-box tag // already masked off (the `obj.prop++` / `PropertyUpdate` codegen path // hands us the bare pointer band, not the full POINTER_TAG value). A Proxy @@ -369,17 +489,19 @@ pub extern "C" fn js_object_set_field_by_name( { let gc_hdr = (raw as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + let o = raw as *mut ObjectHeader; + if try_existing_own_data_overwrite(o, key, value) { + return; + } const LANE_BLOCKING: u16 = crate::gc::OBJ_FLAG_FROZEN | crate::gc::OBJ_FLAG_SEALED | crate::gc::OBJ_FLAG_NO_EXTEND | crate::gc::OBJ_FLAG_HAS_DESCRIPTORS - | crate::gc::OBJ_FLAG_PROTO_OVERRIDE | crate::gc::OBJ_FLAG_NULL_PROTO | crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO; if (*gc_hdr).obj_type == crate::gc::GC_TYPE_OBJECT && (*gc_hdr)._reserved & LANE_BLOCKING == 0 { - let o = raw as *mut ObjectHeader; let class_id = (*o).class_id; // #6595: a per-evaluation CLASS OBJECT must never take // this lane — it skips the #6530 @@ -390,6 +512,7 @@ pub extern "C" fn js_object_set_field_by_name( if (*o).object_type == crate::error::OBJECT_TYPE_REGULAR && class_id != 0 && class_id != NATIVE_MODULE_CLASS_ID + && !super::prototype_chain::object_has_prototype_override(raw) && super::prop_plan::store_plan_check(class_id, key as usize) { let keys = (*o).keys_array; @@ -397,75 +520,6 @@ pub extern "C" fn js_object_set_field_by_name( || (((keys as u64) >> 48) == 0 && crate::value::addr_class::is_above_handle_band(keys as usize)); if keys_ok { - // Overwrite of an EXISTING own key: the keys array - // doesn't change, so the shape-transition cache - // (which stores append EDGES) can never serve it — - // the (keys, key) → index read-plan cache can. - // Miss → one bounded scan populates it; absent own - // key falls to the append-edge lookup below. - if !keys.is_null() { - let mut own_idx = - super::prop_plan::read_plan_lookup(keys as usize, key as usize); - if own_idx.is_none() { - let keys_gc = (keys as *const u8).sub(crate::gc::GC_HEADER_SIZE) - as *const crate::gc::GcHeader; - if (*keys_gc).obj_type == crate::gc::GC_TYPE_ARRAY { - let key_count = - crate::array::keys_array_len_capped_to_capacity(keys); - if key_count <= 4096 { - for i in 0..key_count { - let kv = crate::array::js_array_get(keys, i as u32); - if crate::string::js_string_key_matches(kv, key) { - super::prop_plan::read_plan_record( - keys as usize, - key as usize, - i as u32, - ); - own_idx = Some(i as u32); - break; - } - } - } - } - } - if let Some(idx) = own_idx { - let vbits = value.to_bits(); - let vbits = if (vbits >> 48) == 0x7FFD - && (vbits & 0x0000_FFFF_FFFF_FFFF) == 0 - { - crate::value::TAG_UNDEFINED - } else { - vbits - }; - // Layout safety (#6495 family): the slot's - // pointer-ness may change — degrade the - // layout to full-visit before the store. - super::mark_object_dynamic_shape_unknown(o); - let alloc_limit = std::cmp::max( - (*o).field_count, - crate::object::INLINE_SLOT_FLOOR as u32, - ) - as usize; - if (idx as usize) < alloc_limit { - let fields_ptr = (o as *mut u8) - .add(std::mem::size_of::()) - as *mut JSValue; - let slot = fields_ptr.add(idx as usize); - crate::gc::runtime_store_jsvalue_slot( - o as usize, - slot as usize, - idx as usize, - vbits, - ); - if idx >= (*o).field_count { - (*o).field_count = idx + 1; - } - } else { - overflow_set(o as usize, idx as usize, vbits); - } - return; - } - } if let Some((next_keys, slot_idx)) = transition_cache_lookup(keys as usize, key) { @@ -512,65 +566,13 @@ pub extern "C" fn js_object_set_field_by_name( } } } - // A Buffer is an ordinary object in Node (a Uint8Array), so `buf.foo = v` - // stores an own property — and an own key SHADOWS the same-named prototype - // method. Perry keeps buffers outside the object model (raw BufferHeader, - // no GcHeader), so this write used to be dropped entirely. mysql2's - // `MockBuffer` packet sizer depends on it: it replaces the write methods of - // a zero-length Buffer with a no-op, serializes once to measure, then - // allocates for real. Store into the GC-traced buffer own-prop table (the - // read side and the method-call dispatch both consult it). - if !key.is_null() && crate::buffer::is_registered_buffer(obj as usize) { - unsafe { - let key_ptr = (key as *const u8).add(std::mem::size_of::()); - let key_len = (*key).byte_len as usize; - if let Ok(name) = std::str::from_utf8(std::slice::from_raw_parts(key_ptr, key_len)) { - // Numeric keys are element writes (`buf[0] = 1`) — leave those - // to the index path; only NAMED props become expandos. - if name.parse::().is_err() { - crate::buffer::buffer_set_own_prop(obj as usize, name, value); - return; - } - } - } - } - // #5437: a live Web Stream handle arrives here as its raw id in the - // stream band (the `stream.prop = v` codegen path). React's - // `renderToReadableStream` attaches its shell-ready promise as an - // expando (`stream.allReady = ...`); without a store the write was - // dropped, which stalled the Next.js dynamic-SSR render. Route to the - // stdlib per-stream expando table (GC-traced there). - { - let addr = obj as usize; - if crate::value::addr_class::is_stream_id_band(addr) { - if !key.is_null() { - if let (Some(probe), Some(setter)) = ( - crate::object::stream_handle_probe(), - crate::object::stream_expando_set(), - ) { - if unsafe { probe(addr) } { - if let Some(name) = - unsafe { super::has_own_helpers::str_from_string_header(key) } - { - unsafe { setter(addr, name.as_ptr(), name.len(), value) }; - } - } - } - } - // A stream-band address is a reserved handle id, never a real - // `ObjectHeader`. Stop unconditionally — even when the expando - // write was a no-op (dead/unregistered handle, hooks absent, or a - // non-UTF-8 key). Falling through would reach the ObjectHeader - // path below and deref `addr - GC_HEADER_SIZE` (unmapped) → crash. - // Mirrors the reserved small-handle early-return further down. - return; - } - } // `Object.prototype["2"] = v` (stringified-index write) makes the index // visible through array hole/OOB reads. Cheap gate: one relaxed flag // load, then an address compare against the cached canonical // Object.prototype; the digit scan only runs on a match (test262 - // concat/S15.4.4.4_A3_T3). + // concat/S15.4.4.4_A3_T3). Hoisted above the exotic gauntlet (#6809): + // the canonical prototype IS a genuine ObjectHeader, so it must run + // even when the gauntlet below is skipped. { let raw = (obj as u64 & 0x0000_FFFF_FFFF_FFFF) as usize; if crate::array::object_prototype_addr_matches(raw) && !key.is_null() { @@ -581,199 +583,286 @@ pub extern "C" fn js_object_set_field_by_name( } } } - // A `Temporal.*` value is an opaque, immutable NaN-boxed cell that is NOT - // an `ObjectHeader` — writing an arbitrary property (e.g. test262's - // `instance.constructor = …` subclassing probes) must NOT interpret the - // cell as an `ObjectHeader` and corrupt its boxed payload (which segfaults - // on the next deref). The cell's `temporal_rs` slots are immutable, but a - // user-defined *expando* property is legal and lives in the exotic side - // table (like Date/RegExp). `obj` still carries its NaN-box tag here - // (`0x7FFD…` for a real cell), so route through `exotic_expando_kind_of_value`, - // which checks the tag before masking to the cleaned heap address. - if let Some((addr, kind @ super::exotic_expando::ExoticKind::Temporal)) = - super::exotic_expando::exotic_expando_kind_of_value(f64::from_bits(obj as u64)) - { - if !key.is_null() { - unsafe { - let name_ptr = (key as *const u8).add(std::mem::size_of::()); - let name_len = (*key).byte_len as usize; - let name = String::from_utf8_lossy(std::slice::from_raw_parts(name_ptr, name_len)) - .into_owned(); - let receiver = f64::from_bits(obj as u64); - let _ = - super::exotic_expando::exotic_set_property(addr, kind, &name, value, receiver); - } - } - return; - } - if let Some(addr) = - crate::typedarray_props::typed_array_addr_from_value(f64::from_bits(obj as u64)) - { - unsafe { - crate::typedarray_props::typed_array_set_own_property( - addr as *mut crate::typedarray::TypedArrayHeader, - key, - value, - ); - } - return; - } - - // Issue #618-followup: detect INT32-tagged class ref (top16 == 0x7FFE). - // Drizzle's `((SQL2) => { SQL2.Aliased = Aliased; })(SQL)` pattern sets - // a static property on an imported class — Perry stores classes as - // INT32-tagged class ids, so the receiver here is e.g. 0x7FFE_0000_0000_002A - // not a real ObjectHeader. Route to the CLASS_DYNAMIC_PROPS side-table - // so a later `SQL.Aliased` read can find it. - { + // #6809: header-first receiver classification. A receiver whose GC + // header identifies a genuine `ObjectHeader` (GC_TYPE_OBJECT, not a + // RegExpHeader) can never be a Buffer, Web-Stream handle, Temporal + // cell, typed array, INT32 class ref, primitive, or Date/RegExp — those + // are different header types or non-heap encodings — so the whole + // exotic-receiver gauntlet below is skipped in one header read. The + // write profile (#6759 acceptance) showed the gauntlet's address-keyed + // registry probes (each with its own TLS fetch, several behind locks) + // dominating hot stores. Skipping by HEADER also closes the stale- + // registry misroute where a dead exotic's recycled address, re-tenanted + // by a plain object, could hijack the write. RegExp receivers fail + // `meta_capable_object` (header magic) and keep taking the gauntlet. + let receiver_is_object_header = { let bits = obj as u64; - if (bits >> 48) == 0x7FFE && !key.is_null() { - let class_id = (bits & 0xFFFF_FFFF) as u32; + let cleaned = if bits >> 48 == 0x7FFD { + (bits & crate::value::POINTER_MASK) as usize + } else if bits >> 48 == 0 { + bits as usize + } else { + 0 + }; + cleaned != 0 && unsafe { super::prototype_chain::meta_capable_object(cleaned).is_some() } + }; + 'exotic_gauntlet: { + if receiver_is_object_header { + break 'exotic_gauntlet; + } + // A Buffer is an ordinary object in Node (a Uint8Array), so `buf.foo = v` + // stores an own property — and an own key SHADOWS the same-named prototype + // method. Perry keeps buffers outside the object model (raw BufferHeader, + // no GcHeader), so this write used to be dropped entirely. mysql2's + // `MockBuffer` packet sizer depends on it: it replaces the write methods of + // a zero-length Buffer with a no-op, serializes once to measure, then + // allocates for real. Store into the GC-traced buffer own-prop table (the + // read side and the method-call dispatch both consult it). + if !key.is_null() && crate::buffer::is_registered_buffer(obj as usize) { unsafe { - let name_ptr = (key as *const u8).add(std::mem::size_of::()); - let name_len = (*key).byte_len as usize; - let name = std::str::from_utf8(std::slice::from_raw_parts(name_ptr, name_len)) - .unwrap_or("") - .to_string(); - // Empty-string is a legal accessor key (`set ''(v)`); the - // `!name.is_empty()` guard below skips it, so dispatch a - // prototype-ref instance setter / constructor-ref static setter - // named "" here (Test262 accessor-name-* literal-string-empty). - if name.is_empty() { - let recv = f64::from_bits(bits); - if super::class_prototype_ref_id(recv).is_some() - && super::class_registry::class_instance_setter_apply( - class_id, &name, recv, value, - ) - { + let key_ptr = (key as *const u8).add(std::mem::size_of::()); + let key_len = (*key).byte_len as usize; + if let Ok(name) = std::str::from_utf8(std::slice::from_raw_parts(key_ptr, key_len)) + { + // Numeric keys are element writes (`buf[0] = 1`) — leave those + // to the index path; only NAMED props become expandos. + if name.parse::().is_err() { + crate::buffer::buffer_set_own_prop(obj as usize, name, value); return; } - if super::class_registry::class_static_accessor_setter_apply( - class_id, &name, recv, value, + } + } + } + // #5437: a live Web Stream handle arrives here as its raw id in the + // stream band (the `stream.prop = v` codegen path). React's + // `renderToReadableStream` attaches its shell-ready promise as an + // expando (`stream.allReady = ...`); without a store the write was + // dropped, which stalled the Next.js dynamic-SSR render. Route to the + // stdlib per-stream expando table (GC-traced there). + { + let addr = obj as usize; + if crate::value::addr_class::is_stream_id_band(addr) { + if !key.is_null() { + if let (Some(probe), Some(setter)) = ( + crate::object::stream_handle_probe(), + crate::object::stream_expando_set(), ) { - return; + if unsafe { probe(addr) } { + if let Some(name) = + unsafe { super::has_own_helpers::str_from_string_header(key) } + { + unsafe { setter(addr, name.as_ptr(), name.len(), value) }; + } + } } } - if !name.is_empty() { - if name == "name" - && !super::class_registry::class_is_key_deleted(class_id, &name) - && super::class_registry::lookup_static_method_in_chain(class_id, &name) - .is_none() - { - return; - } - let has_own_data = CLASS_DYNAMIC_PROPS.with(|m| { - m.borrow() - .get(&class_id) - .is_some_and(|props| props.contains_key(&name)) - }); - // `C.prototype[key] = v` where `key` is an instance - // `set key(v)` accessor defined on the prototype: invoke the - // setter with `this` = the prototype ref. The prototype ref - // and the constructor ref are both INT32-tagged class refs; - // distinguish via `class_prototype_ref_id`. Instance setters - // live in the vtable; static accessors (below) live in the - // constructor ref's table (Test262 accessor-name-inst). - if !has_own_data - && super::class_prototype_ref_id(f64::from_bits(bits)).is_some() - && super::class_registry::class_instance_setter_apply( - class_id, - &name, - f64::from_bits(bits), - value, - ) - { - return; - } - if !has_own_data - && super::class_registry::class_static_accessor_setter_apply( - class_id, - &name, - f64::from_bits(bits), - value, - ) - { - return; - } - // Writing `.caller` / `.arguments` on a class constructor - // hits the poison-pill %ThrowTypeError% accessor (which has - // no [[Set]]) on `Function.prototype`, so a strict-mode - // assignment throws. Mirrors the read side in - // get_field_by_name and the ordinary-closure setter path. - // A `defineProperty`-installed own data prop was handled by - // `has_own_data` above; prototype-refs (`C.prototype`) are - // plain objects with no such restriction. - if !has_own_data - && matches!(name.as_str(), "caller" | "arguments") - && super::class_prototype_ref_id(f64::from_bits(bits)).is_none() - { - crate::fs::validate::throw_type_error_with_code( - "Restricted function property access", - "ERR_INVALID_ARG_TYPE", - ); - } - class_dynamic_prop_root_store(class_id, name, value); + // A stream-band address is a reserved handle id, never a real + // `ObjectHeader`. Stop unconditionally — even when the expando + // write was a no-op (dead/unregistered handle, hooks absent, or a + // non-UTF-8 key). Falling through would reach the ObjectHeader + // path below and deref `addr - GC_HEADER_SIZE` (unmapped) → crash. + // Mirrors the reserved small-handle early-return further down. + return; + } + } + // A `Temporal.*` value is an opaque, immutable NaN-boxed cell that is NOT + // an `ObjectHeader` — writing an arbitrary property (e.g. test262's + // `instance.constructor = …` subclassing probes) must NOT interpret the + // cell as an `ObjectHeader` and corrupt its boxed payload (which segfaults + // on the next deref). The cell's `temporal_rs` slots are immutable, but a + // user-defined *expando* property is legal and lives in the exotic side + // table (like Date/RegExp). `obj` still carries its NaN-box tag here + // (`0x7FFD…` for a real cell), so route through `exotic_expando_kind_of_value`, + // which checks the tag before masking to the cleaned heap address. + if let Some((addr, kind @ super::exotic_expando::ExoticKind::Temporal)) = + super::exotic_expando::exotic_expando_kind_of_value(f64::from_bits(obj as u64)) + { + if !key.is_null() { + unsafe { + let name_ptr = + (key as *const u8).add(std::mem::size_of::()); + let name_len = (*key).byte_len as usize; + let name = + String::from_utf8_lossy(std::slice::from_raw_parts(name_ptr, name_len)) + .into_owned(); + let receiver = f64::from_bits(obj as u64); + let _ = super::exotic_expando::exotic_set_property( + addr, kind, &name, value, receiver, + ); } } return; } - } - // Property writes to primitive values operate on temporary wrapper objects - // and do not persist. More importantly for Perry's raw-f64 numbers, they - // must never fall through to the ObjectHeader dereference path below. - { - let bits = obj as u64; - let top16 = bits >> 48; - let jv = JSValue::from_bits(bits); - if (jv.is_number() && top16 != 0) - || jv.is_bool() - || jv.is_any_string() - || jv.is_undefined() - || jv.is_null() - || jv.is_bigint() + if let Some(addr) = + crate::typedarray_props::typed_array_addr_from_value(f64::from_bits(obj as u64)) { + unsafe { + crate::typedarray_props::typed_array_set_own_property( + addr as *mut crate::typedarray::TypedArrayHeader, + key, + value, + ); + } return; } - } - // #2089: a `Date` is a NaN-boxed pointer to an 8-byte `DateCell`, and a - // RegExp is a `RegExpHeader` — neither is an `ObjectHeader`, so a write - // must NOT fall through to the object deref below (memory corruption). - // Expando properties on these exotic instances live in the side table - // (`object::exotic_expando`), honoring accessor descriptors and - // attribute writability installed by `Object.defineProperty`. - { - let bits = obj as u64; - let top16 = bits >> 48; - let addr = if top16 == 0x7FFD { - (bits & 0x0000_FFFF_FFFF_FFFF) as usize - } else if top16 == 0 { - bits as usize - } else { - 0 - }; - if addr != 0 { - if let Some(kind) = super::exotic_expando::exotic_expando_kind(addr) { - if !key.is_null() { - unsafe { - let mut sso = [0u8; crate::value::SHORT_STRING_MAX_LEN]; - if let Some(name_bytes) = crate::string::js_string_key_bytes( - crate::value::JSValue::string_ptr(key as *mut _), - &mut sso, + + // Issue #618-followup: detect INT32-tagged class ref (top16 == 0x7FFE). + // Drizzle's `((SQL2) => { SQL2.Aliased = Aliased; })(SQL)` pattern sets + // a static property on an imported class — Perry stores classes as + // INT32-tagged class ids, so the receiver here is e.g. 0x7FFE_0000_0000_002A + // not a real ObjectHeader. Route to the CLASS_DYNAMIC_PROPS side-table + // so a later `SQL.Aliased` read can find it. + { + let bits = obj as u64; + if (bits >> 48) == 0x7FFE && !key.is_null() { + let class_id = (bits & 0xFFFF_FFFF) as u32; + unsafe { + let name_ptr = + (key as *const u8).add(std::mem::size_of::()); + let name_len = (*key).byte_len as usize; + let name = std::str::from_utf8(std::slice::from_raw_parts(name_ptr, name_len)) + .unwrap_or("") + .to_string(); + // Empty-string is a legal accessor key (`set ''(v)`); the + // `!name.is_empty()` guard below skips it, so dispatch a + // prototype-ref instance setter / constructor-ref static setter + // named "" here (Test262 accessor-name-* literal-string-empty). + if name.is_empty() { + let recv = f64::from_bits(bits); + if super::class_prototype_ref_id(recv).is_some() + && super::class_registry::class_instance_setter_apply( + class_id, &name, recv, value, + ) + { + return; + } + if super::class_registry::class_static_accessor_setter_apply( + class_id, &name, recv, value, ) { - if let Ok(name) = std::str::from_utf8(name_bytes) { - let receiver = f64::from_bits( - crate::value::JSValue::pointer(addr as *const u8).bits(), - ); - let _ = super::exotic_expando::exotic_set_property( - addr, kind, name, value, receiver, - ); - } + return; } } + if !name.is_empty() { + if name == "name" + && !super::class_registry::class_is_key_deleted(class_id, &name) + && super::class_registry::lookup_static_method_in_chain(class_id, &name) + .is_none() + { + return; + } + let has_own_data = CLASS_DYNAMIC_PROPS.with(|m| { + m.borrow() + .get(&class_id) + .is_some_and(|props| props.contains_key(&name)) + }); + // `C.prototype[key] = v` where `key` is an instance + // `set key(v)` accessor defined on the prototype: invoke the + // setter with `this` = the prototype ref. The prototype ref + // and the constructor ref are both INT32-tagged class refs; + // distinguish via `class_prototype_ref_id`. Instance setters + // live in the vtable; static accessors (below) live in the + // constructor ref's table (Test262 accessor-name-inst). + if !has_own_data + && super::class_prototype_ref_id(f64::from_bits(bits)).is_some() + && super::class_registry::class_instance_setter_apply( + class_id, + &name, + f64::from_bits(bits), + value, + ) + { + return; + } + if !has_own_data + && super::class_registry::class_static_accessor_setter_apply( + class_id, + &name, + f64::from_bits(bits), + value, + ) + { + return; + } + // Writing `.caller` / `.arguments` on a class constructor + // hits the poison-pill %ThrowTypeError% accessor (which has + // no [[Set]]) on `Function.prototype`, so a strict-mode + // assignment throws. Mirrors the read side in + // get_field_by_name and the ordinary-closure setter path. + // A `defineProperty`-installed own data prop was handled by + // `has_own_data` above; prototype-refs (`C.prototype`) are + // plain objects with no such restriction. + if !has_own_data + && matches!(name.as_str(), "caller" | "arguments") + && super::class_prototype_ref_id(f64::from_bits(bits)).is_none() + { + crate::fs::validate::throw_type_error_with_code( + "Restricted function property access", + "ERR_INVALID_ARG_TYPE", + ); + } + class_dynamic_prop_root_store(class_id, name, value); + } } return; } } + // Property writes to primitive values operate on temporary wrapper objects + // and do not persist. More importantly for Perry's raw-f64 numbers, they + // must never fall through to the ObjectHeader dereference path below. + { + let bits = obj as u64; + let top16 = bits >> 48; + let jv = JSValue::from_bits(bits); + if (jv.is_number() && top16 != 0) + || jv.is_bool() + || jv.is_any_string() + || jv.is_undefined() + || jv.is_null() + || jv.is_bigint() + { + return; + } + } + // #2089: a `Date` is a NaN-boxed pointer to an 8-byte `DateCell`, and a + // RegExp is a `RegExpHeader` — neither is an `ObjectHeader`, so a write + // must NOT fall through to the object deref below (memory corruption). + // Expando properties on these exotic instances live in the side table + // (`object::exotic_expando`), honoring accessor descriptors and + // attribute writability installed by `Object.defineProperty`. + { + let bits = obj as u64; + let top16 = bits >> 48; + let addr = if top16 == 0x7FFD { + (bits & 0x0000_FFFF_FFFF_FFFF) as usize + } else if top16 == 0 { + bits as usize + } else { + 0 + }; + if addr != 0 { + if let Some(kind) = super::exotic_expando::exotic_expando_kind(addr) { + if !key.is_null() { + unsafe { + let mut sso = [0u8; crate::value::SHORT_STRING_MAX_LEN]; + if let Some(name_bytes) = crate::string::js_string_key_bytes( + crate::value::JSValue::string_ptr(key as *mut _), + &mut sso, + ) { + if let Ok(name) = std::str::from_utf8(name_bytes) { + let receiver = f64::from_bits( + crate::value::JSValue::pointer(addr as *const u8).bits(), + ); + let _ = super::exotic_expando::exotic_set_property( + addr, kind, name, value, receiver, + ); + } + } + } + } + return; + } + } + } } // Strip NaN-boxing tags if present (defensive: handle POINTER_TAG, UNDEFINED, NULL, etc.) let obj = { @@ -910,8 +999,8 @@ pub extern "C" fn js_object_set_field_by_name( // fresh array reusing a freed address (its `_reserved` zeroed at // allocation) skips this lookup and can't fire a previous tenant's // stale accessor. - if st.descriptors.accessors_in_use.get() - && (*gc_header)._reserved & crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS != 0 + if (*gc_header)._reserved & crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS != 0 + && crate::state::state().descriptors.accessors_in_use.get() { if let Some(acc) = get_accessor_descriptor(obj as usize, name) { if acc.set != 0 { @@ -1004,6 +1093,26 @@ pub extern "C" fn js_object_set_field_by_name( return; } + // The disposable-stack `disposed` property is an inherited builtin + // getter with no setter. Its prototype descriptor is installed + // gate-neutrally, and these reserved native class ids are not present + // in the JS class-prototype registry, so reject the write here instead + // of creating an own field. A user `defineProperty` own property still + // shadows the inherited accessor. + if !key.is_null() + && ((*obj).class_id == crate::disposable::CLASS_ID_DISPOSABLE_STACK + || (*obj).class_id == crate::disposable::CLASS_ID_ASYNC_DISPOSABLE_STACK) + { + let key_ptr = (key as *const u8).add(std::mem::size_of::()); + let key_len = (*key).byte_len as usize; + let key_bytes = std::slice::from_raw_parts(key_ptr, key_len); + if key_bytes == b"disposed" + && !super::object_ops::own_key_present(obj as *mut ObjectHeader, key) + { + crate::error::throw_immutable_write(0, "disposed"); + } + } + if (*obj).class_id == NATIVE_MODULE_CLASS_ID && !key.is_null() { let key_ptr = (key as *const u8).add(std::mem::size_of::()); let key_len = (*key).byte_len as usize; @@ -1063,9 +1172,8 @@ pub extern "C" fn js_object_set_field_by_name( // diverging chain (per-instance proto override / null proto) or own // descriptors (an own accessor must dispatch through the short-circuit // below, which a plan hit skips). - const PLAN_BLOCKING_FLAGS: u16 = crate::gc::OBJ_FLAG_PROTO_OVERRIDE - | crate::gc::OBJ_FLAG_NULL_PROTO - | crate::gc::OBJ_FLAG_HAS_DESCRIPTORS; + const PLAN_BLOCKING_FLAGS: u16 = + crate::gc::OBJ_FLAG_NULL_PROTO | crate::gc::OBJ_FLAG_HAS_DESCRIPTORS; let obj_class_id = (*obj).class_id; // #6595: class objects (`OBJECT_TYPE_CLASS`) are excluded — their // writes must always reach the `mirror_class_object_static_write` @@ -1075,7 +1183,8 @@ pub extern "C" fn js_object_set_field_by_name( && obj_class_id != 0 && obj_class_id != NATIVE_MODULE_CLASS_ID && (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR - && (*gc_header)._reserved & PLAN_BLOCKING_FLAGS == 0; + && (*gc_header)._reserved & PLAN_BLOCKING_FLAGS == 0 + && !super::prototype_chain::object_has_prototype_override(obj as usize); let plan_fast = plan_eligible && super::prop_plan::store_plan_check(obj_class_id, interned_key as usize); @@ -1361,9 +1470,7 @@ pub extern "C" fn js_object_set_field_by_name( // (OBJ_FLAG_HAS_DESCRIPTORS is clear — vetted below before the plan is // honored), so the descriptor key string can never be consulted: skip // the per-store String allocation entirely. - let needs_descriptor_key = !plan_fast - && (st.descriptors.accessors_in_use.get() - || st.descriptors.property_attrs_in_use.get()); + let needs_descriptor_key = !plan_fast && has_own_descriptors; let incoming_key_str: Option = if needs_descriptor_key && !key.is_null() { let name_ptr = (key as *const u8).add(std::mem::size_of::()); let name_len = (*key).byte_len as usize; @@ -1390,10 +1497,7 @@ pub extern "C" fn js_object_set_field_by_name( // throw "Cannot assign to read only property" on a plain `{}` (Next.js // app-page-turbo runtime's `exports.Fragment = …`). A fresh allocation // has the flag clear, so it skips the stale lookup entirely. - if !plan_fast - && st.descriptors.accessors_in_use.get() - && super::object_has_descriptors(obj as usize) - { + if !plan_fast && has_own_descriptors { if let Some(ref k) = incoming_key_str { if let Some(acc) = get_accessor_descriptor(obj as usize, k) { if acc.set != 0 { @@ -1602,9 +1706,7 @@ pub extern "C" fn js_object_set_field_by_name( // read only property" on a plain `{}` (Next.js app-page-turbo // runtime's `exports.Fragment = …`). A fresh allocation has the // flag clear, so it skips the lookup entirely. - if st.descriptors.property_attrs_in_use.get() - && super::object_has_descriptors(obj as usize) - { + if has_own_descriptors { if let Some(ref k) = incoming_key_str { if let Some(attrs) = get_property_attrs(obj as usize, k) { if !attrs.writable() { diff --git a/crates/perry-runtime/src/object/global_this/math_temporal.rs b/crates/perry-runtime/src/object/global_this/math_temporal.rs index acd2c92db3..615044c4fe 100644 --- a/crates/perry-runtime/src/object/global_this/math_temporal.rs +++ b/crates/perry-runtime/src/object/global_this/math_temporal.rs @@ -649,19 +649,6 @@ fn install_temporal_getter(proto: *mut ObjectHeader, prop: &str, func_ptr: *cons super::super::object_ops::ensure_key_in_keys_array(proto, key); let getter_bits = crate::value::js_nanbox_pointer(closure as i64).to_bits(); super::super::object_ops::install_builtin_getter(proto, prop, getter_bits); - super::super::set_accessor_descriptor( - proto as usize, - prop.to_string(), - super::super::AccessorDescriptor { - get: getter_bits, - set: 0, - }, - ); - super::super::set_property_attrs( - proto as usize, - prop.to_string(), - super::super::PropertyAttrs::new(true, false, true), - ); super::super::set_builtin_property_attrs( closure as usize, "name".to_string(), diff --git a/crates/perry-runtime/src/object/global_this/proto_methods.rs b/crates/perry-runtime/src/object/global_this/proto_methods.rs index b213b1d146..770cbb76e8 100644 --- a/crates/perry-runtime/src/object/global_this/proto_methods.rs +++ b/crates/perry-runtime/src/object/global_this/proto_methods.rs @@ -199,19 +199,6 @@ pub(crate) fn populate_builtin_prototype_methods(builtin_name: &str, proto_obj: if !getter.is_null() { let getter_bits = crate::value::js_nanbox_pointer(getter as i64).to_bits(); install_builtin_getter(proto_obj, "byteLength", getter_bits); - set_accessor_descriptor( - proto_obj as usize, - "byteLength".to_string(), - AccessorDescriptor { - get: getter_bits, - set: 0, - }, - ); - set_property_attrs( - proto_obj as usize, - "byteLength".to_string(), - PropertyAttrs::new(true, false, true), - ); } } install_noop_proto_methods(proto_obj, OBJECT_PROTO_METHODS); @@ -239,19 +226,6 @@ pub(crate) fn populate_builtin_prototype_methods(builtin_name: &str, proto_obj: if !getter.is_null() { let getter_bits = crate::value::js_nanbox_pointer(getter as i64).to_bits(); install_builtin_getter(proto_obj, "byteLength", getter_bits); - set_accessor_descriptor( - proto_obj as usize, - "byteLength".to_string(), - AccessorDescriptor { - get: getter_bits, - set: 0, - }, - ); - set_property_attrs( - proto_obj as usize, - "byteLength".to_string(), - PropertyAttrs::new(true, false, true), - ); } } set_intrinsic_to_string_tag(proto_obj, "SharedArrayBuffer"); diff --git a/crates/perry-runtime/src/object/mod.rs b/crates/perry-runtime/src/object/mod.rs index b4290d9615..e15034cd1b 100644 --- a/crates/perry-runtime/src/object/mod.rs +++ b/crates/perry-runtime/src/object/mod.rs @@ -164,11 +164,11 @@ pub use class_meta_registry::{ }; pub use descriptor_state::PERRY_CLASS_FIELD_INLINE_GUARD_DISABLED; pub(crate) use descriptor_state::{ - accessor_descriptor_keys_for_obj, class_instance_set_may_intercept, clear_accessor_descriptor, - clear_property_attrs, constructor_accessor_ever_installed, descriptors_in_use, - disable_class_field_inline_guard, get_accessor_descriptor, get_property_attrs, - json_object_getter_value, mark_all_keys, object_has_descriptors, - object_proto_may_intercept_key, owner_may_have_descriptor_entries, + accessor_descriptor_keys_for_obj, class_field_inline_guard_enabled, + class_instance_set_may_intercept, clear_accessor_descriptor, clear_property_attrs, + constructor_accessor_ever_installed, descriptors_in_use, disable_class_field_inline_guard, + get_accessor_descriptor, get_property_attrs, json_object_getter_value, mark_all_keys, + object_has_descriptors, object_proto_may_intercept_key, owner_may_have_descriptor_entries, plain_data_write_may_intercept, prune_dead_descriptor_owner_entries, reflect_getter_closure_bits, set_accessor_descriptor, set_builtin_accessor_descriptor, set_builtin_property_attrs, set_property_attrs, AccessorDescriptor, DescriptorTables, @@ -1619,8 +1619,16 @@ pub struct ObjectMeta { /// Same summary for accessor descriptors (`get`/`set` installs) — the /// `accessor_descriptors` table twin of `attr_key_bits`. pub accessor_key_bits: u64, + /// Object-only state that cannot share `GcHeader._reserved`: every bit in + /// that 16-bit word is already owned by GC layout/age or another object + /// flag. In particular, bit 12 is `GC_OBJ_TYPED_LAYOUT_INTACT`, so using + /// it for prototype divergence made every typed-layout object appear to + /// have a custom prototype. + pub flags: u64, } +pub(crate) const OBJECT_META_FLAG_PROTO_OVERRIDE: u64 = 1; + /// Fetch-or-allocate the per-object meta record. Caller must have already /// established that `obj` is a live, non-RegExp `GC_TYPE_OBJECT` allocation /// (see `prototype_chain::meta_capable_object`). @@ -1649,6 +1657,7 @@ pub(crate) unsafe fn object_meta_ensure(obj: *mut ObjectHeader) -> *mut ObjectMe (*meta).prototype = 0; (*meta).attr_key_bits = 0; (*meta).accessor_key_bits = 0; + (*meta).flags = 0; // GC_STORE_AUDIT(BARRIERED): meta-record edge is a header-slot store // followed by an object-slot barrier, mirroring `set_object_keys_array`. (*obj).meta = meta; diff --git a/crates/perry-runtime/src/object/native_module/callable_exports.rs b/crates/perry-runtime/src/object/native_module/callable_exports.rs index b1e5e85793..15ab80ca61 100644 --- a/crates/perry-runtime/src/object/native_module/callable_exports.rs +++ b/crates/perry-runtime/src/object/native_module/callable_exports.rs @@ -1643,7 +1643,18 @@ pub(crate) fn set_bound_native_closure_name( // `name` is read-only, throwing `Cannot assign to read only property 'name'` // in strict mode (jsonwebtoken → Next.js). Pin the proper descriptor so // enumeration matches reflection. - crate::object::set_property_attrs( + // + // #6809: MUST be the gate-neutral BUILTIN install. This runs during + // `populate_global_this_builtins` for every program that touches a + // builtin global (`console.log` suffices) — the user-install variant + // flipped `GLOBAL_DESCRIPTORS_IN_USE` process-wide at startup, which + // pushed EVERY subsequent dynamic property write onto the descriptor- + // interception slow walk (prototype-chain vetting incl. a dynamic + // `.constructor` read per write; measured as the dominant cost of the + // #6759 write micro). Reflection and enumeration read the descriptor + // table unconditionally, so the builtin variant preserves the + // safe-buffer semantics above. + crate::object::set_builtin_property_attrs( closure as usize, "name".to_string(), crate::object::PropertyAttrs::new(false, false, true), diff --git a/crates/perry-runtime/src/object/prop_plan.rs b/crates/perry-runtime/src/object/prop_plan.rs index fe55090a95..c2b33e1c5c 100644 --- a/crates/perry-runtime/src/object/prop_plan.rs +++ b/crates/perry-runtime/src/object/prop_plan.rs @@ -34,9 +34,8 @@ //! //! Per-OBJECT conditions (frozen/sealed/no-extend, own descriptors, //! per-instance `setPrototypeOf` override, null-proto) are NOT part of the -//! verdict — the caller checks those from header flags before honoring a hit -//! (see `OBJ_FLAG_PROTO_OVERRIDE` / `OBJ_FLAG_NULL_PROTO` gating at the call -//! site). +//! verdict — the caller checks header flags plus the ObjectMeta prototype- +//! override bit before honoring a hit. use std::sync::atomic::{AtomicU64, Ordering}; diff --git a/crates/perry-runtime/src/object/prototype_chain.rs b/crates/perry-runtime/src/object/prototype_chain.rs index f4adb9d7b4..8319d59707 100644 --- a/crates/perry-runtime/src/object/prototype_chain.rs +++ b/crates/perry-runtime/src/object/prototype_chain.rs @@ -112,6 +112,7 @@ fn object_set_static_prototype_impl(obj_ptr: usize, proto_bits: u64, instance_ov }; if obj_type == crate::gc::GC_TYPE_ARRAY || obj_type == crate::gc::GC_TYPE_LAZY_ARRAY { ARRAY_TARGET_PROTO_RECORDED.store(true, Ordering::Relaxed); + crate::array::invalidate_array_index_fast_path(); } } // A per-instance prototype override invalidates class-keyed interception @@ -119,14 +120,6 @@ fn object_set_static_prototype_impl(obj_ptr: usize, proto_bits: u64, instance_ov // object itself must never satisfy a class-keyed plan again. if instance_override { crate::object::prop_plan::prop_plan_epoch_bump(); - unsafe { - if let Some(header) = crate::value::addr_class::try_read_gc_header(obj_ptr) { - if header.obj_type == crate::gc::GC_TYPE_OBJECT { - let header = header as *const crate::gc::GcHeader as *mut crate::gc::GcHeader; - (*header)._reserved |= crate::gc::OBJ_FLAG_PROTO_OVERRIDE; - } - } - } } // #6759 Phase B: shaped objects store the recorded prototype in their // own meta record; only non-object owners fall through to the residual @@ -135,6 +128,9 @@ fn object_set_static_prototype_impl(obj_ptr: usize, proto_bits: u64, instance_ov if let Some(obj) = meta_capable_object(obj_ptr) { let meta = crate::object::object_meta_ensure(obj); (*meta).prototype = proto_bits; + if instance_override { + (*meta).flags |= crate::object::OBJECT_META_FLAG_PROTO_OVERRIDE; + } // GC_STORE_AUDIT(BARRIERED): meta-record prototype slot store — // the record is an arena allocation, so the ordinary object-slot // barrier applies (parent = the meta record). @@ -190,6 +186,21 @@ pub fn object_static_prototype(obj_ptr: usize) -> Option { .and_then(|map| map.get(&obj_ptr).copied()) } +/// True only for a per-instance `Object.setPrototypeOf` / literal +/// `__proto__` override. Class-default prototype links use the same metadata +/// record but deliberately leave this bit clear so class-keyed store plans +/// remain valid. +#[inline] +pub(crate) fn object_has_prototype_override(obj_ptr: usize) -> bool { + unsafe { + let Some(obj) = meta_capable_object(obj_ptr) else { + return false; + }; + let meta = (*obj).meta; + !meta.is_null() && (*meta).flags & crate::object::OBJECT_META_FLAG_PROTO_OVERRIDE != 0 + } +} + pub(crate) fn default_object_prototype_bits() -> Option { let object_ctor = super::js_get_global_this_builtin_value(b"Object".as_ptr(), 6); let ctor_bits = object_ctor.to_bits(); diff --git a/crates/perry-runtime/src/object/regex_proto_thunks.rs b/crates/perry-runtime/src/object/regex_proto_thunks.rs index 662b113526..0c073bb321 100644 --- a/crates/perry-runtime/src/object/regex_proto_thunks.rs +++ b/crates/perry-runtime/src/object/regex_proto_thunks.rs @@ -200,19 +200,6 @@ fn install_getter(proto_obj: *mut ObjectHeader, name: &str, func_ptr: *const u8) super::object_ops::ensure_key_in_keys_array(proto_obj, key); let getter_bits = crate::value::js_nanbox_pointer(closure as i64).to_bits(); super::object_ops::install_builtin_getter(proto_obj, name, getter_bits); - super::set_accessor_descriptor( - proto_obj as usize, - name.to_string(), - super::AccessorDescriptor { - get: getter_bits, - set: 0, - }, - ); - super::set_property_attrs( - proto_obj as usize, - name.to_string(), - super::PropertyAttrs::new(true, false, true), - ); super::set_builtin_property_attrs( closure as usize, "name".to_string(), diff --git a/crates/perry-runtime/src/object/temporal_proto.rs b/crates/perry-runtime/src/object/temporal_proto.rs index 08facfe24f..3267fd22e8 100644 --- a/crates/perry-runtime/src/object/temporal_proto.rs +++ b/crates/perry-runtime/src/object/temporal_proto.rs @@ -91,23 +91,11 @@ fn install_getter(proto_obj: *mut ObjectHeader, name: &str) { super::native_module::set_bound_native_closure_name(closure, &format!("get {name}")); super::native_module::set_builtin_closure_length(closure as usize, 0); super::native_module::set_builtin_closure_non_constructable(closure as usize); - let key = crate::string::js_string_from_bytes(name.as_ptr(), name.len() as u32); - super::object_ops::ensure_key_in_keys_array(proto_obj, key); let getter_bits = crate::value::js_nanbox_pointer(closure as i64).to_bits(); + // #6809: Temporal cell reads use the brand dispatcher and direct + // prototype reads use the per-owner descriptor marker installed by + // this helper. Keep startup gate-neutral. super::object_ops::install_builtin_getter(proto_obj, name, getter_bits); - super::set_accessor_descriptor( - proto_obj as usize, - name.to_string(), - super::AccessorDescriptor { - get: getter_bits, - set: 0, - }, - ); - super::set_property_attrs( - proto_obj as usize, - name.to_string(), - super::PropertyAttrs::new(true, false, true), - ); super::set_builtin_property_attrs( closure as usize, "name".to_string(), diff --git a/crates/perry-runtime/src/object/tests.rs b/crates/perry-runtime/src/object/tests.rs index 24b1ae5a16..8a29517e3f 100644 --- a/crates/perry-runtime/src/object/tests.rs +++ b/crates/perry-runtime/src/object/tests.rs @@ -854,6 +854,27 @@ fn wide_object_index_reads_and_descriptor_writes() { } } +#[test] +fn sloppy_put_value_rejects_disposable_stack_getter_without_own_shadow() { + unsafe { + let stack = crate::disposable::js_disposable_stack_new(); + let key = crate::string::js_string_from_bytes(b"disposed".as_ptr(), 8); + let stack_value = crate::value::js_nanbox_pointer(stack as i64); + let key_value = f64::from_bits(JSValue::string_ptr(key).bits()); + + crate::proxy::js_put_value_set(stack_value, key_value, 1.0, stack_value, 0); + + assert_eq!( + crate::disposable::js_disposable_stack_disposed(stack).to_bits(), + crate::value::TAG_FALSE + ); + assert!( + !own_key_present(stack, key), + "a sloppy write to the inherited getter-only accessor must be a silent no-op" + ); + } +} + /// #5736: `own_key_present` on a wide object (≥257 keys — e.g. a barrel /// `export *` namespace) must use the O(1) wide-key index rather than an O(n) /// keys_array scan, so `Object.values`/`Object.entries` (which re-check every diff --git a/crates/perry-runtime/src/proxy.rs b/crates/perry-runtime/src/proxy.rs index 39fd68b6c4..45b78559c5 100644 --- a/crates/perry-runtime/src/proxy.rs +++ b/crates/perry-runtime/src/proxy.rs @@ -1332,6 +1332,26 @@ fn ordinary_set_with_receiver(target: f64, key: f64, value: f64, receiver: f64) crate::object::prototype_chain::object_static_prototype(addr).is_none() && !crate::object::object_proto_may_intercept_key(key) } else { + // `DisposableStack#disposed` is a getter-only + // builtin accessor on a reserved native prototype. + // Those class ids are intentionally absent from the + // JS class-vtable registry, so the shared + // class-interception plan cannot discover it. + // Keep an own descriptor on the instance eligible + // for the normal walk; otherwise report the + // inherited accessor's rejected [[Set]] here + // (silent in sloppy PutValue, TypeError in strict). + let inherited_disposed_readonly = matches!( + class_id, + crate::disposable::CLASS_ID_DISPOSABLE_STACK + | crate::disposable::CLASS_ID_ASYNC_DISPOSABLE_STACK + ) && property_key_to_rust_string(key) + .as_deref() + == Some("disposed") + && own_set_descriptor(target, key).is_none(); + if inherited_disposed_readonly { + return false; + } // Class instance: the `class_id == 0` guard previously sent // EVERY wide class-instance build down the O(own-key) slow // walk (O(n²)). Safe to fast-path when no inherited accessor / @@ -1349,8 +1369,6 @@ fn ordinary_set_with_receiver(target: f64, key: f64, value: f64, receiver: f64) // class chain — SLOW_FLAGS above already excluded // frozen/sealed/descriptor bits; add the per-instance // divergence flags (setPrototypeOf override / null proto). - const CHAIN_DIVERGE: u16 = - crate::gc::OBJ_FLAG_PROTO_OVERRIDE | crate::gc::OBJ_FLAG_NULL_PROTO; let key_ptr = crate::builtins::js_string_coerce(key) as *const crate::StringHeader; let interned = crate::object::interned_key_ptr(key_ptr); @@ -1367,7 +1385,11 @@ fn ordinary_set_with_receiver(target: f64, key: f64, value: f64, receiver: f64) // statics like bundled zod's `ZodX.create` vanished // from ClassRef static dispatch. Class objects // neither record nor honor store plans. - let plan_eligible = header._reserved & CHAIN_DIVERGE == 0 + let plan_eligible = header._reserved & crate::gc::OBJ_FLAG_NULL_PROTO + == 0 + && !crate::object::prototype_chain::object_has_prototype_override( + addr, + ) && class_id != crate::object::NATIVE_MODULE_CLASS_ID && (*(addr as *const crate::ObjectHeader)).object_type == crate::error::OBJECT_TYPE_REGULAR @@ -1879,4 +1901,99 @@ mod tests { js_proxy_revoke(plain_proxy); assert!(!proxy_wraps_callable(plain_proxy)); } + + fn fnv1a(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325u64, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) + } + + fn boxed_object(obj: *mut crate::ObjectHeader) -> f64 { + f64::from_bits(POINTER_TAG | (obj as u64 & POINTER_MASK)) + } + + fn boxed_interned_key(keys: *mut crate::ArrayHeader, slot: u32, name: &[u8]) -> f64 { + let key = crate::array::js_array_get(keys, slot).as_string_ptr(); + let key = crate::string::js_string_intern(key, fnv1a(name)); + f64::from_bits(crate::value::STRING_TAG | (key as u64 & POINTER_MASK)) + } + + /// #6809: the whole-loop preflight may only publish raw slot indexes when + /// every array element has the same writable data layout. The generated + /// clone performs no checks after this result, so heterogeneous shapes, + /// holes, descriptor flags, and unverified typed layouts must all reject. + #[test] + fn object_array_numeric_write2_guard_requires_complete_uniform_proof() { + let packed = b"a\0b\0c\0d\0"; + let keys = crate::object::js_build_class_keys_array( + 0x6809_01, + 4, + packed.as_ptr(), + packed.len() as u32, + ); + let first = crate::object::js_object_alloc_class_inline_keys(0x6809_01, 0, 4, keys); + let second = crate::object::js_object_alloc_class_inline_keys(0x6809_01, 0, 4, keys); + let values = [boxed_object(first), boxed_object(second)]; + let array = crate::array::js_array_from_f64(values.as_ptr(), values.len() as u32); + let array_box = boxed_object(array.cast()); + let c = boxed_interned_key(keys, 2, b"c"); + let d = boxed_interned_key(keys, 3, b"d"); + + assert_eq!( + put_value::js_object_array_numeric_write2_guard(array_box, c, d, 2), + (4u64 << 32) | 3, + "slots c=2 and d=3 should be published with the non-zero encoding" + ); + + unsafe { + let header = + (second as *mut u8).sub(crate::gc::GC_HEADER_SIZE) as *mut crate::gc::GcHeader; + let original = (*header)._reserved; + + (*header)._reserved = original | crate::gc::OBJ_FLAG_HAS_DESCRIPTORS; + assert_eq!( + put_value::js_object_array_numeric_write2_guard(array_box, c, d, 2), + 0, + "descriptor-bearing receivers must use ordinary [[Set]]" + ); + + (*header)._reserved = original | crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT; + assert_eq!( + put_value::js_object_array_numeric_write2_guard(array_box, c, d, 2), + 0, + "an intact typed layout without raw-f64 target slots must reject" + ); + (*header)._reserved = original; + } + + let hole_values = [boxed_object(first), f64::from_bits(crate::value::TAG_HOLE)]; + let hole_array = + crate::array::js_array_from_f64(hole_values.as_ptr(), hole_values.len() as u32); + assert_eq!( + put_value::js_object_array_numeric_write2_guard( + boxed_object(hole_array.cast()), + c, + d, + 2 + ), + 0, + "a hole cannot be treated as an object receiver" + ); + + let other_keys = crate::object::js_build_class_keys_array( + 0x6809_02, + 4, + packed.as_ptr(), + packed.len() as u32, + ); + let other = crate::object::js_object_alloc_class_inline_keys(0x6809_02, 0, 4, other_keys); + let mixed_values = [boxed_object(first), boxed_object(other)]; + let mixed = + crate::array::js_array_from_f64(mixed_values.as_ptr(), mixed_values.len() as u32); + assert_eq!( + put_value::js_object_array_numeric_write2_guard(boxed_object(mixed.cast()), c, d, 2), + 0, + "content-equal but distinct shape keys arrays must not share raw slots" + ); + } } diff --git a/crates/perry-runtime/src/proxy/put_value.rs b/crates/perry-runtime/src/proxy/put_value.rs index 34424fd9cb..4df3f12cae 100644 --- a/crates/perry-runtime/src/proxy/put_value.rs +++ b/crates/perry-runtime/src/proxy/put_value.rs @@ -107,6 +107,26 @@ pub extern "C" fn js_put_value_set( receiver: f64, strict: i32, ) -> f64 { + // Sloppy script assignment lowers to PutValue rather than the named-field + // setter. Existing own data fields need none of PutValue's rooting, + // ToPropertyKey, Proxy, typed-array, or receiver-aware prototype work. + // Keep this before the handle scope: the helper validates both heap + // headers and only performs a barriered overwrite when the target and + // receiver are the same ordinary object and codegen supplied an interned + // heap-string key. + let target_bits = target.to_bits(); + let key_bits = key.to_bits(); + if target_bits == receiver.to_bits() + && (target_bits & !POINTER_MASK) == POINTER_TAG + && (key_bits & !POINTER_MASK) == crate::value::STRING_TAG + { + let obj = (target_bits & POINTER_MASK) as *mut crate::ObjectHeader; + let key_ptr = (key_bits & POINTER_MASK) as *const crate::StringHeader; + if unsafe { crate::object::try_existing_own_data_overwrite(obj, key_ptr, value) } { + return value; + } + } + let scope = crate::gc::RuntimeHandleScope::new(); let target_handle = scope.root_nanbox_f64(target); let key_handle = scope.root_nanbox_f64(key); @@ -201,7 +221,6 @@ pub extern "C" fn js_put_value_set( } } - let target_bits = target.to_bits(); if target_bits == TAG_NULL || target_bits == TAG_UNDEFINED { let key_name = key_to_rust_string(property_key).unwrap_or_else(|| "property".to_string()); let msg = format!("Cannot set properties of null or undefined (setting '{key_name}')"); @@ -218,3 +237,341 @@ pub extern "C" fn js_put_value_set( } value_handle.get_nanbox_f64() } + +/// Miss path for the codegen-emitted monomorphic PutValue store cache. +/// +/// The full strict/sloppy `[[Set]]` semantics run first. Only a successful +/// ordinary class-instance own-data overwrite may prime `[shape_token, slot]`; +/// every exotic, descriptor-bearing, frozen, class-object, plain-class-zero, +/// overflow, or typed-layout-intact receiver remains permanently on the miss +/// path. The token mirrors the read PIC: a stamped runtime ShapeId is lifted +/// above the pointer range with bit 62; otherwise the shared keys pointer is +/// used. The generated hit path repeats all mutable per-object guards. +#[no_mangle] +pub extern "C" fn js_put_value_set_ic_miss( + target: f64, + key: *const crate::StringHeader, + value: f64, + strict: i32, + cache: *mut [i64; 2], +) -> f64 { + let scope = crate::gc::RuntimeHandleScope::new(); + let target_handle = scope.root_nanbox_f64(target); + let key_handle = scope.root_string_ptr(key); + let value_handle = scope.root_nanbox_f64(value); + let key_value = if key.is_null() { + f64::from_bits(crate::value::TAG_UNDEFINED) + } else { + f64::from_bits(crate::value::js_nanbox_string(key as i64).to_bits()) + }; + let result = js_put_value_set( + target_handle.get_nanbox_f64(), + key_value, + value_handle.get_nanbox_f64(), + target_handle.get_nanbox_f64(), + strict, + ); + + if cache.is_null() { + return result; + } + + unsafe { + let target = target_handle.get_nanbox_f64(); + let target_bits = target.to_bits(); + if (target_bits & !POINTER_MASK) != POINTER_TAG { + return result; + } + let obj_addr = (target_bits & POINTER_MASK) as usize; + let key = key_handle.get_raw_const_ptr::(); + let Some(gc_header) = crate::value::addr_class::try_read_gc_header(obj_addr) else { + return result; + }; + const BLOCKING_FLAGS: u16 = crate::gc::OBJ_FLAG_FROZEN + | crate::gc::OBJ_FLAG_SEALED + | crate::gc::OBJ_FLAG_NO_EXTEND + | crate::gc::OBJ_FLAG_HAS_DESCRIPTORS + | crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO + // A generated hit cannot update/downgrade a typed layout without + // calling the runtime. The miss store clears this bit; prime only + // once that per-object downgrade is visible. + | crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT; + if gc_header.obj_type != crate::gc::GC_TYPE_OBJECT + || gc_header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || gc_header._reserved & BLOCKING_FLAGS != 0 + || key.is_null() + { + return result; + } + + let obj = obj_addr as *mut crate::ObjectHeader; + let class_id = (*obj).class_id; + if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR + || class_id == 0 + || class_id == crate::object::NATIVE_MODULE_CLASS_ID + { + return result; + } + let Some(key_gc) = crate::value::addr_class::try_read_gc_header(key as usize) else { + return result; + }; + if key_gc.obj_type != crate::gc::GC_TYPE_STRING + || key_gc.gc_flags & (crate::gc::GC_FLAG_FORWARDED | crate::gc::GC_FLAG_INTERNED) + != crate::gc::GC_FLAG_INTERNED + { + return result; + } + + let keys = (*obj).keys_array; + if keys.is_null() || (keys as u64) >> 48 != 0 { + return result; + } + let Some(keys_gc) = crate::value::addr_class::try_read_gc_header(keys as usize) else { + return result; + }; + if keys_gc.obj_type != crate::gc::GC_TYPE_ARRAY + || keys_gc.gc_flags & (crate::gc::GC_FLAG_FORWARDED | crate::gc::GC_FLAG_SHAPE_SHARED) + != crate::gc::GC_FLAG_SHAPE_SHARED + { + return result; + } + + let mut own_idx = crate::object::prop_plan::read_plan_lookup(keys as usize, key as usize); + if own_idx.is_none() { + let key_count = crate::array::keys_array_len_capped_to_capacity(keys); + if key_count > 4096 { + return result; + } + for i in 0..key_count { + let candidate = crate::array::js_array_get(keys, i as u32); + if crate::string::js_string_key_matches(candidate, key) { + crate::object::prop_plan::read_plan_record( + keys as usize, + key as usize, + i as u32, + ); + own_idx = Some(i as u32); + break; + } + } + } + let Some(idx) = own_idx else { + return result; + }; + let alloc_limit = + std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize; + if idx as usize >= alloc_limit { + return result; + } + + let parent_class_id = (*obj).parent_class_id; + let shape_token = if crate::object::shapes::is_shape_id(parent_class_id) { + crate::object::shapes::PIC_ID_TOKEN_BIT | parent_class_id as u64 + } else { + keys as u64 + }; + + // Publish the token last conceptually: a zero-initialized or stale + // token cannot hit this slot until it matches this receiver's current + // discriminated shape token. Perry's read PIC uses the same format. + (*cache)[1] = idx as i64; + (*cache)[0] = shape_token as i64; + } + + result +} + +/// Preflight for codegen's call-free nested object-write loop. +/// +/// Returns two existing own-data slot indexes, encoded as +/// `((slot_2 + 1) << 32) | (slot_1 + 1)`, or zero when the generated raw +/// loop must not run. The caller scans once, then performs only finite numeric +/// stores until both loops finish. That call-free interval is load-bearing: +/// no GC can move the array, its elements, their shared keys array, or their +/// typed-layout records after this function validates them. +/// +/// This is intentionally stricter than ordinary `[[Set]]`: every receiver +/// must be a regular, writable object with the exact same shared keys array, +/// both keys must already be own data slots, and typed-layout receivers must +/// prove both slots are raw f64. Any doubt falls back to the existing generic +/// loop before the first observable store. +#[no_mangle] +pub extern "C" fn js_object_array_numeric_write2_guard( + array: f64, + key_1: f64, + key_2: f64, + count: u32, +) -> u64 { + // Reuse the process gate js_gc_init disables for typed-feedback tracing, + // typed-layout verification, and the explicit inline-field escape hatch. + // This loop bypasses the same observations/checks as the class-field + // inline clone and therefore must honor the identical gate. + if count == 0 || !crate::object::class_field_inline_guard_enabled() { + return 0; + } + + let array_bits = array.to_bits(); + if (array_bits & !POINTER_MASK) != POINTER_TAG { + return 0; + } + let array_addr = (array_bits & POINTER_MASK) as usize; + let Some(array_gc) = (unsafe { crate::value::addr_class::try_read_gc_header(array_addr) }) + else { + return 0; + }; + if array_gc.obj_type != crate::gc::GC_TYPE_ARRAY + || array_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || array_gc._reserved & crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS != 0 + || crate::array::PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED + .load(std::sync::atomic::Ordering::Relaxed) + != 0 + { + return 0; + } + + let arr = array_addr as *const crate::array::ArrayHeader; + let (length, capacity) = unsafe { ((*arr).length, (*arr).capacity) }; + if length > 16_000_000 || capacity > 16_000_000 || length > capacity || count > length { + return 0; + } + + let decode_key = |boxed: f64| -> Option<*const crate::StringHeader> { + let bits = boxed.to_bits(); + if (bits & !POINTER_MASK) != crate::value::STRING_TAG { + return None; + } + let ptr = (bits & POINTER_MASK) as *const crate::StringHeader; + let gc = unsafe { crate::value::addr_class::try_read_gc_header(ptr as usize) }?; + (gc.obj_type == crate::gc::GC_TYPE_STRING + && gc.gc_flags & (crate::gc::GC_FLAG_FORWARDED | crate::gc::GC_FLAG_INTERNED) + == crate::gc::GC_FLAG_INTERNED) + .then_some(ptr) + }; + let Some(key_1) = decode_key(key_1) else { + return 0; + }; + let Some(key_2) = decode_key(key_2) else { + return 0; + }; + + const BLOCKING_FLAGS: u16 = crate::gc::OBJ_FLAG_FROZEN + | crate::gc::OBJ_FLAG_SEALED + | crate::gc::OBJ_FLAG_NO_EXTEND + | crate::gc::OBJ_FLAG_HAS_DESCRIPTORS + | crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO; + + unsafe fn validated_object( + bits: u64, + ) -> Option<( + *mut crate::ObjectHeader, + *mut crate::array::ArrayHeader, + u16, + )> { + if (bits & !POINTER_MASK) != POINTER_TAG { + return None; + } + let addr = (bits & POINTER_MASK) as usize; + let gc = crate::value::addr_class::try_read_gc_header(addr)?; + if gc.obj_type != crate::gc::GC_TYPE_OBJECT + || gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || gc._reserved & BLOCKING_FLAGS != 0 + { + return None; + } + let obj = addr as *mut crate::ObjectHeader; + if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR + || (*obj).class_id == 0 + || (*obj).class_id == crate::object::NATIVE_MODULE_CLASS_ID + { + return None; + } + let keys = (*obj).keys_array; + if keys.is_null() || (keys as u64) >> 48 != 0 { + return None; + } + let keys_gc = crate::value::addr_class::try_read_gc_header(keys as usize)?; + if keys_gc.obj_type != crate::gc::GC_TYPE_ARRAY + || keys_gc.gc_flags & (crate::gc::GC_FLAG_FORWARDED | crate::gc::GC_FLAG_SHAPE_SHARED) + != crate::gc::GC_FLAG_SHAPE_SHARED + { + return None; + } + Some((obj, keys, gc._reserved)) + } + + unsafe fn find_slot( + keys: *mut crate::array::ArrayHeader, + key: *const crate::StringHeader, + ) -> Option { + let key_count = crate::array::keys_array_len_capped_to_capacity(keys); + if key_count > 4096 { + return None; + } + for i in 0..key_count { + let candidate = crate::array::js_array_get(keys, i as u32); + if crate::string::js_string_key_matches(candidate, key) { + return Some(i as u32); + } + } + None + } + + let elements = unsafe { + (arr as *const u8).add(std::mem::size_of::()) as *const f64 + }; + let first_bits = unsafe { (*elements).to_bits() }; + if first_bits == crate::value::TAG_HOLE { + return 0; + } + let Some((first, shared_keys, first_flags)) = (unsafe { validated_object(first_bits) }) else { + return 0; + }; + let Some(slot_1) = (unsafe { find_slot(shared_keys, key_1) }) else { + return 0; + }; + let Some(slot_2) = (unsafe { find_slot(shared_keys, key_2) }) else { + return 0; + }; + + let first_limit = unsafe { + std::cmp::max( + (*first).field_count, + crate::object::INLINE_SLOT_FLOOR as u32, + ) + }; + if slot_1 >= first_limit || slot_2 >= first_limit { + return 0; + } + if first_flags & crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT != 0 + && (!crate::gc::layout_typed_raw_f64_slot_for_user(first as usize, slot_1 as usize) + || !crate::gc::layout_typed_raw_f64_slot_for_user(first as usize, slot_2 as usize)) + { + return 0; + } + + for i in 1..count as usize { + let bits = unsafe { (*elements.add(i)).to_bits() }; + if bits == crate::value::TAG_HOLE { + return 0; + } + let Some((obj, keys, flags)) = (unsafe { validated_object(bits) }) else { + return 0; + }; + if keys != shared_keys { + return 0; + } + let limit = + unsafe { std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) }; + if slot_1 >= limit || slot_2 >= limit { + return 0; + } + if flags & crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT != 0 + && (!crate::gc::layout_typed_raw_f64_slot_for_user(obj as usize, slot_1 as usize) + || !crate::gc::layout_typed_raw_f64_slot_for_user(obj as usize, slot_2 as usize)) + { + return 0; + } + } + + (u64::from(slot_2 + 1) << 32) | u64::from(slot_1 + 1) +} diff --git a/crates/perry-runtime/src/typed_feedback.rs b/crates/perry-runtime/src/typed_feedback.rs index 2684116973..fb242f4a6c 100644 --- a/crates/perry-runtime/src/typed_feedback.rs +++ b/crates/perry-runtime/src/typed_feedback.rs @@ -901,6 +901,9 @@ fn observe_property( obj_bits: u64, key: *const crate::StringHeader, ) { + if site_id == 0 || !typed_feedback_enabled() { + return; + } let object_addr = normalize_raw_object_addr(obj_bits); let (shape_addr, class_id, gc_type) = object_shape(object_addr); observe( @@ -1005,6 +1008,12 @@ pub extern "C" fn js_typed_feedback_object_set_field_by_name_fast( key: *const crate::StringHeader, value: f64, ) { + if !typed_feedback_enabled() { + if crate::object::js_object_set_field_by_name_transition_fast(obj, key, value) == 0 { + crate::object::js_object_set_field_by_name(obj, key, value); + } + return; + } let object_addr = normalize_raw_object_addr(obj as u64); let (shape_addr, class_id, gc_type) = object_shape(object_addr); let handled = crate::object::js_object_set_field_by_name_transition_fast(obj, key, value) != 0;