perf(codegen/runtime): #6812 — spill lanes make append-past-capacity arrays clone-eligible (w13 beats node)

changelog.d/6812-spill-lanes.md

@@ -0,0 +1 @@
+perf(codegen/runtime): #6812 — the whole-loop write clone learns spill lanes. The preflight guard classifies each write lane as inline or spill (slot past the inline capacity but covered by the object-owned overflow buffer, uniform across every receiver), and the fast nest stores spill lanes through obj → meta → buffer — two dependent loads, still call-free. Combined with the first-iteration peel, append-past-capacity arrays (`o[7]` added to 5-field objects, then updated in rounds) now run in the clone tier: w13 went from 160 ms (triage baseline; ~155 ms on the spill-buffer build) to 7 ms, beating node (~13 ms).

crates/perry-codegen/src/stmt/loops.rs

@@ -13,7 +13,7 @@ use crate::native_value::{
     BoundedBufferIndex, BoundsProof, BoundsState, BufferAccessMode, LengthSource, LoweredValue,
     MaterializationReason,
 };
-use crate::types::{DOUBLE, I1, I32, I64};
+use crate::types::{DOUBLE, I1, I32, I64, I8};
 
 #[derive(Clone, Copy)]
 enum NumericBulkFillValue {
@@ -2542,13 +2542,18 @@ fn lower_object_array_write_versioned_for(
     let fast_inner_pre_idx = ctx.new_block("object_array_write.loop.fast.inner.preheader");
     let fast_inner_cond_idx = ctx.new_block("object_array_write.loop.fast.inner.cond");
     let fast_inner_body_idx = ctx.new_block("object_array_write.loop.fast.inner.body");
+    // #6812 spill lanes: the per-lane store chain ends in a `done` block, so
+    // the inner back-edge needs a dedicated latch — the counter phi must
+    // name its true predecessor.
+    let fast_inner_latch_idx = ctx.new_block("object_array_write.loop.fast.inner.latch");
     let fast_inner_exit_idx = ctx.new_block("object_array_write.loop.fast.inner.exit");
     let fast_done_idx = ctx.new_block("object_array_write.loop.fast.done");
     let fast_entry_label = ctx.block_label(fast_entry_idx);
     let fast_outer_cond_label = ctx.block_label(fast_outer_cond_idx);
     let fast_inner_pre_label = ctx.block_label(fast_inner_pre_idx);
     let fast_inner_cond_label = ctx.block_label(fast_inner_cond_idx);
     let fast_inner_body_label = ctx.block_label(fast_inner_body_idx);
+    let fast_inner_latch_label = ctx.block_label(fast_inner_latch_idx);
     let fast_inner_exit_label = ctx.block_label(fast_inner_exit_idx);
     let fast_done_label = ctx.block_label(fast_done_idx);
 
@@ -2565,7 +2570,13 @@ fn lower_object_array_write_versioned_for(
                 blk.lshr(I64, &packed_slots, &(index * 16).to_string())
             };
             let encoded = blk.and(I64, &shifted, "65535");
-            slots.push(blk.sub(I64, &encoded, "1"));
+            // #6812 spill lanes: bit 15 of a lane means the store goes
+            // through the object-owned spill buffer (obj → meta → buffer);
+            // the low 15 bits carry slot + 1 (find_slot caps slots at 4096,
+            // so the +1 packing can never carry into the flag).
+            let spill_flag = blk.and(I64, &encoded, "32768");
+            let low = blk.and(I64, &encoded, "32767");
+            slots.push((blk.sub(I64, &low, "1"), spill_flag));
         }
         let array_bits = blk.bitcast_double_to_i64(&array_box);
         let array_handle = blk.and(I64, &array_bits, crate::nanbox::POINTER_MASK_I64);
@@ -2617,7 +2628,7 @@ fn lower_object_array_write_versioned_for(
         I32,
         &[
             ("0", &fast_inner_pre_label),
-            (&inner_next, &fast_inner_body_label),
+            (&inner_next, &fast_inner_latch_label),
         ],
     );
     let inner_more = ctx.block().icmp_slt(I32, &inner, &inner_bound_operand);
@@ -2636,10 +2647,44 @@ fn lower_object_array_write_versioned_for(
         let object_handle = blk.and(I64, &object_bits, crate::nanbox::POINTER_MASK_I64);
         blk.inttoptr(I64, &object_handle)
     };
-    let header_words =
-        (crate::target_layout::object_header_size_bytes(ctx.target_triple) / 8).to_string();
-    for (slot, value) in slots.iter().zip(&matched.values) {
+    let object_header_size = crate::target_layout::object_header_size_bytes(ctx.target_triple);
+    let header_words = (object_header_size / 8).to_string();
+    // `meta` is the LAST ObjectHeader field (a documented invariant of the
+    // header layout): a POINTER-WIDTH field at byte offset
+    // (header_size - pointer_size). On ILP32 (arm64_32) the header is 24
+    // bytes with a 4-byte `meta` at offset 20 — neither 8-byte-word-indexable
+    // nor i64-loadable — so the spill path addresses it by BYTE offset and
+    // loads pointer-width, mirroring the `new.rs` allocator's meta store.
+    let meta_ptr_size: u64 = if crate::target_layout::target_is_ilp32(ctx.target_triple) {
+        4
+    } else {
+        8
+    };
+    let meta_byte_off = (object_header_size - meta_ptr_size).to_string();
+    let meta_load_ty = if meta_ptr_size == 4 { I32 } else { I64 };
+    for (lane_index, ((slot, spill_flag), value)) in slots.iter().zip(&matched.values).enumerate() {
         let value = emit_object_array_write_number(ctx, value, &outer_double, &inner_double);
+        // #6812 spill lanes: the guard proved every receiver holds this
+        // lane's slot on the SAME side (inline vs spill), so the flag is
+        // loop-invariant — LLVM unswitches the branch out of the nest. Both
+        // paths remain call-free raw stores, preserving the guard's no-GC
+        // interval.
+        let spill_idx = ctx.new_block(&format!(
+            "object_array_write.loop.fast.store.spill.{lane_index}"
+        ));
+        let inline_idx = ctx.new_block(&format!(
+            "object_array_write.loop.fast.store.inline.{lane_index}"
+        ));
+        let done_idx = ctx.new_block(&format!(
+            "object_array_write.loop.fast.store.done.{lane_index}"
+        ));
+        let spill_label = ctx.block_label(spill_idx);
+        let inline_label = ctx.block_label(inline_idx);
+        let done_label = ctx.block_label(done_idx);
+        let is_spill = ctx.block().icmp_ne(I64, spill_flag, "0");
+        ctx.block().cond_br(&is_spill, &spill_label, &inline_label);
+
+        ctx.current_block = inline_idx;
         let field_ptr = {
             let blk = ctx.block();
             let field_word = blk.add(I64, slot, &header_words);
@@ -2648,7 +2693,40 @@ fn lower_object_array_write_versioned_for(
         // GC_STORE_AUDIT(POINTER_FREE): the versioned loop emits only numeric
         // values into fields proven numeric by the entry guard.
         ctx.block().store(DOUBLE, &value, &field_ptr);
-    }
+        ctx.block().br(&done_label);
+
+        ctx.current_block = spill_idx;
+        {
+            let blk = ctx.block();
+            let meta_slot_ptr = blk.gep(I8, &object_ptr, &[(I64, &meta_byte_off)]);
+            let meta_loaded = blk.load(meta_load_ty, &meta_slot_ptr);
+            let meta_i64 = if meta_ptr_size == 4 {
+                blk.zext(I32, &meta_loaded, I64)
+            } else {
+                meta_loaded
+            };
+            let meta_ptr = blk.inttoptr(I64, &meta_i64);
+            // ObjectMeta layout word 4 = `spill`; buffer elements start one
+            // word past the 8-byte ArrayHeader. Both offsets are locked by
+            // const assertions next to the runtime structs
+            // (perry-runtime/src/object/mod.rs, #6812 spill lanes).
+            let spill_slot_ptr = blk.gep_inbounds(I64, &meta_ptr, &[(I64, "4")]);
+            let spill_i64 = blk.load(I64, &spill_slot_ptr);
+            let spill_ptr = blk.inttoptr(I64, &spill_i64);
+            let elem_word = blk.add(I64, slot, "1");
+            let elem_ptr = blk.gep_inbounds(I64, &spill_ptr, &[(I64, &elem_word)]);
+            // GC_STORE_AUDIT(POINTER_FREE): finite numeric bits into a
+            // guard-proven live spill slot; numbers create no references,
+            // so no barrier or layout note is needed (same argument as the
+            // inline lane above).
+            blk.store(DOUBLE, &value, &elem_ptr);
+        }
+        ctx.block().br(&done_label);
+
+        ctx.current_block = done_idx;
+    }
+    ctx.block().br(&fast_inner_latch_label);
+    ctx.current_block = fast_inner_latch_idx;
     ctx.block()
         .emit_raw(format!("{} = add i32 {}, 1", inner_next, inner));
     ctx.block().br(&fast_inner_cond_label);

crates/perry-runtime/src/object/mod.rs

@@ -1859,6 +1859,13 @@ pub struct ObjectMeta {
 
 pub(crate) const OBJECT_META_FLAG_PROTO_OVERRIDE: u64 = 1;
 
+// #6812 spill lanes: the versioned write-loop emitter
+// (perry-codegen/src/stmt/loops.rs) addresses `meta.spill` at word 4 of the
+// ObjectMeta record and buffer elements one word past the ArrayHeader. Keep
+// codegen and these structs in lock-step.
+const _: () = assert!(std::mem::offset_of!(ObjectMeta, spill) == 32);
+const _: () = assert!(std::mem::size_of::<crate::array::ArrayHeader>() == 8);
+
 /// Fetch-or-allocate the per-object meta record. Caller must have already
 /// established that `obj` is a live, non-RegExp `GC_TYPE_OBJECT` allocation
 /// (see `prototype_chain::meta_capable_object`).

crates/perry-runtime/src/proxy/put_value.rs

@@ -569,31 +569,79 @@ fn object_array_numeric_write_slots(array: f64, keys: &[f64], count: u32) -> Opt
         )?;
     }
 
+    // #6812 spill lanes: classify each lane as INLINE (slot within the
+    // receiver's inline alloc_limit) or SPILL (slot lives in the
+    // object-owned overflow buffer, `meta.spill`). The FIRST receiver fixes
+    // each lane's mode; every later receiver must be on the SAME side of its
+    // own alloc_limit and pass the same coverage proof, so the emitter's
+    // per-lane store sequence (raw inline store vs the meta → spill
+    // indirection) is uniform across the whole proven prefix. A spill lane
+    // sets bit 15 of its result lane; the caller's `+1` packing cannot carry
+    // into it (slot ≤ 4096).
+    unsafe fn spill_lane_covers(obj: *const crate::ObjectHeader, slot: u32) -> bool {
+        let meta = (*obj).meta;
+        if meta.is_null() {
+            return false;
+        }
+        let spill = (*meta).spill as *const crate::array::ArrayHeader;
+        if spill.is_null() {
+            return false;
+        }
+        // Mirror the shared-keys validation: a live, non-forwarded plain
+        // array whose length high-water covers the slot (a key present in
+        // the shape implies its value slot was written, so length > slot —
+        // verified rather than assumed).
+        let Some(gc) = crate::value::addr_class::try_read_gc_header(spill as usize) else {
+            return false;
+        };
+        if gc.obj_type != crate::gc::GC_TYPE_ARRAY
+            || gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0
+        {
+            return false;
+        }
+        slot < (*spill).length && slot < (*spill).capacity
+    }
+
     let first_limit = unsafe {
         std::cmp::max(
             (*first).field_count,
             crate::object::INLINE_SLOT_FLOOR as u32,
         )
     };
-    if slots[..keys.len()]
-        .iter()
-        .any(|slot| u32::from(*slot) >= first_limit)
-    {
-        trace_object_array_numeric_write_rejection("first receiver target slot is out of bounds");
-        return None;
+    let mut lane_spill = [false; 4];
+    for index in 0..keys.len() {
+        let slot = u32::from(slots[index]);
+        if slot >= first_limit {
+            if !unsafe { spill_lane_covers(first, slot) } {
+                trace_object_array_numeric_write_rejection(
+                    "first receiver target slot is out of bounds",
+                );
+                return None;
+            }
+            lane_spill[index] = true;
+        }
     }
-    if first_flags & crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT != 0
-        && slots[..keys.len()].iter().any(|slot| {
+    if first_flags & crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT != 0 {
+        // Typed-intact receivers keep raw-f64 inline invariants; a spill
+        // lane on one is unexpected — reject conservatively rather than
+        // reason about typed descriptors for out-of-line slots.
+        if lane_spill[..keys.len()].iter().any(|s| *s) {
+            trace_object_array_numeric_write_rejection(
+                "first receiver typed descriptor does not contain every target slot",
+            );
+            return None;
+        }
+        if slots[..keys.len()].iter().any(|slot| {
             !crate::gc::layout_typed_accepts_finite_number_slot_for_user(
                 first as usize,
                 usize::from(*slot),
             )
-        })
-    {
-        trace_object_array_numeric_write_rejection(
-            "first receiver typed descriptor does not contain every target slot",
-        );
-        return None;
+        }) {
+            trace_object_array_numeric_write_rejection(
+                "first receiver typed descriptor does not contain every target slot",
+            );
+            return None;
+        }
     }
 
     for i in 1..count as usize {
@@ -614,30 +662,51 @@ fn object_array_numeric_write_slots(array: f64, keys: &[f64], count: u32) -> Opt
         }
         let limit =
             unsafe { std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) };
-        if slots[..keys.len()]
-            .iter()
-            .any(|slot| u32::from(*slot) >= limit)
-        {
-            trace_object_array_numeric_write_rejection(
-                "receiver prefix contains an out-of-bounds target slot",
-            );
-            return None;
+        for index in 0..keys.len() {
+            let slot = u32::from(slots[index]);
+            if lane_spill[index] {
+                // Mode uniformity: this receiver must ALSO hold the slot in
+                // its spill buffer (a wider receiver holding it inline would
+                // make the emitter's spill store write the wrong memory).
+                if slot < limit || !unsafe { spill_lane_covers(obj, slot) } {
+                    trace_object_array_numeric_write_rejection(
+                        "receiver prefix contains an out-of-bounds target slot",
+                    );
+                    return None;
+                }
+            } else if slot >= limit {
+                trace_object_array_numeric_write_rejection(
+                    "receiver prefix contains an out-of-bounds target slot",
+                );
+                return None;
+            }
         }
-        if flags & crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT != 0
-            && slots[..keys.len()].iter().any(|slot| {
+        if flags & crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT != 0 {
+            if lane_spill[..keys.len()].iter().any(|s| *s) {
+                trace_object_array_numeric_write_rejection(
+                    "receiver typed descriptor does not contain every target slot",
+                );
+                return None;
+            }
+            if slots[..keys.len()].iter().any(|slot| {
                 !crate::gc::layout_typed_accepts_finite_number_slot_for_user(
                     obj as usize,
                     usize::from(*slot),
                 )
-            })
-        {
-            trace_object_array_numeric_write_rejection(
-                "receiver typed descriptor does not contain every target slot",
-            );
-            return None;
+            }) {
+                trace_object_array_numeric_write_rejection(
+                    "receiver typed descriptor does not contain every target slot",
+                );
+                return None;
+            }
         }
     }
 
+    for index in 0..keys.len() {
+        if lane_spill[index] {
+            slots[index] |= 0x8000;
+        }
+    }
     Some(slots)
 }
 

docs/object-write-matrix.md

@@ -47,7 +47,7 @@ Ratio = perry/node median (fill from measurement; `<1` = beating node).
 | w10_poly8 | 8 shapes through one site | PIC exhausted → runtime miss | 27 | 19 | 1.4 | close; megamorphic path is decent |
 | w11_stable_dynkey | `o[k]`, `const k = "c"` | clone (const-string local = static) | 6 | 8 | **0.75** | BEATS node |
 | w12_arb_dynkey | rotating keys from array | generic | 84 | 18 | 4.7 | GAP: GC-safe dynamic-key cache |
-| w13_int_key | `o[7]` on plain object | generic numeric-as-property; append past inline capacity | 160 | 13 | 12.3 | PARTIAL: integer keys are static clone keys and iteration #1 is peeled, so a within-capacity append clones (variant: 5 vs 12 ms, **0.42**, beats node). The canonical cell appends a 6th key past the literal's 5-slot capacity → overflow side-table; needs the object-owned spill (next slice) |
+| w13_int_key | `o[7]` on plain object | *(pre-spill-lanes baseline)* generic → peel + whole-loop clone with spill lanes | 160 → 7 | 13 | 12.3 → **0.54** | BEATS node — integer static keys (#6841), first-iteration peel (#6841), object-owned spill (#6849), and guard/emitter spill lanes make the append-past-capacity array clone-eligible |
 | w15_append_build | fresh `{}` + 6 assigns (builder) | *(pre-#6829 baseline)* generic transitions; `class_id==0` blocks PIC | 1489 → 196 (#6829) | 8 | 186 → 25 | #6829 folds builders into literals; residual tracked below |
 | w16_overflow_slot | writes past inline capacity | *(pre-#6812-w16 baseline)* runtime (PIC bounds reject) → whole-loop clone | 4163 → 3 | 29 | 173 → **0.26** | BEATS node — `{}` per-site classes + learned width + compile-time width hint make builder arrays uniform and clone-eligible. The #6812-w13 peel adds one ordinary outer round (3 → 6 ms; reclaimable with a guard-first second-chance design). Ratios vs each run's own node baseline (2026-07-25 sweeps) |
 | w17_alloc_rhs | allocating RHS (`"s"+i`) | NOT PIC (safepoint-free rule) | 26 | 18 | 1.4 | close; revisit only with receiver-reload design |