From bdee6bf78fd33de0675268f4cdf9369be203f51a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 26 Jul 2026 17:31:18 +0200 Subject: [PATCH 1/3] =?UTF-8?q?perf(codegen/runtime):=20#6812=20=E2=80=94?= =?UTF-8?q?=20spill=20lanes=20in=20the=20whole-loop=20write=20clone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard classifies each lane as INLINE or SPILL from the first receiver (spill = slot past the inline alloc_limit but covered by the object-owned meta.spill buffer: live non-forwarded array, length high-water > slot); every later receiver must sit on the SAME side of its own alloc_limit and pass the same coverage proof, so the emitter's per-lane store sequence is uniform across the proven prefix. Spill lanes set bit 15 of their u16 result lane — the +1 packing cannot carry into it (find_slot caps slots at 4096). Typed-layout-intact receivers reject spill lanes conservatively. The fast nest branches per lane on the loop-invariant flag (LLVM unswitches it): inline lanes store as before; spill lanes store through obj → meta(word header-1) → spill(word 4) → elements(word 1+slot). Both paths stay call-free raw numeric stores, preserving the guard's no-GC interval; const assertions beside the runtime structs lock the codegen offsets. The store chain ends in per-lane done blocks, so the inner back-edge gained a dedicated latch and the counter phi names its true predecessor. This makes append-past-capacity arrays (canonical w13: a 6th key on 5-field literals) clone-eligible after the #6841 peel primes their shapes. Claude-Session: https://claude.ai/code/session_01QJ5mwMDPc63tNLAFPdthAG --- crates/perry-codegen/src/stmt/loops.rs | 75 +++++++++++- crates/perry-runtime/src/object/mod.rs | 7 ++ crates/perry-runtime/src/proxy/put_value.rs | 129 +++++++++++++++----- 3 files changed, 175 insertions(+), 36 deletions(-) diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index 947f525755..140f9f384d 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -2542,6 +2542,10 @@ fn lower_object_array_write_versioned_for( let fast_inner_pre_idx = ctx.new_block("object_array_write.loop.fast.inner.preheader"); let fast_inner_cond_idx = ctx.new_block("object_array_write.loop.fast.inner.cond"); let fast_inner_body_idx = ctx.new_block("object_array_write.loop.fast.inner.body"); + // #6812 spill lanes: the per-lane store chain ends in a `done` block, so + // the inner back-edge needs a dedicated latch — the counter phi must + // name its true predecessor. + let fast_inner_latch_idx = ctx.new_block("object_array_write.loop.fast.inner.latch"); let fast_inner_exit_idx = ctx.new_block("object_array_write.loop.fast.inner.exit"); let fast_done_idx = ctx.new_block("object_array_write.loop.fast.done"); let fast_entry_label = ctx.block_label(fast_entry_idx); @@ -2549,6 +2553,7 @@ fn lower_object_array_write_versioned_for( let fast_inner_pre_label = ctx.block_label(fast_inner_pre_idx); let fast_inner_cond_label = ctx.block_label(fast_inner_cond_idx); let fast_inner_body_label = ctx.block_label(fast_inner_body_idx); + let fast_inner_latch_label = ctx.block_label(fast_inner_latch_idx); let fast_inner_exit_label = ctx.block_label(fast_inner_exit_idx); let fast_done_label = ctx.block_label(fast_done_idx); @@ -2565,7 +2570,13 @@ fn lower_object_array_write_versioned_for( blk.lshr(I64, &packed_slots, &(index * 16).to_string()) }; let encoded = blk.and(I64, &shifted, "65535"); - slots.push(blk.sub(I64, &encoded, "1")); + // #6812 spill lanes: bit 15 of a lane means the store goes + // through the object-owned spill buffer (obj → meta → buffer); + // the low 15 bits carry slot + 1 (find_slot caps slots at 4096, + // so the +1 packing can never carry into the flag). + let spill_flag = blk.and(I64, &encoded, "32768"); + let low = blk.and(I64, &encoded, "32767"); + slots.push((blk.sub(I64, &low, "1"), spill_flag)); } let array_bits = blk.bitcast_double_to_i64(&array_box); let array_handle = blk.and(I64, &array_bits, crate::nanbox::POINTER_MASK_I64); @@ -2617,7 +2628,7 @@ fn lower_object_array_write_versioned_for( I32, &[ ("0", &fast_inner_pre_label), - (&inner_next, &fast_inner_body_label), + (&inner_next, &fast_inner_latch_label), ], ); let inner_more = ctx.block().icmp_slt(I32, &inner, &inner_bound_operand); @@ -2636,10 +2647,34 @@ fn lower_object_array_write_versioned_for( let object_handle = blk.and(I64, &object_bits, crate::nanbox::POINTER_MASK_I64); blk.inttoptr(I64, &object_handle) }; - let header_words = - (crate::target_layout::object_header_size_bytes(ctx.target_triple) / 8).to_string(); - for (slot, value) in slots.iter().zip(&matched.values) { + let header_words_n = crate::target_layout::object_header_size_bytes(ctx.target_triple) / 8; + let header_words = header_words_n.to_string(); + // `meta` is the LAST ObjectHeader field (a documented invariant of the + // header layout), i.e. the word directly before the field region. + let meta_word = (header_words_n - 1).to_string(); + for (lane_index, ((slot, spill_flag), value)) in slots.iter().zip(&matched.values).enumerate() { let value = emit_object_array_write_number(ctx, value, &outer_double, &inner_double); + // #6812 spill lanes: the guard proved every receiver holds this + // lane's slot on the SAME side (inline vs spill), so the flag is + // loop-invariant — LLVM unswitches the branch out of the nest. Both + // paths remain call-free raw stores, preserving the guard's no-GC + // interval. + let spill_idx = ctx.new_block(&format!( + "object_array_write.loop.fast.store.spill.{lane_index}" + )); + let inline_idx = ctx.new_block(&format!( + "object_array_write.loop.fast.store.inline.{lane_index}" + )); + let done_idx = ctx.new_block(&format!( + "object_array_write.loop.fast.store.done.{lane_index}" + )); + let spill_label = ctx.block_label(spill_idx); + let inline_label = ctx.block_label(inline_idx); + let done_label = ctx.block_label(done_idx); + let is_spill = ctx.block().icmp_ne(I64, spill_flag, "0"); + ctx.block().cond_br(&is_spill, &spill_label, &inline_label); + + ctx.current_block = inline_idx; let field_ptr = { let blk = ctx.block(); let field_word = blk.add(I64, slot, &header_words); @@ -2648,7 +2683,35 @@ fn lower_object_array_write_versioned_for( // GC_STORE_AUDIT(POINTER_FREE): the versioned loop emits only numeric // values into fields proven numeric by the entry guard. ctx.block().store(DOUBLE, &value, &field_ptr); - } + ctx.block().br(&done_label); + + ctx.current_block = spill_idx; + { + let blk = ctx.block(); + let meta_slot_ptr = blk.gep_inbounds(I64, &object_ptr, &[(I64, &meta_word)]); + let meta_i64 = blk.load(I64, &meta_slot_ptr); + let meta_ptr = blk.inttoptr(I64, &meta_i64); + // ObjectMeta layout word 4 = `spill`; buffer elements start one + // word past the 8-byte ArrayHeader. Both offsets are locked by + // const assertions next to the runtime structs + // (perry-runtime/src/object/mod.rs, #6812 spill lanes). + let spill_slot_ptr = blk.gep_inbounds(I64, &meta_ptr, &[(I64, "4")]); + let spill_i64 = blk.load(I64, &spill_slot_ptr); + let spill_ptr = blk.inttoptr(I64, &spill_i64); + let elem_word = blk.add(I64, slot, "1"); + let elem_ptr = blk.gep_inbounds(I64, &spill_ptr, &[(I64, &elem_word)]); + // GC_STORE_AUDIT(POINTER_FREE): finite numeric bits into a + // guard-proven live spill slot; numbers create no references, + // so no barrier or layout note is needed (same argument as the + // inline lane above). + blk.store(DOUBLE, &value, &elem_ptr); + } + ctx.block().br(&done_label); + + ctx.current_block = done_idx; + } + ctx.block().br(&fast_inner_latch_label); + ctx.current_block = fast_inner_latch_idx; ctx.block() .emit_raw(format!("{} = add i32 {}, 1", inner_next, inner)); ctx.block().br(&fast_inner_cond_label); diff --git a/crates/perry-runtime/src/object/mod.rs b/crates/perry-runtime/src/object/mod.rs index 042969b9fd..196e21f3b7 100644 --- a/crates/perry-runtime/src/object/mod.rs +++ b/crates/perry-runtime/src/object/mod.rs @@ -1859,6 +1859,13 @@ pub struct ObjectMeta { pub(crate) const OBJECT_META_FLAG_PROTO_OVERRIDE: u64 = 1; +// #6812 spill lanes: the versioned write-loop emitter +// (perry-codegen/src/stmt/loops.rs) addresses `meta.spill` at word 4 of the +// ObjectMeta record and buffer elements one word past the ArrayHeader. Keep +// codegen and these structs in lock-step. +const _: () = assert!(std::mem::offset_of!(ObjectMeta, spill) == 32); +const _: () = assert!(std::mem::size_of::() == 8); + /// Fetch-or-allocate the per-object meta record. Caller must have already /// established that `obj` is a live, non-RegExp `GC_TYPE_OBJECT` allocation /// (see `prototype_chain::meta_capable_object`). diff --git a/crates/perry-runtime/src/proxy/put_value.rs b/crates/perry-runtime/src/proxy/put_value.rs index 774aaec408..6be163e033 100644 --- a/crates/perry-runtime/src/proxy/put_value.rs +++ b/crates/perry-runtime/src/proxy/put_value.rs @@ -569,31 +569,79 @@ fn object_array_numeric_write_slots(array: f64, keys: &[f64], count: u32) -> Opt )?; } + // #6812 spill lanes: classify each lane as INLINE (slot within the + // receiver's inline alloc_limit) or SPILL (slot lives in the + // object-owned overflow buffer, `meta.spill`). The FIRST receiver fixes + // each lane's mode; every later receiver must be on the SAME side of its + // own alloc_limit and pass the same coverage proof, so the emitter's + // per-lane store sequence (raw inline store vs the meta → spill + // indirection) is uniform across the whole proven prefix. A spill lane + // sets bit 15 of its result lane; the caller's `+1` packing cannot carry + // into it (slot ≤ 4096). + unsafe fn spill_lane_covers(obj: *const crate::ObjectHeader, slot: u32) -> bool { + let meta = (*obj).meta; + if meta.is_null() { + return false; + } + let spill = (*meta).spill as *const crate::array::ArrayHeader; + if spill.is_null() { + return false; + } + // Mirror the shared-keys validation: a live, non-forwarded plain + // array whose length high-water covers the slot (a key present in + // the shape implies its value slot was written, so length > slot — + // verified rather than assumed). + let Some(gc) = crate::value::addr_class::try_read_gc_header(spill as usize) else { + return false; + }; + if gc.obj_type != crate::gc::GC_TYPE_ARRAY + || gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + { + return false; + } + slot < (*spill).length && slot < (*spill).capacity + } + let first_limit = unsafe { std::cmp::max( (*first).field_count, crate::object::INLINE_SLOT_FLOOR as u32, ) }; - if slots[..keys.len()] - .iter() - .any(|slot| u32::from(*slot) >= first_limit) - { - trace_object_array_numeric_write_rejection("first receiver target slot is out of bounds"); - return None; + let mut lane_spill = [false; 4]; + for index in 0..keys.len() { + let slot = u32::from(slots[index]); + if slot >= first_limit { + if !unsafe { spill_lane_covers(first, slot) } { + trace_object_array_numeric_write_rejection( + "first receiver target slot is out of bounds", + ); + return None; + } + lane_spill[index] = true; + } } - if first_flags & crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT != 0 - && slots[..keys.len()].iter().any(|slot| { + if first_flags & crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT != 0 { + // Typed-intact receivers keep raw-f64 inline invariants; a spill + // lane on one is unexpected — reject conservatively rather than + // reason about typed descriptors for out-of-line slots. + if lane_spill[..keys.len()].iter().any(|s| *s) { + trace_object_array_numeric_write_rejection( + "first receiver typed descriptor does not contain every target slot", + ); + return None; + } + if slots[..keys.len()].iter().any(|slot| { !crate::gc::layout_typed_accepts_finite_number_slot_for_user( first as usize, usize::from(*slot), ) - }) - { - trace_object_array_numeric_write_rejection( - "first receiver typed descriptor does not contain every target slot", - ); - return None; + }) { + trace_object_array_numeric_write_rejection( + "first receiver typed descriptor does not contain every target slot", + ); + return None; + } } for i in 1..count as usize { @@ -614,30 +662,51 @@ fn object_array_numeric_write_slots(array: f64, keys: &[f64], count: u32) -> Opt } let limit = unsafe { std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) }; - if slots[..keys.len()] - .iter() - .any(|slot| u32::from(*slot) >= limit) - { - trace_object_array_numeric_write_rejection( - "receiver prefix contains an out-of-bounds target slot", - ); - return None; + for index in 0..keys.len() { + let slot = u32::from(slots[index]); + if lane_spill[index] { + // Mode uniformity: this receiver must ALSO hold the slot in + // its spill buffer (a wider receiver holding it inline would + // make the emitter's spill store write the wrong memory). + if slot < limit || !unsafe { spill_lane_covers(obj, slot) } { + trace_object_array_numeric_write_rejection( + "receiver prefix contains an out-of-bounds target slot", + ); + return None; + } + } else if slot >= limit { + trace_object_array_numeric_write_rejection( + "receiver prefix contains an out-of-bounds target slot", + ); + return None; + } } - if flags & crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT != 0 - && slots[..keys.len()].iter().any(|slot| { + if flags & crate::gc::GC_OBJ_TYPED_LAYOUT_INTACT != 0 { + if lane_spill[..keys.len()].iter().any(|s| *s) { + trace_object_array_numeric_write_rejection( + "receiver typed descriptor does not contain every target slot", + ); + return None; + } + if slots[..keys.len()].iter().any(|slot| { !crate::gc::layout_typed_accepts_finite_number_slot_for_user( obj as usize, usize::from(*slot), ) - }) - { - trace_object_array_numeric_write_rejection( - "receiver typed descriptor does not contain every target slot", - ); - return None; + }) { + trace_object_array_numeric_write_rejection( + "receiver typed descriptor does not contain every target slot", + ); + return None; + } } } + for index in 0..keys.len() { + if lane_spill[index] { + slots[index] |= 0x8000; + } + } Some(slots) } From 9999ca5cce618fa5f335c7322f25ae6ccd7d3748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 26 Jul 2026 19:46:30 +0200 Subject: [PATCH 2/3] docs: #6812 spill-lanes changelog fragment + w13 matrix row Claude-Session: https://claude.ai/code/session_01QJ5mwMDPc63tNLAFPdthAG --- changelog.d/6812-spill-lanes.md | 1 + docs/object-write-matrix.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/6812-spill-lanes.md diff --git a/changelog.d/6812-spill-lanes.md b/changelog.d/6812-spill-lanes.md new file mode 100644 index 0000000000..f9e5f5de19 --- /dev/null +++ b/changelog.d/6812-spill-lanes.md @@ -0,0 +1 @@ +perf(codegen/runtime): #6812 — the whole-loop write clone learns spill lanes. The preflight guard classifies each write lane as inline or spill (slot past the inline capacity but covered by the object-owned overflow buffer, uniform across every receiver), and the fast nest stores spill lanes through obj → meta → buffer — two dependent loads, still call-free. Combined with the first-iteration peel, append-past-capacity arrays (`o[7]` added to 5-field objects, then updated in rounds) now run in the clone tier: w13 went from ~155 ms to 7 ms, beating node (~13 ms). diff --git a/docs/object-write-matrix.md b/docs/object-write-matrix.md index 2c7cd19a8a..b0f51fef43 100644 --- a/docs/object-write-matrix.md +++ b/docs/object-write-matrix.md @@ -47,7 +47,7 @@ Ratio = perry/node median (fill from measurement; `<1` = beating node). | w10_poly8 | 8 shapes through one site | PIC exhausted → runtime miss | 27 | 19 | 1.4 | close; megamorphic path is decent | | w11_stable_dynkey | `o[k]`, `const k = "c"` | clone (const-string local = static) | 6 | 8 | **0.75** | BEATS node | | w12_arb_dynkey | rotating keys from array | generic | 84 | 18 | 4.7 | GAP: GC-safe dynamic-key cache | -| w13_int_key | `o[7]` on plain object | generic numeric-as-property; append past inline capacity | 160 | 13 | 12.3 | PARTIAL: integer keys are static clone keys and iteration #1 is peeled, so a within-capacity append clones (variant: 5 vs 12 ms, **0.42**, beats node). The canonical cell appends a 6th key past the literal's 5-slot capacity → overflow side-table; needs the object-owned spill (next slice) | +| w13_int_key | `o[7]` on plain object | *(pre-spill-lanes baseline)* generic → peel + whole-loop clone with spill lanes | 160 → 7 | 13 | 12.3 → **0.54** | BEATS node — integer static keys (#6841), first-iteration peel (#6841), object-owned spill (#6849), and guard/emitter spill lanes make the append-past-capacity array clone-eligible | | w15_append_build | fresh `{}` + 6 assigns (builder) | *(pre-#6829 baseline)* generic transitions; `class_id==0` blocks PIC | 1489 → 196 (#6829) | 8 | 186 → 25 | #6829 folds builders into literals; residual tracked below | | w16_overflow_slot | writes past inline capacity | *(pre-#6812-w16 baseline)* runtime (PIC bounds reject) → whole-loop clone | 4163 → 3 | 29 | 173 → **0.26** | BEATS node — `{}` per-site classes + learned width + compile-time width hint make builder arrays uniform and clone-eligible. The #6812-w13 peel adds one ordinary outer round (3 → 6 ms; reclaimable with a guard-first second-chance design). Ratios vs each run's own node baseline (2026-07-25 sweeps) | | w17_alloc_rhs | allocating RHS (`"s"+i`) | NOT PIC (safepoint-free rule) | 26 | 18 | 1.4 | close; revisit only with receiver-reload design | From 52be25b0600f12dfcac1dcc20dda791fc291a9d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 26 Jul 2026 20:00:16 +0200 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20CodeRabbit=20round=201=20on=20#6856?= =?UTF-8?q?=20=E2=80=94=20ILP32=20meta=20addressing;=20one=20w13=20baselin?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spill store path addressed 'meta' as 8-byte word (header_words - 1), which on ILP32 (arm64_32: 24-byte header, 4-byte meta at offset 20) reads the wrong pointer. Address it by byte offset with a pointer-width load (zext to i64 on ILP32), mirroring the new.rs allocator's meta store. Changelog now cites the same w13 baseline lineage as the matrix doc. Claude-Session: https://claude.ai/code/session_01QJ5mwMDPc63tNLAFPdthAG --- changelog.d/6812-spill-lanes.md | 2 +- crates/perry-codegen/src/stmt/loops.rs | 29 +++++++++++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/changelog.d/6812-spill-lanes.md b/changelog.d/6812-spill-lanes.md index f9e5f5de19..a85cb21bfa 100644 --- a/changelog.d/6812-spill-lanes.md +++ b/changelog.d/6812-spill-lanes.md @@ -1 +1 @@ -perf(codegen/runtime): #6812 — the whole-loop write clone learns spill lanes. The preflight guard classifies each write lane as inline or spill (slot past the inline capacity but covered by the object-owned overflow buffer, uniform across every receiver), and the fast nest stores spill lanes through obj → meta → buffer — two dependent loads, still call-free. Combined with the first-iteration peel, append-past-capacity arrays (`o[7]` added to 5-field objects, then updated in rounds) now run in the clone tier: w13 went from ~155 ms to 7 ms, beating node (~13 ms). +perf(codegen/runtime): #6812 — the whole-loop write clone learns spill lanes. The preflight guard classifies each write lane as inline or spill (slot past the inline capacity but covered by the object-owned overflow buffer, uniform across every receiver), and the fast nest stores spill lanes through obj → meta → buffer — two dependent loads, still call-free. Combined with the first-iteration peel, append-past-capacity arrays (`o[7]` added to 5-field objects, then updated in rounds) now run in the clone tier: w13 went from 160 ms (triage baseline; ~155 ms on the spill-buffer build) to 7 ms, beating node (~13 ms). diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index 140f9f384d..b678f87f9b 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -13,7 +13,7 @@ use crate::native_value::{ BoundedBufferIndex, BoundsProof, BoundsState, BufferAccessMode, LengthSource, LoweredValue, MaterializationReason, }; -use crate::types::{DOUBLE, I1, I32, I64}; +use crate::types::{DOUBLE, I1, I32, I64, I8}; #[derive(Clone, Copy)] enum NumericBulkFillValue { @@ -2647,11 +2647,21 @@ fn lower_object_array_write_versioned_for( let object_handle = blk.and(I64, &object_bits, crate::nanbox::POINTER_MASK_I64); blk.inttoptr(I64, &object_handle) }; - let header_words_n = crate::target_layout::object_header_size_bytes(ctx.target_triple) / 8; - let header_words = header_words_n.to_string(); + let object_header_size = crate::target_layout::object_header_size_bytes(ctx.target_triple); + let header_words = (object_header_size / 8).to_string(); // `meta` is the LAST ObjectHeader field (a documented invariant of the - // header layout), i.e. the word directly before the field region. - let meta_word = (header_words_n - 1).to_string(); + // header layout): a POINTER-WIDTH field at byte offset + // (header_size - pointer_size). On ILP32 (arm64_32) the header is 24 + // bytes with a 4-byte `meta` at offset 20 — neither 8-byte-word-indexable + // nor i64-loadable — so the spill path addresses it by BYTE offset and + // loads pointer-width, mirroring the `new.rs` allocator's meta store. + let meta_ptr_size: u64 = if crate::target_layout::target_is_ilp32(ctx.target_triple) { + 4 + } else { + 8 + }; + let meta_byte_off = (object_header_size - meta_ptr_size).to_string(); + let meta_load_ty = if meta_ptr_size == 4 { I32 } else { I64 }; for (lane_index, ((slot, spill_flag), value)) in slots.iter().zip(&matched.values).enumerate() { let value = emit_object_array_write_number(ctx, value, &outer_double, &inner_double); // #6812 spill lanes: the guard proved every receiver holds this @@ -2688,8 +2698,13 @@ fn lower_object_array_write_versioned_for( ctx.current_block = spill_idx; { let blk = ctx.block(); - let meta_slot_ptr = blk.gep_inbounds(I64, &object_ptr, &[(I64, &meta_word)]); - let meta_i64 = blk.load(I64, &meta_slot_ptr); + let meta_slot_ptr = blk.gep(I8, &object_ptr, &[(I64, &meta_byte_off)]); + let meta_loaded = blk.load(meta_load_ty, &meta_slot_ptr); + let meta_i64 = if meta_ptr_size == 4 { + blk.zext(I32, &meta_loaded, I64) + } else { + meta_loaded + }; let meta_ptr = blk.inttoptr(I64, &meta_i64); // ObjectMeta layout word 4 = `spill`; buffer elements start one // word past the 8-byte ArrayHeader. Both offsets are locked by