From a2c8fd026a008d74fbeeb12b7b7dc9bf5bac25dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 11 Aug 2026 22:26:02 +0200 Subject: [PATCH] =?UTF-8?q?perf(codegen):=20interp=200.843=20->=200.784=20?= =?UTF-8?q?s=20=E2=80=94=20branch=20the=20generic=20property-get=20guard?= =?UTF-8?q?=20instead=20of=20AND-ing=20eight=20predicates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inline monomorphic-IC diamond built one flat `hit` predicate out of eight tests, so LLVM if-converted the whole thing: every receiver executed every load and every compare, including the two epoch loads, even after the first test had already decided the answer. It also tested the two RARE receiver tags (SSO, INT32 class-ref) before the pointer tag, putting two constant materialisations, two compares and two branches in front of every real object read. The guard chain now branches: pget.recv_ok -- is_real_ptr --> pic.recv_hdr (four header predicates, deliberately kept flat: same two cache lines, LLVM fuses them into one `ccmp` chain) pic.recv_hdr -- is_object --> pic.token pic.token -- hit --> pic.hit and the POINTER/STRING tag test goes first, with SSO/INT32 discriminated in a cold `pget.recv_other` (the three tag classes are pairwise disjoint, so the order is free). `pic.miss` recomputes what the polymorphic-way compares need from the same memory rather than taking phis, which would have dragged their materialisation back onto the hot path. Semantics are unchanged: every predicate is still checked, with control flow instead of data flow. `evalNode`'s monomorphic hit path drops from 58 to 46 aarch64 instructions before the field load. Quiet M1 mini, best-of-5, exit-checked, 19-program corpus, both arms linking the same runtime archives: interp 0.8433 -> 0.7843 (-7.0%), iso_miss 1.2310 -> 1.1784 (-4.3%), cycles -2.8%, pipeline -1.9%, churn_read -1.8%, retain_wide1 -1.3%, deeplist -1.3%. Six binaries compile byte-identical and set the run's noise floor at +-0.6%. Claude-Session: https://claude.ai/code/session_012B8z92S82sCfqCrVqrFgS2 --- changelog.d/7887-pget-guard-branches.md | 34 ++++ .../src/expr/property_get/generic_dispatch.rs | 182 +++++++++++++----- .../src/expr/property_get/tests.rs | 169 ++++++++++++++++ 3 files changed, 336 insertions(+), 49 deletions(-) create mode 100644 changelog.d/7887-pget-guard-branches.md diff --git a/changelog.d/7887-pget-guard-branches.md b/changelog.d/7887-pget-guard-branches.md new file mode 100644 index 0000000000..83082d7f41 --- /dev/null +++ b/changelog.d/7887-pget-guard-branches.md @@ -0,0 +1,34 @@ +**`perf(codegen)`: the generic property-get guard branches instead of AND-ing eight predicates — `interp` 0.843 → 0.784 s, `iso_miss` 1.231 → 1.178 s.** + +The inline monomorphic-IC diamond built one flat `hit` predicate out of eight tests, so LLVM +if-converted the whole region: every receiver executed every load and every compare — +including the two epoch loads — even after the first test had already decided the answer. It +also tested the two *rare* receiver tags (SSO, INT32 class-ref) before the pointer tag, +putting two constant materialisations, two compares and two branches in front of every real +object read. + +The chain now branches — `pget.recv_ok` → `pic.recv_hdr` → `pic.token` → `pic.hit` — and the +POINTER/STRING tag test goes first, with SSO/INT32 discriminated in a cold `pget.recv_other` +(the three tag classes are pairwise disjoint, so the order is free). The four header +predicates stay flat on purpose: they are four loads from the same two cache lines and LLVM +fuses their compares into one `ccmp` chain, which beats four branches. `pic.miss` recomputes +what the polymorphic-way compares need from the same memory rather than taking phis, which +would have dragged their `cset`/`csinc` materialisation back onto the hot path. + +Semantics are unchanged — every predicate is still checked, with control flow instead of data +flow. `evalNode`'s monomorphic hit path drops from 58 to 46 aarch64 instructions before the +field load. + +Measured on the quiet M1 mini, best-of-5, exit-checked, 19-program corpus, both arms linking +the same runtime archives: `interp` −7.0%, `iso_miss` −4.3%, `cycles` −2.8%, `pipeline` +−1.9%, `churn_read` −1.8%, `deeplist`/`retain_wide1` −1.3%. Six binaries compile +byte-identical and set the run's noise floor at ±0.6%. + +The guard test walks the def chain rather than asserting presence: it walks the CFG backwards +from the block performing the raw slot load, requires every edge on that path to be the +**true** edge of a `cond_br`, and takes the transitive def closure of those conditions, +requiring each guard to be reachable from a branch condition. Three sabotages were run and all +three went red (constant branch condition, swapped `cond_br` edges, deleted predicate) — a +presence assertion catches none of the first two. The walk must be scoped to a single +function: register names restart at `%r1` in every body, so a global def map resolved the +receiver-tag condition to an unrelated `ptrtoint` in another function. diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 83c2938625..0cd4adaa06 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -128,21 +128,19 @@ pub(crate) fn lower_generic_property_get( // `js_object_get_field_by_name_f64` runtime entry (which // handles `.length` directly from the NaN-box length // byte and returns `undefined` for other keys). - let is_sso = ctx.block().icmp_eq(I64, &obj_tag, "32761"); // 0x7FF9 - // v0.5.747: INT32-tagged class refs (top16 == 0x7FFE) used - // as PropertyGet receivers. Pre-fix these fell through to - // the invalid-recv path (returning undefined) because the - // 0xFFFD-masked tag check (0x7FFE & 0xFFFD = 0x7FFC, not - // 0x7FFD) treated them as non-pointer values. Drizzle's - // `is(value, type)` chain depends on `Cls.kind` reads through - // an Any-typed local. Refs #420 / #618 followup. - // - // Note: this also catches plain int32 numeric values (e.g. - // `(42).property`). The runtime helper's INT32-tag arm at - // js_object_get_field_by_name returns undefined for any - // class_id not registered in CLASS_DYNAMIC_PROPS, matching - // the previous behavior — pure ints have no static fields. - let is_int32_class = ctx.block().icmp_eq(I64, &obj_tag, "32766"); // 0x7FFE + // v0.5.747: INT32-tagged class refs (top16 == 0x7FFE) used + // as PropertyGet receivers. Pre-fix these fell through to + // the invalid-recv path (returning undefined) because the + // 0xFFFD-masked tag check (0x7FFE & 0xFFFD = 0x7FFC, not + // 0x7FFD) treated them as non-pointer values. Drizzle's + // `is(value, type)` chain depends on `Cls.kind` reads through + // an Any-typed local. Refs #420 / #618 followup. + // + // Note: this also catches plain int32 numeric values (e.g. + // `(42).property`). The runtime helper's INT32-tag arm at + // js_object_get_field_by_name returns undefined for any + // class_id not registered in CLASS_DYNAMIC_PROPS, matching + // the previous behavior — pure ints have no static fields. let obj_tag_masked = ctx.block().and(I64, &obj_tag, "65533"); // 0xFFFD let is_valid = ctx.block().icmp_eq(I64, &obj_tag_masked, "32765"); // 0x7FFD let sso_idx = ctx.new_block("pget.recv_sso"); @@ -155,19 +153,28 @@ pub(crate) fn lower_generic_property_get( let invalid_label = ctx.block_label(invalid_idx); let class_ref_label = ctx.block_label(class_ref_idx); let final_merge_label = ctx.block_label(final_merge_idx); - // Three-step branch: first check SSO, then class-ref, then - // pointer-validity. Inverse branches funnel into invalid_idx. - let pic_or_invalid_idx = ctx.new_block("pget.check_ptr"); - let pic_or_invalid_label = ctx.block_label(pic_or_invalid_idx); + // #7883: the POINTER/STRING test goes FIRST, and the two rare tags are + // discriminated in a cold block off its false edge. The three tag classes + // are pairwise disjoint — `is_valid` is `(tag & 0xFFFD) == 0x7FFD`, true + // only for 0x7FFD/0x7FFF, while SSO is 0x7FF9 and an INT32 class ref is + // 0x7FFE — so testing them in any order gives the same routing. The old + // order (SSO, then class-ref, then pointer) put two 16-bit constant + // materialisations, two compares and two branches in front of every real + // object receiver: 13 instructions before the PIC on the path that is + // taken essentially always. Now it is `lshr` + `and` + `cmp` + branch. + let other_idx = ctx.new_block("pget.recv_other"); + let other_label = ctx.block_label(other_idx); let check_class_ref_idx = ctx.new_block("pget.check_class_ref"); let check_class_ref_label = ctx.block_label(check_class_ref_idx); + ctx.block().cond_br(&is_valid, &pic_label, &other_label); + ctx.current_block = other_idx; + let is_sso = ctx.block().icmp_eq(I64, &obj_tag, "32761"); // 0x7FF9 ctx.block() .cond_br(&is_sso, &sso_label, &check_class_ref_label); ctx.current_block = check_class_ref_idx; + let is_int32_class = ctx.block().icmp_eq(I64, &obj_tag, "32766"); // 0x7FFE ctx.block() - .cond_br(&is_int32_class, &class_ref_label, &pic_or_invalid_label); - ctx.current_block = pic_or_invalid_idx; - ctx.block().cond_br(&is_valid, &pic_label, &invalid_label); + .cond_br(&is_int32_class, &class_ref_label, &invalid_label); // Class-ref dispatch: route through the runtime helper which // detects INT32 class-ref bits and consults CLASS_DYNAMIC_PROPS @@ -265,29 +272,48 @@ pub(crate) fn lower_generic_property_get( // // Threshold matches `js_native_call_method`'s small-handle // detection (raw_ptr < 0x100000). + let cache_ref = format!("@{}", cache_name); let is_real_ptr = ctx.block().icmp_ugt(I64, &obj_handle, "1048575"); // 0x100000 - // Sentinel address: the per-site cache global itself — - // always valid, 16-byte aligned, and its bytes don't - // match GC_TYPE_OBJECT (=2) or an active keys_array, so - // the IC will cleanly miss when we substitute it for a - // small handle. - let cache_ref = format!("@{}", cache_name); - let cache_addr = ctx.block().ptrtoint(&cache_ref, I64); - let safe_obj_handle = ctx - .block() - .select(I1, &is_real_ptr, I64, &obj_handle, &cache_addr); + // #7883: the hit/miss/merge blocks are minted here so the guard chain + // below can BRANCH OUT to the miss on the first failing predicate + // instead of AND-ing eight of them into one flat `hit`. LLVM if-converts + // a flat predicate, so every receiver paid every load and every compare — + // including the two epoch loads — even after the very first one had + // already decided the answer. Each group now ends in its own `cond_br`; + // the miss block reconstructs what the polymorphic-way compares need + // through phis (`false`/`0` on the early-exit edges, which is exactly + // what the flat predicate computed there). + let hit_idx = ctx.new_block("pic.hit"); + let miss_idx = ctx.new_block("pic.miss"); + let merge_idx = ctx.new_block("pic.merge"); + let hit_label = ctx.block_label(hit_idx); + let miss_label = ctx.block_label(miss_idx); + let merge_label = ctx.block_label(merge_idx); + let hdr_idx = ctx.new_block("pic.recv_hdr"); + let hdr_label = ctx.block_label(hdr_idx); + let tok_idx = ctx.new_block("pic.token"); + let tok_label = ctx.block_label(tok_idx); + // Small-handle receivers (native-module registry ids) must never be + // dereferenced. Pre-#7883 they were kept out of the loads by selecting a + // sentinel address and AND-ing `is_real_ptr` into `hit`; the branch does + // the same job without putting a `select` (and the sentinel's address + // materialisation) in front of every real object read. The miss path + // still substitutes the sentinel, because the way compares below load + // `field_count` unconditionally. + // (edge labels are no longer needed: the miss block recomputes.) + ctx.block().cond_br(&is_real_ptr, &hdr_label, &miss_label); + ctx.current_block = hdr_idx; // GcHeader sits 8 bytes before the user pointer; obj_type is the // first u8 (GC_TYPE_OBJECT=2). Cost: 1 sub + 1 load i8 + 1 cmp // i8 + 1 and i1 — the cond_br's `is_object` operand is folded // into the existing branch instruction by LLVM. Branch-predicted // taken since real PropertyGet receivers are objects. - let gc_type_addr = ctx.block().sub(I64, &safe_obj_handle, "8"); + let gc_type_addr = ctx.block().sub(I64, &obj_handle, "8"); let gc_type_ptr = ctx.block().inttoptr(I64, &gc_type_addr); let gc_type = ctx.block().load(I8, &gc_type_ptr); - let gc_type_ok = ctx.block().icmp_eq(I8, &gc_type, "2"); - let is_object = ctx.block().and(I1, &is_real_ptr, &gc_type_ok); + let is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); // Issue #618: closures share GC_TYPE_OBJECT but their offset+16 // is a capture slot, not `keys_array`. The PIC's keys_val == @@ -299,7 +325,7 @@ pub(crate) fn lower_generic_property_get( // `js_object_get_field_ic_miss` → `js_object_get_field_by_name`, // which dispatches closure dynamic-prop reads via the // `CLOSURE_DYNAMIC_PROPS` side-table. - let magic_addr = ctx.block().add(I64, &safe_obj_handle, "12"); + let magic_addr = ctx.block().add(I64, &obj_handle, "12"); let magic_ptr = ctx.block().inttoptr(I64, &magic_addr); let magic_val = ctx.block().load(I32, &magic_ptr); // CLOSURE_MAGIC = 0x434C4F53 (4 bytes "CLOS" little-endian). @@ -321,7 +347,7 @@ pub(crate) fn lower_generic_property_get( // Specific repro: `function f(): any { ... return new // RegExp(...) } const r = f(); r.source` — fast path returns // garbage f64 instead of routing through `js_regexp_get_source`. - let object_type_ptr = ctx.block().inttoptr(I64, &safe_obj_handle); + let object_type_ptr = ctx.block().inttoptr(I64, &obj_handle); let object_type = ctx.block().load(I32, &object_type_ptr); let object_type_ok = ctx.block().icmp_eq(I32, &object_type, "1"); let is_object = ctx.block().and(I1, &is_object, &object_type_ok); @@ -338,15 +364,25 @@ pub(crate) fn lower_generic_property_get( // descriptors) whenever it is set. Mirrors the guard in // `class_field_inline_guard.rs`. Cost: 1 sub + load i16 + and + cmp, // folded into the existing `hit` cond_br. - let reserved_addr = ctx.block().sub(I64, &safe_obj_handle, "6"); + let reserved_addr = ctx.block().sub(I64, &obj_handle, "6"); let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); let reserved = ctx.block().load(crate::types::I16, &reserved_ptr); let has_desc = ctx.block().and(crate::types::I16, &reserved, "2048"); // OBJ_FLAG_HAS_DESCRIPTORS (0x800) let no_desc = ctx.block().icmp_eq(crate::types::I16, &has_desc, "0"); let is_object = ctx.block().and(I1, &is_object, &no_desc); + // #7883: first exit. The four header predicates above are kept as one + // flat `and` on purpose — they are four loads from the SAME two cache + // lines and LLVM fuses their compares into a `ccmp` chain, which is + // cheaper than four branches. What was NOT worth folding is everything + // below: the keys load, the token select and the two epoch loads all + // hang off the same predicate, so a non-object receiver used to execute + // them before the flat `hit` could reject it. + ctx.block().cond_br(&is_object, &tok_label, &miss_label); + ctx.current_block = tok_idx; + // Load obj->keys_array at offset 16 of ObjectHeader. - let keys_addr = ctx.block().add(I64, &safe_obj_handle, "16"); + let keys_addr = ctx.block().add(I64, &obj_handle, "16"); let keys_ptr_p = ctx.block().inttoptr(I64, &keys_addr); let keys_val = ctx.block().load(I64, &keys_ptr_p); @@ -361,7 +397,7 @@ pub(crate) fn lower_generic_property_get( // two token kinds can never collide numerically — one compare, no // discriminant word. `parent_class_id` is a u32 at offset 8 on every // target (the four leading u32s precede the pointer fields). - let pcid_addr = ctx.block().add(I64, &safe_obj_handle, "8"); + let pcid_addr = ctx.block().add(I64, &obj_handle, "8"); let pcid_ptr = ctx.block().inttoptr(I64, &pcid_addr); let pcid = ctx.block().load(I32, &pcid_ptr); // In-range test via wrapping add + ult: (pcid - 0x8000_0000) < 0x4000_0000. @@ -389,8 +425,7 @@ pub(crate) fn lower_generic_property_get( // (which resolves inherited props correctly). Id tokens always // carry bit 62, so they are never zero. let token_nonnull = ctx.block().icmp_ne(I64, &token, "0"); - let hit_token = ctx.block().and(I1, &is_object, &token_eq); - let hit = ctx.block().and(I1, &hit_token, &token_nonnull); + let hit = ctx.block().and(I1, &token_eq, &token_nonnull); // #6080a: pointer tokens are only trustworthy within the GC epoch they // were primed in. The `@perry_ic_N` global is invisible to every GC @@ -412,12 +447,6 @@ pub(crate) fn lower_generic_property_get( let epoch_ok = ctx.block().or(I1, &is_stamp, &epoch_eq); let hit = ctx.block().and(I1, &hit, &epoch_ok); - let hit_idx = ctx.new_block("pic.hit"); - let miss_idx = ctx.new_block("pic.miss"); - let merge_idx = ctx.new_block("pic.merge"); - let hit_label = ctx.block_label(hit_idx); - let miss_label = ctx.block_label(miss_idx); - let merge_label = ctx.block_label(merge_idx); ctx.block().cond_br(&hit, &hit_label, &miss_label); // PIC hit: bounds-check the cached slot, then direct field load. @@ -434,7 +463,7 @@ pub(crate) fn lower_generic_property_get( // rule; an out-of-bounds slot falls to the miss path, which reads // the overflow map correctly (and records the guard failure — // `record_guard_pass` only fires after the bounds check passes). - let fc_addr = ctx.block().add(I64, &safe_obj_handle, "12"); + let fc_addr = ctx.block().add(I64, &obj_handle, "12"); let fc_ptr = ctx.block().inttoptr(I64, &fc_addr); let fc = ctx.block().load(I32, &fc_ptr); let fc64 = ctx.block().zext(I32, &fc, I64); @@ -486,6 +515,61 @@ pub(crate) fn lower_generic_property_get( // it was a real miss — the feedback heuristics see an unchanged signal // (the site IS polymorphic; only the cost of that changed). ctx.current_block = miss_idx; + // #7883: the guard chain now branches out at three points, so the values + // the polymorphic way compares consult are no longer live on every edge + // into this block — and phi-ing them would drag their materialisation + // (`cset`/`csinc` per value) back onto the hot path, which is the whole + // point of branching. They are recomputed here instead, from the SAME + // memory with no intervening store, so every one is bit-identical to + // what the pre-#7883 flat predicate computed. This block is cold — every + // path out of it either loads a way slot or calls the miss handler. + // + // The small-handle sentinel substitution lives here for the same reason: + // the way compares load `field_count` unconditionally, and a native + // registry-id receiver reaches this block without ever being a pointer. + let cache_addr = ctx.block().ptrtoint(&cache_ref, I64); + let safe_obj_handle = ctx + .block() + .select(I1, &is_real_ptr, I64, &obj_handle, &cache_addr); + let m_gc_type_addr = ctx.block().sub(I64, &safe_obj_handle, "8"); + let m_gc_type_ptr = ctx.block().inttoptr(I64, &m_gc_type_addr); + let m_gc_type = ctx.block().load(I8, &m_gc_type_ptr); + let m_gc_type_ok = ctx.block().icmp_eq(I8, &m_gc_type, "2"); + let is_object = ctx.block().and(I1, &is_real_ptr, &m_gc_type_ok); + let m_magic_addr = ctx.block().add(I64, &safe_obj_handle, "12"); + let m_magic_ptr = ctx.block().inttoptr(I64, &m_magic_addr); + let m_magic = ctx.block().load(I32, &m_magic_ptr); + let m_is_closure = ctx.block().icmp_eq(I32, &m_magic, "1129268819"); + let m_not_closure = ctx.block().xor(I1, &m_is_closure, "true"); + let is_object = ctx.block().and(I1, &is_object, &m_not_closure); + let m_ot_ptr = ctx.block().inttoptr(I64, &safe_obj_handle); + let m_ot = ctx.block().load(I32, &m_ot_ptr); + let m_ot_ok = ctx.block().icmp_eq(I32, &m_ot, "1"); + let is_object = ctx.block().and(I1, &is_object, &m_ot_ok); + let m_res_addr = ctx.block().sub(I64, &safe_obj_handle, "6"); + let m_res_ptr = ctx.block().inttoptr(I64, &m_res_addr); + let m_res = ctx.block().load(crate::types::I16, &m_res_ptr); + let m_has_desc = ctx.block().and(crate::types::I16, &m_res, "2048"); + let m_no_desc = ctx.block().icmp_eq(crate::types::I16, &m_has_desc, "0"); + let is_object = ctx.block().and(I1, &is_object, &m_no_desc); + let m_keys_addr = ctx.block().add(I64, &safe_obj_handle, "16"); + let m_keys_ptr = ctx.block().inttoptr(I64, &m_keys_addr); + let m_keys = ctx.block().load(I64, &m_keys_ptr); + let m_pcid_addr = ctx.block().add(I64, &safe_obj_handle, "8"); + let m_pcid_ptr = ctx.block().inttoptr(I64, &m_pcid_addr); + let m_pcid = ctx.block().load(I32, &m_pcid_ptr); + let m_pcid_rel = ctx.block().add(I32, &m_pcid, "-2147483648"); + let m_is_stamp = ctx.block().icmp_ult(I32, &m_pcid_rel, "1073741824"); + let m_pcid64 = ctx.block().zext(I32, &m_pcid, I64); + let m_id_token = ctx.block().or(I64, &m_pcid64, "4611686018427387904"); + let token = ctx + .block() + .select(I1, &m_is_stamp, I64, &m_id_token, &m_keys); + let token_nonnull = ctx.block().icmp_ne(I64, &token, "0"); + let m_cache_epoch_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "2")]); + let m_cache_epoch = ctx.block().load(I64, &m_cache_epoch_ptr); + let m_live_epoch = ctx.block().load(I64, "@PERRY_IC_EPOCH"); + let epoch_eq = ctx.block().icmp_eq(I64, &m_cache_epoch, &m_live_epoch); crate::expr::emit_typed_feedback_record_call( ctx.block(), "js_typed_feedback_record_guard_fail", diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index d75d2c484b..92be586635 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -360,3 +360,172 @@ mod nested_namespace_members { ); } } + +/// #7883: the inline PIC's guard chain is a chain of BRANCHES, not one flat +/// `and`, so a presence assertion on the individual predicates is no longer +/// evidence of anything — hard-wiring any of the branches to `true` leaves +/// every predicate in the IR as dead code and a "the mask is emitted" test +/// stays green (round 5's first sabotage failed exactly this way). +/// +/// This walks the CFG **backwards** from the block that performs the raw +/// inline slot load to the PIC entry, and requires that +/// +/// 1. every edge on that path is the **true** edge of a `cond_br` +/// (so swapping a branch's successors turns it red), and +/// 2. the transitive def chain of those branch conditions contains every +/// guard the raw load depends on for safety (so replacing any condition +/// with a constant, or deleting a predicate, turns it red). +#[test] +fn generic_property_get_slot_load_is_reached_only_through_every_guard() { + let ir = emit(false, None); + + // Register names restart at %r1 in every function, so the walk MUST be + // scoped to one function or the def map silently resolves a condition to + // an identically-named register in a different body (this test read a + // string-handle `ptrtoint` as the receiver-tag test before it was fixed). + let func = ir + .split("\ndefine ") + .find(|f| f.contains("pic.hit.load")) + .unwrap_or_else(|| panic!("no function contains a PIC hit load:\n{ir}")) + .to_string(); + + let mut blocks: Vec<(String, Vec)> = Vec::new(); + let mut cur: Option<(String, Vec)> = None; + for line in func.lines() { + let t = line.trim_end(); + if let Some(lbl) = t.strip_suffix(':') { + if !lbl.is_empty() && !t.starts_with(' ') && !t.starts_with('\t') { + if let Some(b) = cur.take() { + blocks.push(b); + } + cur = Some((lbl.to_string(), Vec::new())); + continue; + } + } + if let Some((_, body)) = cur.as_mut() { + body.push(t.to_string()); + } + } + if let Some(b) = cur.take() { + blocks.push(b); + } + let load_label = blocks + .iter() + .find(|(l, _)| l.starts_with("pic.hit.load")) + .map(|(l, _)| l.clone()) + .unwrap_or_else(|| panic!("no `pic.hit.load` block:\n{func}")); + + let mut defs: std::collections::HashMap = std::collections::HashMap::new(); + for (_, body) in &blocks { + for l in body { + if let Some((lhs, rhs)) = l.trim().split_once(" = ") { + if lhs.starts_with('%') { + defs.insert(lhs.to_string(), rhs.to_string()); + } + } + } + } + + // Backwards walk to the entry block, collecting the condition of every + // `cond_br` whose TRUE edge we arrived on. + let mut conds: Vec = Vec::new(); + let mut at = load_label.clone(); + let mut steps = 0; + loop { + steps += 1; + assert!(steps < 32, "runaway CFG walk at `{at}`:\n{func}"); + let preds: Vec<&(String, Vec)> = blocks + .iter() + .filter(|(_, body)| { + body.iter().any(|l| { + l.trim_start().starts_with("br ") && l.contains(&format!("label %{at}")) + }) + }) + .collect(); + if preds.is_empty() { + break; // reached the entry block + } + assert_eq!( + preds.len(), + 1, + "the guard chain must be a chain — `{at}` has {} predecessors:\n{func}", + preds.len() + ); + let (pred_label, pred_body) = preds[0]; + let term = pred_body + .iter() + .rev() + .find(|l| l.trim_start().starts_with("br ")) + .unwrap_or_else(|| panic!("`{pred_label}` has no terminator:\n{func}")); + let t = term.trim(); + if let Some(rest) = t.strip_prefix("br i1 ") { + let parts: Vec<&str> = rest.split(", ").collect(); + assert_eq!(parts.len(), 3, "malformed cond_br in `{pred_label}`: {t}"); + let cond = parts[0].to_string(); + let true_target = parts[1].trim_start_matches("label %").to_string(); + assert_eq!( + true_target, at, + "`{pred_label}` must reach `{at}` on its TRUE edge — a swapped \ + cond_br would run the inline slot load when the guard FAILS:\n{t}" + ); + assert!( + cond.starts_with('%'), + "`{pred_label}`'s branch condition is the constant `{cond}` — the \ + guard decides nothing:\n{func}" + ); + conds.push(cond); + } + at = pred_label.clone(); + } + assert!( + conds.len() >= 5, + "expected at least five guard branches between the PIC entry and the \ + inline slot load, found {}: {conds:?}\n{func}", + conds.len() + ); + + // Transitive def closure of every collected condition. + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut reached: Vec = Vec::new(); + let mut work = conds.clone(); + while let Some(v) = work.pop() { + if !seen.insert(v.clone()) { + continue; + } + let Some(rhs) = defs.get(&v) else { continue }; + reached.push(rhs.clone()); + let chars: Vec = rhs.chars().collect(); + let mut i = 0; + while i < chars.len() { + if chars[i] == '%' { + let mut j = i + 1; + while j < chars.len() + && (chars[j].is_alphanumeric() || chars[j] == '.' || chars[j] == '_') + { + j += 1; + } + work.push(chars[i..j].iter().collect()); + i = j; + } else { + i += 1; + } + } + } + let chain = reached.join("\n"); + for (needle, what) in [ + ("32765", "the POINTER/STRING receiver-tag test"), + ("1048575", "the small-handle (native registry id) test"), + ("icmp eq i8", "the GcHeader obj_type == GC_TYPE_OBJECT test"), + ("1129268819", "the CLOSURE_MAGIC test"), + ("2048", "the OBJ_FLAG_HAS_DESCRIPTORS test"), + ("@PERRY_IC_EPOCH", "the read-PIC epoch gate"), + ("@perry_ic_", "the per-site cached shape-token compare"), + ] { + assert!( + chain.contains(needle), + "the inline slot load must be gated on {what}, but no branch \ + condition on the path to `{load_label}` depends on it.\n\ + conditions: {conds:?}\nreached def chain:\n{chain}\n\nIR:\n{func}" + ); + } +}