diff --git a/changelog.d/7887-pget-guard-branches.md b/changelog.d/7887-pget-guard-branches.md new file mode 100644 index 0000000000..83082d7f41 --- /dev/null +++ b/changelog.d/7887-pget-guard-branches.md @@ -0,0 +1,34 @@ +**`perf(codegen)`: the generic property-get guard branches instead of AND-ing eight predicates — `interp` 0.843 → 0.784 s, `iso_miss` 1.231 → 1.178 s.** + +The inline monomorphic-IC diamond built one flat `hit` predicate out of eight tests, so LLVM +if-converted the whole region: every receiver executed every load and every compare — +including the two epoch loads — even after the first test had already decided the answer. It +also tested the two *rare* receiver tags (SSO, INT32 class-ref) before the pointer tag, +putting two constant materialisations, two compares and two branches in front of every real +object read. + +The chain now branches — `pget.recv_ok` → `pic.recv_hdr` → `pic.token` → `pic.hit` — and the +POINTER/STRING tag test goes first, with SSO/INT32 discriminated in a cold `pget.recv_other` +(the three tag classes are pairwise disjoint, so the order is free). The four header +predicates stay flat on purpose: they are four loads from the same two cache lines and LLVM +fuses their compares into one `ccmp` chain, which beats four branches. `pic.miss` recomputes +what the polymorphic-way compares need from the same memory rather than taking phis, which +would have dragged their `cset`/`csinc` materialisation back onto the hot path. + +Semantics are unchanged — every predicate is still checked, with control flow instead of data +flow. `evalNode`'s monomorphic hit path drops from 58 to 46 aarch64 instructions before the +field load. + +Measured on the quiet M1 mini, best-of-5, exit-checked, 19-program corpus, both arms linking +the same runtime archives: `interp` −7.0%, `iso_miss` −4.3%, `cycles` −2.8%, `pipeline` +−1.9%, `churn_read` −1.8%, `deeplist`/`retain_wide1` −1.3%. Six binaries compile +byte-identical and set the run's noise floor at ±0.6%. + +The guard test walks the def chain rather than asserting presence: it walks the CFG backwards +from the block performing the raw slot load, requires every edge on that path to be the +**true** edge of a `cond_br`, and takes the transitive def closure of those conditions, +requiring each guard to be reachable from a branch condition. Three sabotages were run and all +three went red (constant branch condition, swapped `cond_br` edges, deleted predicate) — a +presence assertion catches none of the first two. The walk must be scoped to a single +function: register names restart at `%r1` in every body, so a global def map resolved the +receiver-tag condition to an unrelated `ptrtoint` in another function. diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 83c2938625..0cd4adaa06 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -128,21 +128,19 @@ pub(crate) fn lower_generic_property_get( // `js_object_get_field_by_name_f64` runtime entry (which // handles `.length` directly from the NaN-box length // byte and returns `undefined` for other keys). - let is_sso = ctx.block().icmp_eq(I64, &obj_tag, "32761"); // 0x7FF9 - // v0.5.747: INT32-tagged class refs (top16 == 0x7FFE) used - // as PropertyGet receivers. Pre-fix these fell through to - // the invalid-recv path (returning undefined) because the - // 0xFFFD-masked tag check (0x7FFE & 0xFFFD = 0x7FFC, not - // 0x7FFD) treated them as non-pointer values. Drizzle's - // `is(value, type)` chain depends on `Cls.kind` reads through - // an Any-typed local. Refs #420 / #618 followup. - // - // Note: this also catches plain int32 numeric values (e.g. - // `(42).property`). The runtime helper's INT32-tag arm at - // js_object_get_field_by_name returns undefined for any - // class_id not registered in CLASS_DYNAMIC_PROPS, matching - // the previous behavior — pure ints have no static fields. - let is_int32_class = ctx.block().icmp_eq(I64, &obj_tag, "32766"); // 0x7FFE + // v0.5.747: INT32-tagged class refs (top16 == 0x7FFE) used + // as PropertyGet receivers. Pre-fix these fell through to + // the invalid-recv path (returning undefined) because the + // 0xFFFD-masked tag check (0x7FFE & 0xFFFD = 0x7FFC, not + // 0x7FFD) treated them as non-pointer values. Drizzle's + // `is(value, type)` chain depends on `Cls.kind` reads through + // an Any-typed local. Refs #420 / #618 followup. + // + // Note: this also catches plain int32 numeric values (e.g. + // `(42).property`). The runtime helper's INT32-tag arm at + // js_object_get_field_by_name returns undefined for any + // class_id not registered in CLASS_DYNAMIC_PROPS, matching + // the previous behavior — pure ints have no static fields. let obj_tag_masked = ctx.block().and(I64, &obj_tag, "65533"); // 0xFFFD let is_valid = ctx.block().icmp_eq(I64, &obj_tag_masked, "32765"); // 0x7FFD let sso_idx = ctx.new_block("pget.recv_sso"); @@ -155,19 +153,28 @@ pub(crate) fn lower_generic_property_get( let invalid_label = ctx.block_label(invalid_idx); let class_ref_label = ctx.block_label(class_ref_idx); let final_merge_label = ctx.block_label(final_merge_idx); - // Three-step branch: first check SSO, then class-ref, then - // pointer-validity. Inverse branches funnel into invalid_idx. - let pic_or_invalid_idx = ctx.new_block("pget.check_ptr"); - let pic_or_invalid_label = ctx.block_label(pic_or_invalid_idx); + // #7883: the POINTER/STRING test goes FIRST, and the two rare tags are + // discriminated in a cold block off its false edge. The three tag classes + // are pairwise disjoint — `is_valid` is `(tag & 0xFFFD) == 0x7FFD`, true + // only for 0x7FFD/0x7FFF, while SSO is 0x7FF9 and an INT32 class ref is + // 0x7FFE — so testing them in any order gives the same routing. The old + // order (SSO, then class-ref, then pointer) put two 16-bit constant + // materialisations, two compares and two branches in front of every real + // object receiver: 13 instructions before the PIC on the path that is + // taken essentially always. Now it is `lshr` + `and` + `cmp` + branch. + let other_idx = ctx.new_block("pget.recv_other"); + let other_label = ctx.block_label(other_idx); let check_class_ref_idx = ctx.new_block("pget.check_class_ref"); let check_class_ref_label = ctx.block_label(check_class_ref_idx); + ctx.block().cond_br(&is_valid, &pic_label, &other_label); + ctx.current_block = other_idx; + let is_sso = ctx.block().icmp_eq(I64, &obj_tag, "32761"); // 0x7FF9 ctx.block() .cond_br(&is_sso, &sso_label, &check_class_ref_label); ctx.current_block = check_class_ref_idx; + let is_int32_class = ctx.block().icmp_eq(I64, &obj_tag, "32766"); // 0x7FFE ctx.block() - .cond_br(&is_int32_class, &class_ref_label, &pic_or_invalid_label); - ctx.current_block = pic_or_invalid_idx; - ctx.block().cond_br(&is_valid, &pic_label, &invalid_label); + .cond_br(&is_int32_class, &class_ref_label, &invalid_label); // Class-ref dispatch: route through the runtime helper which // detects INT32 class-ref bits and consults CLASS_DYNAMIC_PROPS @@ -265,29 +272,48 @@ pub(crate) fn lower_generic_property_get( // // Threshold matches `js_native_call_method`'s small-handle // detection (raw_ptr < 0x100000). + let cache_ref = format!("@{}", cache_name); let is_real_ptr = ctx.block().icmp_ugt(I64, &obj_handle, "1048575"); // 0x100000 - // Sentinel address: the per-site cache global itself — - // always valid, 16-byte aligned, and its bytes don't - // match GC_TYPE_OBJECT (=2) or an active keys_array, so - // the IC will cleanly miss when we substitute it for a - // small handle. - let cache_ref = format!("@{}", cache_name); - let cache_addr = ctx.block().ptrtoint(&cache_ref, I64); - let safe_obj_handle = ctx - .block() - .select(I1, &is_real_ptr, I64, &obj_handle, &cache_addr); + // #7883: the hit/miss/merge blocks are minted here so the guard chain + // below can BRANCH OUT to the miss on the first failing predicate + // instead of AND-ing eight of them into one flat `hit`. LLVM if-converts + // a flat predicate, so every receiver paid every load and every compare — + // including the two epoch loads — even after the very first one had + // already decided the answer. Each group now ends in its own `cond_br`; + // the miss block reconstructs what the polymorphic-way compares need + // through phis (`false`/`0` on the early-exit edges, which is exactly + // what the flat predicate computed there). + let hit_idx = ctx.new_block("pic.hit"); + let miss_idx = ctx.new_block("pic.miss"); + let merge_idx = ctx.new_block("pic.merge"); + let hit_label = ctx.block_label(hit_idx); + let miss_label = ctx.block_label(miss_idx); + let merge_label = ctx.block_label(merge_idx); + let hdr_idx = ctx.new_block("pic.recv_hdr"); + let hdr_label = ctx.block_label(hdr_idx); + let tok_idx = ctx.new_block("pic.token"); + let tok_label = ctx.block_label(tok_idx); + // Small-handle receivers (native-module registry ids) must never be + // dereferenced. Pre-#7883 they were kept out of the loads by selecting a + // sentinel address and AND-ing `is_real_ptr` into `hit`; the branch does + // the same job without putting a `select` (and the sentinel's address + // materialisation) in front of every real object read. The miss path + // still substitutes the sentinel, because the way compares below load + // `field_count` unconditionally. + // (edge labels are no longer needed: the miss block recomputes.) + ctx.block().cond_br(&is_real_ptr, &hdr_label, &miss_label); + ctx.current_block = hdr_idx; // GcHeader sits 8 bytes before the user pointer; obj_type is the // first u8 (GC_TYPE_OBJECT=2). Cost: 1 sub + 1 load i8 + 1 cmp // i8 + 1 and i1 — the cond_br's `is_object` operand is folded // into the existing branch instruction by LLVM. Branch-predicted // taken since real PropertyGet receivers are objects. - let gc_type_addr = ctx.block().sub(I64, &safe_obj_handle, "8"); + let gc_type_addr = ctx.block().sub(I64, &obj_handle, "8"); let gc_type_ptr = ctx.block().inttoptr(I64, &gc_type_addr); let gc_type = ctx.block().load(I8, &gc_type_ptr); - let gc_type_ok = ctx.block().icmp_eq(I8, &gc_type, "2"); - let is_object = ctx.block().and(I1, &is_real_ptr, &gc_type_ok); + let is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); // Issue #618: closures share GC_TYPE_OBJECT but their offset+16 // is a capture slot, not `keys_array`. The PIC's keys_val == @@ -299,7 +325,7 @@ pub(crate) fn lower_generic_property_get( // `js_object_get_field_ic_miss` → `js_object_get_field_by_name`, // which dispatches closure dynamic-prop reads via the // `CLOSURE_DYNAMIC_PROPS` side-table. - let magic_addr = ctx.block().add(I64, &safe_obj_handle, "12"); + let magic_addr = ctx.block().add(I64, &obj_handle, "12"); let magic_ptr = ctx.block().inttoptr(I64, &magic_addr); let magic_val = ctx.block().load(I32, &magic_ptr); // CLOSURE_MAGIC = 0x434C4F53 (4 bytes "CLOS" little-endian). @@ -321,7 +347,7 @@ pub(crate) fn lower_generic_property_get( // Specific repro: `function f(): any { ... return new // RegExp(...) } const r = f(); r.source` — fast path returns // garbage f64 instead of routing through `js_regexp_get_source`. - let object_type_ptr = ctx.block().inttoptr(I64, &safe_obj_handle); + let object_type_ptr = ctx.block().inttoptr(I64, &obj_handle); let object_type = ctx.block().load(I32, &object_type_ptr); let object_type_ok = ctx.block().icmp_eq(I32, &object_type, "1"); let is_object = ctx.block().and(I1, &is_object, &object_type_ok); @@ -338,15 +364,25 @@ pub(crate) fn lower_generic_property_get( // descriptors) whenever it is set. Mirrors the guard in // `class_field_inline_guard.rs`. Cost: 1 sub + load i16 + and + cmp, // folded into the existing `hit` cond_br. - let reserved_addr = ctx.block().sub(I64, &safe_obj_handle, "6"); + let reserved_addr = ctx.block().sub(I64, &obj_handle, "6"); let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); let reserved = ctx.block().load(crate::types::I16, &reserved_ptr); let has_desc = ctx.block().and(crate::types::I16, &reserved, "2048"); // OBJ_FLAG_HAS_DESCRIPTORS (0x800) let no_desc = ctx.block().icmp_eq(crate::types::I16, &has_desc, "0"); let is_object = ctx.block().and(I1, &is_object, &no_desc); + // #7883: first exit. The four header predicates above are kept as one + // flat `and` on purpose — they are four loads from the SAME two cache + // lines and LLVM fuses their compares into a `ccmp` chain, which is + // cheaper than four branches. What was NOT worth folding is everything + // below: the keys load, the token select and the two epoch loads all + // hang off the same predicate, so a non-object receiver used to execute + // them before the flat `hit` could reject it. + ctx.block().cond_br(&is_object, &tok_label, &miss_label); + ctx.current_block = tok_idx; + // Load obj->keys_array at offset 16 of ObjectHeader. - let keys_addr = ctx.block().add(I64, &safe_obj_handle, "16"); + let keys_addr = ctx.block().add(I64, &obj_handle, "16"); let keys_ptr_p = ctx.block().inttoptr(I64, &keys_addr); let keys_val = ctx.block().load(I64, &keys_ptr_p); @@ -361,7 +397,7 @@ pub(crate) fn lower_generic_property_get( // two token kinds can never collide numerically — one compare, no // discriminant word. `parent_class_id` is a u32 at offset 8 on every // target (the four leading u32s precede the pointer fields). - let pcid_addr = ctx.block().add(I64, &safe_obj_handle, "8"); + let pcid_addr = ctx.block().add(I64, &obj_handle, "8"); let pcid_ptr = ctx.block().inttoptr(I64, &pcid_addr); let pcid = ctx.block().load(I32, &pcid_ptr); // In-range test via wrapping add + ult: (pcid - 0x8000_0000) < 0x4000_0000. @@ -389,8 +425,7 @@ pub(crate) fn lower_generic_property_get( // (which resolves inherited props correctly). Id tokens always // carry bit 62, so they are never zero. let token_nonnull = ctx.block().icmp_ne(I64, &token, "0"); - let hit_token = ctx.block().and(I1, &is_object, &token_eq); - let hit = ctx.block().and(I1, &hit_token, &token_nonnull); + let hit = ctx.block().and(I1, &token_eq, &token_nonnull); // #6080a: pointer tokens are only trustworthy within the GC epoch they // were primed in. The `@perry_ic_N` global is invisible to every GC @@ -412,12 +447,6 @@ pub(crate) fn lower_generic_property_get( let epoch_ok = ctx.block().or(I1, &is_stamp, &epoch_eq); let hit = ctx.block().and(I1, &hit, &epoch_ok); - let hit_idx = ctx.new_block("pic.hit"); - let miss_idx = ctx.new_block("pic.miss"); - let merge_idx = ctx.new_block("pic.merge"); - let hit_label = ctx.block_label(hit_idx); - let miss_label = ctx.block_label(miss_idx); - let merge_label = ctx.block_label(merge_idx); ctx.block().cond_br(&hit, &hit_label, &miss_label); // PIC hit: bounds-check the cached slot, then direct field load. @@ -434,7 +463,7 @@ pub(crate) fn lower_generic_property_get( // rule; an out-of-bounds slot falls to the miss path, which reads // the overflow map correctly (and records the guard failure — // `record_guard_pass` only fires after the bounds check passes). - let fc_addr = ctx.block().add(I64, &safe_obj_handle, "12"); + let fc_addr = ctx.block().add(I64, &obj_handle, "12"); let fc_ptr = ctx.block().inttoptr(I64, &fc_addr); let fc = ctx.block().load(I32, &fc_ptr); let fc64 = ctx.block().zext(I32, &fc, I64); @@ -486,6 +515,61 @@ pub(crate) fn lower_generic_property_get( // it was a real miss — the feedback heuristics see an unchanged signal // (the site IS polymorphic; only the cost of that changed). ctx.current_block = miss_idx; + // #7883: the guard chain now branches out at three points, so the values + // the polymorphic way compares consult are no longer live on every edge + // into this block — and phi-ing them would drag their materialisation + // (`cset`/`csinc` per value) back onto the hot path, which is the whole + // point of branching. They are recomputed here instead, from the SAME + // memory with no intervening store, so every one is bit-identical to + // what the pre-#7883 flat predicate computed. This block is cold — every + // path out of it either loads a way slot or calls the miss handler. + // + // The small-handle sentinel substitution lives here for the same reason: + // the way compares load `field_count` unconditionally, and a native + // registry-id receiver reaches this block without ever being a pointer. + let cache_addr = ctx.block().ptrtoint(&cache_ref, I64); + let safe_obj_handle = ctx + .block() + .select(I1, &is_real_ptr, I64, &obj_handle, &cache_addr); + let m_gc_type_addr = ctx.block().sub(I64, &safe_obj_handle, "8"); + let m_gc_type_ptr = ctx.block().inttoptr(I64, &m_gc_type_addr); + let m_gc_type = ctx.block().load(I8, &m_gc_type_ptr); + let m_gc_type_ok = ctx.block().icmp_eq(I8, &m_gc_type, "2"); + let is_object = ctx.block().and(I1, &is_real_ptr, &m_gc_type_ok); + let m_magic_addr = ctx.block().add(I64, &safe_obj_handle, "12"); + let m_magic_ptr = ctx.block().inttoptr(I64, &m_magic_addr); + let m_magic = ctx.block().load(I32, &m_magic_ptr); + let m_is_closure = ctx.block().icmp_eq(I32, &m_magic, "1129268819"); + let m_not_closure = ctx.block().xor(I1, &m_is_closure, "true"); + let is_object = ctx.block().and(I1, &is_object, &m_not_closure); + let m_ot_ptr = ctx.block().inttoptr(I64, &safe_obj_handle); + let m_ot = ctx.block().load(I32, &m_ot_ptr); + let m_ot_ok = ctx.block().icmp_eq(I32, &m_ot, "1"); + let is_object = ctx.block().and(I1, &is_object, &m_ot_ok); + let m_res_addr = ctx.block().sub(I64, &safe_obj_handle, "6"); + let m_res_ptr = ctx.block().inttoptr(I64, &m_res_addr); + let m_res = ctx.block().load(crate::types::I16, &m_res_ptr); + let m_has_desc = ctx.block().and(crate::types::I16, &m_res, "2048"); + let m_no_desc = ctx.block().icmp_eq(crate::types::I16, &m_has_desc, "0"); + let is_object = ctx.block().and(I1, &is_object, &m_no_desc); + let m_keys_addr = ctx.block().add(I64, &safe_obj_handle, "16"); + let m_keys_ptr = ctx.block().inttoptr(I64, &m_keys_addr); + let m_keys = ctx.block().load(I64, &m_keys_ptr); + let m_pcid_addr = ctx.block().add(I64, &safe_obj_handle, "8"); + let m_pcid_ptr = ctx.block().inttoptr(I64, &m_pcid_addr); + let m_pcid = ctx.block().load(I32, &m_pcid_ptr); + let m_pcid_rel = ctx.block().add(I32, &m_pcid, "-2147483648"); + let m_is_stamp = ctx.block().icmp_ult(I32, &m_pcid_rel, "1073741824"); + let m_pcid64 = ctx.block().zext(I32, &m_pcid, I64); + let m_id_token = ctx.block().or(I64, &m_pcid64, "4611686018427387904"); + let token = ctx + .block() + .select(I1, &m_is_stamp, I64, &m_id_token, &m_keys); + let token_nonnull = ctx.block().icmp_ne(I64, &token, "0"); + let m_cache_epoch_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "2")]); + let m_cache_epoch = ctx.block().load(I64, &m_cache_epoch_ptr); + let m_live_epoch = ctx.block().load(I64, "@PERRY_IC_EPOCH"); + let epoch_eq = ctx.block().icmp_eq(I64, &m_cache_epoch, &m_live_epoch); crate::expr::emit_typed_feedback_record_call( ctx.block(), "js_typed_feedback_record_guard_fail", diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index d75d2c484b..92be586635 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -360,3 +360,172 @@ mod nested_namespace_members { ); } } + +/// #7883: the inline PIC's guard chain is a chain of BRANCHES, not one flat +/// `and`, so a presence assertion on the individual predicates is no longer +/// evidence of anything — hard-wiring any of the branches to `true` leaves +/// every predicate in the IR as dead code and a "the mask is emitted" test +/// stays green (round 5's first sabotage failed exactly this way). +/// +/// This walks the CFG **backwards** from the block that performs the raw +/// inline slot load to the PIC entry, and requires that +/// +/// 1. every edge on that path is the **true** edge of a `cond_br` +/// (so swapping a branch's successors turns it red), and +/// 2. the transitive def chain of those branch conditions contains every +/// guard the raw load depends on for safety (so replacing any condition +/// with a constant, or deleting a predicate, turns it red). +#[test] +fn generic_property_get_slot_load_is_reached_only_through_every_guard() { + let ir = emit(false, None); + + // Register names restart at %r1 in every function, so the walk MUST be + // scoped to one function or the def map silently resolves a condition to + // an identically-named register in a different body (this test read a + // string-handle `ptrtoint` as the receiver-tag test before it was fixed). + let func = ir + .split("\ndefine ") + .find(|f| f.contains("pic.hit.load")) + .unwrap_or_else(|| panic!("no function contains a PIC hit load:\n{ir}")) + .to_string(); + + let mut blocks: Vec<(String, Vec)> = Vec::new(); + let mut cur: Option<(String, Vec)> = None; + for line in func.lines() { + let t = line.trim_end(); + if let Some(lbl) = t.strip_suffix(':') { + if !lbl.is_empty() && !t.starts_with(' ') && !t.starts_with('\t') { + if let Some(b) = cur.take() { + blocks.push(b); + } + cur = Some((lbl.to_string(), Vec::new())); + continue; + } + } + if let Some((_, body)) = cur.as_mut() { + body.push(t.to_string()); + } + } + if let Some(b) = cur.take() { + blocks.push(b); + } + let load_label = blocks + .iter() + .find(|(l, _)| l.starts_with("pic.hit.load")) + .map(|(l, _)| l.clone()) + .unwrap_or_else(|| panic!("no `pic.hit.load` block:\n{func}")); + + let mut defs: std::collections::HashMap = std::collections::HashMap::new(); + for (_, body) in &blocks { + for l in body { + if let Some((lhs, rhs)) = l.trim().split_once(" = ") { + if lhs.starts_with('%') { + defs.insert(lhs.to_string(), rhs.to_string()); + } + } + } + } + + // Backwards walk to the entry block, collecting the condition of every + // `cond_br` whose TRUE edge we arrived on. + let mut conds: Vec = Vec::new(); + let mut at = load_label.clone(); + let mut steps = 0; + loop { + steps += 1; + assert!(steps < 32, "runaway CFG walk at `{at}`:\n{func}"); + let preds: Vec<&(String, Vec)> = blocks + .iter() + .filter(|(_, body)| { + body.iter().any(|l| { + l.trim_start().starts_with("br ") && l.contains(&format!("label %{at}")) + }) + }) + .collect(); + if preds.is_empty() { + break; // reached the entry block + } + assert_eq!( + preds.len(), + 1, + "the guard chain must be a chain — `{at}` has {} predecessors:\n{func}", + preds.len() + ); + let (pred_label, pred_body) = preds[0]; + let term = pred_body + .iter() + .rev() + .find(|l| l.trim_start().starts_with("br ")) + .unwrap_or_else(|| panic!("`{pred_label}` has no terminator:\n{func}")); + let t = term.trim(); + if let Some(rest) = t.strip_prefix("br i1 ") { + let parts: Vec<&str> = rest.split(", ").collect(); + assert_eq!(parts.len(), 3, "malformed cond_br in `{pred_label}`: {t}"); + let cond = parts[0].to_string(); + let true_target = parts[1].trim_start_matches("label %").to_string(); + assert_eq!( + true_target, at, + "`{pred_label}` must reach `{at}` on its TRUE edge — a swapped \ + cond_br would run the inline slot load when the guard FAILS:\n{t}" + ); + assert!( + cond.starts_with('%'), + "`{pred_label}`'s branch condition is the constant `{cond}` — the \ + guard decides nothing:\n{func}" + ); + conds.push(cond); + } + at = pred_label.clone(); + } + assert!( + conds.len() >= 5, + "expected at least five guard branches between the PIC entry and the \ + inline slot load, found {}: {conds:?}\n{func}", + conds.len() + ); + + // Transitive def closure of every collected condition. + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut reached: Vec = Vec::new(); + let mut work = conds.clone(); + while let Some(v) = work.pop() { + if !seen.insert(v.clone()) { + continue; + } + let Some(rhs) = defs.get(&v) else { continue }; + reached.push(rhs.clone()); + let chars: Vec = rhs.chars().collect(); + let mut i = 0; + while i < chars.len() { + if chars[i] == '%' { + let mut j = i + 1; + while j < chars.len() + && (chars[j].is_alphanumeric() || chars[j] == '.' || chars[j] == '_') + { + j += 1; + } + work.push(chars[i..j].iter().collect()); + i = j; + } else { + i += 1; + } + } + } + let chain = reached.join("\n"); + for (needle, what) in [ + ("32765", "the POINTER/STRING receiver-tag test"), + ("1048575", "the small-handle (native registry id) test"), + ("icmp eq i8", "the GcHeader obj_type == GC_TYPE_OBJECT test"), + ("1129268819", "the CLOSURE_MAGIC test"), + ("2048", "the OBJ_FLAG_HAS_DESCRIPTORS test"), + ("@PERRY_IC_EPOCH", "the read-PIC epoch gate"), + ("@perry_ic_", "the per-site cached shape-token compare"), + ] { + assert!( + chain.contains(needle), + "the inline slot load must be gated on {what}, but no branch \ + condition on the path to `{load_label}` depends on it.\n\ + conditions: {conds:?}\nreached def chain:\n{chain}\n\nIR:\n{func}" + ); + } +}