From b44a30bc7b39b296f0617aecb883f35cd3573526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 26 Jul 2026 18:52:03 +0200 Subject: [PATCH 1/3] perf(codegen): inline small hot (in-loop) functions via inlinehint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After native Math.imul + typed-array-param reads (#6850), the only residual gap of a tight integer-math kernel vs V8 is function-call overhead: V8 inlines a small hot function into its loop; Perry left it out-of-line. A NaN-boxed bit-mixer (`mix`, ~10 statements) costs ~800 in LLVM's inline model once GC shadow-frame calls + typed-array reads + double<->i32 marshaling are counted — well above the base -O3 threshold — so -O3 kept it as a call. Bias, don't force. Perry keeps `alwaysinline` only for the genuinely tiny (<= 8 statements). This adds a distinct `inlinehint` path for functions that are: - small (9..=SIZE_CAP statements, default cap 20), - called from inside a loop (a whole-module HIR pre-pass collects callee ids with an in-loop call site — approximates "hot" for an AOT compiler), AND - called from few total sites (default <= 4). The linker raises `-inlinehint-threshold` (default 850) so the hinted kernels actually inline; this lifts LLVM's ceiling ONLY for functions Perry stamped `inlinehint` — every other function keeps the base -O3 threshold, so cold code is untouched. Anti-bloat is the whole point, and it rests on two backstops: 1. The loop gate: a function only ever called from cold/straight-line code is never hinted, so cold utility functions can't bloat the binary. 2. The call-site cap: `-inlinehint-threshold` raises the ceiling at EVERY call site of a hinted callee (LLVM can't tell hot from cold per-site through a function attribute), so a hot kernel also called from many cold sites would be duplicated at all of them. Capping total call sites bounds the duplication. Measured binary-size delta (flag OFF vs ON), byte-precise __text section: - real programs (5 test-files): 0 hints, 0.000% delta; - broadly-called helper (300 fns x 40 cold sites): excluded by cap, 0%; - large programs (>6MB IR compile at -Os): flag inert, 0%; - realistic hot-kernel density: +0.34% (25 kernels), +0.79% (50), +1.57% (100); an all-kernels synthetic (250) reaches +3.86% — bounded by the cap (without it the same program's optimized IR grew 5.7x). `mix` 40M-call microbench (min-of-many, contended box load ~27): OFF 1024ms -> ON 904ms. Structural proof (load-independent): the caller loop no longer emits a `bl` to `mix` (0 vs 2 BR26 relocations), and `mix` carries `inlinehint`. Gated behind PERRY_INLINE_HOT_SMALL (default on); PERRY_INLINE_HOT_SMALL_CAP / _THRESHOLD / _MAX_SITES tune the window, hint threshold, and call-site cap. All four are folded into the object cache key. Existing `noinline` cases (try/setjmp/volatile) are respected via to_ir's has_try-first attribute precedence. Adds test_gap_inline_hot_small.ts (byte-identical to Node ON/OFF and under PERRY_GC_FORCE_EVACUATE=1). Stacks on #6850 (merged). Claude-Session: https://claude.ai/code/session_01UGDwjukzhowLDJYsMPFwMv --- crates/perry-codegen/src/codegen/function.rs | 20 ++ crates/perry-codegen/src/codegen/helpers.rs | 75 +++++ crates/perry-codegen/src/codegen/mod.rs | 11 +- crates/perry-codegen/src/codegen/opts.rs | 8 + .../src/collectors/hot_callees.rs | 298 ++++++++++++++++++ crates/perry-codegen/src/collectors/mod.rs | 2 + crates/perry-codegen/src/function.rs | 17 + crates/perry-codegen/src/linker.rs | 14 + .../src/commands/compile/object_cache.rs | 27 ++ .../object_cache/object_cache_tests.rs | 5 + test-files/test_gap_inline_hot_small.ts | 73 +++++ 11 files changed, 549 insertions(+), 1 deletion(-) create mode 100644 crates/perry-codegen/src/collectors/hot_callees.rs create mode 100644 test-files/test_gap_inline_hot_small.ts diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 37f58e19d9..4b973bbab8 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -15,6 +15,7 @@ use crate::strings::StringPool; use crate::types::{LlvmType, DOUBLE, I1, I32, I64, I8, PTR}; use super::helpers::shadow_stack_enabled; +use super::helpers::{inline_hot_small_enabled, inline_hot_small_size_cap, INLINE_HOT_SMALL_MIN}; use super::opts::CrossModuleCtx; use super::typed_abi::{ emit_typed_arg_guard, emit_typed_arg_to_raw, generic_function_body_name, lower_typed_f64_body, @@ -394,6 +395,25 @@ pub(super) fn compile_function( if f.body.len() <= 8 && !f.is_async && !f.is_generator && !f.was_plain_async { lf.force_inline = true; } + // Inline-hot-small (PERRY_INLINE_HOT_SMALL, default ON): bias — do not + // force — LLVM toward inlining a *small* function that has a *hot* (in-loop) + // call site. `inlinehint` only raises LLVM's inline threshold for this + // callee; its `-O3` growth budget still refuses cold/oversized inlines, so + // cold utility functions never bloat the binary (that is the anti-bloat + // property, proved by the binary-size gate). We skip `alwaysinline` + // functions (the hint would be redundant), async/generator forms, and rely + // on `to_ir`'s `has_try`-first attribute precedence to keep any function + // whose body later turns out to need `noinline` (try/setjmp/volatile) out. + if !lf.force_inline + && inline_hot_small_enabled() + && (INLINE_HOT_SMALL_MIN..=inline_hot_small_size_cap()).contains(&f.body.len()) + && !f.is_async + && !f.is_generator + && !f.was_plain_async + && cross_module.hot_loop_callees.contains(&f.id) + { + lf.inline_hint = true; + } let _ = lf.create_block("entry"); let mut boxed_vars = module_boxed_vars.clone(); diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index a4f59b09e4..0674e3f27d 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -74,6 +74,81 @@ pub(super) fn shadow_stack_enabled() -> bool { }) } +/// Inline-hot-small gate. Default ON. When enabled, small functions +/// (`INLINE_HOT_SMALL_MIN ..= SIZE_CAP` statements) that have ≥1 call site +/// inside a loop get LLVM's `inlinehint` — a *bounded* nudge that raises the +/// inline threshold for that callee while LLVM's `-O3` growth budget stays the +/// backstop (unlike `alwaysinline`, which is unconditional). Disable with +/// `PERRY_INLINE_HOT_SMALL=0`/`off`/`false` for bisection / binary-size A/B. +pub(crate) fn inline_hot_small_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + !matches!( + std::env::var("PERRY_INLINE_HOT_SMALL").as_deref(), + Ok("0") | Ok("off") | Ok("false") + ) + }) +} + +/// LLVM `-inlinehint-threshold` used for `inlinehint`-marked callees when the +/// feature is on. The default hint threshold (325) is too low for Perry's +/// NaN-boxed kernels — a ~10-statement bit-mixer costs ~800 in LLVM's inline +/// model once GC shadow-frame calls + typed-array reads + double↔i32 marshaling +/// are counted — so we raise it. Critically this only affects functions Perry +/// stamped `inlinehint` (the small + in-loop-callsite gate); every other +/// function keeps the base `-O3` threshold, so cold code is untouched. +/// Overridable via `PERRY_INLINE_HOT_SMALL_THRESHOLD` for the binary-size A/B. +pub(crate) fn inline_hot_small_hint_threshold() -> u32 { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + std::env::var("PERRY_INLINE_HOT_SMALL_THRESHOLD") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(850) + }) +} + +/// Smallest body-statement count eligible for `inlinehint`. Functions of `<= 8` +/// statements already get unconditional `alwaysinline`, so the hint window +/// starts one above that. +pub(super) const INLINE_HOT_SMALL_MIN: usize = 9; + +/// Largest body-statement count eligible for `inlinehint`. Chosen +/// conservatively and validated against the binary-size regression gate (a +/// larger cap duplicates more code at each hinted site). Overridable via +/// `PERRY_INLINE_HOT_SMALL_CAP` for tuning experiments. +pub(super) fn inline_hot_small_size_cap() -> usize { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + std::env::var("PERRY_INLINE_HOT_SMALL_CAP") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(20) + }) +} + +/// Maximum total (module-wide) direct call sites a function may have and still +/// be hinted. This is the anti-bloat backstop: the raised `-inlinehint-threshold` +/// lifts LLVM's ceiling for a hinted callee at *every* one of its call sites, so +/// without this cap a small hot kernel that is also called from many cold sites +/// would be duplicated at all of them. Capping call sites bounds the added code +/// (≤ this many inlined copies). A tight bit-mixer kernel has 1–2 sites and +/// still qualifies; a broadly-shared helper does not. Overridable via +/// `PERRY_INLINE_HOT_SMALL_MAX_SITES`. +pub(crate) fn inline_hot_small_max_call_sites() -> u32 { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + std::env::var("PERRY_INLINE_HOT_SMALL_MAX_SITES") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(4) + }) +} + pub(super) fn enable_module_init_shadow_frame( func: &mut crate::function::LlFunction, stmts: &[perry_hir::Stmt], diff --git a/crates/perry-codegen/src/codegen/mod.rs b/crates/perry-codegen/src/codegen/mod.rs index 3357053443..307e915e12 100644 --- a/crates/perry-codegen/src/codegen/mod.rs +++ b/crates/perry-codegen/src/codegen/mod.rs @@ -47,7 +47,9 @@ mod closure_collect; mod entry; mod func_registry; mod function; -mod helpers; +// `pub(crate)` so `crate::linker` can read the inline-hot-small policy +// (`inline_hot_small_enabled` / `inline_hot_small_hint_threshold`). +pub(crate) mod helpers; mod i64_spec; mod method; mod method_registry; @@ -1545,6 +1547,13 @@ pub fn compile_module(hir: &HirModule, opts: CompileOptions) -> Result> target_triple: triple.clone(), app_metadata: opts.app_metadata.clone(), module_dispatch: crate::collectors::collect_module_dispatch_facts(hir), + // Inline-hot-small pre-pass (#6850 follow-up): FuncIds with an in-loop + // call site AND few total call sites, so small hot callees can earn + // `inlinehint` while the call-site cap bounds duplication. + hot_loop_callees: crate::collectors::collect_hot_loop_callees( + hir, + crate::codegen::helpers::inline_hot_small_max_call_sites(), + ), clamp3_functions: hir .functions .iter() diff --git a/crates/perry-codegen/src/codegen/opts.rs b/crates/perry-codegen/src/codegen/opts.rs index c005b5cae6..d96b784407 100644 --- a/crates/perry-codegen/src/codegen/opts.rs +++ b/crates/perry-codegen/src/codegen/opts.rs @@ -911,4 +911,12 @@ pub(crate) struct CrossModuleCtx { /// `@__perry_ns_` + populator even when `namespace_entries` /// is empty (side-effect-only modules with no `export`s). pub is_dynamic_import_target: bool, + /// Inline-hot-small pre-pass result: `FuncId`s in THIS module that have + /// ≥1 direct call site inside a loop (`for`/`while`/`do-while`). Consumed + /// by `compile_function` to decide whether a small callee earns LLVM's + /// `inlinehint`. Built once per module via + /// `collectors::collect_hot_loop_callees`. Empty when + /// `PERRY_INLINE_HOT_SMALL` is off (the flag is checked at the decision + /// site, so the set is still populated but simply not consulted). + pub hot_loop_callees: std::collections::HashSet, } diff --git a/crates/perry-codegen/src/collectors/hot_callees.rs b/crates/perry-codegen/src/collectors/hot_callees.rs new file mode 100644 index 0000000000..83c05f7caf --- /dev/null +++ b/crates/perry-codegen/src/collectors/hot_callees.rs @@ -0,0 +1,298 @@ +//! Whole-module pre-pass that finds user functions eligible for the +//! inline-hot-small heuristic: **small** functions that are called from a +//! **loop** and have **few total call sites**. +//! +//! Why all three conditions (see `codegen/function.rs` for the size gate, and +//! `linker.rs` for the raised `-inlinehint-threshold`): +//! +//! * **In a loop** (approx. "hot", since AOT has no profile) — a function only +//! ever called from straight-line / cold code is never hinted, so cold +//! utility functions can't bloat the binary. +//! * **Few call sites** — this is the anti-bloat backstop. `inlinehint` + +//! the raised threshold lifts LLVM's inline ceiling for the callee at *every* +//! one of its call sites (LLVM can't tell hot from cold per-site through a +//! function attribute). A small function called from 1 hot loop **and** 300 +//! cold sites would therefore be duplicated 300×. Capping total call sites +//! bounds the duplication: a hinted function is inlined at most +//! `max_call_sites` times, so the added code is bounded regardless of the +//! raised threshold. A bit-mixer kernel like `mix` has 1 HIR call site, so it +//! qualifies; a shared helper called from dozens of sites does not. +//! +//! Direction of error: **under-inclusion is safe** (a missed call site just +//! forgoes an inlining opportunity or, via undercount, leaves a function +//! eligible when it has slightly more sites than counted — still bounded). We +//! never propagate the in-loop flag into a nested closure body (the closure +//! runs at its own, unknown frequency), so we can't wrongly mark a cold callee +//! hot. The walker covers the common expression containers and uses a +//! non-recursing catch-all for the long tail of runtime-intrinsic variants. + +use std::collections::{HashMap, HashSet}; + +use perry_hir::{CallArg, Expr, Module, Stmt}; + +#[derive(Default)] +struct HotCalleeScan { + /// FuncIds with ≥1 direct call site (`FuncRef` callee) inside a loop. + in_loop: HashSet, + /// Total direct call-site count per FuncId (loop and non-loop). + call_counts: HashMap, +} + +/// Collect the set of `FuncId`s eligible for `inlinehint`: those with ≥1 direct +/// call site inside a loop AND at most `max_call_sites` total direct call sites +/// across the whole module (`init` + every function / constructor / method). +pub fn collect_hot_loop_callees(hir: &Module, max_call_sites: u32) -> HashSet { + let mut scan = HotCalleeScan::default(); + walk_stmts(&hir.init, false, &mut scan); + for f in &hir.functions { + walk_stmts(&f.body, false, &mut scan); + } + for c in &hir.classes { + if let Some(ctor) = &c.constructor { + walk_stmts(&ctor.body, false, &mut scan); + } + for m in &c.methods { + walk_stmts(&m.body, false, &mut scan); + } + } + scan.in_loop + .iter() + .copied() + .filter(|id| scan.call_counts.get(id).copied().unwrap_or(0) <= max_call_sites) + .collect() +} + +fn record_callee(callee: &Expr, in_loop: bool, scan: &mut HotCalleeScan) { + if let Expr::FuncRef(id) = callee { + *scan.call_counts.entry(*id).or_insert(0) += 1; + if in_loop { + scan.in_loop.insert(*id); + } + } +} + +fn walk_stmts(stmts: &[Stmt], in_loop: bool, scan: &mut HotCalleeScan) { + for s in stmts { + walk_stmt(s, in_loop, scan); + } +} + +fn walk_stmt(s: &Stmt, in_loop: bool, scan: &mut HotCalleeScan) { + match s { + Stmt::Let { init: Some(e), .. } => walk_expr(e, in_loop, scan), + Stmt::Let { init: None, .. } => {} + Stmt::Expr(e) | Stmt::Throw(e) => walk_expr(e, in_loop, scan), + Stmt::Return(Some(e)) => walk_expr(e, in_loop, scan), + Stmt::Return(None) => {} + Stmt::If { + condition, + then_branch, + else_branch, + } => { + walk_expr(condition, in_loop, scan); + walk_stmts(then_branch, in_loop, scan); + if let Some(eb) = else_branch { + walk_stmts(eb, in_loop, scan); + } + } + // The dominant guard: everything inside a loop body / its per-iteration + // condition + update is "hot". The `for`-init runs once, so it keeps + // the enclosing `in_loop`. + Stmt::While { condition, body } | Stmt::DoWhile { body, condition } => { + walk_expr(condition, true, scan); + walk_stmts(body, true, scan); + } + Stmt::For { + init, + condition, + update, + body, + } => { + if let Some(init_stmt) = init { + walk_stmt(init_stmt, in_loop, scan); + } + if let Some(c) = condition { + walk_expr(c, true, scan); + } + if let Some(u) = update { + walk_expr(u, true, scan); + } + walk_stmts(body, true, scan); + } + Stmt::Labeled { body, .. } => walk_stmt(body, in_loop, scan), + Stmt::Try { + body, + catch, + finally, + } => { + walk_stmts(body, in_loop, scan); + if let Some(c) = catch { + walk_stmts(&c.body, in_loop, scan); + } + if let Some(f) = finally { + walk_stmts(f, in_loop, scan); + } + } + Stmt::Switch { + discriminant, + cases, + } => { + walk_expr(discriminant, in_loop, scan); + for c in cases { + if let Some(t) = &c.test { + walk_expr(t, in_loop, scan); + } + walk_stmts(&c.body, in_loop, scan); + } + } + Stmt::Break + | Stmt::Continue + | Stmt::LabeledBreak(_) + | Stmt::LabeledContinue(_) + | Stmt::PreallocateBoxes(_) + | Stmt::PreallocateTdzBoxes(_) => {} + } +} + +fn walk_call_args(args: &[CallArg], in_loop: bool, scan: &mut HotCalleeScan) { + for a in args { + match a { + CallArg::Expr(e) | CallArg::Spread(e) => walk_expr(e, in_loop, scan), + } + } +} + +fn walk_expr(e: &Expr, in_loop: bool, scan: &mut HotCalleeScan) { + match e { + // The two forms that resolve to a known local function. Record the + // callee (count + in-loop flag), then keep descending — args can + // themselves contain calls / closures. + Expr::Call { callee, args, .. } => { + record_callee(callee, in_loop, scan); + walk_expr(callee, in_loop, scan); + for a in args { + walk_expr(a, in_loop, scan); + } + } + Expr::CallSpread { callee, args, .. } => { + record_callee(callee, in_loop, scan); + walk_expr(callee, in_loop, scan); + walk_call_args(args, in_loop, scan); + } + + // A closure introduces its own (unknown) invocation frequency: a loop + // that merely *creates* a closure does not make the closure body hot. + // Reset `in_loop` so calls inside the closure are only hot relative to + // loops nested within the closure itself. + Expr::Closure { body, .. } => walk_stmts(body, false, scan), + + // Assignment / update carriers. + Expr::LocalSet(_, value) | Expr::GlobalSet(_, value) => walk_expr(value, in_loop, scan), + + // Arithmetic / logical / comparison trees. + Expr::Binary { left, right, .. } + | Expr::Compare { left, right, .. } + | Expr::Logical { left, right, .. } => { + walk_expr(left, in_loop, scan); + walk_expr(right, in_loop, scan); + } + Expr::Unary { operand, .. } + | Expr::Void(operand) + | Expr::TypeOf(operand) + | Expr::Await(operand) + | Expr::Delete(operand) + | Expr::StringCoerce(operand) + | Expr::ObjectCoerce(operand) + | Expr::BooleanCoerce(operand) + | Expr::NumberCoerce(operand) => walk_expr(operand, in_loop, scan), + + Expr::Conditional { + condition, + then_expr, + else_expr, + } => { + walk_expr(condition, in_loop, scan); + walk_expr(then_expr, in_loop, scan); + walk_expr(else_expr, in_loop, scan); + } + + // Member / index access + writes. + Expr::PropertyGet { object, .. } | Expr::PropertyUpdate { object, .. } => { + walk_expr(object, in_loop, scan) + } + Expr::PropertySet { object, value, .. } => { + walk_expr(object, in_loop, scan); + walk_expr(value, in_loop, scan); + } + Expr::IndexGet { object, index } => { + walk_expr(object, in_loop, scan); + walk_expr(index, in_loop, scan); + } + Expr::IndexSet { + object, + index, + value, + } => { + walk_expr(object, in_loop, scan); + walk_expr(index, in_loop, scan); + walk_expr(value, in_loop, scan); + } + Expr::IndexUpdate { object, index, .. } => { + walk_expr(object, in_loop, scan); + walk_expr(index, in_loop, scan); + } + + // Method-call forms: not `FuncRef` callees, but their receiver/args can + // hold hot calls or closures. + Expr::NativeMethodCall { object, args, .. } => { + if let Some(o) = object { + walk_expr(o, in_loop, scan); + } + for a in args { + walk_expr(a, in_loop, scan); + } + } + Expr::StaticMethodCall { args, .. } => { + for a in args { + walk_expr(a, in_loop, scan); + } + } + + // Aggregates. + Expr::Array(elements) => { + for el in elements { + walk_expr(el, in_loop, scan); + } + } + Expr::ArraySpread(elements) => { + for el in elements { + match el { + perry_hir::ArrayElement::Expr(e) | perry_hir::ArrayElement::Spread(e) => { + walk_expr(e, in_loop, scan) + } + perry_hir::ArrayElement::Hole => {} + } + } + } + Expr::Object(props) => { + for (_, v) in props { + walk_expr(v, in_loop, scan); + } + } + Expr::Sequence(es) => { + for e in es { + walk_expr(e, in_loop, scan); + } + } + Expr::New { args, .. } => { + for a in args { + walk_expr(a, in_loop, scan); + } + } + + // Everything else (literals, refs, and the long tail of runtime + // intrinsics) can't reach a hot `FuncRef` call in the patterns this + // heuristic targets; not descending is a safe under-approximation. + _ => {} + } +} diff --git a/crates/perry-codegen/src/collectors/mod.rs b/crates/perry-codegen/src/collectors/mod.rs index cb76e43f07..8db2763cd6 100644 --- a/crates/perry-codegen/src/collectors/mod.rs +++ b/crates/perry-codegen/src/collectors/mod.rs @@ -13,6 +13,7 @@ mod escape_check; mod escape_news; mod escape_objects; mod hir_facts; +mod hot_callees; mod i32_locals; mod i64_emit; mod index_uses; @@ -42,6 +43,7 @@ pub(crate) use escape_arrays::{const_index, MAX_SCALAR_OBJECT_FIELDS}; pub(crate) use escape_check::{check_escapes_in_stmts, find_new_candidates}; pub(crate) use escape_news::MAX_SCALAR_ARRAY_LEN; pub(crate) use hir_facts::{collect_native_region_fact_graph, NativeRegionFactGraph}; +pub(crate) use hot_callees::collect_hot_loop_callees; pub(crate) use i32_locals::{ collect_integer_let_ids, collect_localset_ids_in_stmts, is_strictly_i32_bounded_expr, is_ushr_zero, diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 1ae184fc12..e0da8c6373 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -37,6 +37,17 @@ pub struct LlFunction { /// function at every call site, exposing integer operations to the /// caller's optimizer context (critical for vectorization of clamp patterns). pub force_inline: bool, + /// When true (and `force_inline` is not), emit the `inlinehint` attribute. + /// Unlike `alwaysinline`, `inlinehint` only *raises* LLVM's inline + /// threshold for this callee — LLVM keeps its `-O3` growth budget and can + /// still decline to inline into cold / many call sites. Set for small + /// functions with a hot (in-loop) call site so a bit-mixer-style kernel + /// gets inlined into its loop without the binary-size blowup an + /// unconditional `alwaysinline` threshold bump causes. See the + /// inline-hot-small heuristic in `codegen/function.rs`. `alwaysinline` + /// already implies the hint, so the two are never emitted together, and + /// `has_try` (noinline) still wins over both in `to_ir`. + pub inline_hint: bool, blocks: Vec, block_counter: u32, reg_counter: Rc, @@ -120,6 +131,7 @@ impl LlFunction { linkage: String::new(), has_try: false, force_inline: false, + inline_hint: false, blocks: Vec::new(), block_counter: 0, reg_counter: Rc::new(RegCounter::new()), @@ -418,9 +430,14 @@ impl LlFunction { }; let attrs = if self.has_try { + // noinline (setjmp/volatile/async-rejecting boundary) always wins, + // even if an inline attribute was optimistically set before body + // lowering discovered the try. " #1" } else if self.force_inline { " alwaysinline" + } else if self.inline_hint { + " inlinehint" } else { "" }; diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index 892bee18f6..972f00490c 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -216,6 +216,20 @@ fn build_clang_compile_plan( clang_args.push("-g".to_string()); } clang_args.push("-fno-math-errno".to_string()); + // Inline-hot-small (#6850 follow-up): raise LLVM's `-inlinehint-threshold` + // so `inlinehint`-marked callees (Perry stamps that ONLY on small functions + // with an in-loop call site — see codegen/function.rs) actually inline into + // their hot loops. The default hint threshold (325) is below the ~800 cost + // of a NaN-boxed bit-mixer kernel. This only lifts the ceiling for hinted + // functions; every other function keeps the base -O3 threshold, so cold + // code is untouched (the anti-bloat property). Only meaningful at -O3. + if opt_flag == "-O3" && crate::codegen::helpers::inline_hot_small_enabled() { + clang_args.push("-mllvm".to_string()); + clang_args.push(format!( + "-inlinehint-threshold={}", + crate::codegen::helpers::inline_hot_small_hint_threshold() + )); + } if let Some(arg) = &native_tuning_arg { clang_args.push(arg.clone()); } diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index 72395f51e3..90bf8a5619 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -902,6 +902,33 @@ fn compute_object_cache_key_with_env( .as_deref() .unwrap_or(""), ); + // Inline-hot-small (#6850 follow-up): the enable flag changes the emitted + // IR (`inlinehint` attribute) AND the clang args (`-inlinehint-threshold`), + // and the size-cap / threshold change which functions get the hint and how + // aggressively they inline — all affect the .o bytes, so a warm cache must + // not serve an object built under a different setting. + h.field( + "env_inline_hot_small", + env_var("PERRY_INLINE_HOT_SMALL").as_deref().unwrap_or(""), + ); + h.field( + "env_inline_hot_small_cap", + env_var("PERRY_INLINE_HOT_SMALL_CAP") + .as_deref() + .unwrap_or(""), + ); + h.field( + "env_inline_hot_small_threshold", + env_var("PERRY_INLINE_HOT_SMALL_THRESHOLD") + .as_deref() + .unwrap_or(""), + ); + h.field( + "env_inline_hot_small_max_sites", + env_var("PERRY_INLINE_HOT_SMALL_MAX_SITES") + .as_deref() + .unwrap_or(""), + ); h.finish() } diff --git a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs index 03aa7c83db..9a66a6630c 100644 --- a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs +++ b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs @@ -604,6 +604,11 @@ fn key_changes_with_codegen_env_vars() { "PERRY_CODEGEN_UNIT_SIZE", "PERRY_SETJMP_VOLATILE", "PERRY_GC_MOVING_LOOP_POLLS", + // Inline-hot-small (#6850 follow-up). + "PERRY_INLINE_HOT_SMALL", + "PERRY_INLINE_HOT_SMALL_CAP", + "PERRY_INLINE_HOT_SMALL_THRESHOLD", + "PERRY_INLINE_HOT_SMALL_MAX_SITES", ] { // Sample state without the var, with the var, and with a different // value — all three keys must be distinct. diff --git a/test-files/test_gap_inline_hot_small.ts b/test-files/test_gap_inline_hot_small.ts new file mode 100644 index 0000000000..28ae6005d9 --- /dev/null +++ b/test-files/test_gap_inline_hot_small.ts @@ -0,0 +1,73 @@ +// Inline-hot-small (PERRY_INLINE_HOT_SMALL, #6850 follow-up). +// +// Perry biases LLVM toward inlining a *small* function that has a *hot* +// (in-loop) call site with few total call sites, by stamping `inlinehint` and +// raising `-inlinehint-threshold`. This must not change observable behavior: +// the result of the inlined callee, its GC shadow-frame correctness across the +// inlined boundary, and the array/pointer ops it feeds must all be byte-for-byte +// identical to Node. +// +// `mix` below is a ~10-statement bit-mixer that reads an Int32Array parameter +// and calls Math.imul (the exact shape that stays out-of-line without the flag +// and gets inlined into its loop with it). It has ONE loop call site, so it is +// hinted; its result is loop-carried and feeds Int32Array writes. +// +// Run byte-for-byte vs `node --experimental-strip-types`, and (mechanism +// permitting) under PERRY_GC_FORCE_EVACUATE=1 — the inlined boundary must not +// drop a GC root. + +function mix(S: Int32Array, x: number): number { + let a = x | 0; + a = (a ^ S[a & 1023]) | 0; + a = Math.imul(a, 0x9e3779b1); + a = (a ^ (a >>> 15)) | 0; + a = (a ^ S[(a >>> 7) & 1023]) | 0; + a = Math.imul(a, 0x85ebca6b); + a = (a ^ (a >>> 13)) | 0; + a = (a ^ S[(a >>> 3) & 1023]) | 0; + a = Math.imul(a, 0xc2b2ae35); + a = (a ^ (a >>> 16)) | 0; + return a | 0; +} + +// A second small hot callee whose result indexes into an array (pointer op). +function idx(n: number): number { + let h = n | 0; + h = (h ^ (h >>> 7)) | 0; + h = Math.imul(h, 0x2545f491); + h = (h ^ (h >>> 11)) | 0; + h = (h + 0x7f4a7c15) | 0; + h = (h ^ (h << 3)) | 0; + h = Math.imul(h, 0x27d4eb2f); + h = (h ^ (h >>> 15)) | 0; + h = (h >>> 0) % 64; + return h | 0; +} + +const S = new Int32Array(1024); +for (let i = 0; i < 1024; i++) S[i] = ((i * 2654435761) ^ (i << 28)) | 0; + +// Loop-carried call into `mix`, results written into a typed array (array ops +// fed by the inlined callee's result). +const OUT = new Int32Array(64); +let acc = 0 | 0; +for (let i = 0; i < 20000; i++) { + acc = (acc ^ mix(S, acc ^ i)) | 0; + // pointer/array op fed by a second inlined hot callee + const slot = idx(acc); + OUT[slot] = (OUT[slot] + (acc & 0xffff)) | 0; +} + +console.log("acc=" + acc); + +// Deterministic checksum over the array the inlined results populated. +let chk = 0 | 0; +for (let i = 0; i < 64; i++) chk = (Math.imul(chk, 31) + (OUT[i] | 0)) | 0; +console.log("chk=" + chk); + +// A few spot values so a miscompile of the inlined boundary shows as a diff. +console.log("OUT[0]=" + OUT[0]); +console.log("OUT[17]=" + OUT[17]); +console.log("OUT[63]=" + OUT[63]); +console.log("mix(S,1)=" + mix(S, 1)); +console.log("idx(123456)=" + idx(123456)); From 99d32120f28cf3cd0eb62a8356703a06629419df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 26 Jul 2026 19:13:18 +0200 Subject: [PATCH 2/3] docs(changelog): add 6855 inline-hot-small fragment Claude-Session: https://claude.ai/code/session_01UGDwjukzhowLDJYsMPFwMv --- changelog.d/6855-inline-hot-small.md | 42 ++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 changelog.d/6855-inline-hot-small.md diff --git a/changelog.d/6855-inline-hot-small.md b/changelog.d/6855-inline-hot-small.md new file mode 100644 index 0000000000..de72d9953d --- /dev/null +++ b/changelog.d/6855-inline-hot-small.md @@ -0,0 +1,42 @@ +### Changed + +- Inline small **hot** (in-loop) functions into their loops — the residual gap + a tight integer-math kernel had vs V8 after #6850 (native `Math.imul` + + typed-array-param reads) closed the codegen gap but left the *call itself* + out-of-line. Perry force-inlines functions `<= 8` statements with + `alwaysinline` (unconditional); a NaN-boxed bit-mixer like `mix` is ~10 + statements and costs ~800 in LLVM's inline model (GC shadow-frame calls + + typed-array reads + double<->i32 marshaling), above `-O3`'s base threshold, so + it stayed a call. + - **Bias, don't force.** A distinct `inlinehint` attribute path (separate from + `alwaysinline`) is now stamped on functions that are *small* + (`9..=SIZE_CAP` statements), *hot* (≥1 call site inside a loop — a + whole-module HIR pre-pass, `collectors/hot_callees.rs`, collects such callee + ids), AND called from *few* total sites. A function only ever called from + cold/straight-line code is never hinted, so cold utilities can't bloat the + binary. + - The linker raises LLVM's `-inlinehint-threshold` (default 850) so hinted + kernels actually inline. This lifts the ceiling **only** for functions Perry + marked `inlinehint`; every other function keeps the base `-O3` threshold. + - **Anti-bloat backstop — the call-site cap.** `-inlinehint-threshold` raises + the ceiling at *every* call site of a hinted callee (LLVM can't tell hot + from cold per-site through a function attribute), so a hot kernel also + called from many cold sites would be duplicated at all of them. Capping + total call sites (default 4) bounds the duplication. A synthetic of 300 fns + × 40 cold sites hinted *without* the cap grew its optimized IR 5.7× + (205K → 1.17M lines); *with* the cap those broadly-called fns are excluded, + for a 0% delta. Measured binary-size deltas (flag OFF vs ON, byte-precise + `__text`): real programs and >4-call-site helpers 0.000%; large programs + (>6MB IR → `-Os`) 0.000% (flag inert); realistic hot-kernel densities +0.34% + (25 kernels) / +0.79% (50) / +1.57% (100). `noinline` cases + (try/setjmp/volatile) are respected via `to_ir`'s `has_try`-first attribute + precedence. + - On the 40M-call `mix` microbench, the caller loop no longer emits a `bl` to + `mix` (0 vs 2 `BR26` relocations) and `mix` carries `inlinehint`; wall-clock + fell from ~1024 ms to ~904 ms (min-of-7, contended box). + - Gated behind `PERRY_INLINE_HOT_SMALL` (default on); + `PERRY_INLINE_HOT_SMALL_CAP` / `_THRESHOLD` / `_MAX_SITES` tune the size + window, hint threshold, and call-site cap. All four are folded into the + object cache key. New `test-files/test_gap_inline_hot_small.ts` asserts + byte-identical output to Node with the flag on, off, and under + `PERRY_GC_FORCE_EVACUATE=1`. From f1c14b1ac5eb634a7a0d99bebf59bac15551267c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 26 Jul 2026 19:36:53 +0200 Subject: [PATCH 3/3] perf(codegen): scan all class-member bodies in hot-callee call-site census CodeRabbit (#6855): collect_hot_loop_callees only walked hir.init, free functions, class constructors, and instance methods. It missed static methods, getters/setters, computed-key members (body + key expr), and instance/static field initializers. The miss matters for the anti-bloat cap, not just hot-callee discovery: `max_call_sites` bounds inlinehint duplication, so any call site the census doesn't see is a call site the cap can't count. A small function whose extra call sites hide in a getter or a field initializer could slip under the cap and get hinted despite being widely used, defeating the size backstop. Field/computed-key exprs are walked with in_loop=false so they feed the count without spuriously marking a callee hot. Verified byte-exact vs Node 26.5.0 on a class kernel whose getter, static method, and field initializer each carry an in-loop call (flag on and off). --- .../src/collectors/hot_callees.rs | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/crates/perry-codegen/src/collectors/hot_callees.rs b/crates/perry-codegen/src/collectors/hot_callees.rs index 83c05f7caf..7bbc4f63bc 100644 --- a/crates/perry-codegen/src/collectors/hot_callees.rs +++ b/crates/perry-codegen/src/collectors/hot_callees.rs @@ -40,7 +40,16 @@ struct HotCalleeScan { /// Collect the set of `FuncId`s eligible for `inlinehint`: those with ≥1 direct /// call site inside a loop AND at most `max_call_sites` total direct call sites -/// across the whole module (`init` + every function / constructor / method). +/// across the whole module (`init` + every function + every executable +/// class-member body: constructor, instance/static methods, getters/setters, +/// computed-key members, and instance/static field initializers). +/// +/// Counting *every* call site matters for the anti-bloat cap, not just for +/// finding hot callees: `max_call_sites` bounds duplication, so a call site the +/// scan misses is a call site the cap can't see — a function with many real +/// call sites hidden in (say) a getter or a field initializer could slip under +/// the cap and get hinted despite being widely used. Scanning all member bodies +/// keeps the count accurate so the cap stays a true upper bound. pub fn collect_hot_loop_callees(hir: &Module, max_call_sites: u32) -> HashSet { let mut scan = HotCalleeScan::default(); walk_stmts(&hir.init, false, &mut scan); @@ -51,9 +60,37 @@ pub fn collect_hot_loop_callees(hir: &Module, max_call_sites: u32) -> HashSet