perf(codegen): inline small hot (in-loop) functions via inlinehint

changelog.d/6855-inline-hot-small.md

@@ -0,0 +1,42 @@
+### Changed
+
+- Inline small **hot** (in-loop) functions into their loops — the residual gap
+  a tight integer-math kernel had vs V8 after #6850 (native `Math.imul` +
+  typed-array-param reads) closed the codegen gap but left the *call itself*
+  out-of-line. Perry force-inlines functions `<= 8` statements with
+  `alwaysinline` (unconditional); a NaN-boxed bit-mixer like `mix` is ~10
+  statements and costs ~800 in LLVM's inline model (GC shadow-frame calls +
+  typed-array reads + double<->i32 marshaling), above `-O3`'s base threshold, so
+  it stayed a call.
+  - **Bias, don't force.** A distinct `inlinehint` attribute path (separate from
+    `alwaysinline`) is now stamped on functions that are *small*
+    (`9..=SIZE_CAP` statements), *hot* (≥1 call site inside a loop — a
+    whole-module HIR pre-pass, `collectors/hot_callees.rs`, collects such callee
+    ids), AND called from *few* total sites. A function only ever called from
+    cold/straight-line code is never hinted, so cold utilities can't bloat the
+    binary.
+  - The linker raises LLVM's `-inlinehint-threshold` (default 850) so hinted
+    kernels actually inline. This lifts the ceiling **only** for functions Perry
+    marked `inlinehint`; every other function keeps the base `-O3` threshold.
+  - **Anti-bloat backstop — the call-site cap.** `-inlinehint-threshold` raises
+    the ceiling at *every* call site of a hinted callee (LLVM can't tell hot
+    from cold per-site through a function attribute), so a hot kernel also
+    called from many cold sites would be duplicated at all of them. Capping
+    total call sites (default 4) bounds the duplication. A synthetic of 300 fns
+    × 40 cold sites hinted *without* the cap grew its optimized IR 5.7×
+    (205K → 1.17M lines); *with* the cap those broadly-called fns are excluded,
+    for a 0% delta. Measured binary-size deltas (flag OFF vs ON, byte-precise
+    `__text`): real programs and >4-call-site helpers 0.000%; large programs
+    (>6MB IR → `-Os`) 0.000% (flag inert); realistic hot-kernel densities +0.34%
+    (25 kernels) / +0.79% (50) / +1.57% (100). `noinline` cases
+    (try/setjmp/volatile) are respected via `to_ir`'s `has_try`-first attribute
+    precedence.
+  - On the 40M-call `mix` microbench, the caller loop no longer emits a `bl` to
+    `mix` (0 vs 2 `BR26` relocations) and `mix` carries `inlinehint`; wall-clock
+    fell from ~1024 ms to ~904 ms (min-of-7, contended box).
+  - Gated behind `PERRY_INLINE_HOT_SMALL` (default on);
+    `PERRY_INLINE_HOT_SMALL_CAP` / `_THRESHOLD` / `_MAX_SITES` tune the size
+    window, hint threshold, and call-site cap. All four are folded into the
+    object cache key. New `test-files/test_gap_inline_hot_small.ts` asserts
+    byte-identical output to Node with the flag on, off, and under
+    `PERRY_GC_FORCE_EVACUATE=1`.

crates/perry-codegen/src/codegen/function.rs

@@ -15,6 +15,7 @@ use crate::strings::StringPool;
 use crate::types::{LlvmType, DOUBLE, I1, I32, I64, I8, PTR};
 
 use super::helpers::shadow_stack_enabled;
+use super::helpers::{inline_hot_small_enabled, inline_hot_small_size_cap, INLINE_HOT_SMALL_MIN};
 use super::opts::CrossModuleCtx;
 use super::typed_abi::{
     emit_typed_arg_guard, emit_typed_arg_to_raw, generic_function_body_name, lower_typed_f64_body,
@@ -394,6 +395,25 @@ pub(super) fn compile_function(
     if f.body.len() <= 8 && !f.is_async && !f.is_generator && !f.was_plain_async {
         lf.force_inline = true;
     }
+    // Inline-hot-small (PERRY_INLINE_HOT_SMALL, default ON): bias — do not
+    // force — LLVM toward inlining a *small* function that has a *hot* (in-loop)
+    // call site. `inlinehint` only raises LLVM's inline threshold for this
+    // callee; its `-O3` growth budget still refuses cold/oversized inlines, so
+    // cold utility functions never bloat the binary (that is the anti-bloat
+    // property, proved by the binary-size gate). We skip `alwaysinline`
+    // functions (the hint would be redundant), async/generator forms, and rely
+    // on `to_ir`'s `has_try`-first attribute precedence to keep any function
+    // whose body later turns out to need `noinline` (try/setjmp/volatile) out.
+    if !lf.force_inline
+        && inline_hot_small_enabled()
+        && (INLINE_HOT_SMALL_MIN..=inline_hot_small_size_cap()).contains(&f.body.len())
+        && !f.is_async
+        && !f.is_generator
+        && !f.was_plain_async
+        && cross_module.hot_loop_callees.contains(&f.id)
+    {
+        lf.inline_hint = true;
+    }
     let _ = lf.create_block("entry");
 
     let mut boxed_vars = module_boxed_vars.clone();

crates/perry-codegen/src/codegen/helpers.rs

@@ -74,6 +74,81 @@ pub(super) fn shadow_stack_enabled() -> bool {
     })
 }
 
+/// Inline-hot-small gate. Default ON. When enabled, small functions
+/// (`INLINE_HOT_SMALL_MIN ..= SIZE_CAP` statements) that have ≥1 call site
+/// inside a loop get LLVM's `inlinehint` — a *bounded* nudge that raises the
+/// inline threshold for that callee while LLVM's `-O3` growth budget stays the
+/// backstop (unlike `alwaysinline`, which is unconditional). Disable with
+/// `PERRY_INLINE_HOT_SMALL=0`/`off`/`false` for bisection / binary-size A/B.
+pub(crate) fn inline_hot_small_enabled() -> bool {
+    use std::sync::OnceLock;
+    static CACHED: OnceLock<bool> = OnceLock::new();
+    *CACHED.get_or_init(|| {
+        !matches!(
+            std::env::var("PERRY_INLINE_HOT_SMALL").as_deref(),
+            Ok("0") | Ok("off") | Ok("false")
+        )
+    })
+}
+
+/// LLVM `-inlinehint-threshold` used for `inlinehint`-marked callees when the
+/// feature is on. The default hint threshold (325) is too low for Perry's
+/// NaN-boxed kernels — a ~10-statement bit-mixer costs ~800 in LLVM's inline
+/// model once GC shadow-frame calls + typed-array reads + double↔i32 marshaling
+/// are counted — so we raise it. Critically this only affects functions Perry
+/// stamped `inlinehint` (the small + in-loop-callsite gate); every other
+/// function keeps the base `-O3` threshold, so cold code is untouched.
+/// Overridable via `PERRY_INLINE_HOT_SMALL_THRESHOLD` for the binary-size A/B.
+pub(crate) fn inline_hot_small_hint_threshold() -> u32 {
+    use std::sync::OnceLock;
+    static CACHED: OnceLock<u32> = OnceLock::new();
+    *CACHED.get_or_init(|| {
+        std::env::var("PERRY_INLINE_HOT_SMALL_THRESHOLD")
+            .ok()
+            .and_then(|s| s.parse::<u32>().ok())
+            .unwrap_or(850)
+    })
+}
+
+/// Smallest body-statement count eligible for `inlinehint`. Functions of `<= 8`
+/// statements already get unconditional `alwaysinline`, so the hint window
+/// starts one above that.
+pub(super) const INLINE_HOT_SMALL_MIN: usize = 9;
+
+/// Largest body-statement count eligible for `inlinehint`. Chosen
+/// conservatively and validated against the binary-size regression gate (a
+/// larger cap duplicates more code at each hinted site). Overridable via
+/// `PERRY_INLINE_HOT_SMALL_CAP` for tuning experiments.
+pub(super) fn inline_hot_small_size_cap() -> usize {
+    use std::sync::OnceLock;
+    static CACHED: OnceLock<usize> = OnceLock::new();
+    *CACHED.get_or_init(|| {
+        std::env::var("PERRY_INLINE_HOT_SMALL_CAP")
+            .ok()
+            .and_then(|s| s.parse::<usize>().ok())
+            .unwrap_or(20)
+    })
+}
+
+/// Maximum total (module-wide) direct call sites a function may have and still
+/// be hinted. This is the anti-bloat backstop: the raised `-inlinehint-threshold`
+/// lifts LLVM's ceiling for a hinted callee at *every* one of its call sites, so
+/// without this cap a small hot kernel that is also called from many cold sites
+/// would be duplicated at all of them. Capping call sites bounds the added code
+/// (≤ this many inlined copies). A tight bit-mixer kernel has 1–2 sites and
+/// still qualifies; a broadly-shared helper does not. Overridable via
+/// `PERRY_INLINE_HOT_SMALL_MAX_SITES`.
+pub(crate) fn inline_hot_small_max_call_sites() -> u32 {
+    use std::sync::OnceLock;
+    static CACHED: OnceLock<u32> = OnceLock::new();
+    *CACHED.get_or_init(|| {
+        std::env::var("PERRY_INLINE_HOT_SMALL_MAX_SITES")
+            .ok()
+            .and_then(|s| s.parse::<u32>().ok())
+            .unwrap_or(4)
+    })
+}
+
 pub(super) fn enable_module_init_shadow_frame(
     func: &mut crate::function::LlFunction,
     stmts: &[perry_hir::Stmt],

crates/perry-codegen/src/codegen/mod.rs

@@ -47,7 +47,9 @@ mod closure_collect;
 mod entry;
 mod func_registry;
 mod function;
-mod helpers;
+// `pub(crate)` so `crate::linker` can read the inline-hot-small policy
+// (`inline_hot_small_enabled` / `inline_hot_small_hint_threshold`).
+pub(crate) mod helpers;
 mod i64_spec;
 mod method;
 mod method_registry;
@@ -1545,6 +1547,13 @@ pub fn compile_module(hir: &HirModule, opts: CompileOptions) -> Result<Vec<u8>>
         target_triple: triple.clone(),
         app_metadata: opts.app_metadata.clone(),
         module_dispatch: crate::collectors::collect_module_dispatch_facts(hir),
+        // Inline-hot-small pre-pass (#6850 follow-up): FuncIds with an in-loop
+        // call site AND few total call sites, so small hot callees can earn
+        // `inlinehint` while the call-site cap bounds duplication.
+        hot_loop_callees: crate::collectors::collect_hot_loop_callees(
+            hir,
+            crate::codegen::helpers::inline_hot_small_max_call_sites(),
+        ),
         clamp3_functions: hir
             .functions
             .iter()

crates/perry-codegen/src/codegen/opts.rs

@@ -911,4 +911,12 @@ pub(crate) struct CrossModuleCtx {
     /// `@__perry_ns_<prefix>` + populator even when `namespace_entries`
     /// is empty (side-effect-only modules with no `export`s).
     pub is_dynamic_import_target: bool,
+    /// Inline-hot-small pre-pass result: `FuncId`s in THIS module that have
+    /// ≥1 direct call site inside a loop (`for`/`while`/`do-while`). Consumed
+    /// by `compile_function` to decide whether a small callee earns LLVM's
+    /// `inlinehint`. Built once per module via
+    /// `collectors::collect_hot_loop_callees`. Empty when
+    /// `PERRY_INLINE_HOT_SMALL` is off (the flag is checked at the decision
+    /// site, so the set is still populated but simply not consulted).
+    pub hot_loop_callees: std::collections::HashSet<u32>,
 }