diff --git a/crates/integration/codesize.json b/crates/integration/codesize.json index 16d0a6913..faeb387a8 100644 --- a/crates/integration/codesize.json +++ b/crates/integration/codesize.json @@ -1,7 +1,7 @@ { "Baseline": 838, "Computation": 2368, - "DivisionArithmetics": 11444, + "DivisionArithmetics": 9484, "ERC20": 18057, "Events": 1614, "FibonacciIterative": 1373, diff --git a/crates/integration/codesize_newyork.json b/crates/integration/codesize_newyork.json index 1301d35d3..a1fe72ddf 100644 --- a/crates/integration/codesize_newyork.json +++ b/crates/integration/codesize_newyork.json @@ -1,10 +1,10 @@ { - "Baseline": 493, - "Computation": 1217, - "DivisionArithmetics": 7370, + "Baseline": 515, + "Computation": 1239, + "DivisionArithmetics": 6845, "ERC20": 8726, - "Events": 909, - "FibonacciIterative": 969, - "Flipper": 1058, + "Events": 930, + "FibonacciIterative": 990, + "Flipper": 1080, "SHA1": 6264 } \ No newline at end of file diff --git a/crates/integration/contracts/DivModMulmodConst.sol b/crates/integration/contracts/DivModMulmodConst.sol new file mode 100644 index 000000000..751e79645 --- /dev/null +++ b/crates/integration/contracts/DivModMulmodConst.sol @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +/// Division, remainder, and mulmod by the SAME compile-time constant in one +/// contract, one function per constant magnitude class. Exercises the +/// constant-divisor code paths (LLVM folding, DivRemPairs, Barrett +/// specialization, and runtime routing all branch by constant size) and +/// guards the bug class where a compiler's constant-specialization pass +/// reuses a per-constant helper across optimization phases. +contract DivModMulmodConst { + function mixThree(uint256 x, uint256 y) public pure returns (uint256 q, uint256 r, uint256 m) { + q = x / 3; + r = x % 3; + m = mulmod(x, y, 3); + } + + function mixTwoPow64PlusOne(uint256 x, uint256 y) public pure returns (uint256 q, uint256 r, uint256 m) { + q = x / (2**64 + 1); + r = x % (2**64 + 1); + m = mulmod(x, y, 2**64 + 1); + } + + function mixTwoPow128MinusOne(uint256 x, uint256 y) public pure returns (uint256 q, uint256 r, uint256 m) { + q = x / (2**128 - 1); + r = x % (2**128 - 1); + m = mulmod(x, y, 2**128 - 1); + } + + function mixTwoPow128PlusOne(uint256 x, uint256 y) public pure returns (uint256 q, uint256 r, uint256 m) { + q = x / (2**128 + 1); + r = x % (2**128 + 1); + m = mulmod(x, y, 2**128 + 1); + } + + function mixTwoPow200(uint256 x, uint256 y) public pure returns (uint256 q, uint256 r, uint256 m) { + q = x / (2**200); + r = x % (2**200); + m = mulmod(x, y, 2**200); + } + + /// 2**255: div/rem fold to shifts and the mulmod modulus is the + /// reciprocal-overflow power of two, taking the inline mask rewrite. + function mixTwoPow255(uint256 x, uint256 y) public pure returns (uint256 q, uint256 r, uint256 m) { + q = x / (2**255); + r = x % (2**255); + m = mulmod(x, y, 2**255); + } + + /// 2**255 - 1: 255 bits, the sharp mulmod eligibility boundary — div/rem + /// are Barrett-eligible while mulmod is not Barrett-rewritten. + function mixTwoPow255MinusOne(uint256 x, uint256 y) public pure returns (uint256 q, uint256 r, uint256 m) { + q = x / (2**255 - 1); + r = x % (2**255 - 1); + m = mulmod(x, y, 2**255 - 1); + } + + function mixTwoPow255PlusThree(uint256 x, uint256 y) public pure returns (uint256 q, uint256 r, uint256 m) { + q = x / (2**255 + 3); + r = x % (2**255 + 3); + m = mulmod(x, y, 2**255 + 3); + } + + function mixMax(uint256 x, uint256 y) public pure returns (uint256 q, uint256 r, uint256 m) { + q = x / type(uint256).max; + r = x % type(uint256).max; + m = mulmod(x, y, type(uint256).max); + } +} diff --git a/crates/integration/src/cases.rs b/crates/integration/src/cases.rs index 74310f7aa..5366306c8 100644 --- a/crates/integration/src/cases.rs +++ b/crates/integration/src/cases.rs @@ -472,6 +472,18 @@ sol!( function modLhsMax(uint256 d) external pure returns (uint256); } + contract DivModMulmodConst { + function mixThree(uint256 x, uint256 y) external pure returns (uint256 q, uint256 r, uint256 m); + function mixTwoPow64PlusOne(uint256 x, uint256 y) external pure returns (uint256 q, uint256 r, uint256 m); + function mixTwoPow128MinusOne(uint256 x, uint256 y) external pure returns (uint256 q, uint256 r, uint256 m); + function mixTwoPow128PlusOne(uint256 x, uint256 y) external pure returns (uint256 q, uint256 r, uint256 m); + function mixTwoPow200(uint256 x, uint256 y) external pure returns (uint256 q, uint256 r, uint256 m); + function mixTwoPow255(uint256 x, uint256 y) external pure returns (uint256 q, uint256 r, uint256 m); + function mixTwoPow255MinusOne(uint256 x, uint256 y) external pure returns (uint256 q, uint256 r, uint256 m); + function mixTwoPow255PlusThree(uint256 x, uint256 y) external pure returns (uint256 q, uint256 r, uint256 m); + function mixMax(uint256 x, uint256 y) external pure returns (uint256 q, uint256 r, uint256 m); + } + contract SmodConst { function smodRhsZero(int256 n) external pure returns (int256); function smodRhsOne(int256 n) external pure returns (int256); @@ -494,6 +506,16 @@ sol!( } ); +case!("DivModMulmodConst.sol", DivModMulmodConst, mixThreeCall, const_mix_three, x: U256, y: U256); +case!("DivModMulmodConst.sol", DivModMulmodConst, mixTwoPow64PlusOneCall, const_mix_two_pow64_plus_one, x: U256, y: U256); +case!("DivModMulmodConst.sol", DivModMulmodConst, mixTwoPow128MinusOneCall, const_mix_two_pow128_minus_one, x: U256, y: U256); +case!("DivModMulmodConst.sol", DivModMulmodConst, mixTwoPow128PlusOneCall, const_mix_two_pow128_plus_one, x: U256, y: U256); +case!("DivModMulmodConst.sol", DivModMulmodConst, mixTwoPow200Call, const_mix_two_pow200, x: U256, y: U256); +case!("DivModMulmodConst.sol", DivModMulmodConst, mixTwoPow255Call, const_mix_two_pow255, x: U256, y: U256); +case!("DivModMulmodConst.sol", DivModMulmodConst, mixTwoPow255MinusOneCall, const_mix_two_pow255_minus_one, x: U256, y: U256); +case!("DivModMulmodConst.sol", DivModMulmodConst, mixTwoPow255PlusThreeCall, const_mix_two_pow255_plus_three, x: U256, y: U256); +case!("DivModMulmodConst.sol", DivModMulmodConst, mixMaxCall, const_mix_max, x: U256, y: U256); + sol!( contract Send { function transfer_self(uint _amount) public payable; diff --git a/crates/integration/src/tests.rs b/crates/integration/src/tests.rs index ef5b337b9..266413c8f 100644 --- a/crates/integration/src/tests.rs +++ b/crates/integration/src/tests.rs @@ -159,6 +159,217 @@ fn ulongrem_fuzz() { run_differential(actions); } +/// Differential fuzz of the unsigned 256-bit `div`/`mod` routing. Non-provable +/// i256 `udiv`/`urem` are rewritten by `lower_wide_division` into calls +/// to the stdlib `__udivrem256` (128-bit-digit Knuth long division), via the +/// `__udiv256`/`__urem256` wrappers. `__udivrem256` has two paths: `full` +/// (divisor >= 2^128, exercising the qhat estimate and its correction steps) +/// and `twoone` (divisor < 2^128, a single 128-bit digit). A divisor of zero +/// is short-circuited to 0 by the caller's guard per EVM semantics. The +/// keccak-derived stream feeds full-width dividends while alternating the two +/// paths, and the fixed vectors pin the boundary cases the random stream never +/// hits: dividend < / = / > divisor, divisors at 2^128 +/- 1, a power of two, +/// and zero. +#[test] +fn div_mod_fuzz() { + use alloy_primitives::keccak256; + + let one = U256::from(1u64); + let two_128 = one << 128; + + let mut cases: Vec<(U256, U256)> = vec![ + (U256::from(7u64), U256::MAX), // dividend < divisor: quotient 0 + (U256::MAX, U256::MAX), // dividend == divisor: quotient 1, remainder 0 + (U256::MAX, two_128 - one), // divisor just below 2^128: twoone boundary + (U256::MAX, two_128), // divisor 2^128: full path, minimal high digit + (U256::MAX, two_128 + one), // divisor just above 2^128: qhat correction + (U256::MAX, one << 200), // power-of-two divisor in the full range + (U256::MAX, U256::ZERO), // divisor zero: guarded, returns 0 + ]; + + for i in 0u64..256 { + let derive = |tag: &[u8]| -> U256 { + let mut buf = Vec::with_capacity(8 + tag.len()); + buf.extend_from_slice(&i.to_be_bytes()); + buf.extend_from_slice(tag); + U256::from_be_bytes::<32>(keccak256(&buf).0) + }; + let n = derive(b"n"); + let d_raw = derive(b"d"); + // Even i: divisor in [2^128, 2^256), the full multi-digit path. + // Odd i: divisor in [0, 2^128), the twoone single-digit path. + let d = if i % 2 == 0 { + d_raw | two_128 + } else { + d_raw >> 128 + }; + cases.push((n, d)); + } + + let mut actions = instantiate("contracts/DivisionArithmetics.sol", "DivisionArithmetics"); + for (n, d) in cases { + for data in [ + Contract::division_arithmetics_div(n, d).calldata, + Contract::division_arithmetics_mod(n, d).calldata, + ] { + actions.push(Call { + origin: TestAddress::Alice, + dest: TestAddress::Instantiated(0), + value: 0, + gas_limit: None, + storage_deposit_limit: None, + data, + }); + } + } + + run_differential(actions); +} + +/// Differential fuzz of the signed 256-bit `sdiv`/`smod` routing. Non-provable +/// i256 `sdiv`/`srem` are rewritten into calls to the stdlib `__sdiv256`/ +/// `__srem256`, which reduce to the unsigned routine via sign-magnitude. Fixed +/// vectors pin the signed edges (`INT_MIN`, the `INT_MIN / -1` overflow, `-1` +/// and `0` divisors, mixed signs, `|n| < |d|`); the keccak stream covers +/// full-width signed operands, alternating the divisor so `|d|` exercises both +/// the `full` and `twoone` paths of the underlying `__udivrem256`. +#[test] +fn sdiv_smod_fuzz() { + use alloy_primitives::keccak256; + + let mut cases: Vec<(I256, I256)> = vec![ + (I256::MIN, I256::MINUS_ONE), // overflow: SDIV = INT_MIN, SMOD = 0 + (I256::MIN, I256::try_from(2).unwrap()), + (I256::MAX, I256::MINUS_ONE), + (I256::try_from(-7).unwrap(), I256::try_from(3).unwrap()), + (I256::try_from(7).unwrap(), I256::try_from(-3).unwrap()), + (I256::try_from(-7).unwrap(), I256::try_from(-3).unwrap()), + (I256::try_from(3).unwrap(), I256::try_from(7).unwrap()), // |n| < |d| + (I256::MIN, I256::MIN), // n == d + (I256::try_from(5).unwrap(), I256::ZERO), // divisor 0: guarded, returns 0 + ]; + + for i in 0u64..256 { + let derive = |tag: &[u8]| -> U256 { + let mut buf = Vec::with_capacity(8 + tag.len()); + buf.extend_from_slice(&i.to_be_bytes()); + buf.extend_from_slice(tag); + U256::from_be_bytes::<32>(keccak256(&buf).0) + }; + let n = I256::from_raw(derive(b"n")); + let d_raw = derive(b"d"); + // Even i: full-width signed divisor (|d| usually >= 2^128, the full + // path). Odd i: small-magnitude divisor (|d| < 2^127, the twoone path), + // sign taken from bit 0 so negative small divisors are covered. + let d = if i % 2 == 0 { + I256::from_raw(d_raw) + } else { + let small = I256::from_raw(d_raw >> 129); + if d_raw.bit(0) { + -small + } else { + small + } + }; + cases.push((n, d)); + } + + let mut actions = instantiate("contracts/DivisionArithmetics.sol", "DivisionArithmetics"); + for (n, d) in cases { + for data in [ + Contract::division_arithmetics_sdiv(n, d).calldata, + Contract::division_arithmetics_smod(n, d).calldata, + ] { + actions.push(Call { + origin: TestAddress::Alice, + dest: TestAddress::Instantiated(0), + value: 0, + gas_limit: None, + storage_deposit_limit: None, + data, + }); + } + } + + run_differential(actions); +} + +/// Differential coverage of division, remainder, and mulmod by the SAME +/// compile-time constant inside one contract, across constant magnitudes the +/// suite otherwise never exercises. Guards the bug class where a constant- +/// specialization pass reuses a per-constant helper across optimization phases +/// (e.g. if `x % C` was to be compiled into an unconditional trap whenever +/// `mulmod(x, y, C)` shares the constant), and pins the sharp eligibility +/// boundaries of the in-tree Barrett rung: 2**255 (power of two: div/rem fold +/// to shifts, mulmod takes the inline mask rewrite) and 2**255 - 1 (255 bits: +/// div/rem Barrett-eligible, mulmod is not Barrett-rewritten). Fixed +/// operands pin 0, 1, C - 1, C, C + 1, and 2^256 - 1 per constant, plus the +/// largest multiple M = C * floor(MAX / C) and M +/- 1 (M deterministically +/// fires the division helpers' single correction); a keccak stream adds +/// full-width pairs. +#[test] +fn div_mod_mulmod_const_mix() { + use alloy_primitives::keccak256; + + /// One helper contract under test: its calldata constructor and the + /// compile-time constant it divides by. + type ConstMixHelper = (fn(U256, U256) -> Contract, U256); + + let one = U256::from(1u64); + let helpers: [ConstMixHelper; 9] = [ + (Contract::const_mix_three, U256::from(3u64)), + (Contract::const_mix_two_pow64_plus_one, (one << 64) + one), + (Contract::const_mix_two_pow128_minus_one, (one << 128) - one), + (Contract::const_mix_two_pow128_plus_one, (one << 128) + one), + (Contract::const_mix_two_pow200, one << 200), + (Contract::const_mix_two_pow255, one << 255), + (Contract::const_mix_two_pow255_minus_one, (one << 255) - one), + ( + Contract::const_mix_two_pow255_plus_three, + (one << 255) + U256::from(3u64), + ), + (Contract::const_mix_max, U256::MAX), + ]; + + let mut actions = instantiate("contracts/DivModMulmodConst.sol", "DivModMulmodConst"); + for (index, (helper, constant)) in helpers.iter().enumerate() { + let largest_multiple = *constant * (U256::MAX / *constant); + let mut operands = vec![ + (U256::ZERO, U256::MAX), + (one, one), + (constant.wrapping_sub(one), U256::MAX), + (*constant, *constant), + (constant.wrapping_add(one), constant.wrapping_sub(one)), + (U256::MAX, U256::MAX), + (largest_multiple, U256::MAX), + (largest_multiple.wrapping_sub(one), U256::MAX), + (largest_multiple.wrapping_add(one), U256::MAX), + ]; + for i in 0u64..8 { + let derive = |tag: &[u8]| -> U256 { + let mut buf = Vec::with_capacity(16 + tag.len()); + buf.extend_from_slice(&(index as u64).to_be_bytes()); + buf.extend_from_slice(&i.to_be_bytes()); + buf.extend_from_slice(tag); + U256::from_be_bytes::<32>(keccak256(&buf).0) + }; + operands.push((derive(b"x"), derive(b"y"))); + } + for (x, y) in operands { + actions.push(Call { + origin: TestAddress::Alice, + dest: TestAddress::Instantiated(0), + value: 0, + gas_limit: None, + storage_deposit_limit: None, + data: helper(x, y).calldata, + }); + } + } + + run_differential(actions); +} + #[test] fn bitwise_byte() { let mut actions = instantiate("contracts/Bitwise.sol", "Bitwise"); @@ -1031,6 +1242,9 @@ fn unsigned_division_half_const() { (U256::ZERO, U256::MAX), (five, two), (one, U256::ZERO), + // 2^256 - 1 over a small constant drives the Barrett quotient + // estimate to q - 1, firing the helper's single correction. + (U256::MAX, five), ]; for (n, d) in pairs { push_call( @@ -1936,7 +2150,7 @@ fn invalid_opcode_works() { /// holds the only `sdiv i256` we emit, and InstCombine rewrites it to /// `udiv i256` (then to `udiv i64`) once it sees the AND-masks prove the /// operands non-negative. The latent risk: revive's -/// `narrow_divrem_instructions` (crates/llvm-context/src/polkavm/context/mod.rs) +/// `lower_wide_division` (crates/llvm-context/src/polkavm/context/mod.rs) /// would, given a surviving `sdiv i256 (and a, M), (and b, M)`, rewrite it /// to `sext (sdiv i64 (trunc..), (trunc..))`, which flips sign for any /// operand whose bit (n-1) is set. If LLVM ever stops folding sdiv → udiv diff --git a/crates/llvm-context/src/polkavm/context/constant_division.rs b/crates/llvm-context/src/polkavm/context/constant_division.rs new file mode 100644 index 000000000..af464e46e --- /dev/null +++ b/crates/llvm-context/src/polkavm/context/constant_division.rs @@ -0,0 +1,475 @@ +//! Barrett specialization of 256-bit unsigned division and remainder by +//! compile-time constants, and of `__mulmod` calls with compile-time moduli. +//! +//! A Barrett reduction replaces a runtime division by a constant `C` with a +//! multiplication by the precomputed reciprocal `floor(2^k / C)` followed by +//! a shift and at most a bounded number of conditional corrections — no +//! runtime division at all. +//! +//! The div/rem specialization runs as part of [`super::Context::lower_wide_division`], +//! AFTER the full LLVM optimization pipeline, for two reasons: +//! +//! - Constants are fully exposed there: IPSCCP and inlining have already +//! propagated compile-time divisors across function boundaries, so one +//! late sweep sees every eligible site. +//! - Helpers generated here are never touched by the pipeline again, so they +//! can never be calling-convention-promoted (e.g. to `fastcc`) between +//! creation and use. If, for instance, a helper were to be reused across +//! optimization phases, `x % C` could be compiled into a silent trap +//! through exactly that promotion. Creating helpers post-pipeline makes +//! that failure mode structurally impossible. +//! +//! The `__mulmod` specialization instead runs primarily BEFORE the pipeline +//! (from [`super::Context::specialize_constant_modulus_mulmod`], behind a +//! small constant-folding pre-pass): waiting until after the pipeline loses +//! the common single-constant-modulus contract, where the inliner dissolves +//! `__mulmod`'s body into callers and no rewritable call site survives. +//! [`super::Context::lower_wide_division`] repeats the sweep post-pipeline +//! for moduli that only become constant during optimization. Running early +//! is safe because the rewrite targets the pre-existing EXTERNAL +//! `__mulmod_barrett`, which the pipeline can neither +//! calling-convention-promote nor argument-strip; the helper-reuse trap +//! above concerns internal helpers generated by the compiler. + +use inkwell::values::InstructionOpcode; +use num::{BigUint, One, Zero}; + +use crate::polkavm::context::{attribute::Attribute, Context}; + +/// Reserved name prefix shared by every generated Barrett helper function. +/// [`super::Context::lower_wide_division`] skips functions carrying this +/// prefix when walking the module, so freshly generated helper bodies are +/// never themselves candidates for rewriting. +pub const HELPER_PREFIX: &str = "__barrett_"; + +/// Operation infix of the generated unsigned division helpers: their names +/// are [`HELPER_PREFIX`] + this + the divisor in lowercase hexadecimal +/// (no leading zeros, so at most 64 digits for an i256 divisor — +/// deterministic and bounded). +pub const UDIV_HELPER_INFIX: &str = "udiv_256_"; + +/// Operation infix of the generated unsigned remainder helpers: their names +/// are [`HELPER_PREFIX`] + this + the divisor in lowercase hexadecimal +/// (no leading zeros, so at most 64 digits for an i256 divisor — +/// deterministic and bounded). +pub const UREM_HELPER_INFIX: &str = "urem_256_"; + +/// The number of bits in one LLVM `ConstantInt` limb. +const LIMB_BITS: u32 = 64; + +/// How an eligible `__mulmod` call site with a compile-time constant modulus +/// is rewritten. +#[derive(Debug)] +pub enum MulModRewrite { + /// The modulus is a non-power-of-two 256-bit constant: the call is + /// redirected to `__mulmod_barrett`, with the low word of the Barrett + /// reciprocal `floor(2^512 / m) - 2^256` supplied as the fourth argument. + BarrettCall(BigUint), + /// The modulus is an exact power of two `2^k` with `k` in `0..=255` + /// (including the `mu_lo`-overflow case `2^255` and the `m = 1` case): + /// the call is replaced inline by `mul` + `and` with the mask `2^k - 1`. + /// Proof: `a*b ≡ (a*b mod 2^256) (mod 2^k)` since `2^k | 2^256`; `k = 0` + /// gives mask `0 -> 0`, correct mod 1. Two instructions, no helper. + PowerOfTwoMask(u32), +} + +impl<'ctx> Context<'ctx> { + /// Returns the divisor of an i256 `udiv`/`urem` instruction if it is a + /// compile-time constant eligible for Barrett specialization. + /// + /// Eligible means `1 < C < 2^256` and `C` not a power of two. Divisors of + /// 0, 1, and `2^k` are LLVM folding territory: instcombine canonicalizes + /// power-of-two div/rem to shifts at O1+ so such sites normally never + /// reach this pass, but `default` runs no instcombine, making the + /// exclusion mandatory rather than hygiene. Excluded sites fall through + /// to the runtime routing, which is correct at any optimization level. + /// A constant dividend with a runtime divisor also stays routed. + pub(crate) fn barrett_eligible_divisor( + builder: &inkwell::builder::Builder<'ctx>, + instruction: &inkwell::values::InstructionValue<'ctx>, + ) -> Option { + if !matches!( + instruction.get_opcode(), + InstructionOpcode::UDiv | InstructionOpcode::URem + ) { + return None; + } + if !instruction.get_type().is_int_type() { + return None; + } + let divisor = instruction.get_operand(1)?.value()?; + if !divisor.is_int_value() { + return None; + } + let divisor = divisor.into_int_value(); + if !divisor.is_const() { + return None; + } + builder.position_before(instruction); + let divisor = Self::wide_unsigned_constant(builder, divisor)?; + if divisor <= BigUint::one() || divisor.count_ones() == 1 { + return None; + } + Some(divisor) + } + + /// Classifies a call instruction as an eligible constant-modulus + /// `__mulmod` rewrite, or `None` to leave the site untouched. + /// + /// The callee is compared by pointer identity against the `__mulmod` + /// function value (`mulmod_callee`), never by name: post-Oz calls to the + /// private `__mulmod` may carry `fastcc`, and pointer identity is immune + /// to calling-convention, tail-call, and direct-vs-indirect form. A + /// hypothetically cloned or specialized `__mulmod` would be missed, which + /// fails safe (a missed optimization, never wrong code). + /// + /// Moduli with fewer than 256 bits (and runtime moduli) stay on + /// `__mulmod`: `a*b <= (2^256-1)^2 < 2^512` is exactly the HAC 14.42 + /// operand bound at `t = 256`, so only 256-bit moduli admit the shared + /// helper without pre-reducing the multiplicands. + pub(crate) fn mulmod_rewrite( + builder: &inkwell::builder::Builder<'ctx>, + instruction: &inkwell::values::InstructionValue<'ctx>, + mulmod_callee: inkwell::values::PointerValue<'ctx>, + ) -> Option { + if instruction.get_opcode() != InstructionOpcode::Call { + return None; + } + // Three arguments plus the callee, which LLVM stores last. + if instruction.get_num_operands() != 4 { + return None; + } + let callee = instruction.get_operand(3)?.value()?; + if !callee.is_pointer_value() || callee.into_pointer_value() != mulmod_callee { + return None; + } + let modulus = instruction.get_operand(2)?.value()?; + if !modulus.is_int_value() { + return None; + } + let modulus = modulus.into_int_value(); + if !modulus.is_const() { + return None; + } + builder.position_before(instruction); + let modulus = Self::wide_unsigned_constant(builder, modulus)?; + if modulus.count_ones() == 1 { + return Some(MulModRewrite::PowerOfTwoMask(modulus.bits() as u32 - 1)); + } + if modulus.bits() == 256 { + return Some(MulModRewrite::BarrettCall(Self::mulmod_reciprocal_low( + &modulus, + ))); + } + None + } + + /// Computes the Barrett reciprocal `mu = floor(2^256 / C)` for a division + /// helper. + /// + /// `C` not a power of two implies `2^256 = mu*C + e` with `1 <= e <= C-1`. + /// The self-check aborts compilation on violation: a compiler bug here + /// must never become a wrong on-chain result. + pub(crate) fn division_reciprocal(divisor: &BigUint) -> BigUint { + let two_pow_256 = BigUint::one() << 256u32; + let reciprocal = &two_pow_256 / divisor; + assert!( + &reciprocal * divisor <= two_pow_256 && (&reciprocal + 1u32) * divisor > two_pow_256, + "Barrett reciprocal self-check failed for divisor 0x{}", + divisor.to_str_radix(16), + ); + reciprocal + } + + /// Computes the low word `mu_lo = floor(2^512 / m) - 2^256` of the Barrett + /// reciprocal for an eligible `__mulmod` modulus. + /// + /// Range proof: for `2^255 < m <= 2^256 - 1` and `m` not a power of two, + /// `mu >= 2^256 + 1` (since `(2^256-1)(2^256+1) < 2^512`) and + /// `mu < 2^257`, so `mu_lo` lies in `[1, 2^256 - 1]`: it always fits an + /// i256 and is never zero. The self-checks abort compilation on + /// violation. + pub(crate) fn mulmod_reciprocal_low(modulus: &BigUint) -> BigUint { + let two_pow_256 = BigUint::one() << 256u32; + let two_pow_512 = BigUint::one() << 512u32; + let reciprocal = &two_pow_512 / modulus; + assert!( + &reciprocal * modulus <= two_pow_512 && (&reciprocal + 1u32) * modulus > two_pow_512, + "Barrett mulmod reciprocal self-check failed for modulus 0x{}", + modulus.to_str_radix(16), + ); + assert!( + reciprocal > two_pow_256, + "Barrett mulmod reciprocal 0x{} out of range for modulus 0x{}", + reciprocal.to_str_radix(16), + modulus.to_str_radix(16), + ); + let reciprocal_low = reciprocal - two_pow_256; + assert!( + reciprocal_low.bits() <= 256, + "Barrett mulmod reciprocal low word overflows i256 for modulus 0x{}", + modulus.to_str_radix(16), + ); + reciprocal_low + } + + /// Reads an arbitrary-width LLVM unsigned integer constant into a + /// [`BigUint`] without printing IR. + /// + /// For each 64-bit limb the builder is asked for `lshr` on two constant + /// operands, which LLVM's `IRBuilder` constant folder folds to a plain + /// constant without inserting an instruction (the builder must merely be + /// positioned); the limb is then read via `const_truncate` to i64 and + /// `get_zero_extended_constant`. + /// + /// Defensive contract: if a shift result is somehow non-constant, the + /// accidental instruction is erased and `None` is returned — the call + /// site falls back to the runtime routing; this function never + /// miscompiles. The final self-check reconstructs the constant with + /// [`Self::biguint_constant`] and compares interned pointers (LLVM + /// interns `ConstantInt`s per context and type, so equality is complete); + /// this also rejects constant expressions that fold limb-wise but are not + /// plain `ConstantInt`s. + pub(crate) fn wide_unsigned_constant( + builder: &inkwell::builder::Builder<'ctx>, + value: inkwell::values::IntValue<'ctx>, + ) -> Option { + let value_type = value.get_type(); + let i64_type = value_type.get_context().i64_type(); + let limb_count = value_type.get_bit_width().div_ceil(LIMB_BITS); + + let mut result = BigUint::zero(); + for limb_index in (0..limb_count).rev() { + let shift_amount = value_type.const_int((limb_index * LIMB_BITS) as u64, false); + let shifted = builder + .build_right_shift(value, shift_amount, false, "") + .ok()?; + if let Some(accidental_instruction) = shifted.as_instruction() { + accidental_instruction.erase_from_basic_block(); + return None; + } + let limb = shifted + .const_truncate(i64_type) + .get_zero_extended_constant()?; + result = (result << LIMB_BITS) | BigUint::from(limb); + } + + let reconstructed = Self::biguint_constant(value_type, &result); + if reconstructed != value { + return None; + } + Some(result) + } + + /// Materializes a [`BigUint`] as an LLVM constant of the given integer + /// type. + /// + /// # Panics + /// If the value does not fit the type. + pub(crate) fn biguint_constant( + int_type: inkwell::types::IntType<'ctx>, + value: &BigUint, + ) -> inkwell::values::IntValue<'ctx> { + let limb_count = int_type.get_bit_width().div_ceil(LIMB_BITS) as usize; + let mut limbs = value.to_u64_digits(); + assert!( + limbs.len() <= limb_count && value.bits() <= int_type.get_bit_width() as u64, + "constant 0x{} does not fit i{}", + value.to_str_radix(16), + int_type.get_bit_width(), + ); + limbs.resize(limb_count, 0); + int_type.const_int_arbitrary_precision(&limbs) + } + + /// Returns the Barrett helper function for the given operation and + /// divisor, generating it on first use. + /// + /// The helper has internal linkage, `noinline` and `nounwind`, a single + /// entry block, and is built with a dedicated builder so the caller's + /// builder position is never clobbered. Shape (`` is + /// `256 + bits(mu)` rounded up to a multiple of 64 — no-wrap proof: + /// `x*mu < 2^256 * 2^bits(mu) <= 2^wide`; range 320 for large `C` to 512 + /// for `C = 3`): + /// + /// ```llvm + /// define internal i256 @__barrett_udiv_256_(i256 %x) { + /// entry: + /// %xw = zext i256 %x to i + /// %p = mul i %xw, ; no wrap (see width proof) + /// %s = lshr i %p, 256 + /// %qh = trunc i %s to i256 ; qh <= q < 2^256: lossless + /// %qc = mul i256 %qh, ; qh*C <= q*C <= x: no wrap + /// %r0 = sub i256 %x, %qc ; r0 in [0, 2C) + /// %f = icmp uge i256 %r0, ; fires iff qh = q-1 + /// ; udiv: %q1 = add i256 %qh, 1 ; %q = select %f, %q1, %qh ; ret %q + /// ; urem: %r1 = sub i256 %r0, ; %r = select %f, %r1, %r0 ; ret %r + /// } + /// ``` + /// + /// Exactly ONE correction. Proof: with `mu = (2^256 - e)/C`, + /// `1 <= e <= C-1`, and `qh = floor(x*mu/2^256)`: upper — + /// `x*mu/2^256 <= x/C` so `qh <= q`; lower — + /// `x*mu/2^256 = x/C - x*e/(C*2^256)` and `x*e/(C*2^256) < e/C < 1`, so + /// `qh >= q-1`. Hence `qh` is in `{q-1, q}`, `r0` is in `[0, 2C)`, one + /// conditional subtract/increment is necessary and sufficient, and + /// `r0 >= C` iff `qh = q-1`. Tightness: `C = 3` or `5` with + /// `x = 2^256-1` drives `qh = q-1`, so the correction is not dead code. + /// A second correction would be provably dead and is deliberately not + /// emitted (dead margin is untestable code; the emission-time `mu` + /// self-check guards the only way the premise could break). Do not + /// harmonize with `__mulmod_barrett`'s two corrections; these are + /// different theorems. + /// + /// Degenerate case `bits(C) == 256` (`mu = 1`): a compare-and-subtract + /// body is emitted instead — `x < 2^256 < 2C` forces `q` in `{0, 1}`, so + /// `q = zext(icmp uge x, C)` and `r = select(uge, x - C, x)`. Same helper + /// name and type; call sites are agnostic to which body was chosen. + /// + /// # Panics + /// If a function with the helper's reserved name exists with a different + /// type: the prefix is reserved, so a mismatch is a compiler invariant + /// violation. + pub(crate) fn barrett_divrem_helper( + &self, + opcode: InstructionOpcode, + divisor: &BigUint, + ) -> inkwell::values::FunctionValue<'ctx> { + let operation_infix = match opcode { + InstructionOpcode::UDiv => UDIV_HELPER_INFIX, + InstructionOpcode::URem => UREM_HELPER_INFIX, + _ => unreachable!("only udiv/urem sites are Barrett-specialized"), + }; + let name = format!( + "{}{}{}", + HELPER_PREFIX, + operation_infix, + divisor.to_str_radix(16) + ); + + let word_type = self.word_type(); + let helper_type = word_type.fn_type(&[word_type.into()], false); + if let Some(function) = self.module().get_function(&name) { + assert!( + function.get_type() == helper_type, + "the reserved Barrett helper symbol `{name}` has a mismatched type", + ); + return function; + } + + let function = self.module().add_function( + &name, + helper_type, + Some(inkwell::module::Linkage::Internal), + ); + for attribute in [Attribute::NoInline, Attribute::NoUnwind] { + function.add_attribute( + inkwell::attributes::AttributeLoc::Function, + self.llvm.create_enum_attribute(attribute as u32, 0), + ); + } + + let builder = self.llvm.create_builder(); + builder.position_at_end(self.llvm.append_basic_block(function, "entry")); + let dividend = function + .get_first_param() + .expect("the Barrett helper takes the dividend") + .into_int_value(); + let divisor_constant = Self::biguint_constant(word_type, divisor); + + if divisor.bits() as u32 == word_type.get_bit_width() { + let correction_fires = builder + .build_int_compare(inkwell::IntPredicate::UGE, dividend, divisor_constant, "") + .expect("the Barrett helper compare is valid"); + let result = match opcode { + InstructionOpcode::UDiv => builder + .build_int_z_extend(correction_fires, word_type, "") + .expect("the Barrett helper zext is valid") + .into(), + InstructionOpcode::URem => { + let reduced = builder + .build_int_sub(dividend, divisor_constant, "") + .expect("the Barrett helper sub is valid"); + builder + .build_select(correction_fires, reduced, dividend, "") + .expect("the Barrett helper select is valid") + } + _ => unreachable!("only udiv/urem sites are Barrett-specialized"), + }; + builder + .build_return(Some(&result)) + .expect("the Barrett helper return is valid"); + return function; + } + + let reciprocal = Self::division_reciprocal(divisor); + let wide_width = (word_type.get_bit_width() as u64 + reciprocal.bits()) + .div_ceil(LIMB_BITS as u64) as u32 + * LIMB_BITS; + let wide_type = self + .llvm + .custom_width_int_type( + std::num::NonZeroU32::new(wide_width).expect("the wide width is non-zero"), + ) + .expect("the wide width is a valid integer width"); + + let dividend_wide = builder + .build_int_z_extend(dividend, wide_type, "") + .expect("the Barrett helper zext is valid"); + let product = builder + .build_int_mul( + dividend_wide, + Self::biguint_constant(wide_type, &reciprocal), + "", + ) + .expect("the Barrett helper multiply is valid"); + let shifted = builder + .build_right_shift( + product, + wide_type.const_int(word_type.get_bit_width() as u64, false), + false, + "", + ) + .expect("the Barrett helper shift is valid"); + let quotient_estimate = builder + .build_int_truncate(shifted, word_type, "") + .expect("the Barrett helper truncate is valid"); + let quotient_times_divisor = builder + .build_int_mul(quotient_estimate, divisor_constant, "") + .expect("the Barrett helper multiply is valid"); + let remainder_estimate = builder + .build_int_sub(dividend, quotient_times_divisor, "") + .expect("the Barrett helper sub is valid"); + let correction_fires = builder + .build_int_compare( + inkwell::IntPredicate::UGE, + remainder_estimate, + divisor_constant, + "", + ) + .expect("the Barrett helper compare is valid"); + let result = match opcode { + InstructionOpcode::UDiv => { + let corrected = builder + .build_int_add(quotient_estimate, word_type.const_int(1, false), "") + .expect("the Barrett helper add is valid"); + builder + .build_select(correction_fires, corrected, quotient_estimate, "") + .expect("the Barrett helper select is valid") + } + InstructionOpcode::URem => { + let corrected = builder + .build_int_sub(remainder_estimate, divisor_constant, "") + .expect("the Barrett helper sub is valid"); + builder + .build_select(correction_fires, corrected, remainder_estimate, "") + .expect("the Barrett helper select is valid") + } + _ => unreachable!("only udiv/urem sites are Barrett-specialized"), + }; + builder + .build_return(Some(&result)) + .expect("the Barrett helper return is valid"); + function + } +} diff --git a/crates/llvm-context/src/polkavm/context/function/llvm_runtime.rs b/crates/llvm-context/src/polkavm/context/function/llvm_runtime.rs index 137835ab0..6f404350f 100644 --- a/crates/llvm-context/src/polkavm/context/function/llvm_runtime.rs +++ b/crates/llvm-context/src/polkavm/context/function/llvm_runtime.rs @@ -31,6 +31,41 @@ impl<'ctx> LLVMRuntime<'ctx> { /// The corresponding runtime function name. pub const FUNCTION_SIGNEXTEND: &'static str = "__signextend"; + /// Name of the unsigned 256-bit division helper in `stdlib.ll` + /// It is not a registered runtime function. It must keep external + /// linkage so it survives optimization until `lower_wide_division` + /// reroutes non-narrowable i256 `udiv` to it, so only its name lives here. + pub const FUNCTION_UDIV256: &'static str = "__udiv256"; + + /// Name of the unsigned 256-bit remainder helper in `stdlib.ll` + /// It is not a registered runtime function. It must keep external + /// linkage so it survives optimization until `lower_wide_division` + /// reroutes non-narrowable i256 `urem` to it, so only its name lives here. + pub const FUNCTION_UREM256: &'static str = "__urem256"; + + /// Name of the signed 256-bit division helper in `stdlib.ll` + /// It is not a registered runtime function. It must keep external + /// linkage so it survives optimization until `lower_wide_division` + /// reroutes non-narrowable i256 `sdiv` to it, so only its name lives here. + pub const FUNCTION_SDIV256: &'static str = "__sdiv256"; + + /// Name of the signed 256-bit remainder helper in `stdlib.ll` + /// It is not a registered runtime function. It must keep external + /// linkage so it survives optimization until `lower_wide_division` + /// reroutes non-narrowable i256 `srem` to it, so only its name lives here. + pub const FUNCTION_SREM256: &'static str = "__srem256"; + + /// Name of the Barrett constant-modulus mulmod helper in `stdlib.ll` + /// It is not a registered runtime function (registration would flip it to + /// private linkage). It must keep external linkage for both rewrite + /// phases: call sites created by `specialize_constant_modulus_mulmod` + /// BEFORE the optimization pipeline must target a function the pipeline + /// can neither erase, argument-strip, nor calling-convention-promote, + /// and the function must still exist when `lower_wide_division` + /// rewrites late-exposed constant-modulus `__mulmod` call sites AFTER + /// the pipeline. Only its name lives here. + pub const FUNCTION_MULMOD_BARRETT: &'static str = "__mulmod_barrett"; + /// A shortcut constructor. pub fn new( llvm: &'ctx inkwell::context::Context, diff --git a/crates/llvm-context/src/polkavm/context/mod.rs b/crates/llvm-context/src/polkavm/context/mod.rs index 84cc6c2dd..a6f29c900 100644 --- a/crates/llvm-context/src/polkavm/context/mod.rs +++ b/crates/llvm-context/src/polkavm/context/mod.rs @@ -51,6 +51,7 @@ pub mod argument; pub mod attribute; pub mod build; pub mod code_type; +mod constant_division; pub mod debug_info; pub mod function; pub mod global; @@ -348,6 +349,19 @@ impl<'ctx> Context<'ctx> { ) })?; + // Rewrite constant-modulus `__mulmod` calls into Barrett form BEFORE + // the pipeline inlines `__mulmod`'s long-division body into callers, + // after which the mulmod-level structure (and with it the chance to + // skip its operand pre-reductions entirely) is unrecoverable. + self.specialize_constant_modulus_mulmod(&target_machine) + .map_err(|error| { + anyhow::anyhow!( + "The contract `{}` mulmod specialization error: {}", + contract_path, + error + ) + })?; + self.optimizer .run(&target_machine, self.module()) .map_err(|error| { @@ -358,11 +372,12 @@ impl<'ctx> Context<'ctx> { ) })?; - // Narrow large integer div/rem where operands provably fit in a smaller - // type. LLVM's i256 div/rem backend expansion produces enormous basic - // blocks that pallet-revive rejects as `BasicBlockTooLarge`; narrowing - // here keeps the expansion compact. - self.narrow_divrem_instructions(); + // Lower wide integer division that LLVM's i256 div/rem backend would otherwise + // expand into enormous basic blocks that pallet-revive rejects as `BasicBlockTooLarge`: + // - Narrow large integer div/rem where operands provably fit a smaller type. + // - Specialize constant divisors and constant `__mulmod` moduli into Barrett reductions + // - Route the rest to the stdlib runtime routines. + self.lower_wide_division(); self.debug_config .dump_llvm_ir_optimized(contract_path, self.module())?; @@ -1765,8 +1780,13 @@ impl<'ctx> Context<'ctx> { ) } - /// Narrows large integer div/rem instructions whose operands provably fit - /// in a smaller type. + /// Lowers wide integer division after the LLVM optimization pipeline: + /// narrows large div/rem whose operands provably fit a smaller type, + /// specializes constant-divisor div/rem into Barrett reductions, routes + /// the remaining i256 div/rem to the stdlib runtime routines, and + /// repeats the constant-modulus `__mulmod` sweep for moduli that only + /// became constant during the pipeline (the primary `__mulmod` rewrite + /// runs beforehand in [`Self::specialize_constant_modulus_mulmod`]). /// /// LLVM's i256 div/rem backend expansion produces enormous basic blocks /// that pallet-revive rejects as `BasicBlockTooLarge`. LLVM's own @@ -1774,12 +1794,44 @@ impl<'ctx> Context<'ctx> { /// functions after inlining. This runs as a safety net after the full /// LLVM optimization pipeline so we operate on the final form where the /// proof sources we accept (LLVM constants, `and` masks, `zext`) are - /// observable across function boundaries after IPSCCP and inlining. - fn narrow_divrem_instructions(&self) { + /// observable across function boundaries after IPSCCP and inlining. The + /// Barrett helpers are generated here, after the pipeline, so they are + /// never calling-convention-promoted between creation and use (see + /// [`constant_division`] for the full argument). + fn lower_wide_division(&self) { let builder = self.llvm.create_builder(); + let stdlib_function = |name: &str| { + self.module() + .get_function(name) + .unwrap_or_else(|| panic!("{name} must be declared in the stdlib module")) + }; + let udiv256 = stdlib_function(LLVMRuntime::FUNCTION_UDIV256); + let urem256 = stdlib_function(LLVMRuntime::FUNCTION_UREM256); + let sdiv256 = stdlib_function(LLVMRuntime::FUNCTION_SDIV256); + let srem256 = stdlib_function(LLVMRuntime::FUNCTION_SREM256); + + // Catch `__mulmod` call sites whose modulus only became a constant + // during the main pipeline; sites already constant beforehand were + // rewritten by `specialize_constant_modulus_mulmod`. + self.rewrite_constant_modulus_mulmod_calls(); for function in self.module().get_functions() { + // Generated Barrett helpers are never candidates for rewriting: + // their bodies contain no eligible operations, and skipping them + // removes any reliance on the module function list tolerating + // appends during iteration (defense in depth). + if function + .get_name() + .to_string_lossy() + .starts_with(constant_division::HELPER_PREFIX) + { + continue; + } + let mut to_narrow = Vec::new(); + let mut to_barrett_divrem: Vec<(inkwell::values::InstructionValue, num::BigUint)> = + Vec::new(); + let mut to_route = Vec::new(); for basic_block in function.get_basic_blocks() { for instruction in basic_block.get_instructions() { @@ -1793,13 +1845,15 @@ impl<'ctx> Context<'ctx> { if !is_divrem { continue; } - if instruction.get_type().into_int_type().get_bit_width() < 256 { + let bit_width = instruction.get_type().into_int_type().get_bit_width(); + if bit_width < 256 { continue; } let lhs = instruction.get_operand(0).and_then(|op| op.value()); let rhs = instruction.get_operand(1).and_then(|op| op.value()); + let mut narrowed = false; if let (Some(lhs), Some(rhs)) = (lhs, rhs) { let lhs_width = Self::provable_bit_width(lhs); let rhs_width = Self::provable_bit_width(rhs); @@ -1819,9 +1873,38 @@ impl<'ctx> Context<'ctx> { !is_signed || narrow_width > max_operand_width; if narrow_width < 256 && signed_sign_bit_safe { to_narrow.push((instruction, narrow_width)); + narrowed = true; } } } + + // Second rung: an i256 udiv/urem by a compile-time constant that cannot + // be narrowed is rewritten into a call to a generated Barrett helper + // (multiplication by a precomputed reciprocal), measurably ~1.5x faster + // per call than the routed long-division runtime routine. + if !narrowed && bit_width == 256 { + if let Some(divisor) = + Self::barrett_eligible_divisor(&builder, &instruction) + { + to_barrett_divrem.push((instruction, divisor)); + continue; + } + } + + // An i256 div/rem we cannot prove narrowable would otherwise be expanded by + // the backend into a ~500-instruction bit-at-a-time loop. Route it to the + // efficient stdlib routines instead: unsigned ops use 128-bit-digit long + // division bottoming out at a hardware-backed 128/64 divide via `__udivti3`. + // Signed ops wrap those with sign-magnitude. + if !narrowed && bit_width == 256 { + match instruction.get_opcode() { + InstructionOpcode::UDiv => to_route.push((instruction, udiv256)), + InstructionOpcode::URem => to_route.push((instruction, urem256)), + InstructionOpcode::SDiv => to_route.push((instruction, sdiv256)), + InstructionOpcode::SRem => to_route.push((instruction, srem256)), + _ => {} + } + } } } @@ -1884,6 +1967,245 @@ impl<'ctx> Context<'ctx> { instruction.replace_all_uses_with(&wide_instruction); instruction.erase_from_basic_block(); } + + for (instruction, divisor) in to_barrett_divrem { + let dividend = instruction + .get_operand(0) + .unwrap() + .value() + .unwrap() + .into_int_value(); + let helper = self.barrett_divrem_helper(instruction.get_opcode(), &divisor); + + builder.position_before(&instruction); + + let call_site = builder.build_call(helper, &[dividend.into()], "").unwrap(); + // One-line insurance: a freshly generated helper is default-CC + // by construction, but a helper reused by name must never be + // called with a mismatched convention. + call_site.set_call_convention(helper.get_call_conventions()); + let call_instruction = call_site + .try_as_basic_value() + .unwrap_basic() + .into_int_value() + .as_instruction() + .unwrap(); + instruction.replace_all_uses_with(&call_instruction); + instruction.erase_from_basic_block(); + } + + for (instruction, target) in to_route { + let lhs = instruction + .get_operand(0) + .unwrap() + .value() + .unwrap() + .into_int_value(); + let rhs = instruction + .get_operand(1) + .unwrap() + .value() + .unwrap() + .into_int_value(); + + builder.position_before(&instruction); + + let call_value = builder + .build_call(target, &[lhs.into(), rhs.into()], "") + .unwrap() + .try_as_basic_value() + .unwrap_basic(); + let call_instruction = call_value.into_int_value().as_instruction().unwrap(); + instruction.replace_all_uses_with(&call_instruction); + instruction.erase_from_basic_block(); + } + } + } + + /// Rewrites `__mulmod` calls with a compile-time-constant modulus into + /// Barrett form ahead of the main optimization pipeline. + /// + /// Constant moduli usually reach `__mulmod` in composed form (solc's code + /// generator emits large literals as `not`/`shl`/`sub` expressions and + /// may hold them in stack variables), so a cheap constant-folding + /// pre-pass runs first to expose them as plain constants. The rewrite + /// must happen before the main pipeline: once the inliner dissolves + /// `__mulmod`'s body into callers, an eligible site decays into two + /// operand pre-reductions plus a 512-by-256 long division, and the + /// Barrett form — which needs no pre-reduction at all, since its operand + /// bound `a*b < 2^512` holds for arbitrary 256-bit factors — is no + /// longer recoverable. Sites whose modulus only becomes constant during + /// the main pipeline are still caught by the sweep in + /// [`Self::lower_wide_division`]. + /// + /// The rewritten calls target the external `__mulmod_barrett`, which the + /// pipeline can neither calling-convention-promote nor argument-strip, + /// so running before optimization is safe (the post-pipeline `fastcc` + /// hazard documented in [`constant_division`] concerns helpers generated after the + /// pipeline, not pre-existing external stdlib routines). + /// + /// Skipped entirely when the module contains no `__mulmod` call or at + /// `-O0` (the pre-pass would violate the no-optimization contract), so + /// the compile-time cost and any phase-ordering influence are confined + /// to modules that actually use `mulmod`. + fn specialize_constant_modulus_mulmod( + &self, + target_machine: &TargetMachine, + ) -> anyhow::Result<()> { + if self.optimizer.settings().level_middle_end == inkwell::OptimizationLevel::None { + return Ok(()); + } + let has_mulmod_calls = self + .module() + .get_function(LLVMRuntime::FUNCTION_MULMOD) + .map(|function| { + function + .as_global_value() + .as_pointer_value() + .get_first_use() + .is_some() + }) + .unwrap_or(false); + if !has_mulmod_calls { + return Ok(()); + } + target_machine + .run_optimization_passes(self.module(), Self::MULMOD_CONSTANT_FOLD_PASSES) + .map_err(|error| { + anyhow::anyhow!("the mulmod constant-fold pre-pass failed: {error}") + })?; + self.rewrite_constant_modulus_mulmod_calls(); + Ok(()) + } + + /// The constant-folding pre-pass run by + /// [`Self::specialize_constant_modulus_mulmod`]: `mem2reg` surfaces + /// moduli held in stack slots, `sccp` folds constants across the + /// branches solc emits around composed literals, and `instcombine` + /// canonicalizes the remainder into plain `ConstantInt` operands. + /// A single `instcombine` iteration suffices for that job, so the + /// fixpoint verification a standalone `instcombine` performs by default + /// is disabled (the main pipeline reruns `instcombine` to fixpoint + /// afterwards anyway). + const MULMOD_CONSTANT_FOLD_PASSES: &'static str = + "function(mem2reg,sccp,instcombine)"; + + /// Rewrites every eligible constant-modulus `__mulmod` call site in the + /// module: 256-bit moduli into `__mulmod_barrett` calls carrying the + /// precomputed reciprocal, power-of-two moduli into inline `mul` + mask. + /// + /// Runs both before the main optimization pipeline (from + /// [`Self::specialize_constant_modulus_mulmod`]) and after it (from + /// [`Self::lower_wide_division`], for moduli the pipeline only then + /// exposed as constants); the classification is idempotent, and sites + /// rewritten by the first sweep no longer call `__mulmod`, so the second + /// sweep cannot see them again. + fn rewrite_constant_modulus_mulmod_calls(&self) { + // `__mulmod` is a registered runtime function with private linkage: + // it always exists at the pre-pipeline sweep, but the pipeline erases + // it from modules that never call mulmod, so the post-pipeline sweep + // must tolerate its absence; without the function there can be no + // eligible call sites. + let Some(mulmod_callee) = self + .module() + .get_function(LLVMRuntime::FUNCTION_MULMOD) + .map(|function| function.as_global_value().as_pointer_value()) + else { + return; + }; + let mulmod_barrett = self + .module() + .get_function(LLVMRuntime::FUNCTION_MULMOD_BARRETT) + .unwrap_or_else(|| { + panic!( + "{} must be declared in the stdlib module", + LLVMRuntime::FUNCTION_MULMOD_BARRETT + ) + }); + let builder = self.llvm.create_builder(); + + for function in self.module().get_functions() { + let mut to_mulmod_rewrite: Vec<( + inkwell::values::InstructionValue, + constant_division::MulModRewrite, + )> = Vec::new(); + + for basic_block in function.get_basic_blocks() { + for instruction in basic_block.get_instructions() { + if instruction.get_opcode() != InstructionOpcode::Call { + continue; + } + if let Some(rewrite) = + Self::mulmod_rewrite(&builder, &instruction, mulmod_callee) + { + to_mulmod_rewrite.push((instruction, rewrite)); + } + } + } + + for (instruction, rewrite) in to_mulmod_rewrite { + let call_operand = |index: u32| { + instruction + .get_operand(index) + .unwrap() + .value() + .unwrap() + .into_int_value() + }; + let factor_one = call_operand(0); + let factor_two = call_operand(1); + let modulus = call_operand(2); + + builder.position_before(&instruction); + + match rewrite { + constant_division::MulModRewrite::BarrettCall(reciprocal_low) => { + let reciprocal_low_constant = + Self::biguint_constant(self.word_type(), &reciprocal_low); + let call_site = builder + .build_call( + mulmod_barrett, + &[ + factor_one.into(), + factor_two.into(), + modulus.into(), + reciprocal_low_constant.into(), + ], + "", + ) + .unwrap(); + // The callee is external with default CC; keep the + // fresh call site in lockstep regardless. + call_site.set_call_convention(mulmod_barrett.get_call_conventions()); + let call_instruction = call_site + .try_as_basic_value() + .unwrap_basic() + .into_int_value() + .as_instruction() + .unwrap(); + instruction.replace_all_uses_with(&call_instruction); + instruction.erase_from_basic_block(); + } + constant_division::MulModRewrite::PowerOfTwoMask(exponent) => { + let mask = (num::BigUint::from(1u32) << exponent) - 1u32; + let mask_constant = Self::biguint_constant(self.word_type(), &mask); + let product = builder.build_int_mul(factor_one, factor_two, "").unwrap(); + let masked = builder.build_and(product, mask_constant, "").unwrap(); + match masked.as_instruction() { + Some(masked_instruction) => { + instruction.replace_all_uses_with(&masked_instruction) + } + // Both factors were constants and the rewrite + // folded away entirely; replace the call's uses + // with the folded constant directly. + None => inkwell::values::IntValue::try_from(instruction) + .expect("the __mulmod call returns an integer") + .replace_all_uses_with(masked), + } + instruction.erase_from_basic_block(); + } + } + } } } diff --git a/crates/llvm-context/src/polkavm/context/tests.rs b/crates/llvm-context/src/polkavm/context/tests.rs index ebb6b0e09..1bc18848a 100644 --- a/crates/llvm-context/src/polkavm/context/tests.rs +++ b/crates/llvm-context/src/polkavm/context/tests.rs @@ -1,7 +1,11 @@ //! The LLVM IR generator context tests. +use inkwell::values::InstructionOpcode; +use num::BigUint; + use crate::optimizer::settings::Settings as OptimizerSettings; use crate::polkavm::context::attribute::Attribute; +use crate::polkavm::context::constant_division; use crate::polkavm::context::Context; use crate::PolkaVMTarget; @@ -150,3 +154,778 @@ pub fn check_attribute_min_size_mode_z() { .attributes(inkwell::attributes::AttributeLoc::Function) .contains(&llvm.create_enum_attribute(Attribute::MinSize as u32, 0))); } + +/// Adds an `i256 name(i256 x parameter_count)` function with an empty entry +/// block to the dummy context's module. +fn add_word_function<'ctx>( + llvm: &'ctx inkwell::context::Context, + context: &Context<'ctx>, + name: &str, + parameter_count: usize, +) -> inkwell::values::FunctionValue<'ctx> { + let word_type = context.word_type(); + let parameter_types = vec![word_type.into(); parameter_count]; + let function = + context + .module() + .add_function(name, word_type.fn_type(¶meter_types, false), None); + llvm.append_basic_block(function, "entry"); + function +} + +/// Counts the call instructions in `function` whose callee is `callee`, +/// compared by pointer identity. +fn count_calls_to<'ctx>( + function: inkwell::values::FunctionValue<'ctx>, + callee: inkwell::values::FunctionValue<'ctx>, +) -> usize { + let callee_pointer = callee.as_global_value().as_pointer_value(); + let mut count = 0; + for basic_block in function.get_basic_blocks() { + for instruction in basic_block.get_instructions() { + if instruction.get_opcode() != InstructionOpcode::Call { + continue; + } + let last_operand = instruction + .get_operand(instruction.get_num_operands() - 1) + .and_then(|operand| operand.value()); + if last_operand.is_some_and(|operand| { + operand.is_pointer_value() && operand.into_pointer_value() == callee_pointer + }) { + count += 1; + } + } + } + count +} + +/// Counts the instructions in `function` with the given opcode; when +/// `bit_width` is given, only instructions of that integer width count. +fn count_opcodes( + function: inkwell::values::FunctionValue, + opcode: InstructionOpcode, + bit_width: Option, +) -> usize { + let mut count = 0; + for basic_block in function.get_basic_blocks() { + for instruction in basic_block.get_instructions() { + if instruction.get_opcode() != opcode { + continue; + } + if let Some(bit_width) = bit_width { + if !instruction.get_type().is_int_type() + || instruction.get_type().into_int_type().get_bit_width() != bit_width + { + continue; + } + } + count += 1; + } + } + count +} + +/// Counts the module functions whose name starts with the reserved Barrett +/// helper prefix. +fn count_barrett_helpers(context: &Context) -> usize { + context + .module() + .get_functions() + .filter(|function| { + function + .get_name() + .to_string_lossy() + .starts_with(constant_division::HELPER_PREFIX) + }) + .count() +} + +/// The secp256k1 field prime: a 256-bit constant with all limbs populated. +fn secp256k1_prime() -> BigUint { + BigUint::parse_bytes( + b"fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f", + 16, + ) + .expect("the secp256k1 prime is valid hexadecimal") +} + +#[test] +fn barrett_rewrites_constant_unsigned_division() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + + // 2^128 + 1: not narrowable, not a power of two, takes the multiply path. + let divisor = (BigUint::from(1u32) << 128u32) + 1u32; + let function = add_word_function(&llvm, &context, "test", 1); + let builder = llvm.create_builder(); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let dividend = function.get_first_param().unwrap().into_int_value(); + let quotient = builder + .build_int_unsigned_div(dividend, Context::biguint_constant(word_type, &divisor), "") + .unwrap(); + builder.build_return(Some("ient)).unwrap(); + + context.lower_wide_division(); + + let helper_name = format!( + "{}{}{}", + constant_division::HELPER_PREFIX, + constant_division::UDIV_HELPER_INFIX, + divisor.to_str_radix(16) + ); + let helper = context + .module() + .get_function(&helper_name) + .expect("the Barrett division helper must have been generated"); + assert_eq!(helper.get_linkage(), inkwell::module::Linkage::Internal); + assert!(helper + .attributes(inkwell::attributes::AttributeLoc::Function) + .contains(&llvm.create_enum_attribute(Attribute::NoInline as u32, 0))); + assert_eq!(count_calls_to(function, helper), 1); + assert_eq!(count_opcodes(function, InstructionOpcode::UDiv, None), 0); + context.verify().unwrap(); +} + +#[test] +fn barrett_rewrites_constant_unsigned_remainder() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + + let divisor = secp256k1_prime(); + let function = add_word_function(&llvm, &context, "test", 1); + let builder = llvm.create_builder(); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let dividend = function.get_first_param().unwrap().into_int_value(); + let remainder = builder + .build_int_unsigned_rem(dividend, Context::biguint_constant(word_type, &divisor), "") + .unwrap(); + builder.build_return(Some(&remainder)).unwrap(); + + context.lower_wide_division(); + + let helper_name = format!( + "{}{}{}", + constant_division::HELPER_PREFIX, + constant_division::UREM_HELPER_INFIX, + divisor.to_str_radix(16) + ); + let helper = context + .module() + .get_function(&helper_name) + .expect("the Barrett remainder helper must have been generated"); + assert_eq!(helper.get_linkage(), inkwell::module::Linkage::Internal); + assert!(helper + .attributes(inkwell::attributes::AttributeLoc::Function) + .contains(&llvm.create_enum_attribute(Attribute::NoInline as u32, 0))); + assert_eq!(count_calls_to(function, helper), 1); + assert_eq!(count_opcodes(function, InstructionOpcode::URem, None), 0); + context.verify().unwrap(); +} + +#[test] +fn barrett_helper_deduplicated_across_functions() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + let builder = llvm.create_builder(); + + let divisor = (BigUint::from(1u32) << 192u32) + 9u32; + let mut functions = Vec::new(); + for name in ["test_one", "test_two"] { + let function = add_word_function(&llvm, &context, name, 1); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let dividend = function.get_first_param().unwrap().into_int_value(); + let quotient = builder + .build_int_unsigned_div(dividend, Context::biguint_constant(word_type, &divisor), "") + .unwrap(); + builder.build_return(Some("ient)).unwrap(); + functions.push(function); + } + + context.lower_wide_division(); + + assert_eq!(count_barrett_helpers(&context), 1); + let helper_name = format!( + "{}{}{}", + constant_division::HELPER_PREFIX, + constant_division::UDIV_HELPER_INFIX, + divisor.to_str_radix(16) + ); + let helper = context.module().get_function(&helper_name).unwrap(); + for function in functions { + assert_eq!(count_calls_to(function, helper), 1); + } + context.verify().unwrap(); +} + +#[test] +fn barrett_skips_powers_of_two_and_trivial_divisors() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + let builder = llvm.create_builder(); + + let divisors = [ + BigUint::from(0u32), + BigUint::from(1u32), + BigUint::from(1u32) << 200u32, + ]; + let mut functions = Vec::new(); + for (index, divisor) in divisors.iter().enumerate() { + let function = add_word_function(&llvm, &context, &format!("test_{index}"), 1); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let dividend = function.get_first_param().unwrap().into_int_value(); + let quotient = builder + .build_int_unsigned_div(dividend, Context::biguint_constant(word_type, divisor), "") + .unwrap(); + builder.build_return(Some("ient)).unwrap(); + functions.push(function); + } + + context.lower_wide_division(); + + assert_eq!(count_barrett_helpers(&context), 0); + let udiv256 = context.module().get_function("__udiv256").unwrap(); + for function in functions { + assert_eq!(count_calls_to(function, udiv256), 1); + } + context.verify().unwrap(); +} + +#[test] +fn barrett_respects_narrowing_precedence() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + + let function = add_word_function(&llvm, &context, "test", 1); + let builder = llvm.create_builder(); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let parameter = function.get_first_param().unwrap().into_int_value(); + let masked = builder + .build_and(parameter, word_type.const_int(u16::MAX as u64, false), "") + .unwrap(); + let quotient = builder + .build_int_unsigned_div(masked, word_type.const_int(5, false), "") + .unwrap(); + builder.build_return(Some("ient)).unwrap(); + + context.lower_wide_division(); + + assert_eq!(count_barrett_helpers(&context), 0); + assert_eq!( + count_opcodes(function, InstructionOpcode::UDiv, Some(256)), + 0 + ); + assert_eq!( + count_opcodes(function, InstructionOpcode::UDiv, Some(16)), + 1 + ); + context.verify().unwrap(); +} + +#[test] +fn barrett_leaves_signed_constant_division_routed() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + let builder = llvm.create_builder(); + + // A 256-bit magnitude defeats narrowing, keeping the sites on the ladder's + // routing rung. + let divisor = Context::biguint_constant(word_type, &secp256k1_prime()); + + let division = add_word_function(&llvm, &context, "test_sdiv", 1); + builder.position_at_end(division.get_first_basic_block().unwrap()); + let dividend = division.get_first_param().unwrap().into_int_value(); + let quotient = builder.build_int_signed_div(dividend, divisor, "").unwrap(); + builder.build_return(Some("ient)).unwrap(); + + let remainder_function = add_word_function(&llvm, &context, "test_srem", 1); + builder.position_at_end(remainder_function.get_first_basic_block().unwrap()); + let dividend = remainder_function + .get_first_param() + .unwrap() + .into_int_value(); + let remainder = builder.build_int_signed_rem(dividend, divisor, "").unwrap(); + builder.build_return(Some(&remainder)).unwrap(); + + context.lower_wide_division(); + + assert_eq!(count_barrett_helpers(&context), 0); + let sdiv256 = context.module().get_function("__sdiv256").unwrap(); + let srem256 = context.module().get_function("__srem256").unwrap(); + assert_eq!(count_calls_to(division, sdiv256), 1); + assert_eq!(count_calls_to(remainder_function, srem256), 1); + context.verify().unwrap(); +} + +#[test] +fn barrett_handles_degenerate_full_width_divisor() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + let builder = llvm.create_builder(); + + // 2^256 - 1: bits(C) == 256, the compare-and-subtract helper body. + let divisor = (BigUint::from(1u32) << 256u32) - 1u32; + + let division = add_word_function(&llvm, &context, "test_udiv", 1); + builder.position_at_end(division.get_first_basic_block().unwrap()); + let dividend = division.get_first_param().unwrap().into_int_value(); + let quotient = builder + .build_int_unsigned_div(dividend, Context::biguint_constant(word_type, &divisor), "") + .unwrap(); + builder.build_return(Some("ient)).unwrap(); + + let remainder_function = add_word_function(&llvm, &context, "test_urem", 1); + builder.position_at_end(remainder_function.get_first_basic_block().unwrap()); + let dividend = remainder_function + .get_first_param() + .unwrap() + .into_int_value(); + let remainder = builder + .build_int_unsigned_rem(dividend, Context::biguint_constant(word_type, &divisor), "") + .unwrap(); + builder.build_return(Some(&remainder)).unwrap(); + + context.lower_wide_division(); + + assert_eq!(count_barrett_helpers(&context), 2); + assert_eq!(count_opcodes(division, InstructionOpcode::UDiv, None), 0); + assert_eq!( + count_opcodes(remainder_function, InstructionOpcode::URem, None), + 0 + ); + context.verify().unwrap(); +} + +#[test] +fn wide_unsigned_constant_round_trips_without_instructions() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + + let function = add_word_function(&llvm, &context, "test", 1); + let entry = function.get_first_basic_block().unwrap(); + let builder = llvm.create_builder(); + builder.position_at_end(entry); + + let expected = secp256k1_prime(); + let constant = Context::biguint_constant(word_type, &expected); + let extracted = Context::wide_unsigned_constant(&builder, constant) + .expect("the constant must be extractable"); + assert_eq!(extracted, expected); + // Pins the fold-only contract: extraction must not insert instructions. + assert_eq!(entry.get_instructions().count(), 0); +} + +#[test] +fn mulmod_constant_modulus_rewrites_to_barrett_call() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + + let modulus = secp256k1_prime(); + let function = add_word_function(&llvm, &context, "test", 2); + let builder = llvm.create_builder(); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let mulmod = context.module().get_function("__mulmod").unwrap(); + let result = builder + .build_call( + mulmod, + &[ + function.get_nth_param(0).unwrap().into(), + function.get_nth_param(1).unwrap().into(), + Context::biguint_constant(word_type, &modulus).into(), + ], + "", + ) + .unwrap() + .try_as_basic_value() + .unwrap_basic(); + builder.build_return(Some(&result)).unwrap(); + + context.lower_wide_division(); + + let mulmod_barrett = context.module().get_function("__mulmod_barrett").unwrap(); + assert_eq!(count_calls_to(function, mulmod), 0); + assert_eq!(count_calls_to(function, mulmod_barrett), 1); + + // The rewritten call must carry floor(2^512 / m) - 2^256 as its fourth + // argument (LLVM interns constants, so pointer equality is complete). + let expected_reciprocal_low = + (BigUint::from(1u32) << 512u32) / &modulus - (BigUint::from(1u32) << 256u32); + let expected_constant = Context::biguint_constant(word_type, &expected_reciprocal_low); + let call = function + .get_basic_blocks() + .into_iter() + .flat_map(|basic_block| basic_block.get_instructions()) + .find(|instruction| instruction.get_opcode() == InstructionOpcode::Call) + .expect("the rewritten call must exist"); + assert_eq!(call.get_num_operands(), 5); + let reciprocal_argument = call + .get_operand(3) + .unwrap() + .value() + .unwrap() + .into_int_value(); + assert_eq!(reciprocal_argument, expected_constant); + context.verify().unwrap(); +} + +#[test] +fn mulmod_small_constant_modulus_stays_routed() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + + // 2^128 + 1: bits == 129 < 256, below the sharp eligibility boundary. + let modulus = (BigUint::from(1u32) << 128u32) + 1u32; + let function = add_word_function(&llvm, &context, "test", 2); + let builder = llvm.create_builder(); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let mulmod = context.module().get_function("__mulmod").unwrap(); + let result = builder + .build_call( + mulmod, + &[ + function.get_nth_param(0).unwrap().into(), + function.get_nth_param(1).unwrap().into(), + Context::biguint_constant(word_type, &modulus).into(), + ], + "", + ) + .unwrap() + .try_as_basic_value() + .unwrap_basic(); + builder.build_return(Some(&result)).unwrap(); + + context.lower_wide_division(); + + assert_eq!(count_calls_to(function, mulmod), 1); + let mulmod_barrett = context.module().get_function("__mulmod_barrett").unwrap(); + assert_eq!(count_calls_to(function, mulmod_barrett), 0); + context.verify().unwrap(); +} + +#[test] +fn mulmod_power_of_two_modulus_becomes_masked_multiply() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + + // 2^255: the mu_lo-overflow case, must take the inline mask rewrite. + let modulus = BigUint::from(1u32) << 255u32; + let function = add_word_function(&llvm, &context, "test", 2); + let builder = llvm.create_builder(); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let mulmod = context.module().get_function("__mulmod").unwrap(); + let result = builder + .build_call( + mulmod, + &[ + function.get_nth_param(0).unwrap().into(), + function.get_nth_param(1).unwrap().into(), + Context::biguint_constant(word_type, &modulus).into(), + ], + "", + ) + .unwrap() + .try_as_basic_value() + .unwrap_basic(); + builder.build_return(Some(&result)).unwrap(); + + context.lower_wide_division(); + + assert_eq!(count_opcodes(function, InstructionOpcode::Call, None), 0); + assert_eq!( + count_opcodes(function, InstructionOpcode::Mul, Some(256)), + 1 + ); + assert_eq!( + count_opcodes(function, InstructionOpcode::And, Some(256)), + 1 + ); + let expected_mask = Context::biguint_constant(word_type, &(modulus - 1u32)); + let and_instruction = function + .get_basic_blocks() + .into_iter() + .flat_map(|basic_block| basic_block.get_instructions()) + .find(|instruction| instruction.get_opcode() == InstructionOpcode::And) + .unwrap(); + let mask_operand = and_instruction + .get_operand(1) + .unwrap() + .value() + .unwrap() + .into_int_value(); + assert_eq!(mask_operand, expected_mask); + context.verify().unwrap(); +} + +#[test] +fn mulmod_runtime_modulus_stays_untouched() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + + let function = add_word_function(&llvm, &context, "test", 3); + let builder = llvm.create_builder(); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let mulmod = context.module().get_function("__mulmod").unwrap(); + let result = builder + .build_call( + mulmod, + &[ + function.get_nth_param(0).unwrap().into(), + function.get_nth_param(1).unwrap().into(), + function.get_nth_param(2).unwrap().into(), + ], + "", + ) + .unwrap() + .try_as_basic_value() + .unwrap_basic(); + builder.build_return(Some(&result)).unwrap(); + + context.lower_wide_division(); + + assert_eq!(count_calls_to(function, mulmod), 1); + assert_eq!(count_barrett_helpers(&context), 0); + context.verify().unwrap(); +} + +#[test] +fn mulmod_unrelated_calls_stay_untouched() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let context = Context::new_dummy(&llvm, OptimizerSettings::cycles()); + let word_type = context.word_type(); + + // Same shape as an eligible __mulmod call, different callee. + let unrelated = context.module().add_function( + "unrelated_three_argument_function", + word_type.fn_type( + &[word_type.into(), word_type.into(), word_type.into()], + false, + ), + None, + ); + let function = add_word_function(&llvm, &context, "test", 2); + let builder = llvm.create_builder(); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let result = builder + .build_call( + unrelated, + &[ + function.get_nth_param(0).unwrap().into(), + function.get_nth_param(1).unwrap().into(), + Context::biguint_constant(word_type, &secp256k1_prime()).into(), + ], + "", + ) + .unwrap() + .try_as_basic_value() + .unwrap_basic(); + builder.build_return(Some(&result)).unwrap(); + + context.lower_wide_division(); + + assert_eq!(count_calls_to(function, unrelated), 1); + let mulmod_barrett = context.module().get_function("__mulmod_barrett").unwrap(); + assert_eq!(count_calls_to(function, mulmod_barrett), 0); + context.verify().unwrap(); +} + +#[test] +fn reciprocal_self_checks_hold_on_fuzz_matrix_constants() { + let one = BigUint::from(1u32); + + // The adversarial mulmod modulus grid from the verification protocol. + for modulus in [ + (&one << 255u32) + 1u32, + (&one << 255u32) + 3u32, + (&one << 256u32) - 1u32, + (&one << 256u32) - 2u32, + (&one << 256u32) - 3u32, + secp256k1_prime(), + (&one << 255u32) | &one, + (&one << 255u32) | ((&one << 128u32) - 1u32), + (&one << 255u32) + (&one << 254u32) + 1u32, + ] { + let reciprocal_low = Context::mulmod_reciprocal_low(&modulus); + assert!(reciprocal_low >= one && reciprocal_low.bits() <= 256); + } + + // The per-constant division/remainder fuzz list. + for divisor in [ + BigUint::from(3u32), + BigUint::from(5u32), + BigUint::from(10u32), + (&one << 64u32) + 1u32, + (&one << 128u32) - 1u32, + (&one << 128u32) + 1u32, + (&one << 192u32) + 9u32, + (&one << 255u32) + 3u32, + (&one << 256u32) - 1u32, + ] { + let reciprocal = Context::division_reciprocal(&divisor); + assert!(reciprocal >= one && reciprocal.bits() <= 256); + } +} + +/// Builds `__mulmod(x, y, xor(load(slot), -1))` in a fresh function, with +/// `not(modulus)` stored to a stack slot: the modulus operand is a real +/// instruction chain rather than a `ConstantInt`, modeling how solc emits +/// large constants in NOT-form. +fn add_composed_modulus_mulmod_function<'ctx>( + llvm: &'ctx inkwell::context::Context, + context: &Context<'ctx>, + modulus: &BigUint, +) -> inkwell::values::FunctionValue<'ctx> { + let word_type = context.word_type(); + let function = add_word_function(llvm, context, "test", 2); + let builder = llvm.create_builder(); + builder.position_at_end(function.get_first_basic_block().unwrap()); + + let all_ones = (BigUint::from(1u32) << 256u32) - 1u32; + let negated_modulus = Context::biguint_constant(word_type, &(&all_ones ^ modulus)); + let slot = builder.build_alloca(word_type, "slot").unwrap(); + builder.build_store(slot, negated_modulus).unwrap(); + let loaded = builder + .build_load(word_type, slot, "") + .unwrap() + .into_int_value(); + let composed_modulus = builder + .build_xor(loaded, word_type.const_all_ones(), "") + .unwrap(); + + let mulmod = context.module().get_function("__mulmod").unwrap(); + let result = builder + .build_call( + mulmod, + &[ + function.get_nth_param(0).unwrap().into(), + function.get_nth_param(1).unwrap().into(), + composed_modulus.into(), + ], + "", + ) + .unwrap() + .try_as_basic_value() + .unwrap_basic(); + builder.build_return(Some(&result)).unwrap(); + function +} + +#[test] +fn mulmod_composed_constant_modulus_specializes_before_pipeline() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let settings = OptimizerSettings::cycles(); + let context = Context::new_dummy(&llvm, settings.clone()); + let function = add_composed_modulus_mulmod_function(&llvm, &context, &secp256k1_prime()); + let mulmod = context.module().get_function("__mulmod").unwrap(); + assert_eq!(count_calls_to(function, mulmod), 1); + + let target_machine = crate::PolkaVMTargetMachine::new(PolkaVMTarget::PVM, &settings, false) + .expect("the test target machine must be creatable"); + context + .specialize_constant_modulus_mulmod(&target_machine) + .expect("the pre-pipeline specialization must succeed"); + + // The constant-fold pre-pass exposes the NOT-form modulus and the sweep + // retargets the call, even though nothing was a `ConstantInt` when the + // module was built. + let mulmod_barrett = context.module().get_function("__mulmod_barrett").unwrap(); + assert_eq!(count_calls_to(function, mulmod), 0); + assert_eq!(count_calls_to(function, mulmod_barrett), 1); +} + +#[test] +fn mulmod_specialization_is_skipped_at_optimization_level_zero() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let settings = OptimizerSettings::none(); + let context = Context::new_dummy(&llvm, settings.clone()); + let function = add_composed_modulus_mulmod_function(&llvm, &context, &secp256k1_prime()); + + let target_machine = crate::PolkaVMTargetMachine::new(PolkaVMTarget::PVM, &settings, false) + .expect("the test target machine must be creatable"); + context + .specialize_constant_modulus_mulmod(&target_machine) + .expect("the pre-pipeline specialization must succeed"); + + // At -O0 the hook returns before doing anything: the call still targets + // `__mulmod` and the constant-fold pre-pass never ran (the stack slot + // `mem2reg` would have promoted is still there). + let mulmod = context.module().get_function("__mulmod").unwrap(); + let mulmod_barrett = context.module().get_function("__mulmod_barrett").unwrap(); + assert_eq!(count_calls_to(function, mulmod), 1); + assert_eq!(count_calls_to(function, mulmod_barrett), 0); + assert_eq!(count_opcodes(function, InstructionOpcode::Alloca, None), 1); +} + +#[test] +fn mulmod_specialization_is_skipped_without_mulmod_calls() { + initialize_llvm(); + + let llvm = inkwell::context::Context::create(); + let settings = OptimizerSettings::cycles(); + let context = Context::new_dummy(&llvm, settings.clone()); + + // A function with foldable memory traffic but no `__mulmod` call. + let word_type = context.word_type(); + let function = add_word_function(&llvm, &context, "test", 1); + let builder = llvm.create_builder(); + builder.position_at_end(function.get_first_basic_block().unwrap()); + let slot = builder.build_alloca(word_type, "slot").unwrap(); + builder + .build_store(slot, function.get_first_param().unwrap().into_int_value()) + .unwrap(); + let loaded = builder + .build_load(word_type, slot, "") + .unwrap() + .into_int_value(); + builder.build_return(Some(&loaded)).unwrap(); + + let target_machine = crate::PolkaVMTargetMachine::new(PolkaVMTarget::PVM, &settings, false) + .expect("the test target machine must be creatable"); + context + .specialize_constant_modulus_mulmod(&target_machine) + .expect("the pre-pipeline specialization must succeed"); + + // `__mulmod` exists in the module (the stdlib is always linked) but has + // no uses, so the hook must skip the pre-pass entirely: the promotable + // stack slot proves no optimization ran. + assert_eq!(count_opcodes(function, InstructionOpcode::Alloca, None), 1); +} diff --git a/crates/stdlib/src/lib.rs b/crates/stdlib/src/lib.rs index 8634a78db..d8b8d1904 100644 --- a/crates/stdlib/src/lib.rs +++ b/crates/stdlib/src/lib.rs @@ -28,5 +28,6 @@ mod tests { let module = crate::module(&context, "stdlib").unwrap(); assert!(module.get_function("__signextend").is_some()); + assert!(module.get_function("__mulmod_barrett").is_some()); } } diff --git a/crates/stdlib/stdlib.ll b/crates/stdlib/stdlib.ll index 178c2b345..aca6b6ef4 100644 --- a/crates/stdlib/stdlib.ll +++ b/crates/stdlib/stdlib.ll @@ -3,6 +3,8 @@ target datalayout = "e-m:e-p:32:64-p1:32:64-i64:64-i128:128-n32:64-S64" target triple = "riscv64-unknown-none-elf" +declare i128 @llvm.ctlz.i128(i128, i1) + define i256 @__addmod(i256 %arg1, i256 %arg2, i256 %modulo) #0 { entry: %is_zero = icmp eq i256 %modulo, 0 @@ -28,144 +30,478 @@ return: ret i256 %value } -define private i256 @__clz(i256 %v) #0 { +; Efficient unsigned 256-bit division / remainder and 512-by-256 reduction. +; Core divide primitive is a 128/64 udiv (efficient __udivti3 libcall), +; never a raw i256 udiv/urem (LLVM expands those into a ~500-instruction +; bit-at-a-time loop). Algorithm: Knuth Algorithm D. +; Assumption at every entry: divisor / modulus != 0 (guarded by the caller). +; +; __udiv_qrnnd_128: divide the 256-bit value (uh:ul) by the 128-bit divisor v. +; Returns { q, r } with q = (uh:ul) / v and r = (uh:ul) mod v, both 128 bits. +; +; Notation: b = 2^64 (this routine's digit base); (x:y) means x*2^w + y where +; w is the bit width of y; / is floor division. +; Preconditions: v != 0 and uh < v, so that q fits in 128 bits. +; +; Knuth TAOCP 4.3.1 Algorithm D / Hacker's Delight 9-4 divlu in base b: a +; 4-digit dividend over a 2-digit divisor, unrolled into two digit steps. +; +; 1. Normalize: s = ctlz(v); vn = v << s (top bit set). Split vn = (vn1:vn0) +; into 64-bit digits. +; 2. un = (uh:ul) << s, exact in 256-bit arithmetic (uh < v keeps the top s +; bits free). Take u2 = un >> 128 (the top two digits as one 128-bit +; value) and the 64-bit digits u1 = (un >> 64) mod b, u0 = un mod b. +; +; First digit step, q1 = (u2:u1) / vn (three digits over two): +; 3. Estimate qhat = min(u2 / vn1, b-1) and rhat = u2 - qhat*vn1 (computed +; with the capped qhat; exact in 128 bits, may exceed b). +; 4. Correct, exactly two branchless steps (Knuth Thm. 4.3.1B: a normalized +; divisor never needs more than two): +; if qhat*vn0 > (rhat:u1) { qhat -= 1; rhat += vn1 } +; if qhat*vn0 > (rhat:u1) { qhat -= 1 } (rhat is dead afterwards) +; The qhat*vn0 products are exact in 128 bits; tests and rhat updates run +; in 256-bit arithmetic, so an oversized rhat correctly fails the test. +; q1 = qhat. +; 5. Partial remainder: u21 = (u2:u1) - q1*vn, computed mod 2^128; exact +; because the true value is a remainder mod vn, hence < 2^128. +; +; Second digit step, q0 = (u21:u0) / vn, identical shape: +; 6. Estimate qhat = min(u21 / vn1, b-1) and rhat = u21 - qhat*vn1. +; 7. The same two branchless corrections, against (rhat:u0). q0 = qhat. +; +; 8. Remainder: r = ((u21:u0) - q0*vn) >> s, computed in 256-bit arithmetic; +; less than v after the shift, so it truncates to 128 bits losslessly. +; 9. Return { (q1:q0), r }. +define private { i128, i128 } @__udiv_qrnnd_128(i128 %uh, i128 %ul, i128 %v) noinline #0 { +entry: + %v1 = call i128 @llvm.ctlz.i128(i128 %v, i1 false) + %v2 = shl i128 %v, %v1 + %v3 = lshr i128 %v2, 64 + %v4 = and i128 %v2, 18446744073709551615 + %v5 = zext i128 %uh to i256 + %v6 = zext i128 %ul to i256 + %v7 = shl i256 %v5, 128 + %v8 = or i256 %v7, %v6 + %v9 = zext i128 %v1 to i256 + %v10 = shl i256 %v8, %v9 + %v11 = lshr i256 %v10, 128 + %v12 = trunc i256 %v11 to i128 + %v13 = lshr i256 %v10, 64 + %v14 = and i256 %v13, 18446744073709551615 + %v15 = trunc i256 %v14 to i128 + %v16 = and i256 %v10, 18446744073709551615 + %v17 = trunc i256 %v16 to i128 + %v18 = udiv i128 %v12, %v3 + %v19 = icmp ugt i128 %v18, 18446744073709551615 + %v20 = select i1 %v19, i128 18446744073709551615, i128 %v18 + %v21 = mul i128 %v20, %v3 + %v22 = sub i128 %v12, %v21 + %v23 = zext i128 %v22 to i256 + %v24 = zext i128 %v3 to i256 + %v25 = zext i128 %v15 to i256 + %v26 = mul i128 %v20, %v4 + %v27 = zext i128 %v26 to i256 + %v28 = shl i256 %v23, 64 + %v29 = add i256 %v28, %v25 + %v30 = icmp ugt i256 %v27, %v29 + %v31 = sub i128 %v20, 1 + %v32 = add i256 %v23, %v24 + %v33 = select i1 %v30, i128 %v31, i128 %v20 + %v34 = select i1 %v30, i256 %v32, i256 %v23 + %v35 = mul i128 %v33, %v4 + %v36 = zext i128 %v35 to i256 + %v37 = shl i256 %v34, 64 + %v38 = add i256 %v37, %v25 + %v39 = icmp ugt i256 %v36, %v38 + %v40 = sub i128 %v33, 1 + %v42 = select i1 %v39, i128 %v40, i128 %v33 + %v44 = shl i128 %v12, 64 + %v45 = add i128 %v44, %v15 + %v46 = mul i128 %v42, %v2 + %v47 = sub i128 %v45, %v46 + %v48 = udiv i128 %v47, %v3 + %v49 = icmp ugt i128 %v48, 18446744073709551615 + %v50 = select i1 %v49, i128 18446744073709551615, i128 %v48 + %v51 = mul i128 %v50, %v3 + %v52 = sub i128 %v47, %v51 + %v53 = zext i128 %v52 to i256 + %v54 = zext i128 %v3 to i256 + %v55 = zext i128 %v17 to i256 + %v56 = mul i128 %v50, %v4 + %v57 = zext i128 %v56 to i256 + %v58 = shl i256 %v53, 64 + %v59 = add i256 %v58, %v55 + %v60 = icmp ugt i256 %v57, %v59 + %v61 = sub i128 %v50, 1 + %v62 = add i256 %v53, %v54 + %v63 = select i1 %v60, i128 %v61, i128 %v50 + %v64 = select i1 %v60, i256 %v62, i256 %v53 + %v65 = mul i128 %v63, %v4 + %v66 = zext i128 %v65 to i256 + %v67 = shl i256 %v64, 64 + %v68 = add i256 %v67, %v55 + %v69 = icmp ugt i256 %v66, %v68 + %v70 = sub i128 %v63, 1 + %v72 = select i1 %v69, i128 %v70, i128 %v63 + %v74 = zext i128 %v47 to i256 + %v75 = shl i256 %v74, 64 + %v76 = zext i128 %v17 to i256 + %v77 = add i256 %v75, %v76 + %v78 = zext i128 %v72 to i256 + %v79 = zext i128 %v2 to i256 + %v80 = mul i256 %v78, %v79 + %v81 = sub i256 %v77, %v80 + %v82 = lshr i256 %v81, %v9 + %v83 = trunc i256 %v82 to i128 + %v84 = shl i128 %v42, 64 + %v85 = or i128 %v84, %v72 + %v86 = insertvalue { i128, i128 } undef, i128 %v85, 0 + %v87 = insertvalue { i128, i128 } %v86, i128 %v83, 1 + ret { i128, i128 } %v87 +} + +; One quotient-digit step of Knuth TAOCP 4.3.1 Algorithm D in base B = 2^128. +; Returns q = (uhi:ulo:unext) / vn, the next 128-bit quotient digit of a +; running division by the normalized 256-bit divisor vn. +; +; Notation: B = 2^128; (x:y) = x*B + y, (x:y:z) = x*B^2 + y*B + z; / is +; floor division. +; Preconditions: vn is normalized (bit 255 set) and (uhi:ulo) < vn, which +; guarantees q <= B-1. +; Result: the exact digit q; quotient only. Callers re-derive the partial +; remainder as (uhi:ulo:unext) - q*vn themselves. +; +; 1. Split vn = (vn1:vn0) into 128-bit digits; vn1 has its top bit set. +; 2. Estimate qhat = min((uhi:ulo) / vn1, B-1): __udiv_qrnnd_128 computes +; (uhi:ulo) / vn1; when uhi >= vn1 (only uhi == vn1 is possible under the +; precondition) that quotient would be >= B, so qhat is forced to B-1. +; Normalization bounds the estimate: qhat - 2 <= q <= qhat. +; 3. rhat = (uhi:ulo) - qhat*vn1, exact in 256-bit arithmetic (below 2^129, +; below 2^130 even after the correction's rhat += vn1; may exceed B when +; qhat was capped). +; 4. Correct, exactly two branchless steps (Knuth Thm. 4.3.1B: at most two +; are ever needed for a normalized divisor): +; if qhat*vn0 > (rhat:unext) { qhat -= 1; rhat += vn1 } +; if qhat*vn0 > (rhat:unext) { qhat -= 1 } (rhat is dead afterwards) +; Each test compares qhat*vn0 (exact 256-bit product, zero-extended) +; against rhat*B + unext, exact in 384-bit arithmetic; an rhat >= B +; correctly fails the test. +; 5. Return qhat. +define private i128 @__digit_quot(i128 %uhi, i128 %ulo, i256 %vn, i128 %unext) noinline #0 { entry: - %vs128 = lshr i256 %v, 128 - %vs128nz = icmp ne i256 %vs128, 0 - %n128 = select i1 %vs128nz, i256 128, i256 256 - %va128 = select i1 %vs128nz, i256 %vs128, i256 %v - %vs64 = lshr i256 %va128, 64 - %vs64nz = icmp ne i256 %vs64, 0 - %clza64 = sub i256 %n128, 64 - %n64 = select i1 %vs64nz, i256 %clza64, i256 %n128 - %va64 = select i1 %vs64nz, i256 %vs64, i256 %va128 - %vs32 = lshr i256 %va64, 32 - %vs32nz = icmp ne i256 %vs32, 0 - %clza32 = sub i256 %n64, 32 - %n32 = select i1 %vs32nz, i256 %clza32, i256 %n64 - %va32 = select i1 %vs32nz, i256 %vs32, i256 %va64 - %vs16 = lshr i256 %va32, 16 - %vs16nz = icmp ne i256 %vs16, 0 - %clza16 = sub i256 %n32, 16 - %n16 = select i1 %vs16nz, i256 %clza16, i256 %n32 - %va16 = select i1 %vs16nz, i256 %vs16, i256 %va32 - %vs8 = lshr i256 %va16, 8 - %vs8nz = icmp ne i256 %vs8, 0 - %clza8 = sub i256 %n16, 8 - %n8 = select i1 %vs8nz, i256 %clza8, i256 %n16 - %va8 = select i1 %vs8nz, i256 %vs8, i256 %va16 - %vs4 = lshr i256 %va8, 4 - %vs4nz = icmp ne i256 %vs4, 0 - %clza4 = sub i256 %n8, 4 - %n4 = select i1 %vs4nz, i256 %clza4, i256 %n8 - %va4 = select i1 %vs4nz, i256 %vs4, i256 %va8 - %vs2 = lshr i256 %va4, 2 - %vs2nz = icmp ne i256 %vs2, 0 - %clza2 = sub i256 %n4, 2 - %n2 = select i1 %vs2nz, i256 %clza2, i256 %n4 - %va2 = select i1 %vs2nz, i256 %vs2, i256 %va4 - %vs1 = lshr i256 %va2, 1 - %vs1nz = icmp ne i256 %vs1, 0 - %clza1 = sub i256 %n2, 2 - %clzax = sub i256 %n2, %va2 - %result = select i1 %vs1nz, i256 %clza1, i256 %clzax - ret i256 %result + %v1 = lshr i256 %vn, 128 + %v2 = trunc i256 %v1 to i128 + %v3 = trunc i256 %vn to i128 + %v4 = call { i128, i128 } @__udiv_qrnnd_128(i128 %uhi, i128 %ulo, i128 %v2) + %v5 = extractvalue { i128, i128 } %v4, 0 + %v6 = icmp uge i128 %uhi, %v2 + %v7 = select i1 %v6, i128 340282366920938463463374607431768211455, i128 %v5 + %v8 = zext i128 %uhi to i256 + %v9 = zext i128 %ulo to i256 + %v10 = shl i256 %v8, 128 + %v11 = or i256 %v10, %v9 + %v12 = zext i128 %v7 to i256 + %v13 = mul i256 %v12, %v1 + %v14 = sub i256 %v11, %v13 + %v15 = zext i128 %v2 to i256 + %v16 = zext i128 %unext to i384 + %v17 = zext i128 %v7 to i256 + %v18 = zext i128 %v3 to i256 + %v19 = mul i256 %v17, %v18 + %v20 = zext i256 %v19 to i384 + %v21 = zext i256 %v14 to i384 + %v22 = shl i384 %v21, 128 + %v23 = add i384 %v22, %v16 + %v24 = icmp ugt i384 %v20, %v23 + %v25 = sub i128 %v7, 1 + %v26 = add i256 %v14, %v15 + %v27 = select i1 %v24, i128 %v25, i128 %v7 + %v28 = select i1 %v24, i256 %v26, i256 %v14 + %v29 = zext i128 %v27 to i256 + %v30 = zext i128 %v3 to i256 + %v31 = mul i256 %v29, %v30 + %v32 = zext i256 %v31 to i384 + %v33 = zext i256 %v28 to i384 + %v34 = shl i384 %v33, 128 + %v35 = add i384 %v34, %v16 + %v36 = icmp ugt i384 %v32, %v35 + %v37 = sub i128 %v27, 1 + %v39 = select i1 %v36, i128 %v37, i128 %v27 + ret i128 %v39 } -define private i256 @__ulongrem(i256 %0, i256 %1, i256 %2) #0 { - %.not = icmp ult i256 %1, %2 - br i1 %.not, label %4, label %51 - -4: - %5 = tail call i256 @__clz(i256 %2) - %.not61 = icmp eq i256 %5, 0 - br i1 %.not61, label %13, label %6 - -6: - %7 = shl i256 %2, %5 - %8 = shl i256 %1, %5 - %9 = sub nuw nsw i256 256, %5 - %10 = lshr i256 %0, %9 - %11 = or i256 %10, %8 - %12 = shl i256 %0, %5 - br label %13 - -13: - %.054 = phi i256 [ %7, %6 ], [ %2, %4 ] - %.053 = phi i256 [ %11, %6 ], [ %1, %4 ] - %.052 = phi i256 [ %12, %6 ], [ %0, %4 ] - %14 = lshr i256 %.054, 128 - %15 = udiv i256 %.053, %14 - %16 = urem i256 %.053, %14 - %17 = and i256 %.054, 340282366920938463463374607431768211455 - %18 = lshr i256 %.052, 128 - br label %19 - -19: - %.056 = phi i256 [ %15, %13 ], [ %25, %.critedge ] - %.055 = phi i256 [ %16, %13 ], [ %26, %.critedge ] - %.not62 = icmp ult i256 %.056, 340282366920938463463374607431768211456 - br i1 %.not62, label %20, label %.critedge - -20: - %21 = mul nuw i256 %.056, %17 - %22 = shl nuw i256 %.055, 128 - %23 = or i256 %22, %18 - %24 = icmp ugt i256 %21, %23 - br i1 %24, label %.critedge, label %27 - -.critedge: - %25 = add i256 %.056, -1 - %26 = add i256 %.055, %14 - %.not65 = icmp ult i256 %26, 340282366920938463463374607431768211456 - br i1 %.not65, label %19, label %27 - -27: - %.157 = phi i256 [ %25, %.critedge ], [ %.056, %20 ] - %28 = shl i256 %.053, 128 - %29 = or i256 %18, %28 - %30 = and i256 %.157, 340282366920938463463374607431768211455 - %31 = mul i256 %30, %.054 - %32 = sub i256 %29, %31 - %33 = udiv i256 %32, %14 - %34 = urem i256 %32, %14 - %35 = and i256 %.052, 340282366920938463463374607431768211455 - br label %36 - -36: - %.2 = phi i256 [ %33, %27 ], [ %42, %.critedge1 ] - %.1 = phi i256 [ %34, %27 ], [ %43, %.critedge1 ] - %.not63 = icmp ult i256 %.2, 340282366920938463463374607431768211456 - br i1 %.not63, label %37, label %.critedge1 - -37: - %38 = mul nuw i256 %.2, %17 - %39 = shl i256 %.1, 128 - %40 = or i256 %39, %35 - %41 = icmp ugt i256 %38, %40 - br i1 %41, label %.critedge1, label %44 - -.critedge1: - %42 = add i256 %.2, -1 - %43 = add i256 %.1, %14 - %.not64 = icmp ult i256 %43, 340282366920938463463374607431768211456 - br i1 %.not64, label %36, label %44 - -44: - %.3 = phi i256 [ %42, %.critedge1 ], [ %.2, %37 ] - %45 = shl i256 %32, 128 - %46 = or i256 %45, %35 - %47 = and i256 %.3, 340282366920938463463374607431768211455 - %48 = mul i256 %47, %.054 - %49 = sub i256 %46, %48 - %50 = lshr i256 %49, %5 - br label %51 - -51: - %.0 = phi i256 [ %50, %44 ], [ -1, %3 ] - ret i256 %.0 +; Unsigned 256-bit division. Returns { q, r } with q = u / v and r = u mod v. +; +; Notation: B = 2^128; (x:y) = x*B + y; / is floor division. +; Precondition: v != 0. +; +; 1. Early exit: if u < v, return { 0, u }. Besides the trivial win, this +; makes __mulmod's argument pre-reductions nearly free when the +; arguments are already reduced (< modulus), the common case in +; modular-arithmetic-heavy code. +; 2. Split u = (u1:u0) and v = (v1:v0) into 128-bit digits. +; +; "twoone" path, v1 == 0 (v < 2^128): two digits over one, schoolbook: +; 3. High digit: q1 = u1 / v0, with carry k = u1 - q1*v0 = u1 mod v0 +; (so k < v0). +; 4. Low digit: (q0, r) = __udiv_qrnnd_128(k, u0, v0), dividing (k:u0) by +; v0; k < v0 satisfies its precondition. +; 5. Return { (q1:q0), r }. +; +; "full" path, v1 != 0 (v >= 2^128): the quotient fits in a single digit +; because u / v < 2^256 / 2^128 = B, so one Algorithm D digit step suffices: +; 6. Normalize: s = ctlz(v1) (0..127); vn = v << s, exact in 256 bits. +; 7. un = u << s, exact in 384-bit arithmetic; split into digits +; (un2:un1:un0). u < 2^256 <= v*B implies (un2:un1) < vn, +; __digit_quot's precondition. +; 8. q = __digit_quot(un2, un1, vn, un0) = un / vn = u / v. +; 9. r = (un - q*vn) >> s, exact in 384-bit arithmetic; the result is < v, +; so it truncates to 256 bits losslessly. +; 10. Return { q, r }. +define { i256, i256 } @__udivrem256(i256 %u, i256 %v) #0 { +entry: + ; Early exit: u < v yields quotient 0, remainder u. Makes the __mulmod + ; argument pre-reductions nearly free when the arguments are already + ; reduced (< modulus), the common case in modular-arithmetic-heavy code. + %small = icmp ult i256 %u, %v + br i1 %small, label %early, label %split +early: + %e1 = insertvalue { i256, i256 } undef, i256 0, 0 + %e2 = insertvalue { i256, i256 } %e1, i256 %u, 1 + ret { i256, i256 } %e2 +split: + %v1 = lshr i256 %v, 128 + %v2 = trunc i256 %v1 to i128 + %v3 = trunc i256 %v to i128 + %v4 = lshr i256 %u, 128 + %v5 = trunc i256 %v4 to i128 + %v6 = trunc i256 %u to i128 + %v7 = icmp ne i128 %v2, 0 + br i1 %v7, label %full, label %twoone +twoone: + %v8 = udiv i128 %v5, %v3 + %v9 = mul i128 %v8, %v3 + %v10 = sub i128 %v5, %v9 + %v11 = call { i128, i128 } @__udiv_qrnnd_128(i128 %v10, i128 %v6, i128 %v3) + %v12 = extractvalue { i128, i128 } %v11, 0 + %v13 = extractvalue { i128, i128 } %v11, 1 + %v14 = zext i128 %v8 to i256 + %v15 = shl i256 %v14, 128 + %v16 = zext i128 %v12 to i256 + %v17 = or i256 %v15, %v16 + %v18 = zext i128 %v13 to i256 + %v19 = insertvalue { i256, i256 } undef, i256 %v17, 0 + %v20 = insertvalue { i256, i256 } %v19, i256 %v18, 1 + ret { i256, i256 } %v20 +full: + %v21 = call i128 @llvm.ctlz.i128(i128 %v2, i1 false) + %v22 = zext i128 %v21 to i256 + %v23 = shl i256 %v, %v22 + %v24 = zext i256 %u to i384 + %v25 = zext i128 %v21 to i384 + %v26 = shl i384 %v24, %v25 + %v27 = lshr i384 %v26, 256 + %v28 = trunc i384 %v27 to i128 + %v29 = lshr i384 %v26, 128 + %v30 = trunc i384 %v29 to i256 + %v31 = trunc i256 %v30 to i128 + %v32 = trunc i384 %v26 to i128 + %v33 = call i128 @__digit_quot(i128 %v28, i128 %v31, i256 %v23, i128 %v32) + %v34 = zext i128 %v33 to i256 + %v35 = zext i128 %v28 to i384 + %v36 = zext i128 %v31 to i384 + %v37 = zext i128 %v32 to i384 + %v38 = shl i384 %v35, 256 + %v39 = shl i384 %v36, 128 + %v40 = add i384 %v38, %v39 + %v41 = add i384 %v40, %v37 + %v42 = zext i128 %v33 to i384 + %v43 = zext i256 %v23 to i384 + %v44 = mul i384 %v42, %v43 + %v45 = sub i384 %v41, %v44 + %v46 = lshr i384 %v45, %v25 + %v47 = trunc i384 %v46 to i256 + %v48 = insertvalue { i256, i256 } undef, i256 %v34, 0 + %v49 = insertvalue { i256, i256 } %v48, i256 %v47, 1 + ret { i256, i256 } %v49 } +; Remainder of the 512-bit value (phi:plo) modulo m. +; Remainder only: quotient digits are computed to reduce, never assembled. +; +; Notation: B = 2^128; (x:y) is concatenation at the operands' named widths; +; / is floor division. +; Preconditions: phi < m and m >= 2^128. There is deliberately NO +; small-modulus path: the sole caller is __mulmod's 512-bit branch, taken +; only for m >= 2^128 (smaller moduli use __mulmod's 256-bit fast path), and +; it always passes phi < m because phi <= (m-1)^2 / 2^256 < m. +; Result: (phi*2^256 + plo) mod m. +; +; Knuth TAOCP 4.3.1 Algorithm D in base B: a 4-digit dividend over a 2-digit +; divisor, unrolled into two digit steps: +; 1. Normalize: m1 = m >> 128 (nonzero since m >= 2^128); s = ctlz(m1); +; mn = m << s, exact in 256 bits. +; 2. pn = (phi:plo) << s, exact in 512-bit arithmetic (phi < m keeps the top +; s bits free). Split into four digits (pn3:pn2:pn1:pn0). +; 3. First digit step: q1 = __digit_quot(pn3, pn2, mn, pn1) +; = (pn3:pn2:pn1) / mn; its precondition (pn3:pn2) < mn follows from +; phi < m. Reduce: r1 = (pn3:pn2:pn1) - q1*mn, exact in 384-bit +; arithmetic; r1 < mn, so it truncates to 256 bits losslessly. +; 4. Second digit step: split r1 = (r1hi:r1lo) into 128-bit digits; +; q0 = __digit_quot(r1hi, r1lo, mn, pn0) = (r1:pn0) / mn (its +; precondition r1 < mn holds by construction). Reduce: +; r2 = (r1:pn0) - q0*mn, exact in 384-bit arithmetic; r2 < mn, truncated +; to 256 bits. +; 5. Return r2 >> s = (phi:plo) mod m. +define i256 @__urem512by256(i256 %plo, i256 %phi, i256 %m) #0 { +entry: + %v1 = lshr i256 %m, 128 + %v2 = trunc i256 %v1 to i128 + %v20 = call i128 @llvm.ctlz.i128(i128 %v2, i1 false) + %v21 = zext i128 %v20 to i256 + %v22 = shl i256 %m, %v21 + %v23 = zext i256 %phi to i512 + %v24 = zext i256 %plo to i512 + %v25 = shl i512 %v23, 256 + %v26 = or i512 %v25, %v24 + %v27 = zext i128 %v20 to i512 + %v28 = shl i512 %v26, %v27 + %v29 = lshr i512 %v28, 384 + %v30 = trunc i512 %v29 to i128 + %v31 = lshr i512 %v28, 256 + %v32 = and i512 %v31, 340282366920938463463374607431768211455 + %v33 = trunc i512 %v32 to i128 + %v34 = lshr i512 %v28, 128 + %v35 = and i512 %v34, 340282366920938463463374607431768211455 + %v36 = trunc i512 %v35 to i128 + %v37 = and i512 %v28, 340282366920938463463374607431768211455 + %v38 = trunc i512 %v37 to i128 + %v39 = zext i256 %v22 to i384 + %v40 = call i128 @__digit_quot(i128 %v30, i128 %v33, i256 %v22, i128 %v36) + %v41 = zext i128 %v30 to i384 + %v42 = zext i128 %v33 to i384 + %v43 = zext i128 %v36 to i384 + %v44 = shl i384 %v41, 256 + %v45 = shl i384 %v42, 128 + %v46 = add i384 %v44, %v45 + %v47 = add i384 %v46, %v43 + %v48 = zext i128 %v40 to i384 + %v49 = mul i384 %v48, %v39 + %v50 = sub i384 %v47, %v49 + %v51 = trunc i384 %v50 to i256 + %v52 = lshr i256 %v51, 128 + %v53 = trunc i256 %v52 to i128 + %v54 = trunc i256 %v51 to i128 + %v55 = call i128 @__digit_quot(i128 %v53, i128 %v54, i256 %v22, i128 %v38) + %v56 = zext i256 %v51 to i384 + %v57 = shl i384 %v56, 128 + %v58 = zext i128 %v38 to i384 + %v59 = add i384 %v57, %v58 + %v60 = zext i128 %v55 to i384 + %v61 = mul i384 %v60, %v39 + %v62 = sub i384 %v59, %v61 + %v63 = trunc i384 %v62 to i256 + %v64 = lshr i256 %v63, %v21 + ret i256 %v64 +} + +; Unsigned 256-bit division, quotient only: returns u / v (floor). +; Precondition: v != 0, inherited from the raw i256 udiv this replaces. +; Thin wrapper: the quotient half of __udivrem256. revive emits raw i256 +; udiv/urem; the lower_wide_division pass rewrites the non-narrowable +; ones into calls to these wrappers. Kept external so they survive +; optimization until that late pass; unused copies are dropped by the final +; linker --gc-sections. +define i256 @__udiv256(i256 %u, i256 %v) #0 { + %qr = call { i256, i256 } @__udivrem256(i256 %u, i256 %v) + %q = extractvalue { i256, i256 } %qr, 0 + ret i256 %q +} + +; Unsigned 256-bit remainder, remainder only: returns u mod v. +; Precondition: v != 0, inherited from the raw i256 urem this replaces. +; Thin wrapper: the remainder half of __udivrem256 (see __udiv256 for why +; these wrappers exist). +define i256 @__urem256(i256 %u, i256 %v) #0 { + %qr = call { i256, i256 } @__udivrem256(i256 %u, i256 %v) + %r = extractvalue { i256, i256 } %qr, 1 + ret i256 %r +} + +; Signed 256-bit division with EVM SDIV semantics -- the quotient truncates +; toward zero. Sign-magnitude wrapper over the unsigned divider. +; +; Preconditions (established by the guard code emitted around the call): +; b != 0, and not (a == -2^255 and b == -1), the two's complement overflow +; pair. Result: q = a / b truncated toward zero. +; +; 1. Sign masks: sign_a = a >>s 255, sign_b = b >>s 255 (arithmetic shift: +; 0 for a nonnegative operand, all-ones i.e. -1 for a negative one). +; 2. Magnitudes: |x| = (x ^ sign_x) - sign_x, the branchless conditional +; negate (x^0 - 0 = x; x^-1 - (-1) = ~x + 1 = -x). Well defined for +; every input: |-2^255| = 2^255 fits unsigned. +; 3. q_mag = |a| / |b| via __udiv256; flooring the magnitude quotient is +; what makes the signed result truncate toward zero. +; 4. Reapply the sign: sign_q = sign_a ^ sign_b (negative iff exactly one +; operand was); q = (q_mag ^ sign_q) - sign_q. +define i256 @__sdiv256(i256 %a, i256 %b) #0 { + %sign_a = ashr i256 %a, 255 + %sign_b = ashr i256 %b, 255 + %xa = xor i256 %a, %sign_a + %abs_a = sub i256 %xa, %sign_a + %xb = xor i256 %b, %sign_b + %abs_b = sub i256 %xb, %sign_b + %abs_q = call i256 @__udiv256(i256 %abs_a, i256 %abs_b) + %sign_q = xor i256 %sign_a, %sign_b + %xq = xor i256 %abs_q, %sign_q + %q = sub i256 %xq, %sign_q + ret i256 %q +} + +; Signed 256-bit remainder with EVM SMOD semantics. The remainder takes +; the sign of the dividend, so a == b*(a sdiv b) + (a smod b). +; Sign-magnitude wrapper over the unsigned remainder. +; +; Preconditions (established by the guard code emitted around the call): +; b != 0, and not (a == -2^255 and b == -1) -- the non-UB envelope of the +; raw srem this replaces (the emitted SMOD guard actually swaps a -1 divisor +; for 1, so b == -1 never reaches here). +; Result: r with |r| = |a| mod |b| and sign(r) = sign(a), or r == 0. +; +; 1. Sign masks: sign_a = a >>s 255, sign_b = b >>s 255 (0 or all-ones). +; 2. Magnitudes: |a| = (a ^ sign_a) - sign_a, |b| = (b ^ sign_b) - sign_b. +; 3. r_mag = |a| mod |b| via __urem256. +; 4. Reapply the dividend's sign: r = (r_mag ^ sign_a) - sign_a. +define i256 @__srem256(i256 %a, i256 %b) #0 { + %sign_a = ashr i256 %a, 255 + %sign_b = ashr i256 %b, 255 + %xa = xor i256 %a, %sign_a + %abs_a = sub i256 %xa, %sign_a + %xb = xor i256 %b, %sign_b + %abs_b = sub i256 %xb, %sign_b + %abs_r = call i256 @__urem256(i256 %abs_a, i256 %abs_b) + %xr = xor i256 %abs_r, %sign_a + %r = sub i256 %xr, %sign_a + ret i256 %r +} + +; EVM MULMOD -- (a * b) mod m with the product taken at full width +; (not mod 2^256), and MULMOD(a, b, 0) = 0. +; +; Preconditions: none (m == 0 is handled here). +; Result: 0 if m == 0, otherwise (a*b) mod m with the exact 512-bit product. +; +; 1. If m == 0, return 0. +; 2. Pre-reduce both arguments: am = a mod m, bm = b mod m via __urem256 +; (its u < v early exit makes this nearly free when the arguments are +; already reduced, the common case). The residue is unchanged and the +; product bound shrinks to (m-1)^2. +; 3. Fast path, m < 2^128: am*bm <= (m-1)^2 < 2^256 is exact in 256-bit +; arithmetic; return (am*bm) mod m via __urem256. +; 4. Slow path, m >= 2^128: p = am*bm computed exactly in 512-bit +; arithmetic; split p = (phi:plo) into 256-bit halves and return +; __urem512by256(plo, phi, m). Its preconditions hold: m >= 2^128 from +; the branch, and phi <= (m-1)^2 / 2^256 < m. define i256 @__mulmod(i256 %arg1, i256 %arg2, i256 %modulo) #0 { entry: %cccond = icmp eq i256 %modulo, 0 @@ -173,22 +509,78 @@ entry: ccret: ret i256 0 entrycont: - %arg1m = urem i256 %arg1, %modulo - %arg2m = urem i256 %arg2, %modulo - %less_then_2_128 = icmp ult i256 %modulo, 340282366920938463463374607431768211456 - br i1 %less_then_2_128, label %fast, label %slow + %arg1m = call i256 @__urem256(i256 %arg1, i256 %modulo) + %arg2m = call i256 @__urem256(i256 %arg2, i256 %modulo) + %less = icmp ult i256 %modulo, 340282366920938463463374607431768211456 + br i1 %less, label %fast, label %slow fast: %prod = mul i256 %arg1m, %arg2m - %prodm = urem i256 %prod, %modulo + %prodm = call i256 @__urem256(i256 %prod, i256 %modulo) ret i256 %prodm slow: - %arg1e = zext i256 %arg1m to i512 - %arg2e = zext i256 %arg2m to i512 - %prode = mul i512 %arg1e, %arg2e + %a1e = zext i256 %arg1m to i512 + %a2e = zext i256 %arg2m to i512 + %prode = mul i512 %a1e, %a2e %prodl = trunc i512 %prode to i256 %prodeh = lshr i512 %prode, 256 %prodh = trunc i512 %prodeh to i256 - %res = call i256 @__ulongrem(i256 %prodl, i256 %prodh, i256 %modulo) + %res = call i256 @__urem512by256(i256 %prodl, i256 %prodh, i256 %modulo) + ret i256 %res +} + +; (a*b) mod m via Barrett reduction (HAC 14.42 with b=2, k=t=256), for +; compile-time-constant 256-bit moduli. The compiler rewrites eligible +; __mulmod call sites to this and supplies the reciprocal -- before the +; optimization pipeline for moduli that are already constant (the common +; case), after it for moduli the pipeline exposes as constant. +; +; Preconditions (guaranteed at every rewritten call site; violations return +; garbage but never trap -- the body is straight-line, no udiv/urem/br): +; 2^255 < m < 2^256, m not a power of two, +; mu_lo = floor(2^512/m) - 2^256, so mu = 2^256 + mu_lo with +; 2^256 < mu < 2^257 (m <= 2^256-1 => mu >= 2^256+1 since +; (2^256-1)(2^256+1) < 2^512; m > 2^255 => mu < 2^257). +; +; a and b need NOT be pre-reduced: a*b <= (2^256-1)^2 < 2^512 = b^(2k) is +; exactly HAC's operand bound at t = 256, which is why only 256-bit moduli +; are eligible (smaller moduli stay on __mulmod). +; +; Quotient bound: upper -- q3 <= (x/2^255)(2^512/m)/2^257 = x/m, so q3 <= q +; and r0 >= 0; lower -- q1 > x/2^255 - 1 and mu > 2^512/m - 1 give +; q1*mu/2^257 > x/m - x/2^512 - 2^255/m + 2^-257 > x/m - 2, so q3 >= q - 2. +define i256 @__mulmod_barrett(i256 %a, i256 %b, i256 %m, i256 %mu_lo) noinline #0 { +entry: + %aw = zext i256 %a to i512 + %bw = zext i256 %b to i512 + %x = mul i512 %aw, %bw ; x < 2^512: no wrap + %q1 = lshr i512 %x, 255 ; q1 = floor(x/2^255) < 2^257 + ; q2 = q1*mu reconstructed as (q1 << 256) + q1*mu_lo in i576: + ; q1 << 256 < 2^513, q1*mu_lo < 2^513, q2 < 2^514 < 2^576: no wrap. + %q1w = zext i512 %q1 to i576 + %muw = zext i256 %mu_lo to i576 + %hi = shl i576 %q1w, 256 + %lo = mul i576 %q1w, %muw + %q2 = add i576 %hi, %lo + %q3w = lshr i576 %q2, 257 ; q3 <= floor(x/m), q3 < 2^257 + %q3 = trunc i576 %q3w to i320 ; lossless + ; r0 = x - q3*m computed mod 2^320: the true value lies in [0, 3m) with + ; 3m < 2^258, so 320-bit wrapping arithmetic reproduces it exactly. + %m3 = zext i256 %m to i320 + %x3 = trunc i512 %x to i320 + %q3m = mul i320 %q3, %m3 + %r0 = sub i320 %x3, %q3m + ; Exactly two conditional corrections (HAC note 14.44: 0 <= q - q3 <= 2): + ; r0 in [0,3m) -> r1 in [0,2m) -> r2 in [0,m). The bound is tight for this + ; parameterization (the two deficit terms x/2^512 and 2^255/m each approach + ; 1), so two is required and sufficient -- do not harmonize with the + ; division helpers' single correction; these are different theorems. + %c1 = icmp uge i320 %r0, %m3 + %s1 = sub i320 %r0, %m3 + %r1 = select i1 %c1, i320 %s1, i320 %r0 + %c2 = icmp uge i320 %r1, %m3 + %s2 = sub i320 %r1, %m3 + %r2 = select i1 %c2, i320 %s2, i320 %r1 + %res = trunc i320 %r2 to i256 ; r2 < m < 2^256: lossless ret i256 %res }