From 482faea2891da061fb336e1ec888b903e0ee164d Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 5 Aug 2026 15:06:12 +0100 Subject: [PATCH 1/8] Implement f64<->i64/u64 conversions, with native instructions on NEON and AVX-512 and scalar fallback everywhere else --- fearless_simd/src/generated/avx2.rs | 120 ++++++++++ fearless_simd/src/generated/avx512.rs | 226 ++++++++++++++++++ fearless_simd/src/generated/fallback.rs | 96 ++++++++ fearless_simd/src/generated/neon.rs | 120 ++++++++++ fearless_simd/src/generated/simd_trait.rs | 53 +++- fearless_simd/src/generated/simd_types.rs | 114 +++++++++ fearless_simd/src/generated/sse2.rs | 96 ++++++++ fearless_simd/src/generated/sse4_2.rs | 96 ++++++++ fearless_simd/src/generated/wasm.rs | 96 ++++++++ fearless_simd_gen/src/mk_simd_trait.rs | 15 +- fearless_simd_gen/src/mk_simd_types.rs | 34 +-- fearless_simd_gen/src/mk_wasm.rs | 3 + fearless_simd_gen/src/mk_x86.rs | 79 ++++++ fearless_simd_gen/src/ops.rs | 78 +++++- fearless_simd_tests/tests/generics.rs | 24 ++ .../tests/harness/ops/cvt_f64.rs | 74 ++++++ .../tests/harness/ops/cvt_i64.rs | 33 +++ .../tests/harness/ops/cvt_i64_precise.rs | 42 ++++ .../tests/harness/ops/cvt_u64.rs | 42 ++++ .../tests/harness/ops/cvt_u64_precise.rs | 42 ++++ fearless_simd_tests/tests/harness/ops/mod.rs | 5 + 21 files changed, 1456 insertions(+), 32 deletions(-) create mode 100644 fearless_simd_tests/tests/harness/ops/cvt_f64.rs create mode 100644 fearless_simd_tests/tests/harness/ops/cvt_i64.rs create mode 100644 fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs create mode 100644 fearless_simd_tests/tests/harness/ops/cvt_u64.rs create mode 100644 fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 266be45c..0f8c44c7 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -4793,6 +4793,22 @@ impl Simd for Avx2 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_u64_f64x2(self, a: f64x2) -> u64x2 { + [a[0usize] as u64, a[1usize] as u64].simd_into(self) + } + #[inline(always)] + fn cvt_u64_precise_f64x2(self, a: f64x2) -> u64x2 { + [a[0usize] as u64, a[1usize] as u64].simd_into(self) + } + #[inline(always)] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { + [a[0usize] as i64, a[1usize] as i64].simd_into(self) + } + #[inline(always)] + fn cvt_i64_precise_f64x2(self, a: f64x2) -> i64x2 { + [a[0usize] as i64, a[1usize] as i64].simd_into(self) + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -5214,6 +5230,10 @@ impl Simd for Avx2 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_f64_i64x2(self, a: i64x2) -> f64x2 { + [a[0usize] as f64, a[1usize] as f64].simd_into(self) + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5622,6 +5642,10 @@ impl Simd for Avx2 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_f64_u64x2(self, a: u64x2) -> f64x2 { + [a[0usize] as f64, a[1usize] as f64].simd_into(self) + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -10837,6 +10861,46 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn cvt_u64_f64x4(self, a: f64x4) -> u64x4 { + [ + a[0usize] as u64, + a[1usize] as u64, + a[2usize] as u64, + a[3usize] as u64, + ] + .simd_into(self) + } + #[inline(always)] + fn cvt_u64_precise_f64x4(self, a: f64x4) -> u64x4 { + [ + a[0usize] as u64, + a[1usize] as u64, + a[2usize] as u64, + a[3usize] as u64, + ] + .simd_into(self) + } + #[inline(always)] + fn cvt_i64_f64x4(self, a: f64x4) -> i64x4 { + [ + a[0usize] as i64, + a[1usize] as i64, + a[2usize] as i64, + a[3usize] as i64, + ] + .simd_into(self) + } + #[inline(always)] + fn cvt_i64_precise_f64x4(self, a: f64x4) -> i64x4 { + [ + a[0usize] as i64, + a[1usize] as i64, + a[2usize] as i64, + a[3usize] as i64, + ] + .simd_into(self) + } + #[inline(always)] fn splat_i64x4(self, val: i64) -> i64x4 { crate::kernel!( #[inline(always)] @@ -11275,6 +11339,16 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn cvt_f64_i64x4(self, a: i64x4) -> f64x4 { + [ + a[0usize] as f64, + a[1usize] as f64, + a[2usize] as f64, + a[3usize] as f64, + ] + .simd_into(self) + } + #[inline(always)] fn splat_u64x4(self, val: u64) -> u64x4 { crate::kernel!( #[inline(always)] @@ -11697,6 +11771,16 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn cvt_f64_u64x4(self, a: u64x4) -> f64x4 { + [ + a[0usize] as f64, + a[1usize] as f64, + a[2usize] as f64, + a[3usize] as f64, + ] + .simd_into(self) + } + #[inline(always)] fn splat_mask64x4(self, val: bool) -> mask64x4 { crate::kernel!( #[inline(always)] @@ -15706,6 +15790,32 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn cvt_u64_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4(self.cvt_u64_f64x4(a0), self.cvt_u64_f64x4(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4( + self.cvt_u64_precise_f64x4(a0), + self.cvt_u64_precise_f64x4(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4(self.cvt_i64_f64x4(a0), self.cvt_i64_f64x4(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4( + self.cvt_i64_precise_f64x4(a0), + self.cvt_i64_precise_f64x4(a1), + ) + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -16024,6 +16134,11 @@ impl Simd for Avx2 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn cvt_f64_i64x8(self, a: i64x8) -> f64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_f64x4(self.cvt_f64_i64x4(a0), self.cvt_f64_i64x4(a1)) + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -16337,6 +16452,11 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn cvt_f64_u64x8(self, a: u64x8) -> f64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_f64x4(self.cvt_f64_u64x4(a0), self.cvt_f64_u64x4(a1)) + } + #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index a391c70d..1ca9885e 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -4851,6 +4851,60 @@ impl Simd for Avx512 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_u64_f64x2(self, a: f64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> u64x2 { + _mm_cvttpd_epu64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u64_precise_f64x2(self, a: f64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> u64x2 { + let a = _mm_max_pd(a.into(), _mm_setzero_pd()); + let mut converted = _mm_cvttpd_epu64(a); + let exceeds_unsigned_range = + _mm_cmp_pd_mask::<17i32>(_mm_set1_pd(18446744073709549568.0), a); + converted = _mm_mask_blend_epi64( + exceeds_unsigned_range, + converted, + _mm_set1_epi64x(u64::MAX.cast_signed()), + ); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> i64x2 { + _mm_cvttpd_epi64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i64_precise_f64x2(self, a: f64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2) -> i64x2 { + let a = a.into(); + let in_range = _mm_cmp_pd_mask::<17i32>(a, _mm_set1_pd(9223372036854775808.0)); + let mut converted = _mm_mask_cvttpd_epi64(_mm_set1_epi64x(i64::MAX), in_range, a); + let is_not_nan = _mm_cmp_pd_mask::<7i32>(a, a); + converted = _mm_mask_blend_epi64(is_not_nan, _mm_setzero_si128(), converted); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -5297,6 +5351,16 @@ impl Simd for Avx512 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_f64_i64x2(self, a: i64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2) -> f64x2 { + _mm_cvtepi64_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5733,6 +5797,16 @@ impl Simd for Avx512 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_f64_u64x2(self, a: u64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2) -> f64x2 { + _mm_cvtepu64_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { mask64x2 { val: (if val { 3u64 } else { 0 }) as _, @@ -10908,6 +10982,62 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn cvt_u64_f64x4(self, a: f64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> u64x4 { + _mm256_cvttpd_epu64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u64_precise_f64x4(self, a: f64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> u64x4 { + let a = _mm256_max_pd(a.into(), _mm256_setzero_pd()); + let mut converted = _mm256_cvttpd_epu64(a); + let exceeds_unsigned_range = + _mm256_cmp_pd_mask::<17i32>(_mm256_set1_pd(18446744073709549568.0), a); + converted = _mm256_mask_blend_epi64( + exceeds_unsigned_range, + converted, + _mm256_set1_epi64x(u64::MAX.cast_signed()), + ); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i64_f64x4(self, a: f64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> i64x4 { + _mm256_cvttpd_epi64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i64_precise_f64x4(self, a: f64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> i64x4 { + let a = a.into(); + let in_range = + _mm256_cmp_pd_mask::<17i32>(a, _mm256_set1_pd(9223372036854775808.0)); + let mut converted = + _mm256_mask_cvttpd_epi64(_mm256_set1_epi64x(i64::MAX), in_range, a); + let is_not_nan = _mm256_cmp_pd_mask::<7i32>(a, a); + converted = _mm256_mask_blend_epi64(is_not_nan, _mm256_setzero_si256(), converted); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_i64x4(self, val: i64) -> i64x4 { crate::kernel!( #[inline(always)] @@ -11379,6 +11509,16 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn cvt_f64_i64x4(self, a: i64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4) -> f64x4 { + _mm256_cvtepi64_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_u64x4(self, val: u64) -> u64x4 { crate::kernel!( #[inline(always)] @@ -11840,6 +11980,16 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn cvt_f64_u64x4(self, a: u64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4) -> f64x4 { + _mm256_cvtepu64_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_mask64x4(self, val: bool) -> mask64x4 { mask64x4 { val: (if val { 15u64 } else { 0 }) as _, @@ -17585,6 +17735,62 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn cvt_u64_f64x8(self, a: f64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> u64x8 { + _mm512_cvttpd_epu64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u64_precise_f64x8(self, a: f64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> u64x8 { + let a = _mm512_max_pd(a.into(), _mm512_setzero_pd()); + let mut converted = _mm512_cvttpd_epu64(a); + let exceeds_unsigned_range = + _mm512_cmp_pd_mask::<17i32>(_mm512_set1_pd(18446744073709549568.0), a); + converted = _mm512_mask_blend_epi64( + exceeds_unsigned_range, + converted, + _mm512_set1_epi64(u64::MAX.cast_signed()), + ); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i64_f64x8(self, a: f64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> i64x8 { + _mm512_cvttpd_epi64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i64_precise_f64x8(self, a: f64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8) -> i64x8 { + let a = a.into(); + let in_range = + _mm512_cmp_pd_mask::<17i32>(a, _mm512_set1_pd(9223372036854775808.0)); + let mut converted = + _mm512_mask_cvttpd_epi64(_mm512_set1_epi64(i64::MAX), in_range, a); + let is_not_nan = _mm512_cmp_pd_mask::<7i32>(a, a); + converted = _mm512_mask_blend_epi64(is_not_nan, _mm512_setzero_si512(), converted); + converted.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { crate::kernel!( #[inline(always)] @@ -18079,6 +18285,16 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn cvt_f64_i64x8(self, a: i64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8) -> f64x8 { + _mm512_cvtepi64_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { crate::kernel!( #[inline(always)] @@ -18563,6 +18779,16 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn cvt_f64_u64x8(self, a: u64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8) -> f64x8 { + _mm512_cvtepu64_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { mask64x8 { val: (if val { 255u64 } else { 0 }) as _, diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index a8fcaba6..00295a8a 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -5678,6 +5678,22 @@ impl Simd for Fallback { ]; } #[inline(always)] + fn cvt_u64_f64x2(self, a: f64x2) -> u64x2 { + [a[0usize] as u64, a[1usize] as u64].simd_into(self) + } + #[inline(always)] + fn cvt_u64_precise_f64x2(self, a: f64x2) -> u64x2 { + [a[0usize] as u64, a[1usize] as u64].simd_into(self) + } + #[inline(always)] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { + [a[0usize] as i64, a[1usize] as i64].simd_into(self) + } + #[inline(always)] + fn cvt_i64_precise_f64x2(self, a: f64x2) -> i64x2 { + [a[0usize] as i64, a[1usize] as i64].simd_into(self) + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { [val; 2usize].simd_into(self) } @@ -6042,6 +6058,10 @@ impl Simd for Fallback { ]; } #[inline(always)] + fn cvt_f64_i64x2(self, a: i64x2) -> f64x2 { + [a[0usize] as f64, a[1usize] as f64].simd_into(self) + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { [val; 2usize].simd_into(self) } @@ -6402,6 +6422,10 @@ impl Simd for Fallback { ]; } #[inline(always)] + fn cvt_f64_u64x2(self, a: u64x2) -> f64x2 { + [a[0usize] as f64, a[1usize] as f64].simd_into(self) + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { let val: i64 = if val { !0 } else { 0 }; [val; 2usize].simd_into(self) @@ -9609,6 +9633,32 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] + fn cvt_u64_f64x4(self, a: f64x4) -> u64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_u64x2(self.cvt_u64_f64x2(a0), self.cvt_u64_f64x2(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x4(self, a: f64x4) -> u64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_u64x2( + self.cvt_u64_precise_f64x2(a0), + self.cvt_u64_precise_f64x2(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x4(self, a: f64x4) -> i64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_i64x2(self.cvt_i64_f64x2(a0), self.cvt_i64_f64x2(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x4(self, a: f64x4) -> i64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_i64x2( + self.cvt_i64_precise_f64x2(a0), + self.cvt_i64_precise_f64x2(a1), + ) + } + #[inline(always)] fn splat_i64x4(self, val: i64) -> i64x4 { let half = self.splat_i64x2(val); self.combine_i64x2(half, half) @@ -9889,6 +9939,11 @@ impl Simd for Fallback { self.combine_i64x2(self.neg_i64x2(a0), self.neg_i64x2(a1)) } #[inline(always)] + fn cvt_f64_i64x4(self, a: i64x4) -> f64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_f64x2(self.cvt_f64_i64x2(a0), self.cvt_f64_i64x2(a1)) + } + #[inline(always)] fn splat_u64x4(self, val: u64) -> u64x4 { let half = self.splat_u64x2(val); self.combine_u64x2(half, half) @@ -10164,6 +10219,11 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] + fn cvt_f64_u64x4(self, a: u64x4) -> f64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_f64x2(self.cvt_f64_u64x2(a0), self.cvt_f64_u64x2(a1)) + } + #[inline(always)] fn splat_mask64x4(self, val: bool) -> mask64x4 { let half = self.splat_mask64x2(val); self.combine_mask64x2(half, half) @@ -13786,6 +13846,32 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] + fn cvt_u64_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4(self.cvt_u64_f64x4(a0), self.cvt_u64_f64x4(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4( + self.cvt_u64_precise_f64x4(a0), + self.cvt_u64_precise_f64x4(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4(self.cvt_i64_f64x4(a0), self.cvt_i64_f64x4(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4( + self.cvt_i64_precise_f64x4(a0), + self.cvt_i64_precise_f64x4(a1), + ) + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -14075,6 +14161,11 @@ impl Simd for Fallback { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn cvt_f64_i64x8(self, a: i64x8) -> f64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_f64x4(self.cvt_f64_i64x4(a0), self.cvt_f64_i64x4(a1)) + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -14359,6 +14450,11 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] + fn cvt_f64_u64x8(self, a: u64x8) -> f64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_f64x4(self.cvt_f64_u64x4(a0), self.cvt_f64_u64x4(a1)) + } + #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index 54207fdc..38b18aa6 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -4140,6 +4140,34 @@ impl Simd for Neon { } } #[inline(always)] + fn cvt_u64_f64x2(self, a: f64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f64x2) -> u64x2 { + vcvtq_u64_f64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_u64_precise_f64x2(self, a: f64x2) -> u64x2 { + self.cvt_u64_f64x2(a) + } + #[inline(always)] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f64x2) -> i64x2 { + vcvtq_s64_f64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn cvt_i64_precise_f64x2(self, a: f64x2) -> i64x2 { + self.cvt_i64_f64x2(a) + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4532,6 +4560,16 @@ impl Simd for Neon { } } #[inline(always)] + fn cvt_f64_i64x2(self, a: i64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2) -> f64x2 { + vcvtq_f64_s64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -4914,6 +4952,16 @@ impl Simd for Neon { } } #[inline(always)] + fn cvt_f64_u64x2(self, a: u64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2) -> f64x2 { + vcvtq_f64_u64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -8518,6 +8566,32 @@ impl Simd for Neon { ) } #[inline(always)] + fn cvt_u64_f64x4(self, a: f64x4) -> u64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_u64x2(self.cvt_u64_f64x2(a0), self.cvt_u64_f64x2(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x4(self, a: f64x4) -> u64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_u64x2( + self.cvt_u64_precise_f64x2(a0), + self.cvt_u64_precise_f64x2(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x4(self, a: f64x4) -> i64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_i64x2(self.cvt_i64_f64x2(a0), self.cvt_i64_f64x2(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x4(self, a: f64x4) -> i64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_i64x2( + self.cvt_i64_precise_f64x2(a0), + self.cvt_i64_precise_f64x2(a1), + ) + } + #[inline(always)] fn splat_i64x4(self, val: i64) -> i64x4 { let half = self.splat_i64x2(val); self.combine_i64x2(half, half) @@ -8840,6 +8914,11 @@ impl Simd for Neon { self.combine_i64x2(self.neg_i64x2(a0), self.neg_i64x2(a1)) } #[inline(always)] + fn cvt_f64_i64x4(self, a: i64x4) -> f64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_f64x2(self.cvt_f64_i64x2(a0), self.cvt_f64_i64x2(a1)) + } + #[inline(always)] fn splat_u64x4(self, val: u64) -> u64x4 { let half = self.splat_u64x2(val); self.combine_u64x2(half, half) @@ -9157,6 +9236,11 @@ impl Simd for Neon { ) } #[inline(always)] + fn cvt_f64_u64x4(self, a: u64x4) -> f64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_f64x2(self.cvt_f64_u64x2(a0), self.cvt_f64_u64x2(a1)) + } + #[inline(always)] fn splat_mask64x4(self, val: bool) -> mask64x4 { let half = self.splat_mask64x2(val); self.combine_mask64x2(half, half) @@ -13297,6 +13381,32 @@ impl Simd for Neon { ) } #[inline(always)] + fn cvt_u64_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4(self.cvt_u64_f64x4(a0), self.cvt_u64_f64x4(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4( + self.cvt_u64_precise_f64x4(a0), + self.cvt_u64_precise_f64x4(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4(self.cvt_i64_f64x4(a0), self.cvt_i64_f64x4(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4( + self.cvt_i64_precise_f64x4(a0), + self.cvt_i64_precise_f64x4(a1), + ) + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -13648,6 +13758,11 @@ impl Simd for Neon { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn cvt_f64_i64x8(self, a: i64x8) -> f64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_f64x4(self.cvt_f64_i64x4(a0), self.cvt_f64_i64x4(a1)) + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -13994,6 +14109,11 @@ impl Simd for Neon { ) } #[inline(always)] + fn cvt_f64_u64x8(self, a: u64x8) -> f64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_f64x4(self.cvt_f64_u64x4(a0), self.cvt_f64_u64x4(a1)) + } + #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index 3be38a63..83f57c26 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -67,7 +67,8 @@ pub trait Simd: Block = f64x2, Mask = Self::mask64s, ByteVector = Self::u8s, - >; + > + SimdCvtFloat + + SimdCvtFloat; #[doc = r" A native-width SIMD vector of [`u8`]s."] type u8s: SimdInt< Self, @@ -124,7 +125,7 @@ pub trait Simd: Block = u64x2, Mask = Self::mask64s, ByteVector = Self::u8s, - >; + > + SimdCvtTruncate; #[doc = r" A native-width SIMD vector of [`i64`]s."] type i64s: SimdInt< Self, @@ -132,7 +133,8 @@ pub trait Simd: Block = i64x2, Mask = Self::mask64s, ByteVector = Self::u8s, - > + core::ops::Neg; + > + SimdCvtTruncate + + core::ops::Neg; #[doc = r" A native-width SIMD mask with 8-bit lanes."] type mask8s: SimdMask + Select @@ -1050,6 +1052,14 @@ pub trait Simd: vectors: [f64x2; 4usize], dest: &mut [f64; 8usize], ) -> (); + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + fn cvt_u64_f64x2(self, a: f64x2) -> u64x2; + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + fn cvt_u64_precise_f64x2(self, a: f64x2) -> u64x2; + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2; + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + fn cvt_i64_precise_f64x2(self, a: f64x2) -> i64x2; #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_i64x2(self, val: i64) -> i64x2; #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] @@ -1144,6 +1154,8 @@ pub trait Simd: vectors: [i64x2; 4usize], dest: &mut [i64; 8usize], ) -> (); + #[doc = "Convert each signed 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + fn cvt_f64_i64x2(self, a: i64x2) -> f64x2; #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u64x2(self, val: u64) -> u64x2; #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] @@ -1236,6 +1248,8 @@ pub trait Simd: vectors: [u64x2; 4usize], dest: &mut [u64; 8usize], ) -> (); + #[doc = "Convert each unsigned 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + fn cvt_f64_u64x2(self, a: u64x2) -> f64x2; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask64x2(self, val: bool) -> mask64x2; #[doc = "Create a SIMD mask from a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are ignored."] @@ -2112,6 +2126,14 @@ pub trait Simd: fn combine_f64x4(self, a: f64x4, b: f64x4) -> f64x8; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_f64x4(self, a: f64x4) -> (f64x2, f64x2); + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + fn cvt_u64_f64x4(self, a: f64x4) -> u64x4; + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + fn cvt_u64_precise_f64x4(self, a: f64x4) -> u64x4; + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + fn cvt_i64_f64x4(self, a: f64x4) -> i64x4; + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + fn cvt_i64_precise_f64x4(self, a: f64x4) -> i64x4; #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_i64x4(self, val: i64) -> i64x4; #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] @@ -2198,6 +2220,8 @@ pub trait Simd: fn split_i64x4(self, a: i64x4) -> (i64x2, i64x2); #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i64x4(self, a: i64x4) -> i64x4; + #[doc = "Convert each signed 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + fn cvt_f64_i64x4(self, a: i64x4) -> f64x4; #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u64x4(self, val: u64) -> u64x4; #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] @@ -2282,6 +2306,8 @@ pub trait Simd: fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u64x4(self, a: u64x4) -> (u64x2, u64x2); + #[doc = "Convert each unsigned 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + fn cvt_f64_u64x4(self, a: u64x4) -> f64x4; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask64x4(self, val: bool) -> mask64x4; #[doc = "Create a SIMD mask from a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are ignored."] @@ -3148,6 +3174,14 @@ pub trait Simd: fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4); + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + fn cvt_u64_f64x8(self, a: f64x8) -> u64x8; + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + fn cvt_u64_precise_f64x8(self, a: f64x8) -> u64x8; + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + fn cvt_i64_f64x8(self, a: f64x8) -> i64x8; + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + fn cvt_i64_precise_f64x8(self, a: f64x8) -> i64x8; #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_i64x8(self, val: i64) -> i64x8; #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] @@ -3232,6 +3266,8 @@ pub trait Simd: fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4); #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i64x8(self, a: i64x8) -> i64x8; + #[doc = "Convert each signed 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + fn cvt_f64_i64x8(self, a: i64x8) -> f64x8; #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u64x8(self, val: u64) -> u64x8; #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] @@ -3314,6 +3350,8 @@ pub trait Simd: fn max_u64x8(self, a: u64x8, b: u64x8) -> u64x8; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4); + #[doc = "Convert each unsigned 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + fn cvt_f64_u64x8(self, a: u64x8) -> f64x8; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask64x8(self, val: bool) -> mask64x8; #[doc = "Create a SIMD mask from a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are ignored."] @@ -3636,8 +3674,8 @@ pub trait SimdFloat: { #[doc = r" Convert this floating-point type to an integer. This is a convenience method that"] #[doc = r" delegates to [`SimdCvtTruncate::truncate_from`], and can only be called if there"] - #[doc = r" actually exists a target type of the same bit width (currently, only `u32` and"] - #[doc = r" `i32`)."] + #[doc = r" actually exists a target type of the same bit width (`u32`/`i32` for `f32`, or"] + #[doc = r" `u64`/`i64` for `f64`)."] #[doc = r""] #[doc = r" For more information about the semantics of this specific conversion, see the"] #[doc = r" concrete `SimdCvtTruncate` implementations for integer types."] @@ -3648,7 +3686,8 @@ pub trait SimdFloat: #[doc = r" Convert this floating-point type to an integer, saturating on overflow and returning"] #[doc = r" 0 for NaN. This is a convenience method that delegates to"] #[doc = r" [`SimdCvtTruncate::truncate_from_precise`], and can only be called if there actually"] - #[doc = r" exists a target type of the same bit width (currently, only `u32` and `i32`)."] + #[doc = r" exists a target type of the same bit width (`u32`/`i32` for `f32`, or `u64`/`i64`"] + #[doc = r" for `f64`)."] #[doc = r""] #[doc = r" For more information about the semantics of this specific conversion, see the"] #[doc = r" concrete `SimdCvtTruncate` implementations for integer types."] @@ -3749,7 +3788,7 @@ pub trait SimdInt: { #[doc = r" Convert this integer type to a floating-point type. This is a convenience method"] #[doc = r" that delegates to [`SimdCvtFloat::float_from`], and can only be called if there"] - #[doc = r" actually exists a target type of the same bit width (currently, only `f32`)."] + #[doc = r" actually exists a target type of the same bit width (`f32` or `f64`)."] #[inline(always)] fn to_float>(self) -> T { T::float_from(self) diff --git a/fearless_simd/src/generated/simd_types.rs b/fearless_simd/src/generated/simd_types.rs index 5782c2f7..8a55cea7 100644 --- a/fearless_simd/src/generated/simd_types.rs +++ b/fearless_simd/src/generated/simd_types.rs @@ -2260,6 +2260,20 @@ impl crate::SimdFloat for f64x2 { self.simd.trunc_f64x2(self) } } +impl SimdCvtFloat> for f64x2 { + #[doc = "Convert each unsigned 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + #[inline(always)] + fn float_from(x: u64x2) -> Self { + x.simd.cvt_f64_u64x2(x) + } +} +impl SimdCvtFloat> for f64x2 { + #[doc = "Convert each signed 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + #[inline(always)] + fn float_from(x: i64x2) -> Self { + x.simd.cvt_f64_i64x2(x) + } +} impl crate::SimdCombine for f64x2 { type Combined = f64x4; #[inline(always)] @@ -2481,6 +2495,18 @@ impl crate::SimdInt for i64x2 { self.simd.max_i64x2(self, rhs.simd_into(self.simd)) } } +impl SimdCvtTruncate> for i64x2 { + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + #[inline(always)] + fn truncate_from(x: f64x2) -> Self { + x.simd.cvt_i64_f64x2(x) + } + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + #[inline(always)] + fn truncate_from_precise(x: f64x2) -> Self { + x.simd.cvt_i64_precise_f64x2(x) + } +} impl crate::SimdCombine for i64x2 { type Combined = i64x4; #[inline(always)] @@ -2702,6 +2728,18 @@ impl crate::SimdInt for u64x2 { self.simd.max_u64x2(self, rhs.simd_into(self.simd)) } } +impl SimdCvtTruncate> for u64x2 { + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + #[inline(always)] + fn truncate_from(x: f64x2) -> Self { + x.simd.cvt_u64_f64x2(x) + } + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + #[inline(always)] + fn truncate_from_precise(x: f64x2) -> Self { + x.simd.cvt_u64_precise_f64x2(x) + } +} impl crate::SimdCombine for u64x2 { type Combined = u64x4; #[inline(always)] @@ -5193,6 +5231,20 @@ impl crate::SimdFloat for f64x4 { self.simd.trunc_f64x4(self) } } +impl SimdCvtFloat> for f64x4 { + #[doc = "Convert each unsigned 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + #[inline(always)] + fn float_from(x: u64x4) -> Self { + x.simd.cvt_f64_u64x4(x) + } +} +impl SimdCvtFloat> for f64x4 { + #[doc = "Convert each signed 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + #[inline(always)] + fn float_from(x: i64x4) -> Self { + x.simd.cvt_f64_i64x4(x) + } +} impl crate::SimdSplit for f64x4 { type Split = f64x2; #[inline(always)] @@ -5421,6 +5473,18 @@ impl crate::SimdInt for i64x4 { self.simd.max_i64x4(self, rhs.simd_into(self.simd)) } } +impl SimdCvtTruncate> for i64x4 { + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + #[inline(always)] + fn truncate_from(x: f64x4) -> Self { + x.simd.cvt_i64_f64x4(x) + } + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + #[inline(always)] + fn truncate_from_precise(x: f64x4) -> Self { + x.simd.cvt_i64_precise_f64x4(x) + } +} impl crate::SimdSplit for i64x4 { type Split = i64x2; #[inline(always)] @@ -5649,6 +5713,18 @@ impl crate::SimdInt for u64x4 { self.simd.max_u64x4(self, rhs.simd_into(self.simd)) } } +impl SimdCvtTruncate> for u64x4 { + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + #[inline(always)] + fn truncate_from(x: f64x4) -> Self { + x.simd.cvt_u64_f64x4(x) + } + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + #[inline(always)] + fn truncate_from_precise(x: f64x4) -> Self { + x.simd.cvt_u64_precise_f64x4(x) + } +} impl crate::SimdSplit for u64x4 { type Split = u64x2; #[inline(always)] @@ -8244,6 +8320,20 @@ impl crate::SimdFloat for f64x8 { self.simd.trunc_f64x8(self) } } +impl SimdCvtFloat> for f64x8 { + #[doc = "Convert each unsigned 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + #[inline(always)] + fn float_from(x: u64x8) -> Self { + x.simd.cvt_f64_u64x8(x) + } +} +impl SimdCvtFloat> for f64x8 { + #[doc = "Convert each signed 64-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] + #[inline(always)] + fn float_from(x: i64x8) -> Self { + x.simd.cvt_f64_i64x8(x) + } +} impl crate::SimdSplit for f64x8 { type Split = f64x4; #[inline(always)] @@ -8478,6 +8568,18 @@ impl crate::SimdInt for i64x8 { self.simd.max_i64x8(self, rhs.simd_into(self.simd)) } } +impl SimdCvtTruncate> for i64x8 { + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + #[inline(always)] + fn truncate_from(x: f64x8) -> Self { + x.simd.cvt_i64_f64x8(x) + } + #[doc = "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + #[inline(always)] + fn truncate_from_precise(x: f64x8) -> Self { + x.simd.cvt_i64_precise_f64x8(x) + } +} impl crate::SimdSplit for i64x8 { type Split = i64x4; #[inline(always)] @@ -8712,6 +8814,18 @@ impl crate::SimdInt for u64x8 { self.simd.max_u64x8(self, rhs.simd_into(self.simd)) } } +impl SimdCvtTruncate> for u64x8 { + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results."] + #[inline(always)] + fn truncate_from(x: f64x8) -> Self { + x.simd.cvt_u64_f64x8(x) + } + #[doc = "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] + #[inline(always)] + fn truncate_from_precise(x: f64x8) -> Self { + x.simd.cvt_u64_precise_f64x8(x) + } +} impl crate::SimdSplit for u64x8 { type Split = u64x4; #[inline(always)] diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index 860b9741..acd248d0 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -5471,6 +5471,22 @@ impl Simd for Sse2 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_u64_f64x2(self, a: f64x2) -> u64x2 { + [a[0usize] as u64, a[1usize] as u64].simd_into(self) + } + #[inline(always)] + fn cvt_u64_precise_f64x2(self, a: f64x2) -> u64x2 { + [a[0usize] as u64, a[1usize] as u64].simd_into(self) + } + #[inline(always)] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { + [a[0usize] as i64, a[1usize] as i64].simd_into(self) + } + #[inline(always)] + fn cvt_i64_precise_f64x2(self, a: f64x2) -> i64x2 { + [a[0usize] as i64, a[1usize] as i64].simd_into(self) + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -5947,6 +5963,10 @@ impl Simd for Sse2 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_f64_i64x2(self, a: i64x2) -> f64x2 { + [a[0usize] as f64, a[1usize] as f64].simd_into(self) + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -6415,6 +6435,10 @@ impl Simd for Sse2 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_f64_u64x2(self, a: u64x2) -> f64x2 { + [a[0usize] as f64, a[1usize] as f64].simd_into(self) + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -9791,6 +9815,32 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn cvt_u64_f64x4(self, a: f64x4) -> u64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_u64x2(self.cvt_u64_f64x2(a0), self.cvt_u64_f64x2(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x4(self, a: f64x4) -> u64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_u64x2( + self.cvt_u64_precise_f64x2(a0), + self.cvt_u64_precise_f64x2(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x4(self, a: f64x4) -> i64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_i64x2(self.cvt_i64_f64x2(a0), self.cvt_i64_f64x2(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x4(self, a: f64x4) -> i64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_i64x2( + self.cvt_i64_precise_f64x2(a0), + self.cvt_i64_precise_f64x2(a1), + ) + } + #[inline(always)] fn splat_i64x4(self, val: i64) -> i64x4 { let half = self.splat_i64x2(val); self.combine_i64x2(half, half) @@ -10085,6 +10135,11 @@ impl Simd for Sse2 { self.combine_i64x2(self.neg_i64x2(a0), self.neg_i64x2(a1)) } #[inline(always)] + fn cvt_f64_i64x4(self, a: i64x4) -> f64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_f64x2(self.cvt_f64_i64x2(a0), self.cvt_f64_i64x2(a1)) + } + #[inline(always)] fn splat_u64x4(self, val: u64) -> u64x4 { let half = self.splat_u64x2(val); self.combine_u64x2(half, half) @@ -10374,6 +10429,11 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn cvt_f64_u64x4(self, a: u64x4) -> f64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_f64x2(self.cvt_f64_u64x2(a0), self.cvt_f64_u64x2(a1)) + } + #[inline(always)] fn splat_mask64x4(self, val: bool) -> mask64x4 { let half = self.splat_mask64x2(val); self.combine_mask64x2(half, half) @@ -14137,6 +14197,32 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn cvt_u64_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4(self.cvt_u64_f64x4(a0), self.cvt_u64_f64x4(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4( + self.cvt_u64_precise_f64x4(a0), + self.cvt_u64_precise_f64x4(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4(self.cvt_i64_f64x4(a0), self.cvt_i64_f64x4(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4( + self.cvt_i64_precise_f64x4(a0), + self.cvt_i64_precise_f64x4(a1), + ) + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -14440,6 +14526,11 @@ impl Simd for Sse2 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn cvt_f64_i64x8(self, a: i64x8) -> f64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_f64x4(self.cvt_f64_i64x4(a0), self.cvt_f64_i64x4(a1)) + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -14738,6 +14829,11 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn cvt_f64_u64x8(self, a: u64x8) -> f64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_f64x4(self.cvt_f64_u64x4(a0), self.cvt_f64_u64x4(a1)) + } + #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 5b0b6195..46843d1c 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -4755,6 +4755,22 @@ impl Simd for Sse4_2 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_u64_f64x2(self, a: f64x2) -> u64x2 { + [a[0usize] as u64, a[1usize] as u64].simd_into(self) + } + #[inline(always)] + fn cvt_u64_precise_f64x2(self, a: f64x2) -> u64x2 { + [a[0usize] as u64, a[1usize] as u64].simd_into(self) + } + #[inline(always)] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { + [a[0usize] as i64, a[1usize] as i64].simd_into(self) + } + #[inline(always)] + fn cvt_i64_precise_f64x2(self, a: f64x2) -> i64x2 { + [a[0usize] as i64, a[1usize] as i64].simd_into(self) + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -5174,6 +5190,10 @@ impl Simd for Sse4_2 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_f64_i64x2(self, a: i64x2) -> f64x2 { + [a[0usize] as f64, a[1usize] as f64].simd_into(self) + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5585,6 +5605,10 @@ impl Simd for Sse4_2 { kernel(self, vectors, dest); } #[inline(always)] + fn cvt_f64_u64x2(self, a: u64x2) -> f64x2 { + [a[0usize] as f64, a[1usize] as f64].simd_into(self) + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -9079,6 +9103,32 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn cvt_u64_f64x4(self, a: f64x4) -> u64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_u64x2(self.cvt_u64_f64x2(a0), self.cvt_u64_f64x2(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x4(self, a: f64x4) -> u64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_u64x2( + self.cvt_u64_precise_f64x2(a0), + self.cvt_u64_precise_f64x2(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x4(self, a: f64x4) -> i64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_i64x2(self.cvt_i64_f64x2(a0), self.cvt_i64_f64x2(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x4(self, a: f64x4) -> i64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_i64x2( + self.cvt_i64_precise_f64x2(a0), + self.cvt_i64_precise_f64x2(a1), + ) + } + #[inline(always)] fn splat_i64x4(self, val: i64) -> i64x4 { let half = self.splat_i64x2(val); self.combine_i64x2(half, half) @@ -9388,6 +9438,11 @@ impl Simd for Sse4_2 { self.combine_i64x2(self.neg_i64x2(a0), self.neg_i64x2(a1)) } #[inline(always)] + fn cvt_f64_i64x4(self, a: i64x4) -> f64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_f64x2(self.cvt_f64_i64x2(a0), self.cvt_f64_i64x2(a1)) + } + #[inline(always)] fn splat_u64x4(self, val: u64) -> u64x4 { let half = self.splat_u64x2(val); self.combine_u64x2(half, half) @@ -9692,6 +9747,11 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn cvt_f64_u64x4(self, a: u64x4) -> f64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_f64x2(self.cvt_f64_u64x2(a0), self.cvt_f64_u64x2(a1)) + } + #[inline(always)] fn splat_mask64x4(self, val: bool) -> mask64x4 { let half = self.splat_mask64x2(val); self.combine_mask64x2(half, half) @@ -13496,6 +13556,32 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn cvt_u64_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4(self.cvt_u64_f64x4(a0), self.cvt_u64_f64x4(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4( + self.cvt_u64_precise_f64x4(a0), + self.cvt_u64_precise_f64x4(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4(self.cvt_i64_f64x4(a0), self.cvt_i64_f64x4(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4( + self.cvt_i64_precise_f64x4(a0), + self.cvt_i64_precise_f64x4(a1), + ) + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -13799,6 +13885,11 @@ impl Simd for Sse4_2 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn cvt_f64_i64x8(self, a: i64x8) -> f64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_f64x4(self.cvt_f64_i64x4(a0), self.cvt_f64_i64x4(a1)) + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -14097,6 +14188,11 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn cvt_f64_u64x8(self, a: u64x8) -> f64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_f64x4(self.cvt_f64_u64x4(a0), self.cvt_f64_u64x4(a1)) + } + #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 037a4f16..f9031017 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -2934,6 +2934,22 @@ impl Simd for WasmSimd128 { crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); } #[inline(always)] + fn cvt_u64_f64x2(self, a: f64x2) -> u64x2 { + [a[0usize] as u64, a[1usize] as u64].simd_into(self) + } + #[inline(always)] + fn cvt_u64_precise_f64x2(self, a: f64x2) -> u64x2 { + [a[0usize] as u64, a[1usize] as u64].simd_into(self) + } + #[inline(always)] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { + [a[0usize] as i64, a[1usize] as i64].simd_into(self) + } + #[inline(always)] + fn cvt_i64_precise_f64x2(self, a: f64x2) -> i64x2 { + [a[0usize] as i64, a[1usize] as i64].simd_into(self) + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { i64x2_splat(val).simd_into(self) } @@ -3197,6 +3213,10 @@ impl Simd for WasmSimd128 { crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); } #[inline(always)] + fn cvt_f64_i64x2(self, a: i64x2) -> f64x2 { + [a[0usize] as f64, a[1usize] as f64].simd_into(self) + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { u64x2_splat(val).simd_into(self) } @@ -3476,6 +3496,10 @@ impl Simd for WasmSimd128 { crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); } #[inline(always)] + fn cvt_f64_u64x2(self, a: u64x2) -> f64x2 { + [a[0usize] as f64, a[1usize] as f64].simd_into(self) + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { let val: i64 = if val { !0 } else { 0 }; i64x2_splat(val).simd_into(self) @@ -6792,6 +6816,32 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn cvt_u64_f64x4(self, a: f64x4) -> u64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_u64x2(self.cvt_u64_f64x2(a0), self.cvt_u64_f64x2(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x4(self, a: f64x4) -> u64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_u64x2( + self.cvt_u64_precise_f64x2(a0), + self.cvt_u64_precise_f64x2(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x4(self, a: f64x4) -> i64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_i64x2(self.cvt_i64_f64x2(a0), self.cvt_i64_f64x2(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x4(self, a: f64x4) -> i64x4 { + let (a0, a1) = self.split_f64x4(a); + self.combine_i64x2( + self.cvt_i64_precise_f64x2(a0), + self.cvt_i64_precise_f64x2(a1), + ) + } + #[inline(always)] fn splat_i64x4(self, val: i64) -> i64x4 { let half = self.splat_i64x2(val); self.combine_i64x2(half, half) @@ -7090,6 +7140,11 @@ impl Simd for WasmSimd128 { self.combine_i64x2(self.neg_i64x2(a0), self.neg_i64x2(a1)) } #[inline(always)] + fn cvt_f64_i64x4(self, a: i64x4) -> f64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_f64x2(self.cvt_f64_i64x2(a0), self.cvt_f64_i64x2(a1)) + } + #[inline(always)] fn splat_u64x4(self, val: u64) -> u64x4 { let half = self.splat_u64x2(val); self.combine_u64x2(half, half) @@ -7383,6 +7438,11 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn cvt_f64_u64x4(self, a: u64x4) -> f64x4 { + let (a0, a1) = self.split_u64x4(a); + self.combine_f64x2(self.cvt_f64_u64x2(a0), self.cvt_f64_u64x2(a1)) + } + #[inline(always)] fn splat_mask64x4(self, val: bool) -> mask64x4 { let half = self.splat_mask64x2(val); self.combine_mask64x2(half, half) @@ -11123,6 +11183,32 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn cvt_u64_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4(self.cvt_u64_f64x4(a0), self.cvt_u64_f64x4(a1)) + } + #[inline(always)] + fn cvt_u64_precise_f64x8(self, a: f64x8) -> u64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_u64x4( + self.cvt_u64_precise_f64x4(a0), + self.cvt_u64_precise_f64x4(a1), + ) + } + #[inline(always)] + fn cvt_i64_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4(self.cvt_i64_f64x4(a0), self.cvt_i64_f64x4(a1)) + } + #[inline(always)] + fn cvt_i64_precise_f64x8(self, a: f64x8) -> i64x8 { + let (a0, a1) = self.split_f64x8(a); + self.combine_i64x4( + self.cvt_i64_precise_f64x4(a0), + self.cvt_i64_precise_f64x4(a1), + ) + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -11425,6 +11511,11 @@ impl Simd for WasmSimd128 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn cvt_f64_i64x8(self, a: i64x8) -> f64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_f64x4(self.cvt_f64_i64x4(a0), self.cvt_f64_i64x4(a1)) + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -11722,6 +11813,11 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn cvt_f64_u64x8(self, a: u64x8) -> f64x8 { + let (a0, a1) = self.split_u64x8(a); + self.combine_f64x4(self.cvt_f64_u64x4(a0), self.cvt_f64_u64x4(a1)) + } + #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd_gen/src/mk_simd_trait.rs b/fearless_simd_gen/src/mk_simd_trait.rs index c39e7a51..04028d45 100644 --- a/fearless_simd_gen/src/mk_simd_trait.rs +++ b/fearless_simd_gen/src/mk_simd_trait.rs @@ -72,7 +72,7 @@ pub(crate) fn mk_simd_trait() -> TokenStream { /// A native-width SIMD vector of [`f32`]s. type f32s: SimdFloat, Mask = Self::mask32s, ByteVector = Self::u8s> + SimdCvtFloat + SimdCvtFloat; /// A native-width SIMD vector of [`f64`]s. - type f64s: SimdFloat, Mask = Self::mask64s, ByteVector = Self::u8s>; + type f64s: SimdFloat, Mask = Self::mask64s, ByteVector = Self::u8s> + SimdCvtFloat + SimdCvtFloat; /// A native-width SIMD vector of [`u8`]s. type u8s: SimdInt, Mask = Self::mask8s, ByteVector = Self::u8s>; /// A native-width SIMD vector of [`i8`]s. @@ -87,9 +87,9 @@ pub(crate) fn mk_simd_trait() -> TokenStream { type i32s: SimdInt, Mask = Self::mask32s, ByteVector = Self::u8s> + SimdCvtTruncate + core::ops::Neg; /// A native-width SIMD vector of [`u64`]s. - type u64s: SimdInt, Mask = Self::mask64s, ByteVector = Self::u8s>; + type u64s: SimdInt, Mask = Self::mask64s, ByteVector = Self::u8s> + SimdCvtTruncate; /// A native-width SIMD vector of [`i64`]s. - type i64s: SimdInt, Mask = Self::mask64s, ByteVector = Self::u8s> + type i64s: SimdInt, Mask = Self::mask64s, ByteVector = Self::u8s> + SimdCvtTruncate + core::ops::Neg; /// A native-width SIMD mask with 8-bit lanes. type mask8s: SimdMask + Select + Select + Select; @@ -286,8 +286,8 @@ fn mk_simd_float() -> TokenStream { { /// Convert this floating-point type to an integer. This is a convenience method that /// delegates to [`SimdCvtTruncate::truncate_from`], and can only be called if there - /// actually exists a target type of the same bit width (currently, only `u32` and - /// `i32`). + /// actually exists a target type of the same bit width (`u32`/`i32` for `f32`, or + /// `u64`/`i64` for `f64`). /// /// For more information about the semantics of this specific conversion, see the /// concrete `SimdCvtTruncate` implementations for integer types. @@ -297,7 +297,8 @@ fn mk_simd_float() -> TokenStream { /// Convert this floating-point type to an integer, saturating on overflow and returning /// 0 for NaN. This is a convenience method that delegates to /// [`SimdCvtTruncate::truncate_from_precise`], and can only be called if there actually - /// exists a target type of the same bit width (currently, only `u32` and `i32`). + /// exists a target type of the same bit width (`u32`/`i32` for `f32`, or `u64`/`i64` + /// for `f64`). /// /// For more information about the semantics of this specific conversion, see the /// concrete `SimdCvtTruncate` implementations for integer types. @@ -326,7 +327,7 @@ fn mk_simd_int() -> TokenStream { { /// Convert this integer type to a floating-point type. This is a convenience method /// that delegates to [`SimdCvtFloat::float_from`], and can only be called if there - /// actually exists a target type of the same bit width (currently, only `f32`). + /// actually exists a target type of the same bit width (`f32` or `f64`). #[inline(always)] fn to_float>(self) -> T { T::float_from(self) } diff --git a/fearless_simd_gen/src/mk_simd_types.rs b/fearless_simd_gen/src/mk_simd_types.rs index ea915255..079c46ef 100644 --- a/fearless_simd_gen/src/mk_simd_types.rs +++ b/fearless_simd_gen/src/mk_simd_types.rs @@ -7,8 +7,9 @@ use quote::{format_ident, quote}; use crate::{ generic::{generic_op_name, unrolled_array}, ops::{ - F32_TO_I32, F32_TO_I32_PRECISE, F32_TO_U32, F32_TO_U32_PRECISE, I32_TO_F32, Op, OpSig, - TyFlavor, U32_TO_F32, vec_trait_ops_for, + F32_TO_I32, F32_TO_I32_PRECISE, F32_TO_U32, F32_TO_U32_PRECISE, F64_TO_I64, + F64_TO_I64_PRECISE, F64_TO_U64, F64_TO_U64_PRECISE, I32_TO_F32, I64_TO_F64, Op, OpSig, + TyFlavor, U32_TO_F32, U64_TO_F64, vec_trait_ops_for, }, types::{SIMD_TYPES, ScalarType, VecType}, }; @@ -118,9 +119,8 @@ pub(crate) fn mk_simd_types() -> TokenStream { }; let impl_block = simd_vec_impl(ty); let mut conditional_impls = Vec::new(); - // TODO: Relax `if` clauses once 64-bit integer or 16-bit floats vectors are implemented match ty.scalar { - ScalarType::Float if ty.scalar_bits == 32 => { + ScalarType::Float if matches!(ty.scalar_bits, 32 | 64) => { for src_scalar in [ScalarType::Unsigned, ScalarType::Int] { let src_ty = ty.cast(src_scalar); let method = format_ident!( @@ -129,9 +129,11 @@ pub(crate) fn mk_simd_types() -> TokenStream { src_ty.rust_name() ); let src_ty = src_ty.rust(); - let op = match src_scalar { - ScalarType::Unsigned => U32_TO_F32, - ScalarType::Int => I32_TO_F32, + let op = match (src_scalar, ty.scalar_bits) { + (ScalarType::Unsigned, 32) => U32_TO_F32, + (ScalarType::Int, 32) => I32_TO_F32, + (ScalarType::Unsigned, 64) => U64_TO_F64, + (ScalarType::Int, 64) => I64_TO_F64, _ => unreachable!(), }; let doc = op.format_docstring(TyFlavor::VecImpl); @@ -146,16 +148,18 @@ pub(crate) fn mk_simd_types() -> TokenStream { }); } } - ScalarType::Int | ScalarType::Unsigned if ty.scalar_bits == 32 => { + ScalarType::Int | ScalarType::Unsigned if matches!(ty.scalar_bits, 32 | 64) => { let src_ty = ty.cast(ScalarType::Float); let method = format_ident!( "cvt_{}_{}", ty.scalar.rust_name(ty.scalar_bits), src_ty.rust_name() ); - let op = match ty.scalar { - ScalarType::Unsigned => F32_TO_U32, - ScalarType::Int => F32_TO_I32, + let op = match (ty.scalar, ty.scalar_bits) { + (ScalarType::Unsigned, 32) => F32_TO_U32, + (ScalarType::Int, 32) => F32_TO_I32, + (ScalarType::Unsigned, 64) => F64_TO_U64, + (ScalarType::Int, 64) => F64_TO_I64, _ => unreachable!(), }; let doc = op.format_docstring(TyFlavor::VecImpl); @@ -164,9 +168,11 @@ pub(crate) fn mk_simd_types() -> TokenStream { ty.scalar.rust_name(ty.scalar_bits), src_ty.rust_name() ); - let op_precise = match ty.scalar { - ScalarType::Unsigned => F32_TO_U32_PRECISE, - ScalarType::Int => F32_TO_I32_PRECISE, + let op_precise = match (ty.scalar, ty.scalar_bits) { + (ScalarType::Unsigned, 32) => F32_TO_U32_PRECISE, + (ScalarType::Int, 32) => F32_TO_I32_PRECISE, + (ScalarType::Unsigned, 64) => F64_TO_U64_PRECISE, + (ScalarType::Int, 64) => F64_TO_I64_PRECISE, _ => unreachable!(), }; let doc_precise = op_precise.format_docstring(TyFlavor::VecImpl); diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index a6251648..6c697327 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -551,6 +551,9 @@ impl Level for WasmSimd128 { scalar_bits, precise, } => { + if scalar_bits == 64 { + return fallback_method(op, vec_ty); + } let (op, uses_relaxed) = match (vec_ty.scalar, target_ty, precise) { (ScalarType::Float, ScalarType::Int | ScalarType::Unsigned, false) => { ("relaxed_trunc", true) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 19e05a18..60ce1d94 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2885,6 +2885,85 @@ impl X86 { "we currently only support converting between types of the same width" ); + if vec_ty.scalar_bits == 64 && *self != Self::Avx512 { + return fallback_method(op, vec_ty); + } + + if vec_ty.scalar_bits == 64 { + let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); + return self.kernel_method(op, vec_ty, |token| match (vec_ty.scalar, target_scalar) { + (ScalarType::Int, ScalarType::Float) => { + let convert = simple_intrinsic("cvtepi64", &target_ty); + quote! { + #convert(a.into()).simd_into(#token) + } + } + (ScalarType::Unsigned, ScalarType::Float) => { + let convert = simple_intrinsic("cvtepu64", &target_ty); + quote! { + #convert(a.into()).simd_into(#token) + } + } + (ScalarType::Float, ScalarType::Int) => { + let convert = simple_intrinsic("cvttpd", &target_ty); + if precise { + let masked_convert = + intrinsic_ident("mask_cvttpd", "epi64", vec_ty.n_bits()); + let cmp = intrinsic_ident("cmp", "pd_mask", vec_ty.n_bits()); + let blend = avx512_mask_blend_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let set0_int = + intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); + let lt = avx512_float_compare_predicate("simd_lt"); + let ord = avx512_float_compare_predicate("ord"); + quote! { + let a = a.into(); + let in_range = #cmp::<#lt>(a, #set1_float(9223372036854775808.0)); + let mut converted = #masked_convert(#set1_int(i64::MAX), in_range, a); + let is_not_nan = #cmp::<#ord>(a, a); + converted = #blend(is_not_nan, #set0_int(), converted); + converted.simd_into(#token) + } + } else { + quote! { + #convert(a.into()).simd_into(#token) + } + } + } + (ScalarType::Float, ScalarType::Unsigned) => { + let convert = simple_intrinsic("cvttpd", &target_ty); + if precise { + let max = simple_intrinsic("max", vec_ty); + let cmp = intrinsic_ident("cmp", "pd_mask", vec_ty.n_bits()); + let blend = avx512_mask_blend_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let set0_float = + intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); + let lt = avx512_float_compare_predicate("simd_lt"); + quote! { + let a = #max(a.into(), #set0_float()); + let mut converted = #convert(a); + let exceeds_unsigned_range = + #cmp::<#lt>(#set1_float(18446744073709549568.0), a); + converted = #blend( + exceeds_unsigned_range, + converted, + #set1_int(u64::MAX.cast_signed()), + ); + converted.simd_into(#token) + } + } else { + quote! { + #convert(a.into()).simd_into(#token) + } + } + } + _ => unimplemented!(), + }); + } + if *self == Self::Sse2 && (precise || vec_ty.scalar == ScalarType::Unsigned diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index e401c55b..34357c7a 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -1238,6 +1238,72 @@ pub(crate) const I32_TO_F32: Op = Op::new( "Convert each signed 32-bit integer element to a floating-point value.\n\n\ Values that cannot be exactly represented are rounded to the nearest representable value.", ); +pub(crate) const F64_TO_U64: Op = Op::new( + "cvt_u64", + OpKind::OwnTrait, + OpSig::Cvt { + target_ty: ScalarType::Unsigned, + scalar_bits: 64, + precise: false, + }, + "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\n\ + Out-of-range values or NaN will produce implementation-defined results.", +); +pub(crate) const F64_TO_U64_PRECISE: Op = Op::new( + "cvt_u64_precise", + OpKind::OwnTrait, + OpSig::Cvt { + target_ty: ScalarType::Unsigned, + scalar_bits: 64, + precise: true, + }, + "Convert each floating-point element to an unsigned 64-bit integer, truncating towards zero.\n\n\ + Out-of-range values are saturated to the closest in-range value. NaN becomes 0.", +); +pub(crate) const F64_TO_I64: Op = Op::new( + "cvt_i64", + OpKind::OwnTrait, + OpSig::Cvt { + target_ty: ScalarType::Int, + scalar_bits: 64, + precise: false, + }, + "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\n\ + Out-of-range values or NaN will produce implementation-defined results.", +); +pub(crate) const F64_TO_I64_PRECISE: Op = Op::new( + "cvt_i64_precise", + OpKind::OwnTrait, + OpSig::Cvt { + target_ty: ScalarType::Int, + scalar_bits: 64, + precise: true, + }, + "Convert each floating-point element to a signed 64-bit integer, truncating towards zero.\n\n\ + Out-of-range values are saturated to the closest in-range value. NaN becomes 0.", +); +pub(crate) const U64_TO_F64: Op = Op::new( + "cvt_f64", + OpKind::OwnTrait, + OpSig::Cvt { + target_ty: ScalarType::Float, + scalar_bits: 64, + precise: false, + }, + "Convert each unsigned 64-bit integer element to a floating-point value.\n\n\ + Values that cannot be exactly represented are rounded to the nearest representable value.", +); +pub(crate) const I64_TO_F64: Op = Op::new( + "cvt_f64", + OpKind::OwnTrait, + OpSig::Cvt { + target_ty: ScalarType::Float, + scalar_bits: 64, + precise: false, + }, + "Convert each signed 64-bit integer element to a floating-point value.\n\n\ + Values that cannot be exactly represented are rounded to the nearest representable value.", +); pub(crate) fn ops_for_type(ty: &VecType) -> Vec { let base = match ty.scalar { @@ -1316,10 +1382,6 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { )); } - if ty.scalar == ScalarType::Float && ty.scalar_bits == 64 { - return ops; - } - if matches!(ty.scalar, ScalarType::Unsigned) { if let Some(target_ty) = ty.widened() { ops.push(Op::new( @@ -1349,6 +1411,14 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { } (ScalarType::Unsigned, 32) => ops.push(U32_TO_F32), (ScalarType::Int, 32) => ops.push(I32_TO_F32), + (ScalarType::Float, 64) => { + ops.push(F64_TO_U64); + ops.push(F64_TO_U64_PRECISE); + ops.push(F64_TO_I64); + ops.push(F64_TO_I64_PRECISE); + } + (ScalarType::Unsigned, 64) => ops.push(U64_TO_F64), + (ScalarType::Int, 64) => ops.push(I64_TO_F64), _ => (), } diff --git a/fearless_simd_tests/tests/generics.rs b/fearless_simd_tests/tests/generics.rs index 416d64b2..21f437e8 100644 --- a/fearless_simd_tests/tests/generics.rs +++ b/fearless_simd_tests/tests/generics.rs @@ -12,6 +12,30 @@ fn generic_cast(x: S::f32s) -> S::u32s { x.to_int() } +fn generic_f64_to_u64(x: S::f64s) -> S::u64s { + x.to_int() +} + +fn generic_f64_to_i64(x: S::f64s) -> S::i64s { + x.to_int() +} + +fn generic_f64_to_u64_precise(x: S::f64s) -> S::u64s { + x.to_int_precise() +} + +fn generic_f64_to_i64_precise(x: S::f64s) -> S::i64s { + x.to_int_precise() +} + +fn generic_u64_to_f64(x: S::u64s) -> S::f64s { + x.to_float() +} + +fn generic_i64_to_f64(x: S::i64s) -> S::f64s { + x.to_float() +} + // Ensure that a generic vector's byte representation is itself a same-token // byte vector whose byte representation is idempotent. fn generic_bytes>(value: V) -> V { diff --git a/fearless_simd_tests/tests/harness/ops/cvt_f64.rs b/fearless_simd_tests/tests/harness/ops/cvt_f64.rs new file mode 100644 index 00000000..acc2bf28 --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/cvt_f64.rs @@ -0,0 +1,74 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +#[simd_test] +fn cvt_f64_i64x2(simd: S) { + let values = [i64::MIN, i64::MAX]; + let a = i64x2::from_slice(simd, &values); + assert_eq!(*a.to_float::>(), values.map(|x| x as f64)); +} + +#[simd_test] +fn cvt_f64_u64x2(simd: S) { + let values = [0, u64::MAX]; + let a = u64x2::from_slice(simd, &values); + assert_eq!(*a.to_float::>(), values.map(|x| x as f64)); +} + +#[simd_test] +fn cvt_f64_i64x4(simd: S) { + let values = [ + i64::MIN, + -9_007_199_254_740_993, + 9_007_199_254_740_993, + i64::MAX, + ]; + let a = i64x4::from_slice(simd, &values); + assert_eq!(*a.to_float::>(), values.map(|x| x as f64)); +} + +#[simd_test] +fn cvt_f64_u64x4(simd: S) { + let values = [0, 9_007_199_254_740_993, 1 << 63, u64::MAX]; + let a = u64x4::from_slice(simd, &values); + assert_eq!(*a.to_float::>(), values.map(|x| x as f64)); +} + +#[simd_test] +fn cvt_f64_i64x8(simd: S) { + use fearless_simd::SimdCvtFloat; + let values = [ + i64::MIN, + -9_007_199_254_740_993, + -42, + -1, + 0, + 1, + 9_007_199_254_740_993, + i64::MAX, + ]; + let a = i64x8::from_slice(simd, &values); + let result = f64x8::float_from(a); + assert_eq!(*result, values.map(|x| x as f64)); +} + +#[simd_test] +fn cvt_f64_u64x8(simd: S) { + use fearless_simd::SimdCvtFloat; + let values = [ + 0, + 1, + 42, + 1 << 53, + (1 << 53) + 1, + 1 << 63, + u64::MAX - 1, + u64::MAX, + ]; + let a = u64x8::from_slice(simd, &values); + let result = f64x8::float_from(a); + assert_eq!(*result, values.map(|x| x as f64)); +} diff --git a/fearless_simd_tests/tests/harness/ops/cvt_i64.rs b/fearless_simd_tests/tests/harness/ops/cvt_i64.rs new file mode 100644 index 00000000..8cd7e186 --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/cvt_i64.rs @@ -0,0 +1,33 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +#[simd_test] +fn cvt_i64_f64x2(simd: S) { + let values = [-42.9, 42.9]; + let a = f64x2::from_slice(simd, &values); + assert_eq!(*a.to_int::>(), values.map(|x| x as i64)); +} + +#[simd_test] +fn cvt_i64_f64x4(simd: S) { + let values = [ + -0.0, + 0.0, + -9_223_372_036_854_775_808.0, + 9_223_372_036_854_774_784.0, + ]; + let a = f64x4::from_slice(simd, &values); + assert_eq!(*a.to_int::>(), values.map(|x| x as i64)); +} + +#[simd_test] +fn cvt_i64_f64x8(simd: S) { + use fearless_simd::SimdCvtTruncate; + let values = [-1234.75, -1.99, -0.99, -0.0, 0.0, 0.99, 1.99, 1234.75]; + let a = f64x8::from_slice(simd, &values); + let result = i64x8::truncate_from(a); + assert_eq!(*result, values.map(|x| x as i64)); +} diff --git a/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs b/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs new file mode 100644 index 00000000..14fa4687 --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs @@ -0,0 +1,42 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +#[simd_test] +fn cvt_i64_precise_f64x2(simd: S) { + let values = [f64::NAN, f64::INFINITY]; + let a = f64x2::from_slice(simd, &values); + assert_eq!(*a.to_int_precise::>(), values.map(|x| x as i64)); +} + +#[simd_test] +fn cvt_i64_precise_f64x4(simd: S) { + let values = [ + f64::NEG_INFINITY, + -9_223_372_036_854_775_808.0, + 9_223_372_036_854_774_784.0, + 9_223_372_036_854_775_808.0, + ]; + let a = f64x4::from_slice(simd, &values); + assert_eq!(*a.to_int_precise::>(), values.map(|x| x as i64)); +} + +#[simd_test] +fn cvt_i64_precise_f64x8(simd: S) { + use fearless_simd::SimdCvtTruncate; + let values = [ + f64::NAN, + f64::NEG_INFINITY, + f64::INFINITY, + -1e300, + 1e300, + -42.9, + -0.0, + 42.9, + ]; + let a = f64x8::from_slice(simd, &values); + let result = i64x8::truncate_from_precise(a); + assert_eq!(*result, values.map(|x| x as i64)); +} diff --git a/fearless_simd_tests/tests/harness/ops/cvt_u64.rs b/fearless_simd_tests/tests/harness/ops/cvt_u64.rs new file mode 100644 index 00000000..1ec82910 --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/cvt_u64.rs @@ -0,0 +1,42 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +#[simd_test] +fn cvt_u64_f64x2(simd: S) { + let values = [0.0, 42.9]; + let a = f64x2::from_slice(simd, &values); + assert_eq!(*a.to_int::>(), values.map(|x| x as u64)); +} + +#[simd_test] +fn cvt_u64_f64x4(simd: S) { + let values = [ + -0.0, + 0.99, + 9_223_372_036_854_775_808.0, + 18_446_744_073_709_549_568.0, + ]; + let a = f64x4::from_slice(simd, &values); + assert_eq!(*a.to_int::>(), values.map(|x| x as u64)); +} + +#[simd_test] +fn cvt_u64_f64x8(simd: S) { + use fearless_simd::SimdCvtTruncate; + let values = [ + 0.0, + -0.0, + 0.99, + 1.99, + 42.9, + 1e9, + 1e15, + 18_446_744_073_709_549_568.0, + ]; + let a = f64x8::from_slice(simd, &values); + let result = u64x8::truncate_from(a); + assert_eq!(*result, values.map(|x| x as u64)); +} diff --git a/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs b/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs new file mode 100644 index 00000000..71dd6b80 --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs @@ -0,0 +1,42 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +#[simd_test] +fn cvt_u64_precise_f64x2(simd: S) { + let values = [f64::NAN, f64::INFINITY]; + let a = f64x2::from_slice(simd, &values); + assert_eq!(*a.to_int_precise::>(), values.map(|x| x as u64)); +} + +#[simd_test] +fn cvt_u64_precise_f64x4(simd: S) { + let values = [ + f64::NEG_INFINITY, + -1.0, + 18_446_744_073_709_549_568.0, + 18_446_744_073_709_551_616.0, + ]; + let a = f64x4::from_slice(simd, &values); + assert_eq!(*a.to_int_precise::>(), values.map(|x| x as u64)); +} + +#[simd_test] +fn cvt_u64_precise_f64x8(simd: S) { + use fearless_simd::SimdCvtTruncate; + let values = [ + f64::NAN, + f64::NEG_INFINITY, + f64::INFINITY, + -1e300, + 1e300, + -0.0, + 42.9, + 18_446_744_073_709_549_568.0, + ]; + let a = f64x8::from_slice(simd, &values); + let result = u64x8::truncate_from_precise(a); + assert_eq!(*result, values.map(|x| x as u64)); +} diff --git a/fearless_simd_tests/tests/harness/ops/mod.rs b/fearless_simd_tests/tests/harness/ops/mod.rs index 0e6f6adf..927cc94c 100644 --- a/fearless_simd_tests/tests/harness/ops/mod.rs +++ b/fearless_simd_tests/tests/harness/ops/mod.rs @@ -20,10 +20,15 @@ mod ceil; mod combine; mod copysign; mod cvt_f32; +mod cvt_f64; mod cvt_i32; mod cvt_i32_precise; +mod cvt_i64; +mod cvt_i64_precise; mod cvt_u32; mod cvt_u32_precise; +mod cvt_u64; +mod cvt_u64_precise; mod deinterleave; mod div; mod floor; From f0faf1ea30943f792bf0e9181294e669f1967465 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 5 Aug 2026 15:26:52 +0100 Subject: [PATCH 2/8] Refactor x86 handle_cvt to use one toplevel match --- fearless_simd_gen/src/mk_x86.rs | 523 +++++++++++++++++--------------- 1 file changed, 271 insertions(+), 252 deletions(-) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 60ce1d94..f8f0a24f 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2885,26 +2885,37 @@ impl X86 { "we currently only support converting between types of the same width" ); - if vec_ty.scalar_bits == 64 && *self != Self::Avx512 { - return fallback_method(op, vec_ty); - } - - if vec_ty.scalar_bits == 64 { - let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); - return self.kernel_method(op, vec_ty, |token| match (vec_ty.scalar, target_scalar) { - (ScalarType::Int, ScalarType::Float) => { + match ( + *self, + vec_ty.scalar, + target_scalar, + vec_ty.scalar_bits, + vec_ty.n_bits(), + precise, + ) { + (Self::Sse2 | Self::Sse4_2, _, _, 64, 128, _) + | (Self::Avx2, _, _, 64, 128 | 256, _) => fallback_method(op, vec_ty), // no hardware support + (Self::Avx512, ScalarType::Int, ScalarType::Float, 64, 128 | 256 | 512, _) => { + let target_ty = vec_ty.cast(target_scalar); + self.kernel_method(op, vec_ty, |token| { let convert = simple_intrinsic("cvtepi64", &target_ty); quote! { #convert(a.into()).simd_into(#token) } - } - (ScalarType::Unsigned, ScalarType::Float) => { + }) + } + (Self::Avx512, ScalarType::Unsigned, ScalarType::Float, 64, 128 | 256 | 512, _) => { + let target_ty = vec_ty.cast(target_scalar); + self.kernel_method(op, vec_ty, |token| { let convert = simple_intrinsic("cvtepu64", &target_ty); quote! { #convert(a.into()).simd_into(#token) } - } - (ScalarType::Float, ScalarType::Int) => { + }) + } + (Self::Avx512, ScalarType::Float, ScalarType::Int, 64, 128 | 256 | 512, precise) => { + let target_ty = vec_ty.cast(target_scalar); + self.kernel_method(op, vec_ty, |token| { let convert = simple_intrinsic("cvttpd", &target_ty); if precise { let masked_convert = @@ -2930,8 +2941,18 @@ impl X86 { #convert(a.into()).simd_into(#token) } } - } - (ScalarType::Float, ScalarType::Unsigned) => { + }) + } + ( + Self::Avx512, + ScalarType::Float, + ScalarType::Unsigned, + 64, + 128 | 256 | 512, + precise, + ) => { + let target_ty = vec_ty.cast(target_scalar); + self.kernel_method(op, vec_ty, |token| { let convert = simple_intrinsic("cvttpd", &target_ty); if precise { let max = simple_intrinsic("max", vec_ty); @@ -2959,251 +2980,247 @@ impl X86 { #convert(a.into()).simd_into(#token) } } - } - _ => unimplemented!(), - }); - } - - if *self == Self::Sse2 - && (precise - || vec_ty.scalar == ScalarType::Unsigned - || target_scalar == ScalarType::Unsigned) - { - return fallback_method(op, vec_ty); - } - - if *self == Self::Avx512 - && vec_ty.scalar == ScalarType::Float - && target_scalar == ScalarType::Unsigned - { - let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); - let convert = intrinsic_ident("cvttps", "epu32", vec_ty.n_bits()); - return self.kernel_method(op, vec_ty, |token| { - if precise { - let max = simple_intrinsic("max", vec_ty); - let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); - let blend = avx512_mask_blend_intrinsic(&target_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); - let set0_float = - intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); - let lt = avx512_float_compare_predicate("simd_lt"); - quote! { - let a = #max(a.into(), #set0_float()); - let mut converted = #convert(a); - let exceeds_unsigned_range = #cmp::<#lt>(#set1_float(4294967040.0), a); - converted = #blend( - exceeds_unsigned_range, - converted, - #set1_int(u32::MAX.cast_signed()), - ); - converted.simd_into(#token) - } - } else { - quote! { - #convert(a.into()).simd_into(#token) - } - } - }); - } - - if *self == Self::Avx512 - && vec_ty.scalar == ScalarType::Float - && target_scalar == ScalarType::Int - && vec_ty.scalar_bits == 32 - { - let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); - let convert = intrinsic_ident("cvttps", "epi32", vec_ty.n_bits()); - return self.kernel_method(op, vec_ty, |token| { - if precise { - let masked_convert = intrinsic_ident("mask_cvttps", "epi32", vec_ty.n_bits()); - let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); - let blend = avx512_mask_blend_intrinsic(&target_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); - let set0_int = - intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); - let lt = avx512_float_compare_predicate("simd_lt"); - let ord = avx512_float_compare_predicate("ord"); - quote! { - let a = a.into(); - let in_range = #cmp::<#lt>(a, #set1_float(2147483648.0)); - let mut converted = #masked_convert(#set1_int(i32::MAX), in_range, a); - let is_not_nan = #cmp::<#ord>(a, a); - converted = #blend(is_not_nan, #set0_int(), converted); - converted.simd_into(#token) + }) + } + (Self::Sse2, _, _, 32, 128, true) + | (Self::Sse2, ScalarType::Unsigned, _, 32, 128, _) + | (Self::Sse2, _, ScalarType::Unsigned, 32, 128, _) => fallback_method(op, vec_ty), + ( + Self::Avx512, + ScalarType::Float, + ScalarType::Unsigned, + 32, + 128 | 256 | 512, + precise, + ) => { + let target_ty = vec_ty.cast(target_scalar); + let convert = intrinsic_ident("cvttps", "epu32", vec_ty.n_bits()); + self.kernel_method(op, vec_ty, |token| { + if precise { + let max = simple_intrinsic("max", vec_ty); + let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); + let blend = avx512_mask_blend_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let set0_float = + intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); + let lt = avx512_float_compare_predicate("simd_lt"); + quote! { + let a = #max(a.into(), #set0_float()); + let mut converted = #convert(a); + let exceeds_unsigned_range = #cmp::<#lt>(#set1_float(4294967040.0), a); + converted = #blend( + exceeds_unsigned_range, + converted, + #set1_int(u32::MAX.cast_signed()), + ); + converted.simd_into(#token) + } + } else { + quote! { + #convert(a.into()).simd_into(#token) + } } - } else { - quote! { - #convert(a.into()).simd_into(#token) + }) + } + (Self::Avx512, ScalarType::Float, ScalarType::Int, 32, 128 | 256 | 512, precise) => { + let target_ty = vec_ty.cast(target_scalar); + let convert = intrinsic_ident("cvttps", "epi32", vec_ty.n_bits()); + self.kernel_method(op, vec_ty, |token| { + if precise { + let masked_convert = + intrinsic_ident("mask_cvttps", "epi32", vec_ty.n_bits()); + let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); + let blend = avx512_mask_blend_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let set0_int = + intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); + let lt = avx512_float_compare_predicate("simd_lt"); + let ord = avx512_float_compare_predicate("ord"); + quote! { + let a = a.into(); + let in_range = #cmp::<#lt>(a, #set1_float(2147483648.0)); + let mut converted = #masked_convert(#set1_int(i32::MAX), in_range, a); + let is_not_nan = #cmp::<#ord>(a, a); + converted = #blend(is_not_nan, #set0_int(), converted); + converted.simd_into(#token) + } + } else { + quote! { + #convert(a.into()).simd_into(#token) + } } - } - }); - } - - if *self == Self::Avx512 - && matches!(vec_ty.n_bits(), 128 | 256) - && vec_ty.scalar == ScalarType::Unsigned - && target_scalar == ScalarType::Float - && vec_ty.scalar_bits == 32 - { - // We cannot emit the intrinsics for the conversion instructions - // because the required intrinsics are mysteriously absent from stdarch: - // https://github.com/rust-lang/rust/issues/158196 - // Fortunately LLVM optimizes this sequence into the single instruction we're after. - // TODO: switch to intrinsics once they're added, stabilized, and our MSRV is high enough. - let bits = vec_ty.n_bits(); - let zext = format_ident!("_mm512_zextsi{bits}_si512"); - let convert = intrinsic_ident("cvtepu32", "ps", 512); - let cast = format_ident!("_mm512_castps512_ps{bits}"); - return self.kernel_method(op, vec_ty, |token| { - quote! { - #cast(#convert(#zext(a.into()))).simd_into(#token) - } - }); - } - - if *self == Self::Avx512 && vec_ty.n_bits() == 512 { - let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); - return self.kernel_method(op, vec_ty, |token| match (vec_ty.scalar, target_scalar) { - (ScalarType::Int, ScalarType::Float) => { - let intrinsic = simple_intrinsic("cvtepi32", &target_ty); + }) + } + (Self::Avx512, ScalarType::Unsigned, ScalarType::Float, 32, bits @ (128 | 256), _) => { + // We cannot emit the intrinsics for the conversion instructions + // because the required intrinsics are mysteriously absent from stdarch: + // https://github.com/rust-lang/rust/issues/158196 + // Fortunately LLVM optimizes this sequence into the single instruction we're after. + // TODO: switch to intrinsics once they're added, stabilized, and our MSRV is high enough. + let zext = format_ident!("_mm512_zextsi{bits}_si512"); + let convert = intrinsic_ident("cvtepu32", "ps", 512); + let cast = format_ident!("_mm512_castps512_ps{bits}"); + self.kernel_method(op, vec_ty, |token| { quote! { - #intrinsic(a.into()).simd_into(#token) + #cast(#convert(#zext(a.into()))).simd_into(#token) } - } - (ScalarType::Unsigned, ScalarType::Float) => { - let intrinsic = simple_intrinsic("cvtepu32", &target_ty); + }) + } + ( + Self::Avx512, + source @ (ScalarType::Int | ScalarType::Unsigned), + ScalarType::Float, + 32, + 512, + _, + ) => { + let target_ty = vec_ty.cast(target_scalar); + self.kernel_method(op, vec_ty, |token| { + let intrinsic = match source { + ScalarType::Int => simple_intrinsic("cvtepi32", &target_ty), + ScalarType::Unsigned => simple_intrinsic("cvtepu32", &target_ty), + _ => unreachable!(), + }; quote! { #intrinsic(a.into()).simd_into(#token) } - } - _ => unimplemented!(), - }); - } - - self.kernel_method(op, vec_ty, |token| match (vec_ty.scalar, target_scalar) { - (ScalarType::Float, ScalarType::Int | ScalarType::Unsigned) => { - let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); - let max = simple_intrinsic("max", vec_ty); - let set0 = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); - let cmplt = float_compare_method("simd_lt", vec_ty); - let cmpord = float_compare_method("ord", vec_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); - let movemask = simple_intrinsic("movemask", vec_ty); - let all_ones = match (vec_ty.n_bits(), vec_ty.scalar_bits) { - (128, 32) => quote! { 0b1111 }, - (256, 32) => quote! { 0b11111111 }, - _ => unimplemented!(), - }; - let convert = simple_sign_unaware_intrinsic("cvttps", &target_ty); - let cast_to_int = cast_ident( - vec_ty.scalar, - target_scalar, - vec_ty.scalar_bits, - vec_ty.scalar_bits, - vec_ty.n_bits(), - ); - let blend = intrinsic_ident("blendv", "epi8", vec_ty.n_bits()); - let and = intrinsic_ident("and", coarse_type(&target_ty), vec_ty.n_bits()); - let andnot = simple_intrinsic("andnot", vec_ty); - let add_int = simple_sign_unaware_intrinsic("add", &target_ty); - let sub_float = simple_intrinsic("sub", vec_ty); - - match (target_scalar, precise) { - (ScalarType::Int, false) => { - quote! { - #convert(a.into()).simd_into(#token) + }) + } + ( + Self::Sse2 | Self::Sse4_2, + ScalarType::Float, + target @ (ScalarType::Int | ScalarType::Unsigned), + 32, + 128, + precise, + ) + | ( + Self::Avx2, + ScalarType::Float, + target @ (ScalarType::Int | ScalarType::Unsigned), + 32, + 128 | 256, + precise, + ) => { + self.kernel_method(op, vec_ty, |token| { + let target_ty = vec_ty.cast(target); + let max = simple_intrinsic("max", vec_ty); + let set0 = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); + let cmplt = float_compare_method("simd_lt", vec_ty); + let cmpord = float_compare_method("ord", vec_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let movemask = simple_intrinsic("movemask", vec_ty); + let all_ones = match (vec_ty.n_bits(), vec_ty.scalar_bits) { + (128, 32) => quote! { 0b1111 }, + (256, 32) => quote! { 0b11111111 }, + _ => unimplemented!(), + }; + let convert = simple_sign_unaware_intrinsic("cvttps", &target_ty); + let cast_to_int = cast_ident( + vec_ty.scalar, + target_scalar, + vec_ty.scalar_bits, + vec_ty.scalar_bits, + vec_ty.n_bits(), + ); + let blend = intrinsic_ident("blendv", "epi8", vec_ty.n_bits()); + let and = intrinsic_ident("and", coarse_type(&target_ty), vec_ty.n_bits()); + let andnot = simple_intrinsic("andnot", vec_ty); + let add_int = simple_sign_unaware_intrinsic("add", &target_ty); + let sub_float = simple_intrinsic("sub", vec_ty); + + match (target, precise) { + (ScalarType::Int, false) => { + quote! { + #convert(a.into()).simd_into(#token) + } } - } - (ScalarType::Unsigned, false) => { - quote! { - let mut converted = #convert(a.into()); - - // In the common case where everything is in range of an i32, we don't need to do anything else. - let in_range = #cmplt(a.into(), #set1_float(2147483648.0)); - let all_in_range = #movemask(in_range) == #all_ones; - - if !all_in_range { - // Add any excess (beyond the maximum value) - let excess = #sub_float(a.into(), #set1_float(2147483648.0)); - let excess_converted = #convert(#andnot(in_range, excess)); - converted = #add_int(converted, excess_converted); + (ScalarType::Unsigned, false) => { + quote! { + let mut converted = #convert(a.into()); + + // In the common case where everything is in range of an i32, we don't need to do anything else. + let in_range = #cmplt(a.into(), #set1_float(2147483648.0)); + let all_in_range = #movemask(in_range) == #all_ones; + + if !all_in_range { + // Add any excess (beyond the maximum value) + let excess = #sub_float(a.into(), #set1_float(2147483648.0)); + let excess_converted = #convert(#andnot(in_range, excess)); + converted = #add_int(converted, excess_converted); + } + + converted.simd_into(#token) } - - converted.simd_into(#token) } - } - (ScalarType::Int, true) => { - quote! { - let a = a.into(); - - let mut converted = #convert(a); - - // In the common case where everything is in range, we don't need to do anything else. - let in_range = #cmplt(a, #set1_float(2147483648.0)); - let all_in_range = #movemask(in_range) == #all_ones; - - if !all_in_range { - // If we are above i32::MAX (2147483647), clamp to it. - converted = #blend(#set1_int(i32::MAX), converted, #cast_to_int(in_range)); - // Set NaN to 0. Using `and` seems slightly faster than `blend`. - let is_not_nan = #cast_to_int(#cmpord(a, a)); - converted = #and(converted, is_not_nan); - // We don't need to handle negative overflow because Intel's "invalid result" sentinel - // value is -2147483648, which is what we want anyway. + (ScalarType::Int, true) => { + quote! { + let a = a.into(); + + let mut converted = #convert(a); + + // In the common case where everything is in range, we don't need to do anything else. + let in_range = #cmplt(a, #set1_float(2147483648.0)); + let all_in_range = #movemask(in_range) == #all_ones; + + if !all_in_range { + // If we are above i32::MAX (2147483647), clamp to it. + converted = #blend(#set1_int(i32::MAX), converted, #cast_to_int(in_range)); + // Set NaN to 0. Using `and` seems slightly faster than `blend`. + let is_not_nan = #cast_to_int(#cmpord(a, a)); + converted = #and(converted, is_not_nan); + // We don't need to handle negative overflow because Intel's "invalid result" sentinel + // value is -2147483648, which is what we want anyway. + } + + converted.simd_into(#token) } - - converted.simd_into(#token) } - } - (ScalarType::Unsigned, true) => { - quote! { - // Clamp out-of-range values (and NaN) to 0. Intel's `_mm_max_ps` always takes the second - // operand if the first is NaN. - let a = #max(a.into(), #set0()); - let mut converted = #convert(a); - - // In the common case where everything is in range of an i32, we don't need to do anything else. - let in_range = #cmplt(a, #set1_float(2147483648.0)); - let all_in_range = #movemask(in_range) == #all_ones; - - if !all_in_range { - let exceeds_unsigned_range = #cast_to_int(#cmplt(#set1_float(4294967040.0), a)); - // Add any excess (beyond the maximum value) - let excess = #sub_float(a, #set1_float(2147483648.0)); - let excess_converted = #convert(#andnot(in_range, excess)); - - // Clamp to u32::MAX. - converted = #add_int(converted, excess_converted); - converted = #blend(converted, #set1_int(u32::MAX.cast_signed()), exceeds_unsigned_range); + (ScalarType::Unsigned, true) => { + quote! { + // Clamp out-of-range values (and NaN) to 0. Intel's `_mm_max_ps` always takes the second + // operand if the first is NaN. + let a = #max(a.into(), #set0()); + let mut converted = #convert(a); + + // In the common case where everything is in range of an i32, we don't need to do anything else. + let in_range = #cmplt(a, #set1_float(2147483648.0)); + let all_in_range = #movemask(in_range) == #all_ones; + + if !all_in_range { + let exceeds_unsigned_range = #cast_to_int(#cmplt(#set1_float(4294967040.0), a)); + // Add any excess (beyond the maximum value) + let excess = #sub_float(a, #set1_float(2147483648.0)); + let excess_converted = #convert(#andnot(in_range, excess)); + + // Clamp to u32::MAX. + converted = #add_int(converted, excess_converted); + converted = #blend(converted, #set1_int(u32::MAX.cast_signed()), exceeds_unsigned_range); + } + + converted.simd_into(#token) } - - converted.simd_into(#token) } + _ => unreachable!(), } - _ => unreachable!(), - } + }) } - (ScalarType::Int, ScalarType::Float) => { - assert_eq!( - vec_ty.scalar_bits, 32, - "i64 to f64 conversions do not exist until AVX-512 and require special consideration" - ); - let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); - let intrinsic = simple_intrinsic("cvtepi32", &target_ty); - quote! { - #intrinsic(a.into()).simd_into(#token) - } + (Self::Sse2 | Self::Sse4_2, ScalarType::Int, ScalarType::Float, 32, 128, _) + | (Self::Avx2 | Self::Avx512, ScalarType::Int, ScalarType::Float, 32, 128 | 256, _) => { + let target_ty = vec_ty.cast(target_scalar); + self.kernel_method(op, vec_ty, |token| { + let intrinsic = simple_intrinsic("cvtepi32", &target_ty); + quote! { + #intrinsic(a.into()).simd_into(#token) + } + }) } - (ScalarType::Unsigned, ScalarType::Float) => { - assert_eq!( - vec_ty.scalar_bits, 32, - "u64 to f64 conversions do not exist until AVX-512 and require special consideration" - ); - - let target_ty = vec_ty.reinterpret(target_scalar, target_scalar_bits); + (Self::Sse4_2, ScalarType::Unsigned, ScalarType::Float, 32, 128, _) + | (Self::Avx2, ScalarType::Unsigned, ScalarType::Float, 32, 128 | 256, _) => { + let target_ty = vec_ty.cast(target_scalar); let set1_int = set1_intrinsic(vec_ty); let set1_float = set1_intrinsic(&target_ty); let add_float = simple_intrinsic("add", &target_ty); @@ -3221,19 +3238,21 @@ impl X86 { // Magical mystery algorithm taken from LLVM: // https://github.com/llvm/llvm-project/blob/6f8e87b9d097c5ef631f24d2eb2f34eb31b54d3b/llvm/lib/Target/X86/X86ISelLowering.cpp // (The file is too big for GitHub to show a preview, so no line numbers.) - quote! { - let a = a.into(); - let lo = #blend::<0xAA>(a, #set1_int(0x4B000000)); - let hi = #blend::<0xAA>(#srli::<16>(a), #set1_int(0x53000000)); + self.kernel_method(op, vec_ty, |token| { + quote! { + let a = a.into(); + let lo = #blend::<0xAA>(a, #set1_int(0x4B000000)); + let hi = #blend::<0xAA>(#srli::<16>(a), #set1_int(0x53000000)); - let fhi = #sub_float(#cast_to_float(hi), #set1_float(f32::from_bits(0x53000080))); - let result = #add_float(#cast_to_float(lo), fhi); + let fhi = #sub_float(#cast_to_float(hi), #set1_float(f32::from_bits(0x53000080))); + let result = #add_float(#cast_to_float(lo), fhi); - result.simd_into(#token) - } + result.simd_into(#token) + } + }) } - _ => unimplemented!(), - }) + _ => unreachable!(), + } } pub(crate) fn handle_mask_reduce( From 22f9738f6529bf811dc42d625d9249f05c88d4cd Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 5 Aug 2026 15:46:35 +0100 Subject: [PATCH 3/8] Further refactoring of handle_cvt --- fearless_simd_gen/src/mk_x86.rs | 555 ++++++++++++++++---------------- 1 file changed, 275 insertions(+), 280 deletions(-) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index f8f0a24f..092e8a98 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -25,6 +25,12 @@ pub(crate) enum X86 { Avx512, } +#[derive(Clone, Copy, PartialEq, Eq)] +enum Precision { + Approx, + Precise, +} + pub(crate) const SSE2_FEATURES: &str = "fxsr,sse,sse2"; pub(crate) const SSE4_2_FEATURES: &str = "fxsr,sse4.2,cmpxchg16b,popcnt"; pub(crate) const AVX2_FEATURES: &str = @@ -2880,68 +2886,95 @@ impl X86 { target_scalar_bits: usize, precise: bool, ) -> TokenStream { + use Precision::{Approx, Precise}; + assert_eq!( vec_ty.scalar_bits, target_scalar_bits, "we currently only support converting between types of the same width" ); - match ( + let precision = if precise { Precise } else { Approx }; + + let conversion = ( *self, vec_ty.scalar, target_scalar, vec_ty.scalar_bits, vec_ty.n_bits(), - precise, - ) { + precision, + ); + + if matches!( + conversion, (Self::Sse2 | Self::Sse4_2, _, _, 64, 128, _) - | (Self::Avx2, _, _, 64, 128 | 256, _) => fallback_method(op, vec_ty), // no hardware support - (Self::Avx512, ScalarType::Int, ScalarType::Float, 64, 128 | 256 | 512, _) => { + | (Self::Avx2, _, _, 64, 128 | 256, _) + | (Self::Sse2, _, _, 32, 128, Precise) + | (Self::Sse2, ScalarType::Unsigned, _, 32, 128, _) + | (Self::Sse2, _, ScalarType::Unsigned, 32, 128, _) + ) { + // These conversions have no hardware support, or their native implementation is + // slower than the scalar fallback. + return fallback_method(op, vec_ty); + } + + self.kernel_method(op, vec_ty, |token| match conversion { + (Self::Avx512, ScalarType::Unsigned, ScalarType::Float, 32, bits @ (128 | 256), _) => { + // We cannot emit the intrinsics for the conversion instructions + // because the required intrinsics are mysteriously absent from stdarch: + // https://github.com/rust-lang/rust/issues/158196 + // Fortunately LLVM optimizes this sequence into the single instruction we're after. + // TODO: switch to intrinsics once they're added, stabilized, and our MSRV is high enough. + let zext = format_ident!("_mm512_zextsi{bits}_si512"); + let convert = intrinsic_ident("cvtepu32", "ps", 512); + let cast = format_ident!("_mm512_castps512_ps{bits}"); + quote! { + #cast(#convert(#zext(a.into()))).simd_into(#token) + } + } + ( + Self::Avx512, + source @ (ScalarType::Int | ScalarType::Unsigned), + ScalarType::Float, + scalar_bits @ (32 | 64), + 128 | 256 | 512, + _, + ) => { + // native AVX-512 conversions with available intrinsics let target_ty = vec_ty.cast(target_scalar); - self.kernel_method(op, vec_ty, |token| { - let convert = simple_intrinsic("cvtepi64", &target_ty); - quote! { - #convert(a.into()).simd_into(#token) - } - }) + let convert = simple_intrinsic( + &format!("cvtep{}{scalar_bits}", source.prefix()), + &target_ty, + ); + quote! { + #convert(a.into()).simd_into(#token) + } } - (Self::Avx512, ScalarType::Unsigned, ScalarType::Float, 64, 128 | 256 | 512, _) => { + (Self::Avx512, ScalarType::Float, ScalarType::Int, 64, 128 | 256 | 512, Precise) => { let target_ty = vec_ty.cast(target_scalar); - self.kernel_method(op, vec_ty, |token| { - let convert = simple_intrinsic("cvtepu64", &target_ty); - quote! { - #convert(a.into()).simd_into(#token) - } - }) + let masked_convert = intrinsic_ident("mask_cvttpd", "epi64", vec_ty.n_bits()); + let cmp = intrinsic_ident("cmp", "pd_mask", vec_ty.n_bits()); + let blend = avx512_mask_blend_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let set0_int = + intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); + let lt = avx512_float_compare_predicate("simd_lt"); + let ord = avx512_float_compare_predicate("ord"); + quote! { + let a = a.into(); + let in_range = #cmp::<#lt>(a, #set1_float(9223372036854775808.0)); + let mut converted = #masked_convert(#set1_int(i64::MAX), in_range, a); + let is_not_nan = #cmp::<#ord>(a, a); + converted = #blend(is_not_nan, #set0_int(), converted); + converted.simd_into(#token) + } } - (Self::Avx512, ScalarType::Float, ScalarType::Int, 64, 128 | 256 | 512, precise) => { + (Self::Avx512, ScalarType::Float, ScalarType::Int, 64, 128 | 256 | 512, Approx) => { let target_ty = vec_ty.cast(target_scalar); - self.kernel_method(op, vec_ty, |token| { - let convert = simple_intrinsic("cvttpd", &target_ty); - if precise { - let masked_convert = - intrinsic_ident("mask_cvttpd", "epi64", vec_ty.n_bits()); - let cmp = intrinsic_ident("cmp", "pd_mask", vec_ty.n_bits()); - let blend = avx512_mask_blend_intrinsic(&target_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); - let set0_int = - intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); - let lt = avx512_float_compare_predicate("simd_lt"); - let ord = avx512_float_compare_predicate("ord"); - quote! { - let a = a.into(); - let in_range = #cmp::<#lt>(a, #set1_float(9223372036854775808.0)); - let mut converted = #masked_convert(#set1_int(i64::MAX), in_range, a); - let is_not_nan = #cmp::<#ord>(a, a); - converted = #blend(is_not_nan, #set0_int(), converted); - converted.simd_into(#token) - } - } else { - quote! { - #convert(a.into()).simd_into(#token) - } - } - }) + let convert = simple_intrinsic("cvttpd", &target_ty); + quote! { + #convert(a.into()).simd_into(#token) + } } ( Self::Avx512, @@ -2949,144 +2982,112 @@ impl X86 { ScalarType::Unsigned, 64, 128 | 256 | 512, - precise, + Precise, ) => { let target_ty = vec_ty.cast(target_scalar); - self.kernel_method(op, vec_ty, |token| { - let convert = simple_intrinsic("cvttpd", &target_ty); - if precise { - let max = simple_intrinsic("max", vec_ty); - let cmp = intrinsic_ident("cmp", "pd_mask", vec_ty.n_bits()); - let blend = avx512_mask_blend_intrinsic(&target_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); - let set0_float = - intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); - let lt = avx512_float_compare_predicate("simd_lt"); - quote! { - let a = #max(a.into(), #set0_float()); - let mut converted = #convert(a); - let exceeds_unsigned_range = - #cmp::<#lt>(#set1_float(18446744073709549568.0), a); - converted = #blend( - exceeds_unsigned_range, - converted, - #set1_int(u64::MAX.cast_signed()), - ); - converted.simd_into(#token) - } - } else { - quote! { - #convert(a.into()).simd_into(#token) - } - } - }) + let convert = simple_intrinsic("cvttpd", &target_ty); + let max = simple_intrinsic("max", vec_ty); + let cmp = intrinsic_ident("cmp", "pd_mask", vec_ty.n_bits()); + let blend = avx512_mask_blend_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let set0_float = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); + let lt = avx512_float_compare_predicate("simd_lt"); + quote! { + let a = #max(a.into(), #set0_float()); + let mut converted = #convert(a); + let exceeds_unsigned_range = + #cmp::<#lt>(#set1_float(18446744073709549568.0), a); + converted = #blend( + exceeds_unsigned_range, + converted, + #set1_int(u64::MAX.cast_signed()), + ); + converted.simd_into(#token) + } } - (Self::Sse2, _, _, 32, 128, true) - | (Self::Sse2, ScalarType::Unsigned, _, 32, 128, _) - | (Self::Sse2, _, ScalarType::Unsigned, 32, 128, _) => fallback_method(op, vec_ty), ( Self::Avx512, ScalarType::Float, ScalarType::Unsigned, - 32, + 64, 128 | 256 | 512, - precise, + Approx, ) => { let target_ty = vec_ty.cast(target_scalar); - let convert = intrinsic_ident("cvttps", "epu32", vec_ty.n_bits()); - self.kernel_method(op, vec_ty, |token| { - if precise { - let max = simple_intrinsic("max", vec_ty); - let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); - let blend = avx512_mask_blend_intrinsic(&target_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); - let set0_float = - intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); - let lt = avx512_float_compare_predicate("simd_lt"); - quote! { - let a = #max(a.into(), #set0_float()); - let mut converted = #convert(a); - let exceeds_unsigned_range = #cmp::<#lt>(#set1_float(4294967040.0), a); - converted = #blend( - exceeds_unsigned_range, - converted, - #set1_int(u32::MAX.cast_signed()), - ); - converted.simd_into(#token) - } - } else { - quote! { - #convert(a.into()).simd_into(#token) - } - } - }) + let convert = simple_intrinsic("cvttpd", &target_ty); + quote! { + #convert(a.into()).simd_into(#token) + } } - (Self::Avx512, ScalarType::Float, ScalarType::Int, 32, 128 | 256 | 512, precise) => { + ( + Self::Avx512, + ScalarType::Float, + ScalarType::Unsigned, + 32, + 128 | 256 | 512, + Precise, + ) => { let target_ty = vec_ty.cast(target_scalar); - let convert = intrinsic_ident("cvttps", "epi32", vec_ty.n_bits()); - self.kernel_method(op, vec_ty, |token| { - if precise { - let masked_convert = - intrinsic_ident("mask_cvttps", "epi32", vec_ty.n_bits()); - let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); - let blend = avx512_mask_blend_intrinsic(&target_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); - let set0_int = - intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); - let lt = avx512_float_compare_predicate("simd_lt"); - let ord = avx512_float_compare_predicate("ord"); - quote! { - let a = a.into(); - let in_range = #cmp::<#lt>(a, #set1_float(2147483648.0)); - let mut converted = #masked_convert(#set1_int(i32::MAX), in_range, a); - let is_not_nan = #cmp::<#ord>(a, a); - converted = #blend(is_not_nan, #set0_int(), converted); - converted.simd_into(#token) - } - } else { - quote! { - #convert(a.into()).simd_into(#token) - } - } - }) - } - (Self::Avx512, ScalarType::Unsigned, ScalarType::Float, 32, bits @ (128 | 256), _) => { - // We cannot emit the intrinsics for the conversion instructions - // because the required intrinsics are mysteriously absent from stdarch: - // https://github.com/rust-lang/rust/issues/158196 - // Fortunately LLVM optimizes this sequence into the single instruction we're after. - // TODO: switch to intrinsics once they're added, stabilized, and our MSRV is high enough. - let zext = format_ident!("_mm512_zextsi{bits}_si512"); - let convert = intrinsic_ident("cvtepu32", "ps", 512); - let cast = format_ident!("_mm512_castps512_ps{bits}"); - self.kernel_method(op, vec_ty, |token| { - quote! { - #cast(#convert(#zext(a.into()))).simd_into(#token) - } - }) + let convert = intrinsic_ident("cvttps", "epu32", vec_ty.n_bits()); + let max = simple_intrinsic("max", vec_ty); + let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); + let blend = avx512_mask_blend_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let set0_float = + intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); + let lt = avx512_float_compare_predicate("simd_lt"); + quote! { + let a = #max(a.into(), #set0_float()); + let mut converted = #convert(a); + let exceeds_unsigned_range = #cmp::<#lt>(#set1_float(4294967040.0), a); + converted = #blend( + exceeds_unsigned_range, + converted, + #set1_int(u32::MAX.cast_signed()), + ); + converted.simd_into(#token) + } } ( Self::Avx512, - source @ (ScalarType::Int | ScalarType::Unsigned), ScalarType::Float, + ScalarType::Unsigned, 32, - 512, - _, + 128 | 256 | 512, + Approx, ) => { + let convert = intrinsic_ident("cvttps", "epu32", vec_ty.n_bits()); + quote! { + #convert(a.into()).simd_into(#token) + } + } + (Self::Avx512, ScalarType::Float, ScalarType::Int, 32, 128 | 256 | 512, Precise) => { let target_ty = vec_ty.cast(target_scalar); - self.kernel_method(op, vec_ty, |token| { - let intrinsic = match source { - ScalarType::Int => simple_intrinsic("cvtepi32", &target_ty), - ScalarType::Unsigned => simple_intrinsic("cvtepu32", &target_ty), - _ => unreachable!(), - }; - quote! { - #intrinsic(a.into()).simd_into(#token) - } - }) + let masked_convert = intrinsic_ident("mask_cvttps", "epi32", vec_ty.n_bits()); + let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); + let blend = avx512_mask_blend_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let set0_int = + intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); + let lt = avx512_float_compare_predicate("simd_lt"); + let ord = avx512_float_compare_predicate("ord"); + quote! { + let a = a.into(); + let in_range = #cmp::<#lt>(a, #set1_float(2147483648.0)); + let mut converted = #masked_convert(#set1_int(i32::MAX), in_range, a); + let is_not_nan = #cmp::<#ord>(a, a); + converted = #blend(is_not_nan, #set0_int(), converted); + converted.simd_into(#token) + } + } + (Self::Avx512, ScalarType::Float, ScalarType::Int, 32, 128 | 256 | 512, Approx) => { + let convert = intrinsic_ident("cvttps", "epi32", vec_ty.n_bits()); + quote! { + #convert(a.into()).simd_into(#token) + } } ( Self::Sse2 | Self::Sse4_2, @@ -3094,7 +3095,7 @@ impl X86 { target @ (ScalarType::Int | ScalarType::Unsigned), 32, 128, - precise, + precision, ) | ( Self::Avx2, @@ -3102,121 +3103,117 @@ impl X86 { target @ (ScalarType::Int | ScalarType::Unsigned), 32, 128 | 256, - precise, + precision, ) => { - self.kernel_method(op, vec_ty, |token| { - let target_ty = vec_ty.cast(target); - let max = simple_intrinsic("max", vec_ty); - let set0 = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); - let cmplt = float_compare_method("simd_lt", vec_ty); - let cmpord = float_compare_method("ord", vec_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); - let movemask = simple_intrinsic("movemask", vec_ty); - let all_ones = match (vec_ty.n_bits(), vec_ty.scalar_bits) { - (128, 32) => quote! { 0b1111 }, - (256, 32) => quote! { 0b11111111 }, - _ => unimplemented!(), - }; - let convert = simple_sign_unaware_intrinsic("cvttps", &target_ty); - let cast_to_int = cast_ident( - vec_ty.scalar, - target_scalar, - vec_ty.scalar_bits, - vec_ty.scalar_bits, - vec_ty.n_bits(), - ); - let blend = intrinsic_ident("blendv", "epi8", vec_ty.n_bits()); - let and = intrinsic_ident("and", coarse_type(&target_ty), vec_ty.n_bits()); - let andnot = simple_intrinsic("andnot", vec_ty); - let add_int = simple_sign_unaware_intrinsic("add", &target_ty); - let sub_float = simple_intrinsic("sub", vec_ty); - - match (target, precise) { - (ScalarType::Int, false) => { - quote! { - #convert(a.into()).simd_into(#token) - } + let target_ty = vec_ty.cast(target); + let max = simple_intrinsic("max", vec_ty); + let set0 = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); + let cmplt = float_compare_method("simd_lt", vec_ty); + let cmpord = float_compare_method("ord", vec_ty); + let set1_float = set1_intrinsic(vec_ty); + let set1_int = set1_intrinsic(&target_ty); + let movemask = simple_intrinsic("movemask", vec_ty); + let all_ones = match (vec_ty.n_bits(), vec_ty.scalar_bits) { + (128, 32) => quote! { 0b1111 }, + (256, 32) => quote! { 0b11111111 }, + _ => unimplemented!(), + }; + let convert = simple_sign_unaware_intrinsic("cvttps", &target_ty); + let cast_to_int = cast_ident( + vec_ty.scalar, + target_scalar, + vec_ty.scalar_bits, + vec_ty.scalar_bits, + vec_ty.n_bits(), + ); + let blend = intrinsic_ident("blendv", "epi8", vec_ty.n_bits()); + let and = intrinsic_ident("and", coarse_type(&target_ty), vec_ty.n_bits()); + let andnot = simple_intrinsic("andnot", vec_ty); + let add_int = simple_sign_unaware_intrinsic("add", &target_ty); + let sub_float = simple_intrinsic("sub", vec_ty); + + match (target, precision) { + (ScalarType::Int, Approx) => { + quote! { + #convert(a.into()).simd_into(#token) } - (ScalarType::Unsigned, false) => { - quote! { - let mut converted = #convert(a.into()); - - // In the common case where everything is in range of an i32, we don't need to do anything else. - let in_range = #cmplt(a.into(), #set1_float(2147483648.0)); - let all_in_range = #movemask(in_range) == #all_ones; - - if !all_in_range { - // Add any excess (beyond the maximum value) - let excess = #sub_float(a.into(), #set1_float(2147483648.0)); - let excess_converted = #convert(#andnot(in_range, excess)); - converted = #add_int(converted, excess_converted); - } - - converted.simd_into(#token) + } + (ScalarType::Unsigned, Approx) => { + quote! { + let mut converted = #convert(a.into()); + + // In the common case where everything is in range of an i32, we don't need to do anything else. + let in_range = #cmplt(a.into(), #set1_float(2147483648.0)); + let all_in_range = #movemask(in_range) == #all_ones; + + if !all_in_range { + // Add any excess (beyond the maximum value) + let excess = #sub_float(a.into(), #set1_float(2147483648.0)); + let excess_converted = #convert(#andnot(in_range, excess)); + converted = #add_int(converted, excess_converted); } + + converted.simd_into(#token) } - (ScalarType::Int, true) => { - quote! { - let a = a.into(); - - let mut converted = #convert(a); - - // In the common case where everything is in range, we don't need to do anything else. - let in_range = #cmplt(a, #set1_float(2147483648.0)); - let all_in_range = #movemask(in_range) == #all_ones; - - if !all_in_range { - // If we are above i32::MAX (2147483647), clamp to it. - converted = #blend(#set1_int(i32::MAX), converted, #cast_to_int(in_range)); - // Set NaN to 0. Using `and` seems slightly faster than `blend`. - let is_not_nan = #cast_to_int(#cmpord(a, a)); - converted = #and(converted, is_not_nan); - // We don't need to handle negative overflow because Intel's "invalid result" sentinel - // value is -2147483648, which is what we want anyway. - } - - converted.simd_into(#token) + } + (ScalarType::Int, Precise) => { + quote! { + let a = a.into(); + + let mut converted = #convert(a); + + // In the common case where everything is in range, we don't need to do anything else. + let in_range = #cmplt(a, #set1_float(2147483648.0)); + let all_in_range = #movemask(in_range) == #all_ones; + + if !all_in_range { + // If we are above i32::MAX (2147483647), clamp to it. + converted = #blend(#set1_int(i32::MAX), converted, #cast_to_int(in_range)); + // Set NaN to 0. Using `and` seems slightly faster than `blend`. + let is_not_nan = #cast_to_int(#cmpord(a, a)); + converted = #and(converted, is_not_nan); + // We don't need to handle negative overflow because Intel's "invalid result" sentinel + // value is -2147483648, which is what we want anyway. } + + converted.simd_into(#token) } - (ScalarType::Unsigned, true) => { - quote! { - // Clamp out-of-range values (and NaN) to 0. Intel's `_mm_max_ps` always takes the second - // operand if the first is NaN. - let a = #max(a.into(), #set0()); - let mut converted = #convert(a); - - // In the common case where everything is in range of an i32, we don't need to do anything else. - let in_range = #cmplt(a, #set1_float(2147483648.0)); - let all_in_range = #movemask(in_range) == #all_ones; - - if !all_in_range { - let exceeds_unsigned_range = #cast_to_int(#cmplt(#set1_float(4294967040.0), a)); - // Add any excess (beyond the maximum value) - let excess = #sub_float(a, #set1_float(2147483648.0)); - let excess_converted = #convert(#andnot(in_range, excess)); - - // Clamp to u32::MAX. - converted = #add_int(converted, excess_converted); - converted = #blend(converted, #set1_int(u32::MAX.cast_signed()), exceeds_unsigned_range); - } - - converted.simd_into(#token) + } + (ScalarType::Unsigned, Precise) => { + quote! { + // Clamp out-of-range values (and NaN) to 0. Intel's `_mm_max_ps` always takes the second + // operand if the first is NaN. + let a = #max(a.into(), #set0()); + let mut converted = #convert(a); + + // In the common case where everything is in range of an i32, we don't need to do anything else. + let in_range = #cmplt(a, #set1_float(2147483648.0)); + let all_in_range = #movemask(in_range) == #all_ones; + + if !all_in_range { + let exceeds_unsigned_range = #cast_to_int(#cmplt(#set1_float(4294967040.0), a)); + // Add any excess (beyond the maximum value) + let excess = #sub_float(a, #set1_float(2147483648.0)); + let excess_converted = #convert(#andnot(in_range, excess)); + + // Clamp to u32::MAX. + converted = #add_int(converted, excess_converted); + converted = #blend(converted, #set1_int(u32::MAX.cast_signed()), exceeds_unsigned_range); } + + converted.simd_into(#token) } - _ => unreachable!(), } - }) + _ => unreachable!(), + } } (Self::Sse2 | Self::Sse4_2, ScalarType::Int, ScalarType::Float, 32, 128, _) - | (Self::Avx2 | Self::Avx512, ScalarType::Int, ScalarType::Float, 32, 128 | 256, _) => { + | (Self::Avx2, ScalarType::Int, ScalarType::Float, 32, 128 | 256, _) => { let target_ty = vec_ty.cast(target_scalar); - self.kernel_method(op, vec_ty, |token| { - let intrinsic = simple_intrinsic("cvtepi32", &target_ty); - quote! { - #intrinsic(a.into()).simd_into(#token) - } - }) + let intrinsic = simple_intrinsic("cvtepi32", &target_ty); + quote! { + #intrinsic(a.into()).simd_into(#token) + } } (Self::Sse4_2, ScalarType::Unsigned, ScalarType::Float, 32, 128, _) | (Self::Avx2, ScalarType::Unsigned, ScalarType::Float, 32, 128 | 256, _) => { @@ -3238,21 +3235,19 @@ impl X86 { // Magical mystery algorithm taken from LLVM: // https://github.com/llvm/llvm-project/blob/6f8e87b9d097c5ef631f24d2eb2f34eb31b54d3b/llvm/lib/Target/X86/X86ISelLowering.cpp // (The file is too big for GitHub to show a preview, so no line numbers.) - self.kernel_method(op, vec_ty, |token| { - quote! { - let a = a.into(); - let lo = #blend::<0xAA>(a, #set1_int(0x4B000000)); - let hi = #blend::<0xAA>(#srli::<16>(a), #set1_int(0x53000000)); + quote! { + let a = a.into(); + let lo = #blend::<0xAA>(a, #set1_int(0x4B000000)); + let hi = #blend::<0xAA>(#srli::<16>(a), #set1_int(0x53000000)); - let fhi = #sub_float(#cast_to_float(hi), #set1_float(f32::from_bits(0x53000080))); - let result = #add_float(#cast_to_float(lo), fhi); + let fhi = #sub_float(#cast_to_float(hi), #set1_float(f32::from_bits(0x53000080))); + let result = #add_float(#cast_to_float(lo), fhi); - result.simd_into(#token) - } - }) + result.simd_into(#token) + } } _ => unreachable!(), - } + }) } pub(crate) fn handle_mask_reduce( From 39adc0e9870d2763637b48699b0a4c0f364d479d Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 5 Aug 2026 15:56:38 +0100 Subject: [PATCH 4/8] Optimize AVX-512 Float->Unsigned conversion --- fearless_simd/src/generated/avx512.rs | 77 +++++++-------------------- fearless_simd_gen/src/mk_x86.rs | 47 ++++++---------- 2 files changed, 34 insertions(+), 90 deletions(-) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index 1ca9885e..c77142cb 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -919,15 +919,9 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f32x4) -> u32x4 { - let a = _mm_max_ps(a.into(), _mm_setzero_ps()); - let mut converted = _mm_cvttps_epu32(a); - let exceeds_unsigned_range = _mm_cmp_ps_mask::<17i32>(_mm_set1_ps(4294967040.0), a); - converted = _mm_mask_blend_epi32( - exceeds_unsigned_range, - converted, - _mm_set1_epi32(u32::MAX.cast_signed()), - ); - converted.simd_into(token) + let a = a.into(); + let positive = _mm_cmp_ps_mask::<17i32>(_mm_setzero_ps(), a); + _mm_maskz_cvttps_epu32(positive, a).simd_into(token) } ); kernel(self, a) @@ -4865,16 +4859,9 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f64x2) -> u64x2 { - let a = _mm_max_pd(a.into(), _mm_setzero_pd()); - let mut converted = _mm_cvttpd_epu64(a); - let exceeds_unsigned_range = - _mm_cmp_pd_mask::<17i32>(_mm_set1_pd(18446744073709549568.0), a); - converted = _mm_mask_blend_epi64( - exceeds_unsigned_range, - converted, - _mm_set1_epi64x(u64::MAX.cast_signed()), - ); - converted.simd_into(token) + let a = a.into(); + let positive = _mm_cmp_pd_mask::<17i32>(_mm_setzero_pd(), a); + _mm_maskz_cvttpd_epu64(positive, a).simd_into(token) } ); kernel(self, a) @@ -6513,16 +6500,9 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f32x8) -> u32x8 { - let a = _mm256_max_ps(a.into(), _mm256_setzero_ps()); - let mut converted = _mm256_cvttps_epu32(a); - let exceeds_unsigned_range = - _mm256_cmp_ps_mask::<17i32>(_mm256_set1_ps(4294967040.0), a); - converted = _mm256_mask_blend_epi32( - exceeds_unsigned_range, - converted, - _mm256_set1_epi32(u32::MAX.cast_signed()), - ); - converted.simd_into(token) + let a = a.into(); + let positive = _mm256_cmp_ps_mask::<17i32>(_mm256_setzero_ps(), a); + _mm256_maskz_cvttps_epu32(positive, a).simd_into(token) } ); kernel(self, a) @@ -10996,16 +10976,9 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f64x4) -> u64x4 { - let a = _mm256_max_pd(a.into(), _mm256_setzero_pd()); - let mut converted = _mm256_cvttpd_epu64(a); - let exceeds_unsigned_range = - _mm256_cmp_pd_mask::<17i32>(_mm256_set1_pd(18446744073709549568.0), a); - converted = _mm256_mask_blend_epi64( - exceeds_unsigned_range, - converted, - _mm256_set1_epi64x(u64::MAX.cast_signed()), - ); - converted.simd_into(token) + let a = a.into(); + let positive = _mm256_cmp_pd_mask::<17i32>(_mm256_setzero_pd(), a); + _mm256_maskz_cvttpd_epu64(positive, a).simd_into(token) } ); kernel(self, a) @@ -12759,16 +12732,9 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f32x16) -> u32x16 { - let a = _mm512_max_ps(a.into(), _mm512_setzero_ps()); - let mut converted = _mm512_cvttps_epu32(a); - let exceeds_unsigned_range = - _mm512_cmp_ps_mask::<17i32>(_mm512_set1_ps(4294967040.0), a); - converted = _mm512_mask_blend_epi32( - exceeds_unsigned_range, - converted, - _mm512_set1_epi32(u32::MAX.cast_signed()), - ); - converted.simd_into(token) + let a = a.into(); + let positive = _mm512_cmp_ps_mask::<17i32>(_mm512_setzero_ps(), a); + _mm512_maskz_cvttps_epu32(positive, a).simd_into(token) } ); kernel(self, a) @@ -17749,16 +17715,9 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f64x8) -> u64x8 { - let a = _mm512_max_pd(a.into(), _mm512_setzero_pd()); - let mut converted = _mm512_cvttpd_epu64(a); - let exceeds_unsigned_range = - _mm512_cmp_pd_mask::<17i32>(_mm512_set1_pd(18446744073709549568.0), a); - converted = _mm512_mask_blend_epi64( - exceeds_unsigned_range, - converted, - _mm512_set1_epi64(u64::MAX.cast_signed()), - ); - converted.simd_into(token) + let a = a.into(); + let positive = _mm512_cmp_pd_mask::<17i32>(_mm512_setzero_pd(), a); + _mm512_maskz_cvttpd_epu64(positive, a).simd_into(token) } ); kernel(self, a) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 092e8a98..64226f97 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2984,26 +2984,18 @@ impl X86 { 128 | 256 | 512, Precise, ) => { - let target_ty = vec_ty.cast(target_scalar); - let convert = simple_intrinsic("cvttpd", &target_ty); - let max = simple_intrinsic("max", vec_ty); + // The ordered `0 < a` comparison excludes negative values, both zeroes, and NaN, + // so zero-masking supplies the result required by Rust casts for those lanes. + // Positive overflow stays active: VCVTTPD2UQQ's unsigned indefinite result is + // `u64::MAX`, which is also the required saturating result. let cmp = intrinsic_ident("cmp", "pd_mask", vec_ty.n_bits()); - let blend = avx512_mask_blend_intrinsic(&target_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); + let convert = intrinsic_ident("maskz_cvttpd", "epu64", vec_ty.n_bits()); let set0_float = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); let lt = avx512_float_compare_predicate("simd_lt"); quote! { - let a = #max(a.into(), #set0_float()); - let mut converted = #convert(a); - let exceeds_unsigned_range = - #cmp::<#lt>(#set1_float(18446744073709549568.0), a); - converted = #blend( - exceeds_unsigned_range, - converted, - #set1_int(u64::MAX.cast_signed()), - ); - converted.simd_into(#token) + let a = a.into(); + let positive = #cmp::<#lt>(#set0_float(), a); + #convert(positive, a).simd_into(#token) } } ( @@ -3028,26 +3020,19 @@ impl X86 { 128 | 256 | 512, Precise, ) => { - let target_ty = vec_ty.cast(target_scalar); - let convert = intrinsic_ident("cvttps", "epu32", vec_ty.n_bits()); - let max = simple_intrinsic("max", vec_ty); + // The ordered `0 < a` comparison excludes negative values, both zeroes, and NaN, + // so zero-masking supplies the result required by Rust casts for those lanes. + // Positive overflow stays active: VCVTTPS2UDQ's unsigned indefinite result is + // `u32::MAX`, which is also the required saturating result. let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); - let blend = avx512_mask_blend_intrinsic(&target_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); + let convert = intrinsic_ident("maskz_cvttps", "epu32", vec_ty.n_bits()); let set0_float = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); let lt = avx512_float_compare_predicate("simd_lt"); quote! { - let a = #max(a.into(), #set0_float()); - let mut converted = #convert(a); - let exceeds_unsigned_range = #cmp::<#lt>(#set1_float(4294967040.0), a); - converted = #blend( - exceeds_unsigned_range, - converted, - #set1_int(u32::MAX.cast_signed()), - ); - converted.simd_into(#token) + let a = a.into(); + let positive = #cmp::<#lt>(#set0_float(), a); + #convert(positive, a).simd_into(#token) } } ( From 451f8ee487311976fbcdb65e94ceb6e64186105b Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 5 Aug 2026 15:59:48 +0100 Subject: [PATCH 5/8] Replace qualified paths for ScalarType with use to make the match more readable --- fearless_simd_gen/src/mk_x86.rs | 59 +++++++++++++++++---------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 64226f97..0678399f 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2887,6 +2887,7 @@ impl X86 { precise: bool, ) -> TokenStream { use Precision::{Approx, Precise}; + use ScalarType::{Float, Int, Unsigned}; assert_eq!( vec_ty.scalar_bits, target_scalar_bits, @@ -2909,8 +2910,8 @@ impl X86 { (Self::Sse2 | Self::Sse4_2, _, _, 64, 128, _) | (Self::Avx2, _, _, 64, 128 | 256, _) | (Self::Sse2, _, _, 32, 128, Precise) - | (Self::Sse2, ScalarType::Unsigned, _, 32, 128, _) - | (Self::Sse2, _, ScalarType::Unsigned, 32, 128, _) + | (Self::Sse2, Unsigned, _, 32, 128, _) + | (Self::Sse2, _, Unsigned, 32, 128, _) ) { // These conversions have no hardware support, or their native implementation is // slower than the scalar fallback. @@ -2918,7 +2919,7 @@ impl X86 { } self.kernel_method(op, vec_ty, |token| match conversion { - (Self::Avx512, ScalarType::Unsigned, ScalarType::Float, 32, bits @ (128 | 256), _) => { + (Self::Avx512, Unsigned, Float, 32, bits @ (128 | 256), _) => { // We cannot emit the intrinsics for the conversion instructions // because the required intrinsics are mysteriously absent from stdarch: // https://github.com/rust-lang/rust/issues/158196 @@ -2933,8 +2934,8 @@ impl X86 { } ( Self::Avx512, - source @ (ScalarType::Int | ScalarType::Unsigned), - ScalarType::Float, + source @ (Int | Unsigned), + Float, scalar_bits @ (32 | 64), 128 | 256 | 512, _, @@ -2949,7 +2950,7 @@ impl X86 { #convert(a.into()).simd_into(#token) } } - (Self::Avx512, ScalarType::Float, ScalarType::Int, 64, 128 | 256 | 512, Precise) => { + (Self::Avx512, Float, Int, 64, 128 | 256 | 512, Precise) => { let target_ty = vec_ty.cast(target_scalar); let masked_convert = intrinsic_ident("mask_cvttpd", "epi64", vec_ty.n_bits()); let cmp = intrinsic_ident("cmp", "pd_mask", vec_ty.n_bits()); @@ -2969,7 +2970,7 @@ impl X86 { converted.simd_into(#token) } } - (Self::Avx512, ScalarType::Float, ScalarType::Int, 64, 128 | 256 | 512, Approx) => { + (Self::Avx512, Float, Int, 64, 128 | 256 | 512, Approx) => { let target_ty = vec_ty.cast(target_scalar); let convert = simple_intrinsic("cvttpd", &target_ty); quote! { @@ -2978,8 +2979,8 @@ impl X86 { } ( Self::Avx512, - ScalarType::Float, - ScalarType::Unsigned, + Float, + Unsigned, 64, 128 | 256 | 512, Precise, @@ -3000,8 +3001,8 @@ impl X86 { } ( Self::Avx512, - ScalarType::Float, - ScalarType::Unsigned, + Float, + Unsigned, 64, 128 | 256 | 512, Approx, @@ -3014,8 +3015,8 @@ impl X86 { } ( Self::Avx512, - ScalarType::Float, - ScalarType::Unsigned, + Float, + Unsigned, 32, 128 | 256 | 512, Precise, @@ -3037,8 +3038,8 @@ impl X86 { } ( Self::Avx512, - ScalarType::Float, - ScalarType::Unsigned, + Float, + Unsigned, 32, 128 | 256 | 512, Approx, @@ -3048,7 +3049,7 @@ impl X86 { #convert(a.into()).simd_into(#token) } } - (Self::Avx512, ScalarType::Float, ScalarType::Int, 32, 128 | 256 | 512, Precise) => { + (Self::Avx512, Float, Int, 32, 128 | 256 | 512, Precise) => { let target_ty = vec_ty.cast(target_scalar); let masked_convert = intrinsic_ident("mask_cvttps", "epi32", vec_ty.n_bits()); let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); @@ -3068,7 +3069,7 @@ impl X86 { converted.simd_into(#token) } } - (Self::Avx512, ScalarType::Float, ScalarType::Int, 32, 128 | 256 | 512, Approx) => { + (Self::Avx512, Float, Int, 32, 128 | 256 | 512, Approx) => { let convert = intrinsic_ident("cvttps", "epi32", vec_ty.n_bits()); quote! { #convert(a.into()).simd_into(#token) @@ -3076,16 +3077,16 @@ impl X86 { } ( Self::Sse2 | Self::Sse4_2, - ScalarType::Float, - target @ (ScalarType::Int | ScalarType::Unsigned), + Float, + target @ (Int | Unsigned), 32, 128, precision, ) | ( Self::Avx2, - ScalarType::Float, - target @ (ScalarType::Int | ScalarType::Unsigned), + Float, + target @ (Int | Unsigned), 32, 128 | 256, precision, @@ -3118,12 +3119,12 @@ impl X86 { let sub_float = simple_intrinsic("sub", vec_ty); match (target, precision) { - (ScalarType::Int, Approx) => { + (Int, Approx) => { quote! { #convert(a.into()).simd_into(#token) } } - (ScalarType::Unsigned, Approx) => { + (Unsigned, Approx) => { quote! { let mut converted = #convert(a.into()); @@ -3141,7 +3142,7 @@ impl X86 { converted.simd_into(#token) } } - (ScalarType::Int, Precise) => { + (Int, Precise) => { quote! { let a = a.into(); @@ -3164,7 +3165,7 @@ impl X86 { converted.simd_into(#token) } } - (ScalarType::Unsigned, Precise) => { + (Unsigned, Precise) => { quote! { // Clamp out-of-range values (and NaN) to 0. Intel's `_mm_max_ps` always takes the second // operand if the first is NaN. @@ -3192,16 +3193,16 @@ impl X86 { _ => unreachable!(), } } - (Self::Sse2 | Self::Sse4_2, ScalarType::Int, ScalarType::Float, 32, 128, _) - | (Self::Avx2, ScalarType::Int, ScalarType::Float, 32, 128 | 256, _) => { + (Self::Sse2 | Self::Sse4_2, Int, Float, 32, 128, _) + | (Self::Avx2, Int, Float, 32, 128 | 256, _) => { let target_ty = vec_ty.cast(target_scalar); let intrinsic = simple_intrinsic("cvtepi32", &target_ty); quote! { #intrinsic(a.into()).simd_into(#token) } } - (Self::Sse4_2, ScalarType::Unsigned, ScalarType::Float, 32, 128, _) - | (Self::Avx2, ScalarType::Unsigned, ScalarType::Float, 32, 128 | 256, _) => { + (Self::Sse4_2, Unsigned, Float, 32, 128, _) + | (Self::Avx2, Unsigned, Float, 32, 128 | 256, _) => { let target_ty = vec_ty.cast(target_scalar); let set1_int = set1_intrinsic(vec_ty); let set1_float = set1_intrinsic(&target_ty); From 3309d08555e2304554da306ee9e2b62df5f63837 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 5 Aug 2026 16:07:32 +0100 Subject: [PATCH 6/8] Consolidate AVX-512 paths in the generator between 32-bit and 64-bit conversions --- fearless_simd_gen/src/mk_x86.rs | 126 ++++++++++++++------------------ 1 file changed, 55 insertions(+), 71 deletions(-) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 0678399f..e29cfba8 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2950,65 +2950,64 @@ impl X86 { #convert(a.into()).simd_into(#token) } } - (Self::Avx512, Float, Int, 64, 128 | 256 | 512, Precise) => { + ( + Self::Avx512, + Float, + Int, + scalar_bits @ (32 | 64), + 128 | 256 | 512, + Precise, + ) => { let target_ty = vec_ty.cast(target_scalar); - let masked_convert = intrinsic_ident("mask_cvttpd", "epi64", vec_ty.n_bits()); - let cmp = intrinsic_ident("cmp", "pd_mask", vec_ty.n_bits()); + let float_suffix = op_suffix(Float, scalar_bits, true); + let int_suffix = op_suffix(Int, scalar_bits, true); + let masked_convert = intrinsic_ident( + &format!("mask_cvtt{float_suffix}"), + int_suffix, + vec_ty.n_bits(), + ); + let cmp = intrinsic_ident( + "cmp", + &format!("{float_suffix}_mask"), + vec_ty.n_bits(), + ); let blend = avx512_mask_blend_intrinsic(&target_ty); let set1_float = set1_intrinsic(vec_ty); let set1_int = set1_intrinsic(&target_ty); let set0_int = intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); + let upper_bound = match scalar_bits { + 32 => quote! { 2147483648.0 }, + 64 => quote! { 9223372036854775808.0 }, + _ => unreachable!(), + }; + let int_max = match scalar_bits { + 32 => quote! { i32::MAX }, + 64 => quote! { i64::MAX }, + _ => unreachable!(), + }; let lt = avx512_float_compare_predicate("simd_lt"); let ord = avx512_float_compare_predicate("ord"); quote! { let a = a.into(); - let in_range = #cmp::<#lt>(a, #set1_float(9223372036854775808.0)); - let mut converted = #masked_convert(#set1_int(i64::MAX), in_range, a); + let in_range = #cmp::<#lt>(a, #set1_float(#upper_bound)); + let mut converted = #masked_convert(#set1_int(#int_max), in_range, a); let is_not_nan = #cmp::<#ord>(a, a); converted = #blend(is_not_nan, #set0_int(), converted); converted.simd_into(#token) } } - (Self::Avx512, Float, Int, 64, 128 | 256 | 512, Approx) => { - let target_ty = vec_ty.cast(target_scalar); - let convert = simple_intrinsic("cvttpd", &target_ty); - quote! { - #convert(a.into()).simd_into(#token) - } - } - ( - Self::Avx512, - Float, - Unsigned, - 64, - 128 | 256 | 512, - Precise, - ) => { - // The ordered `0 < a` comparison excludes negative values, both zeroes, and NaN, - // so zero-masking supplies the result required by Rust casts for those lanes. - // Positive overflow stays active: VCVTTPD2UQQ's unsigned indefinite result is - // `u64::MAX`, which is also the required saturating result. - let cmp = intrinsic_ident("cmp", "pd_mask", vec_ty.n_bits()); - let convert = intrinsic_ident("maskz_cvttpd", "epu64", vec_ty.n_bits()); - let set0_float = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); - let lt = avx512_float_compare_predicate("simd_lt"); - quote! { - let a = a.into(); - let positive = #cmp::<#lt>(#set0_float(), a); - #convert(positive, a).simd_into(#token) - } - } ( Self::Avx512, Float, - Unsigned, - 64, + Int, + scalar_bits @ (32 | 64), 128 | 256 | 512, Approx, ) => { let target_ty = vec_ty.cast(target_scalar); - let convert = simple_intrinsic("cvttpd", &target_ty); + let float_suffix = op_suffix(Float, scalar_bits, true); + let convert = simple_intrinsic(&format!("cvtt{float_suffix}"), &target_ty); quote! { #convert(a.into()).simd_into(#token) } @@ -3017,18 +3016,27 @@ impl X86 { Self::Avx512, Float, Unsigned, - 32, + scalar_bits @ (32 | 64), 128 | 256 | 512, Precise, ) => { // The ordered `0 < a` comparison excludes negative values, both zeroes, and NaN, // so zero-masking supplies the result required by Rust casts for those lanes. - // Positive overflow stays active: VCVTTPS2UDQ's unsigned indefinite result is - // `u32::MAX`, which is also the required saturating result. - let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); - let convert = intrinsic_ident("maskz_cvttps", "epu32", vec_ty.n_bits()); - let set0_float = - intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); + // Positive overflow stays active: AVX-512's unsigned indefinite result is the + // destination type's maximum, which is also the required saturating result. + let float_suffix = op_suffix(Float, scalar_bits, true); + let int_suffix = op_suffix(Unsigned, scalar_bits, true); + let cmp = intrinsic_ident( + "cmp", + &format!("{float_suffix}_mask"), + vec_ty.n_bits(), + ); + let convert = intrinsic_ident( + &format!("maskz_cvtt{float_suffix}"), + int_suffix, + vec_ty.n_bits(), + ); + let set0_float = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); let lt = avx512_float_compare_predicate("simd_lt"); quote! { let a = a.into(); @@ -3040,37 +3048,13 @@ impl X86 { Self::Avx512, Float, Unsigned, - 32, + scalar_bits @ (32 | 64), 128 | 256 | 512, Approx, ) => { - let convert = intrinsic_ident("cvttps", "epu32", vec_ty.n_bits()); - quote! { - #convert(a.into()).simd_into(#token) - } - } - (Self::Avx512, Float, Int, 32, 128 | 256 | 512, Precise) => { let target_ty = vec_ty.cast(target_scalar); - let masked_convert = intrinsic_ident("mask_cvttps", "epi32", vec_ty.n_bits()); - let cmp = intrinsic_ident("cmp", "ps_mask", vec_ty.n_bits()); - let blend = avx512_mask_blend_intrinsic(&target_ty); - let set1_float = set1_intrinsic(vec_ty); - let set1_int = set1_intrinsic(&target_ty); - let set0_int = - intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); - let lt = avx512_float_compare_predicate("simd_lt"); - let ord = avx512_float_compare_predicate("ord"); - quote! { - let a = a.into(); - let in_range = #cmp::<#lt>(a, #set1_float(2147483648.0)); - let mut converted = #masked_convert(#set1_int(i32::MAX), in_range, a); - let is_not_nan = #cmp::<#ord>(a, a); - converted = #blend(is_not_nan, #set0_int(), converted); - converted.simd_into(#token) - } - } - (Self::Avx512, Float, Int, 32, 128 | 256 | 512, Approx) => { - let convert = intrinsic_ident("cvttps", "epi32", vec_ty.n_bits()); + let float_suffix = op_suffix(Float, scalar_bits, true); + let convert = simple_intrinsic(&format!("cvtt{float_suffix}"), &target_ty); quote! { #convert(a.into()).simd_into(#token) } From 70c3211d817694a17f28298bd5b5fd4950573fa5 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 5 Aug 2026 17:48:47 +0100 Subject: [PATCH 7/8] Add comments to explain what the hell is happening in handle_cvt --- fearless_simd_gen/src/mk_x86.rs | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index e29cfba8..74983b0f 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2889,13 +2889,21 @@ impl X86 { use Precision::{Approx, Precise}; use ScalarType::{Float, Int, Unsigned}; + // Conversion methods preserve the number of lanes, so their source and destination + // scalars must have the same width. Widening and narrowing conversions are generated by + // separate handlers. assert_eq!( vec_ty.scalar_bits, target_scalar_bits, "we currently only support converting between types of the same width" ); + // `Precise` float-to-integer conversions reproduce Rust's saturating `as` semantics, + // including zero for NaN. `Approx` conversions may return the hardware-defined result for + // NaN and out-of-range inputs. Precision does not affect integer-to-float conversions. let precision = if precise { Precise } else { Approx }; + // Keep every property that affects instruction selection in one key. The exhaustive match + // below then makes additions to the backend or type matrix fail loudly until handled. let conversion = ( *self, vec_ty.scalar, @@ -2905,6 +2913,9 @@ impl X86 { precision, ); + // Deal with whole-method scalar fallbacks before constructing an intrinsic-backed kernel. + // Pre-AVX-512 x86 has no packed 64-bit conversion instructions. Some SSE2 32-bit cases are + // also absent or slower than converting the lanes individually. if matches!( conversion, (Self::Sse2 | Self::Sse4_2, _, _, 64, 128, _) @@ -2920,6 +2931,7 @@ impl X86 { self.kernel_method(op, vec_ty, |token| match conversion { (Self::Avx512, Unsigned, Float, 32, bits @ (128 | 256), _) => { + // AVX-512 unsigned integer -> float for narrow vectors. // We cannot emit the intrinsics for the conversion instructions // because the required intrinsics are mysteriously absent from stdarch: // https://github.com/rust-lang/rust/issues/158196 @@ -2958,6 +2970,9 @@ impl X86 { 128 | 256 | 512, Precise, ) => { + // The truncating signed conversion already returns iN::MIN for negative overflow. + // Mask positive overflow to iN::MAX, then replace NaN with zero to complete Rust's + // float-to-integer cast semantics. let target_ty = vec_ty.cast(target_scalar); let float_suffix = op_suffix(Float, scalar_bits, true); let int_suffix = op_suffix(Int, scalar_bits, true); @@ -2976,9 +2991,14 @@ impl X86 { let set1_int = set1_intrinsic(&target_ty); let set0_int = intrinsic_ident("setzero", coarse_type(&target_ty), target_ty.n_bits()); + // `2^(N - 1)` is exactly representable in both f32 and f64, and is the first + // positive value outside the corresponding signed integer range. + // Lanes at or above this threshold retain the merge value `iN::MAX`. + // Negative overflow can use the truncating instruction's signed indefinite result, `iN::MIN`; + // NaN is replaced with zero below. let upper_bound = match scalar_bits { - 32 => quote! { 2147483648.0 }, - 64 => quote! { 9223372036854775808.0 }, + 32 => quote! { 2147483648.0 }, // 2^31 + 64 => quote! { 9223372036854775808.0 }, // 2^63 _ => unreachable!(), }; let int_max = match scalar_bits { @@ -3020,10 +3040,9 @@ impl X86 { 128 | 256 | 512, Precise, ) => { - // The ordered `0 < a` comparison excludes negative values, both zeroes, and NaN, - // so zero-masking supplies the result required by Rust casts for those lanes. - // Positive overflow stays active: AVX-512's unsigned indefinite result is the - // destination type's maximum, which is also the required saturating result. + // AVX-512's unsigned indefinite result is already uN::MAX, so positive overflow + // needs no correction. Masking the conversion with an ordered positive comparison + // makes every negative, zero, and NaN lane zero as required by Rust casts. let float_suffix = op_suffix(Float, scalar_bits, true); let int_suffix = op_suffix(Unsigned, scalar_bits, true); let cmp = intrinsic_ident( @@ -3075,6 +3094,8 @@ impl X86 { 128 | 256, precision, ) => { + // SSE and AVX2 only provide packed f32 -> i32 conversion. + // Build unsigned and precise semantics around that signed primitive. let target_ty = vec_ty.cast(target); let max = simple_intrinsic("max", vec_ty); let set0 = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); From 41988bd098be566d3ab1413241ef37e96bafe49d Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Fri, 7 Aug 2026 21:07:52 +0100 Subject: [PATCH 8/8] Add tests that excercise normal values in all lanes --- .../tests/harness/ops/cvt_i32.rs | 38 +++++++++++++ .../tests/harness/ops/cvt_i32_precise.rs | 38 +++++++++++++ .../tests/harness/ops/cvt_i64.rs | 53 ++++++++++++++++++ .../tests/harness/ops/cvt_i64_precise.rs | 56 +++++++++++++++++++ .../tests/harness/ops/cvt_u32.rs | 37 ++++++++++++ .../tests/harness/ops/cvt_u32_precise.rs | 37 ++++++++++++ .../tests/harness/ops/cvt_u64.rs | 53 ++++++++++++++++++ .../tests/harness/ops/cvt_u64_precise.rs | 56 +++++++++++++++++++ 8 files changed, 368 insertions(+) diff --git a/fearless_simd_tests/tests/harness/ops/cvt_i32.rs b/fearless_simd_tests/tests/harness/ops/cvt_i32.rs index a5459c14..a128fd7a 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_i32.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_i32.rs @@ -6,6 +6,44 @@ use fearless_simd_dev_macros::simd_test; // One concrete test row per supported vector type. +#[simd_test] +fn cvt_i32_f32x4_regular(simd: S) { + let a = f32x4::from_slice(simd, &[-101.75, -22.5, 33.875, 404.25]); + assert_eq!(*a.to_int::>(), [-101, -22, 33, 404]); +} + +#[simd_test] +fn cvt_i32_f32x8_regular(simd: S) { + let a = f32x8::from_slice( + simd, + &[ + -801.75, -702.5, -603.25, -504.875, 405.125, 306.5, 207.75, 108.25, + ], + ); + assert_eq!( + *a.to_int::>(), + [-801, -702, -603, -504, 405, 306, 207, 108] + ); +} + +#[simd_test] +fn cvt_i32_f32x16_regular(simd: S) { + let a = f32x16::from_slice( + simd, + &[ + -1601.75, -1502.5, -1403.25, -1304.875, -1205.125, -1106.5, -1007.75, -908.25, 809.5, + 710.75, 611.125, 512.875, 413.25, 314.5, 215.75, 116.25, + ], + ); + assert_eq!( + *a.to_int::>(), + [ + -1601, -1502, -1403, -1304, -1205, -1106, -1007, -908, 809, 710, 611, 512, 413, 314, + 215, 116, + ] + ); +} + #[simd_test] fn cvt_i32_f32x4_rounding(simd: S) { let a = f32x4::from_slice(simd, &[0.0, 0.49, 0.51, 0.99]); diff --git a/fearless_simd_tests/tests/harness/ops/cvt_i32_precise.rs b/fearless_simd_tests/tests/harness/ops/cvt_i32_precise.rs index 38f2cf82..db3b1733 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_i32_precise.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_i32_precise.rs @@ -6,6 +6,44 @@ use fearless_simd_dev_macros::simd_test; // One concrete test row per supported vector type. +#[simd_test] +fn cvt_i32_precise_f32x4_regular(simd: S) { + let a = f32x4::from_slice(simd, &[-101.75, -22.5, 33.875, 404.25]); + assert_eq!(*a.to_int_precise::>(), [-101, -22, 33, 404]); +} + +#[simd_test] +fn cvt_i32_precise_f32x8_regular(simd: S) { + let a = f32x8::from_slice( + simd, + &[ + -801.75, -702.5, -603.25, -504.875, 405.125, 306.5, 207.75, 108.25, + ], + ); + assert_eq!( + *a.to_int_precise::>(), + [-801, -702, -603, -504, 405, 306, 207, 108] + ); +} + +#[simd_test] +fn cvt_i32_precise_f32x16_regular(simd: S) { + let a = f32x16::from_slice( + simd, + &[ + -1601.75, -1502.5, -1403.25, -1304.875, -1205.125, -1106.5, -1007.75, -908.25, 809.5, + 710.75, 611.125, 512.875, 413.25, 314.5, 215.75, 116.25, + ], + ); + assert_eq!( + *a.to_int_precise::>(), + [ + -1601, -1502, -1403, -1304, -1205, -1106, -1007, -908, 809, 710, 611, 512, 413, 314, + 215, 116, + ] + ); +} + #[simd_test] fn cvt_i32_precise_f32x4_sat(simd: S) { let a = f32x4::from_slice(simd, &[-10.3, f32::NAN, 5e9, -5e9]); diff --git a/fearless_simd_tests/tests/harness/ops/cvt_i64.rs b/fearless_simd_tests/tests/harness/ops/cvt_i64.rs index 8cd7e186..a975b5c4 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_i64.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_i64.rs @@ -4,6 +4,59 @@ use fearless_simd::*; use fearless_simd_dev_macros::simd_test; +#[simd_test] +fn cvt_i64_f64x2_regular(simd: S) { + let a = f64x2::from_slice(simd, &[-9_876_543_210.75, 1_234_567_890.25]); + assert_eq!(*a.to_int::>(), [-9_876_543_210, 1_234_567_890]); +} + +#[simd_test] +fn cvt_i64_f64x4_regular(simd: S) { + let a = f64x4::from_slice( + simd, + &[ + -9_876_543_210.75, + -1_234_567_890.25, + 2_345_678_901.5, + 8_765_432_109.875, + ], + ); + assert_eq!( + *a.to_int::>(), + [-9_876_543_210, -1_234_567_890, 2_345_678_901, 8_765_432_109] + ); +} + +#[simd_test] +fn cvt_i64_f64x8_regular(simd: S) { + let a = f64x8::from_slice( + simd, + &[ + -9_876_543_210.75, + -8_765_432_109.625, + -7_654_321_098.5, + -6_543_210_987.375, + 5_432_109_876.25, + 4_321_098_765.125, + 3_210_987_654.875, + 2_109_876_543.75, + ], + ); + assert_eq!( + *a.to_int::>(), + [ + -9_876_543_210, + -8_765_432_109, + -7_654_321_098, + -6_543_210_987, + 5_432_109_876, + 4_321_098_765, + 3_210_987_654, + 2_109_876_543, + ] + ); +} + #[simd_test] fn cvt_i64_f64x2(simd: S) { let values = [-42.9, 42.9]; diff --git a/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs b/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs index 14fa4687..857a41de 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs @@ -4,6 +4,62 @@ use fearless_simd::*; use fearless_simd_dev_macros::simd_test; +#[simd_test] +fn cvt_i64_precise_f64x2_regular(simd: S) { + let a = f64x2::from_slice(simd, &[-9_876_543_210.75, 1_234_567_890.25]); + assert_eq!( + *a.to_int_precise::>(), + [-9_876_543_210, 1_234_567_890] + ); +} + +#[simd_test] +fn cvt_i64_precise_f64x4_regular(simd: S) { + let a = f64x4::from_slice( + simd, + &[ + -9_876_543_210.75, + -1_234_567_890.25, + 2_345_678_901.5, + 8_765_432_109.875, + ], + ); + assert_eq!( + *a.to_int_precise::>(), + [-9_876_543_210, -1_234_567_890, 2_345_678_901, 8_765_432_109] + ); +} + +#[simd_test] +fn cvt_i64_precise_f64x8_regular(simd: S) { + let a = f64x8::from_slice( + simd, + &[ + -9_876_543_210.75, + -8_765_432_109.625, + -7_654_321_098.5, + -6_543_210_987.375, + 5_432_109_876.25, + 4_321_098_765.125, + 3_210_987_654.875, + 2_109_876_543.75, + ], + ); + assert_eq!( + *a.to_int_precise::>(), + [ + -9_876_543_210, + -8_765_432_109, + -7_654_321_098, + -6_543_210_987, + 5_432_109_876, + 4_321_098_765, + 3_210_987_654, + 2_109_876_543, + ] + ); +} + #[simd_test] fn cvt_i64_precise_f64x2(simd: S) { let values = [f64::NAN, f64::INFINITY]; diff --git a/fearless_simd_tests/tests/harness/ops/cvt_u32.rs b/fearless_simd_tests/tests/harness/ops/cvt_u32.rs index 58139476..54f08f13 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_u32.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_u32.rs @@ -6,6 +6,43 @@ use fearless_simd_dev_macros::simd_test; // One concrete test row per supported vector type. +#[simd_test] +fn cvt_u32_f32x4_regular(simd: S) { + let a = f32x4::from_slice(simd, &[101.75, 22.5, 33.875, 404.25]); + assert_eq!(*a.to_int::>(), [101, 22, 33, 404]); +} + +#[simd_test] +fn cvt_u32_f32x8_regular(simd: S) { + let a = f32x8::from_slice( + simd, + &[ + 801.75, 702.5, 603.25, 504.875, 405.125, 306.5, 207.75, 108.25, + ], + ); + assert_eq!( + *a.to_int::>(), + [801, 702, 603, 504, 405, 306, 207, 108] + ); +} + +#[simd_test] +fn cvt_u32_f32x16_regular(simd: S) { + let a = f32x16::from_slice( + simd, + &[ + 1601.75, 1502.5, 1403.25, 1304.875, 1205.125, 1106.5, 1007.75, 908.25, 809.5, 710.75, + 611.125, 512.875, 413.25, 314.5, 215.75, 116.25, + ], + ); + assert_eq!( + *a.to_int::>(), + [ + 1601, 1502, 1403, 1304, 1205, 1106, 1007, 908, 809, 710, 611, 512, 413, 314, 215, 116, + ] + ); +} + #[simd_test] fn cvt_u32_f32x4(simd: S) { let a = f32x4::from_slice(simd, &[1.0, 42.7, 3e9, -0.3]); diff --git a/fearless_simd_tests/tests/harness/ops/cvt_u32_precise.rs b/fearless_simd_tests/tests/harness/ops/cvt_u32_precise.rs index 7150ca3a..3f1763b0 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_u32_precise.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_u32_precise.rs @@ -6,6 +6,43 @@ use fearless_simd_dev_macros::simd_test; // One concrete test row per supported vector type. +#[simd_test] +fn cvt_u32_precise_f32x4_regular(simd: S) { + let a = f32x4::from_slice(simd, &[101.75, 22.5, 33.875, 404.25]); + assert_eq!(*a.to_int_precise::>(), [101, 22, 33, 404]); +} + +#[simd_test] +fn cvt_u32_precise_f32x8_regular(simd: S) { + let a = f32x8::from_slice( + simd, + &[ + 801.75, 702.5, 603.25, 504.875, 405.125, 306.5, 207.75, 108.25, + ], + ); + assert_eq!( + *a.to_int_precise::>(), + [801, 702, 603, 504, 405, 306, 207, 108] + ); +} + +#[simd_test] +fn cvt_u32_precise_f32x16_regular(simd: S) { + let a = f32x16::from_slice( + simd, + &[ + 1601.75, 1502.5, 1403.25, 1304.875, 1205.125, 1106.5, 1007.75, 908.25, 809.5, 710.75, + 611.125, 512.875, 413.25, 314.5, 215.75, 116.25, + ], + ); + assert_eq!( + *a.to_int_precise::>(), + [ + 1601, 1502, 1403, 1304, 1205, 1106, 1007, 908, 809, 710, 611, 512, 413, 314, 215, 116, + ] + ); +} + #[simd_test] fn cvt_u32_precise_f32x4(simd: S) { let a = f32x4::from_slice(simd, &[-1.0, 42.7, 5e9, f32::NAN]); diff --git a/fearless_simd_tests/tests/harness/ops/cvt_u64.rs b/fearless_simd_tests/tests/harness/ops/cvt_u64.rs index 1ec82910..ca3081b5 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_u64.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_u64.rs @@ -4,6 +4,59 @@ use fearless_simd::*; use fearless_simd_dev_macros::simd_test; +#[simd_test] +fn cvt_u64_f64x2_regular(simd: S) { + let a = f64x2::from_slice(simd, &[1_234_567_890.25, 9_876_543_210.75]); + assert_eq!(*a.to_int::>(), [1_234_567_890, 9_876_543_210]); +} + +#[simd_test] +fn cvt_u64_f64x4_regular(simd: S) { + let a = f64x4::from_slice( + simd, + &[ + 1_234_567_890.25, + 2_345_678_901.5, + 8_765_432_109.875, + 9_876_543_210.75, + ], + ); + assert_eq!( + *a.to_int::>(), + [1_234_567_890, 2_345_678_901, 8_765_432_109, 9_876_543_210] + ); +} + +#[simd_test] +fn cvt_u64_f64x8_regular(simd: S) { + let a = f64x8::from_slice( + simd, + &[ + 9_876_543_210.75, + 8_765_432_109.625, + 7_654_321_098.5, + 6_543_210_987.375, + 5_432_109_876.25, + 4_321_098_765.125, + 3_210_987_654.875, + 2_109_876_543.75, + ], + ); + assert_eq!( + *a.to_int::>(), + [ + 9_876_543_210, + 8_765_432_109, + 7_654_321_098, + 6_543_210_987, + 5_432_109_876, + 4_321_098_765, + 3_210_987_654, + 2_109_876_543, + ] + ); +} + #[simd_test] fn cvt_u64_f64x2(simd: S) { let values = [0.0, 42.9]; diff --git a/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs b/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs index 71dd6b80..b56e7a33 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs @@ -4,6 +4,62 @@ use fearless_simd::*; use fearless_simd_dev_macros::simd_test; +#[simd_test] +fn cvt_u64_precise_f64x2_regular(simd: S) { + let a = f64x2::from_slice(simd, &[1_234_567_890.25, 9_876_543_210.75]); + assert_eq!( + *a.to_int_precise::>(), + [1_234_567_890, 9_876_543_210] + ); +} + +#[simd_test] +fn cvt_u64_precise_f64x4_regular(simd: S) { + let a = f64x4::from_slice( + simd, + &[ + 1_234_567_890.25, + 2_345_678_901.5, + 8_765_432_109.875, + 9_876_543_210.75, + ], + ); + assert_eq!( + *a.to_int_precise::>(), + [1_234_567_890, 2_345_678_901, 8_765_432_109, 9_876_543_210] + ); +} + +#[simd_test] +fn cvt_u64_precise_f64x8_regular(simd: S) { + let a = f64x8::from_slice( + simd, + &[ + 9_876_543_210.75, + 8_765_432_109.625, + 7_654_321_098.5, + 6_543_210_987.375, + 5_432_109_876.25, + 4_321_098_765.125, + 3_210_987_654.875, + 2_109_876_543.75, + ], + ); + assert_eq!( + *a.to_int_precise::>(), + [ + 9_876_543_210, + 8_765_432_109, + 7_654_321_098, + 6_543_210_987, + 5_432_109_876, + 4_321_098_765, + 3_210_987_654, + 2_109_876_543, + ] + ); +} + #[simd_test] fn cvt_u64_precise_f64x2(simd: S) { let values = [f64::NAN, f64::INFINITY];