From b811ec663798f00677013d041658aaa521460fae Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 5 Aug 2026 01:13:30 +0100 Subject: [PATCH] Optimize interleaved loads by removing the AVX2 specialization. Yup. --- fearless_simd/src/generated/avx2.rs | 93 ++++++++++++++++------------- fearless_simd_gen/src/mk_x86.rs | 46 ++------------ 2 files changed, 56 insertions(+), 83 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 266be45c..c0194759 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -4732,23 +4732,26 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, src: &[f64; 8usize]) -> [f64x2; 4usize] { - let (chunks, []) = src.as_chunks::<4>() else { + let (chunks, []) = src.as_chunks::<2usize>() else { unreachable!() }; - let v0: __m256d = - crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[0]); - let v1: __m256d = - crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[1]); - let lo = _mm256_unpacklo_pd(v0, v1); - let hi = _mm256_unpackhi_pd(v0, v1); - let out0 = _mm256_permute2f128_pd::<0x20>(lo, hi); - let out1 = _mm256_permute2f128_pd::<0x31>(lo, hi); - let outputs: [__m128d; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]); + let v0: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]); + let v1: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]); + let v2: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]); + let v3: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]); + let out0 = _mm_unpacklo_pd(v0, v2); + let out1 = _mm_unpackhi_pd(v0, v2); + let out2 = _mm_unpacklo_pd(v1, v3); + let out3 = _mm_unpackhi_pd(v1, v3); [ - outputs[0].simd_into(token), - outputs[1].simd_into(token), - outputs[2].simd_into(token), - outputs[3].simd_into(token), + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), ] } ); @@ -5153,23 +5156,26 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, src: &[i64; 8usize]) -> [i64x2; 4usize] { - let (chunks, []) = src.as_chunks::<4>() else { + let (chunks, []) = src.as_chunks::<2usize>() else { unreachable!() }; - let v0: __m256i = - crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[0]); - let v1: __m256i = - crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[1]); - let lo = _mm256_unpacklo_epi64(v0, v1); - let hi = _mm256_unpackhi_epi64(v0, v1); - let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); - let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); - let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]); + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); [ - outputs[0].simd_into(token), - outputs[1].simd_into(token), - outputs[2].simd_into(token), - outputs[3].simd_into(token), + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), ] } ); @@ -5561,23 +5567,26 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, src: &[u64; 8usize]) -> [u64x2; 4usize] { - let (chunks, []) = src.as_chunks::<4>() else { + let (chunks, []) = src.as_chunks::<2usize>() else { unreachable!() }; - let v0: __m256i = - crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[0]); - let v1: __m256i = - crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[1]); - let lo = _mm256_unpacklo_epi64(v0, v1); - let hi = _mm256_unpackhi_epi64(v0, v1); - let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); - let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); - let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]); + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); [ - outputs[0].simd_into(token), - outputs[1].simd_into(token), - outputs[2].simd_into(token), - outputs[3].simd_into(token), + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), ] } ); diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 19e05a18..2732f0c1 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -3252,58 +3252,22 @@ impl X86 { if *self == Self::Sse2 && matches!(vec_ty.scalar_bits, 8 | 16) { return fallback_method(op, vec_ty); } + // SSE4.2 intrinsics are used on both SSE4.2 and AVX2, + // because AVX2 cross-lane shuffles have higher latency than this SSE4.2 formulation, + // and for load operations latency matters a lot more than throughput + // since data processing cannot start until the data is loaded. match vec_ty.scalar_bits { 64 | 32 | 16 | 8 => { - let avx2_64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; - let block_len = if avx2_64 { - 4 - } else { - block_size as usize / vec_ty.scalar_bits - }; + let block_len = block_size as usize / vec_ty.scalar_bits; let block_ty = VecType::new(vec_ty.scalar, vec_ty.scalar_bits, block_len); let scalar_ty = block_ty.scalar.rust(block_ty.scalar_bits); let native_ty = self.arch_ty(&block_ty); - let native_block_ty = self.arch_ty(vec_ty); let vec_32 = block_ty.reinterpret(block_ty.scalar, 32); let unpacklo_32 = simple_sign_unaware_intrinsic("unpacklo", &vec_32); let unpackhi_32 = simple_sign_unaware_intrinsic("unpackhi", &vec_32); let vec_64 = block_ty.reinterpret(block_ty.scalar, 64); let unpacklo_64 = simple_sign_unaware_intrinsic("unpacklo", &vec_64); let unpackhi_64 = simple_sign_unaware_intrinsic("unpackhi", &vec_64); - let permute_128 = intrinsic_ident( - match vec_ty.scalar { - ScalarType::Float => "permute2f128", - _ => "permute2x128", - }, - coarse_type(&block_ty), - 256, - ); - - if avx2_64 { - return self.kernel_method(op, vec_ty, |token| { - quote! { - let (chunks, []) = src.as_chunks::<4>() else { - unreachable!() - }; - let v0: #native_ty = crate::transmute::checked_transmute_copy::<[#scalar_ty; 4], #native_ty>(&chunks[0]); - let v1: #native_ty = crate::transmute::checked_transmute_copy::<[#scalar_ty; 4], #native_ty>(&chunks[1]); - - let lo = #unpacklo_64(v0, v1); // [0,4,2,6] - let hi = #unpackhi_64(v0, v1); // [1,5,3,7] - let out0 = #permute_128::<0x20>(lo, hi); // [0,4,1,5] - let out1 = #permute_128::<0x31>(lo, hi); // [2,6,3,7] - let outputs: [#native_block_ty; 4] = - crate::transmute::checked_transmute_copy(&[out0, out1]); - - [ - outputs[0].simd_into(#token), - outputs[1].simd_into(#token), - outputs[2].simd_into(#token), - outputs[3].simd_into(#token), - ] - } - }); - } let init_shuffle = match vec_ty.scalar_bits { 16 => Some(quote! {