Skip to content

Commit 31df44c

Browse files
Neiko2002Neiko2002
committed
fix(distance): replace non-standard _mm_fmadd_ps and enforce sequential accumulation in SSE/scalar math
1 parent b115c64 commit 31df44c

7 files changed

Lines changed: 37 additions & 31 deletions

File tree

‎cpp/CMakeLists.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@ if (FORCE_AVX2)
4040
if(MSVC)
4141
check_cxx_compiler_flag("/arch:AVX2" COMPILER_SUPPORTS_AVX2)
4242
if(COMPILER_SUPPORTS_AVX2)
43-
target_compile_options(compile-options INTERFACE /O2 /arch:AVX2 /W1 /EHsc)
43+
target_compile_options(compile-options INTERFACE /O2 /arch:AVX2 /W1 /EHsc /fp:precise)
4444
else()
45-
target_compile_options(compile-options INTERFACE /O2 /W1 /EHsc)
45+
target_compile_options(compile-options INTERFACE /O2 /W1 /EHsc /fp:precise)
4646
endif()
4747
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
4848
check_cxx_compiler_flag("-mavx2" COMPILER_SUPPORTS_AVX2)
@@ -78,7 +78,7 @@ else() # Native build
7878
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
7979
target_compile_options(compile-options INTERFACE -O2 -DNDEBUG -march=native -fpic)
8080
elseif(MSVC)
81-
target_compile_options(compile-options INTERFACE /W4 /EHsc $<$<CONFIG:Release>:/O2>)
81+
target_compile_options(compile-options INTERFACE /W4 /EHsc /fp:precise $<$<CONFIG:Release>:/O2>)
8282
if(SUPPORT_AVX512F)
8383
target_compile_options(compile-options INTERFACE /arch:AVX512)
8484
elseif(SUPPORT_AVX2)

‎cpp/deglib/include/distance/fp32_ip.h‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ namespace deglib::distances::fp32_ip {
1414
return 1.f - ip_naive(pVect1v, pVect2v, qty_ptr);
1515
}
1616

17+
#if defined(__GNUC__) || defined(__clang__)
18+
__attribute__((optimize("no-tree-vectorize")))
19+
#endif
1720
inline static float ip_naive(const void *pVect1v, const void *pVect2v, const void *qty_ptr)
1821
{
1922
float *a = (float *) pVect1v;
@@ -31,7 +34,7 @@ namespace deglib::distances::fp32_ip {
3134
dot1 = a[1] * b[1];
3235
dot2 = a[2] * b[2];
3336
dot3 = a[3] * b[3];
34-
result += dot0 + dot1 + dot2 + dot3;
37+
result = (((result + dot0) + dot1) + dot2) + dot3;
3538
a += 4;
3639
b += 4;
3740
}
@@ -126,16 +129,16 @@ namespace deglib::distances::fp32_ip {
126129

127130
__m128 sum128 = _mm_setzero_ps();
128131
while (a < last) {
129-
sum128 = _mm_fmadd_ps(_mm_loadu_ps(a), _mm_loadu_ps(b), sum128);
132+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(a), _mm_loadu_ps(b)));
130133
a += 4;
131134
b += 4;
132-
sum128 = _mm_fmadd_ps(_mm_loadu_ps(a), _mm_loadu_ps(b), sum128);
135+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(a), _mm_loadu_ps(b)));
133136
a += 4;
134137
b += 4;
135-
sum128 = _mm_fmadd_ps(_mm_loadu_ps(a), _mm_loadu_ps(b), sum128);
138+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(a), _mm_loadu_ps(b)));
136139
a += 4;
137140
b += 4;
138-
sum128 = _mm_fmadd_ps(_mm_loadu_ps(a), _mm_loadu_ps(b), sum128);
141+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(a), _mm_loadu_ps(b)));
139142
a += 4;
140143
b += 4;
141144
}
@@ -190,10 +193,10 @@ namespace deglib::distances::fp32_ip {
190193

191194
__m128 sum128 = _mm_setzero_ps();
192195
while (a < last) {
193-
sum128 = _mm_fmadd_ps(_mm_loadu_ps(a), _mm_loadu_ps(b), sum128);
196+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(a), _mm_loadu_ps(b)));
194197
a += 4;
195198
b += 4;
196-
sum128 = _mm_fmadd_ps(_mm_loadu_ps(a), _mm_loadu_ps(b), sum128);
199+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(a), _mm_loadu_ps(b)));
197200
a += 4;
198201
b += 4;
199202
}
@@ -222,7 +225,7 @@ namespace deglib::distances::fp32_ip {
222225

223226
__m128 sum128 = _mm_setzero_ps();
224227
while (a < last) {
225-
sum128 = _mm_fmadd_ps(_mm_loadu_ps(a), _mm_loadu_ps(b), sum128);
228+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(a), _mm_loadu_ps(b)));
226229
a += 4;
227230
b += 4;
228231
}

‎cpp/deglib/include/distance/fp32_l2.h‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ namespace deglib::distances::fp32_l2 {
1010
// Scalar fallback — no SIMD required.
1111
class L2Float {
1212
public:
13+
#if defined(__GNUC__) || defined(__clang__)
14+
__attribute__((optimize("no-tree-vectorize")))
15+
#endif
1316
inline static float compare(const void *pVect1v, const void *pVect2v, const void *qty_ptr)
1417
{
1518
float *a = (float *) pVect1v;
@@ -27,11 +30,11 @@ namespace deglib::distances::fp32_l2 {
2730
diff1 = a[1] - b[1];
2831
diff2 = a[2] - b[2];
2932
diff3 = a[3] - b[3];
30-
result += diff0 * diff0 + diff1 * diff1 + diff2 * diff2 + diff3 * diff3;
33+
result = (((result + diff0 * diff0) + diff1 * diff1) + diff2 * diff2) + diff3 * diff3;
3134
a += 4;
3235
b += 4;
3336
}
34-
// Process last 0-3 elements. Not needed for standard vector lengths.
37+
// Process last 0-3 elements. Not needed for standard vector lengths.
3538
while (a < last) {
3639
diff0 = *a++ - *b++;
3740
result += diff0 * diff0;
@@ -136,19 +139,19 @@ namespace deglib::distances::fp32_l2 {
136139
__m128 v;
137140
while (a < last) {
138141
v = _mm_sub_ps(_mm_loadu_ps(a), _mm_loadu_ps(b));
139-
sum128 = _mm_fmadd_ps(v, v, sum128);
142+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(v, v));
140143
a += 4;
141144
b += 4;
142145
v = _mm_sub_ps(_mm_loadu_ps(a), _mm_loadu_ps(b));
143-
sum128 = _mm_fmadd_ps(v, v, sum128);
146+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(v, v));
144147
a += 4;
145148
b += 4;
146149
v = _mm_sub_ps(_mm_loadu_ps(a), _mm_loadu_ps(b));
147-
sum128 = _mm_fmadd_ps(v, v, sum128);
150+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(v, v));
148151
a += 4;
149152
b += 4;
150153
v = _mm_sub_ps(_mm_loadu_ps(a), _mm_loadu_ps(b));
151-
sum128 = _mm_fmadd_ps(v, v, sum128);
154+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(v, v));
152155
a += 4;
153156
b += 4;
154157
}
@@ -199,11 +202,11 @@ namespace deglib::distances::fp32_l2 {
199202
__m128 v;
200203
while (a < last) {
201204
v = _mm_sub_ps(_mm_loadu_ps(a), _mm_loadu_ps(b));
202-
sum128 = _mm_fmadd_ps(v, v, sum128);
205+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(v, v));
203206
a += 4;
204207
b += 4;
205208
v = _mm_sub_ps(_mm_loadu_ps(a), _mm_loadu_ps(b));
206-
sum128 = _mm_fmadd_ps(v, v, sum128);
209+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(v, v));
207210
a += 4;
208211
b += 4;
209212
}
@@ -230,7 +233,7 @@ namespace deglib::distances::fp32_l2 {
230233
__m128 v;
231234
while (a < last) {
232235
v = _mm_sub_ps(_mm_loadu_ps(a), _mm_loadu_ps(b));
233-
sum128 = _mm_fmadd_ps(v, v, sum128);
236+
sum128 = _mm_add_ps(sum128, _mm_mul_ps(v, v));
234237
a += 4;
235238
b += 4;
236239
}

‎cpp/test/src/regression/builder/test_l2_builder_regression.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ TEST(DeglibBuilderRegressionL2, Benchmark_LowLID)
1717

1818
auto gt_data = compute_groundtruth_l2(base_data, base_count, query_data, query_count, dim, 10);
1919

20-
run_regression_test("LowLID", deglib::Metric::L2, 30000.0, 9.3, 0.99,
20+
run_regression_test("LowLID", deglib::Metric::L2, 27000.0, 9.3, 0.988,
2121
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
2222
deglib::distances::fp32_l2::L2Float{}, 100,
2323
deglib::builder::OptimizationTarget::LowLID);
@@ -36,7 +36,7 @@ TEST(DeglibBuilderRegressionL2, Benchmark_HighLID)
3636

3737
auto gt_data = compute_groundtruth_l2(base_data, base_count, query_data, query_count, dim, 10);
3838

39-
run_regression_test("HighLID", deglib::Metric::L2, 16000.0, 10.0, 0.918,
39+
run_regression_test("HighLID", deglib::Metric::L2, 16000.0, 10.0, 0.887,
4040
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
4141
deglib::distances::fp32_l2::L2Float{}, 100,
4242
deglib::builder::OptimizationTarget::HighLID);
@@ -55,7 +55,7 @@ TEST(DeglibBuilderRegressionL2, Benchmark_StreamingData)
5555

5656
auto gt_data = compute_groundtruth_l2(base_data, base_count, query_data, query_count, dim, 10);
5757

58-
run_regression_test("StreamingData", deglib::Metric::L2, 23000.0, 23.8, 0.95,
58+
run_regression_test("StreamingData", deglib::Metric::L2, 24000.0, 23.8, 0.95,
5959
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
6060
deglib::distances::fp32_l2::L2Float{}, 100,
6161
deglib::builder::OptimizationTarget::StreamingData);

‎cpp/test/src/regression/metric/test_innerproduct_regression.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ TEST(DeglibRegressionIP, Benchmark_SSE_16Ext)
6767

6868
auto gt_data = compute_groundtruth_innerproduct(base_data, base_count, query_data, query_count, dim, 10);
6969

70-
run_regression_test("SSE_16Ext", deglib::Metric::InnerProduct, 16000.0, 11.8, 0.82,
70+
run_regression_test("SSE_16Ext", deglib::Metric::InnerProduct, 16000.0, 11.8, 0.81,
7171
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
7272
deglib::distances::fp32_ip::InnerProductFloat16Ext_SSE{}, 50);
7373
#else
@@ -88,7 +88,7 @@ TEST(DeglibRegressionIP, Benchmark_Scalar)
8888

8989
auto gt_data = compute_groundtruth_innerproduct(base_data, base_count, query_data, query_count, dim, 10);
9090

91-
run_regression_test("Scalar", deglib::Metric::InnerProduct, 13500.0, 15.8, 0.8,
91+
run_regression_test("Scalar", deglib::Metric::InnerProduct, 13500.0, 15.8, 0.83,
9292
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
9393
deglib::distances::fp32_ip::InnerProductFloat{}, 50);
9494
}

‎cpp/test/src/regression/metric/test_l2_regression.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ TEST(DeglibRegressionL2, Benchmark_SSE_16Ext)
6767

6868
auto gt_data = compute_groundtruth_l2(base_data, base_count, query_data, query_count, dim, 10);
6969

70-
run_regression_test("SSE_16Ext", deglib::Metric::L2, 33000.0, 7.0, 0.93,
70+
run_regression_test("SSE_16Ext", deglib::Metric::L2, 33000.0, 7.0, 0.94,
7171
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
7272
deglib::distances::fp32_l2::L2Float16Ext_SSE{}, 100);
7373
#else
@@ -88,7 +88,7 @@ TEST(DeglibRegressionL2, Benchmark_Scalar)
8888

8989
auto gt_data = compute_groundtruth_l2(base_data, base_count, query_data, query_count, dim, 10);
9090

91-
run_regression_test("Scalar", deglib::Metric::L2, 29000.0, 9.3, 0.99,
91+
run_regression_test("Scalar", deglib::Metric::L2, 29000.0, 9.3, 0.988,
9292
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
9393
deglib::distances::fp32_l2::L2Float{}, 100);
9494
}

‎cpp/test/src/regression/metric/test_l2_uint8_regression.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ TEST(DeglibRegressionL2Uint8, Benchmark_AVX512_Ext32)
1717

1818
auto gt_data = compute_groundtruth_l2_uint8(base_data, base_count, query_data, query_count, dim, 10);
1919

20-
run_regression_test("AVX512_Ext32", deglib::Metric::L2_Uint8, 77000.0, 4.6, 0.98,
20+
run_regression_test("AVX512_Ext32", deglib::Metric::L2_Uint8, 78000.0, 4.6, 0.98,
2121
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
2222
deglib::distances::uint8_l2::L2Uint8Ext32_AVX512{}, 500);
2323
#else
@@ -42,7 +42,7 @@ TEST(DeglibRegressionL2Uint8, Benchmark_AVX2_Ext32)
4242

4343
auto gt_data = compute_groundtruth_l2_uint8(base_data, base_count, query_data, query_count, dim, 10);
4444

45-
run_regression_test("AVX2_Ext32", deglib::Metric::L2_Uint8, 78000.0, 4.6, 0.98,
45+
run_regression_test("AVX2_Ext32", deglib::Metric::L2_Uint8, 789000.0, 4.6, 0.98,
4646
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
4747
deglib::distances::uint8_l2::L2Uint8Ext32_AVX2{}, 500);
4848
#else
@@ -67,7 +67,7 @@ TEST(DeglibRegressionL2Uint8, Benchmark_SSE_Ext32)
6767

6868
auto gt_data = compute_groundtruth_l2_uint8(base_data, base_count, query_data, query_count, dim, 10);
6969

70-
run_regression_test("SSE_Ext32", deglib::Metric::L2_Uint8, 73000.0, 5.1, 0.98,
70+
run_regression_test("SSE_Ext32", deglib::Metric::L2_Uint8, 74000.0, 5.1, 0.98,
7171
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
7272
deglib::distances::uint8_l2::L2Uint8Ext32_SSE{}, 500);
7373
#else
@@ -88,7 +88,7 @@ TEST(DeglibRegressionL2Uint8, Benchmark_Scalar)
8888

8989
auto gt_data = compute_groundtruth_l2_uint8(base_data, base_count, query_data, query_count, dim, 10);
9090

91-
run_regression_test("Scalar", deglib::Metric::L2_Uint8, 66000.0, 5.5, 0.98,
91+
run_regression_test("Scalar", deglib::Metric::L2_Uint8, 67000.0, 5.5, 0.98,
9292
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
9393
deglib::distances::uint8_l2::L2Uint8{}, 500);
9494
}

0 commit comments

Comments
 (0)