Implement more intrinsics.

common/math_private.h

@@ -28,6 +28,11 @@
 #elif (defined __clang__ || defined __GNUC__) && (defined __i386__ || defined __amd64__)
 #include <x86intrin.h>
 #define USE_SSE 1
+#elif defined(__aarch64__) || defined(_M_ARM64)
+#include <arm_neon.h>
+#define USE_NEON 1
+#elif (defined(__ARM_FP) && ((__ARM_FP & 0XC) == 0xC))
+#define USE_VFP3 1
 #endif
 #endif
 

double/e_sqrt.c

@@ -93,6 +93,19 @@ __ieee754_sqrt(double x)
 	double ret;
 	_mm_store_sd(&ret, _mm_sqrt_pd(_mm_load_sd(&x)));
 	return ret;
+#elif defined USE_NEON
+	double ret;
+    float64x1_t a = {0};
+    a = vset_lane_f64(x, a, 0);
+    float64x1_t r = vsqrt_f64(a);
+    ret = vget_lane_f64(r, 0);
+	return ret;
+#elif defined USE_VFP3
+	double ret;
+    __asm__ ("vsqrt.f64 %0, %1" 
+             : "=w" (ret) 
+             : "w" (x));
+	return ret;
 #else
 	double z;
 	int32_t sign = (int)0x80000000;

double/s_fabs.c

@@ -20,8 +20,20 @@
 double
 gm_fabs(double x)
 {
+#if defined USE_SSE && (defined __amd64__ || defined _M_AMD64)
+	// Testing shows this only improves performance on x86_64 builds
+	double ret;
+	const union {
+		unsigned long long __ui;
+		double __uf;
+	} sign_mask = { 0x7FFFFFFFFFFFFFFF };
+
+	_mm_store_sd(&ret, _mm_and_pd(_mm_load_sd(&sign_mask.__uf), _mm_load_sd(&x)));
+	return ret;
+#else
 	uint32_t high;
 	GET_HIGH_WORD(high,x);
 	SET_HIGH_WORD(x,high&0x7fffffff);
         return x;
+#endif
 }

float/e_sqrtf.c

@@ -25,6 +25,19 @@ __ieee754_sqrtf(float x)
 	float ret;
 	_mm_store_ss(&ret, _mm_sqrt_ps(_mm_load_ss(&x)));
 	return ret;
+#elif defined USE_NEON
+	float ret;
+    float32x2_t a = {0};
+    a = vset_lane_f32(x, a, 0);
+    float32x2_t r = vsqrt_f32(a);
+    ret = vget_lane_f32(r, 0);
+	return ret;
+#elif defined USE_VFP3
+	float ret;
+    __asm__ ("vsqrt.f32 %0, %1" 
+             : "=w" (ret) 
+             : "w" (x));
+	return ret;
 #else
 	float z;
 	int32_t sign = (int)0x80000000;

float/s_fabsf.c

@@ -23,8 +23,20 @@
 float
 gm_fabsf(float x)
 {
+#if defined USE_SSE && (defined __amd64__ || defined _M_AMD64)
+	// Testing shows this only improves performance on x86_64 builds
+	float ret;
+	const union {
+		unsigned __ui;
+		float __uf;
+	} sign_mask = { 0x7FFFFFFF };
+
+	_mm_store_ss(&ret, _mm_and_ps(_mm_load_ss(&sign_mask.__uf), _mm_load_ss(&x)));
+	return ret;
+#else
 	uint32_t ix;
 	GET_FLOAT_WORD(ix,x);
 	SET_FLOAT_WORD(x,ix&0x7fffffff);
         return x;
+#endif
 }