From 2b9ef62e7a484ceddaeee4609252bf61dd2f9582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Branimir=20Karad=C5=BEi=C4=87?= Date: Wed, 2 Sep 2026 19:13:20 +0000 Subject: [PATCH] Added BX_CONFIG_FMA, SIMD madd/msub/nmsub, f32 rounding and integer div/mod. (#421) --- include/bx/config.h | 16 + include/bx/inline/math.inl | 239 ++++++++++-- include/bx/inline/simd128_neon.inl | 56 ++- include/bx/inline/simd128_ref.inl | 59 ++- include/bx/inline/simd128_sse.inl | 146 ++++++- include/bx/inline/simd128_wasm.inl | 30 ++ include/bx/inline/simd256_avx.inl | 150 ++++++- include/bx/inline/simd256_ref.inl | 119 ++++-- include/bx/inline/simd32_ref.inl | 82 +++- include/bx/inline/simd64_ref.inl | 48 +-- include/bx/inline/simd_impl.inl | 608 ++++++++++++++++++++++++++++- include/bx/inline/simd_ni.inl | 566 +++++++++++++++++++++++++-- include/bx/macros.h | 23 ++ include/bx/math.h | 3 +- include/bx/simd_t.h | 218 +++++++++++ scripts/toolchain.lua | 1 - tests/math_test.cpp | 147 ++++++- tests/simd_test.cpp | 519 ++++++++++++++++++++++++ tests/vsnprintf_test.cpp | 2 - 19 files changed, 2858 insertions(+), 174 deletions(-) diff --git a/include/bx/config.h b/include/bx/config.h index 7a22c15..cbe205a 100644 --- a/include/bx/config.h +++ b/include/bx/config.h @@ -20,4 +20,20 @@ ) #endif // BX_CONFIG_SUPPORTS_THREADING +/// Use fused multiply-add instructions for bx::mad and simd_f32_madd/msub/nmsub. +#ifndef BX_CONFIG_FMA +# if BX_CPU_X86 \ + && (defined(__FMA__) || (BX_COMPILER_MSVC && defined(__AVX2__) ) ) +# define BX_CONFIG_FMA 1 +# elif BX_CPU_ARM \ + && defined(__ARM_FEATURE_FMA) \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__) ) +# define BX_CONFIG_FMA 1 +# elif BX_CPU_ARM && BX_ARCH_64BIT && BX_COMPILER_MSVC +# define BX_CONFIG_FMA 1 +# else +# define BX_CONFIG_FMA 0 +# endif +#endif // BX_CONFIG_FMA + #endif // BX_CONFIG_H_HEADER_GUARD diff --git a/include/bx/inline/math.inl b/include/bx/inline/math.inl index 7ab877f..d2d171d 100644 --- a/include/bx/inline/math.inl +++ b/include/bx/inline/math.inl @@ -109,27 +109,159 @@ namespace bx return tmp == kDoubleExponentMask; } - inline BX_CONSTEXPR_FUNC float floor(float _a) + inline BX_CONSTEXPR_FUNC float truncRef(float _a) { - if (_a < 0.0f) - { - const float fr = fract(-_a); - const float tr = trunc(-_a); + const uint32_t bits = floatToBits(_a); + const uint32_t sign = bits & kFloatSignMask; + const uint32_t magnitude = bits & ~kFloatSignMask; - return -tr - float(0.0f != fr); + if (0x4b000000u <= magnitude) + { + return _a; } - return _a - fract(_a); + const float tr = float(int(_a) ); + const uint32_t trBits = floatToBits(tr) | sign; + const float result = bitsToFloat(trBits); + return result; + } + + inline BX_CONSTEXPR_FUNC float floorRef(float _a) + { + const uint32_t bits = floatToBits(_a); + const uint32_t sign = bits & kFloatSignMask; + const uint32_t magnitude = bits & ~kFloatSignMask; + + if (0x4b000000u <= magnitude) + { + return _a; + } + + const float tr = float(int(_a) ); + const float fl = tr > _a ? tr - 1.0f : tr; + const uint32_t flBits = floatToBits(fl) | sign; + const float result = bitsToFloat(flBits); + + return result; + } + + inline BX_CONSTEXPR_FUNC float ceilRef(float _a) + { + const float na = -_a; + const float fl = floorRef(na); + const float result = -fl; + return result; + } + +#if BX_SIMD_SUPPORTED + inline BX_CONST_FUNC float truncSimd(float _a) + { + const simd128_t aa = simd_splat(_a); + const simd128_t result = simd_f32_trunc(aa); + + float out = 0.0f; + simd_x32_st1(&out, result); + + return out; + } + + inline BX_CONST_FUNC float floorSimd(float _a) + { + const simd128_t aa = simd_splat(_a); + const simd128_t result = simd_f32_floor(aa); + + float out = 0.0f; + simd_x32_st1(&out, result); + + return out; + } + + inline BX_CONST_FUNC float ceilSimd(float _a) + { + const simd128_t aa = simd_splat(_a); + const simd128_t result = simd_f32_ceil(aa); + + float out = 0.0f; + simd_x32_st1(&out, result); + + return out; + } +#endif // BX_SIMD_SUPPORTED + + inline BX_CONSTEXPR_FUNC float floor(float _a) + { +#if BX_SIMD_SUPPORTED + if (isConstantEvaluated() ) + { + return floorRef(_a); + } + + return floorSimd(_a); +#else + return floorRef(_a); +#endif // BX_SIMD_SUPPORTED } inline BX_CONSTEXPR_FUNC float ceil(float _a) { - return -floor(-_a); +#if BX_SIMD_SUPPORTED + if (isConstantEvaluated() ) + { + return ceilRef(_a); + } + + return ceilSimd(_a); +#else + return ceilRef(_a); +#endif // BX_SIMD_SUPPORTED } + inline BX_CONSTEXPR_FUNC float roundRef(float _a) + { + const uint32_t bits = floatToBits(_a); + const uint32_t sign = bits & kFloatSignMask; + const uint32_t magnitude = bits & ~kFloatSignMask; + + if (0x4b000000u <= magnitude) + { + return _a; + } + + const float fl = floorRef(_a); + const float fr = _a - fl; + const bool odd = 0 != (int32_t(fl) & 1); + const bool up = fr > 0.5f || (fr == 0.5f && odd); + const float rd = up ? fl + 1.0f : fl; + const uint32_t rdBits = floatToBits(rd) | sign; + const float result = bitsToFloat(rdBits); + return result; + } + +#if BX_SIMD_SUPPORTED + inline BX_CONST_FUNC float roundSimd(float _a) + { + const simd128_t aa = simd_splat(_a); + const simd128_t result = simd_f32_round(aa); + + float out = 0.0f; + simd_x32_st1(&out, result); + + return out; + } +#endif // BX_SIMD_SUPPORTED + inline BX_CONSTEXPR_FUNC float round(float _a) { - return floor(_a + 0.5f); +#if BX_SIMD_SUPPORTED + if (isConstantEvaluated() ) + { + return roundRef(_a); + } + + return roundSimd(_a); +#else + return roundRef(_a); +#endif // BX_SIMD_SUPPORTED } inline BX_CONSTEXPR_FUNC float lerp(float _a, float _b, float _t) @@ -153,13 +285,18 @@ namespace bx inline BX_CONSTEXPR_FUNC bool signBit(float _a) { - return -0.0f == _a ? 0.0f != _a : 0.0f > _a; + const uint32_t bits = floatToBits(_a); + return 0 != (bits & kFloatSignMask); } inline BX_CONSTEXPR_FUNC float copySign(float _value, float _sign) { #if BX_COMPILER_MSVC - return signBit(_value) != signBit(_sign) ? -_value : _value; + const uint32_t magnitude = floatToBits(_value) & ~kFloatSignMask; + const uint32_t sign = floatToBits(_sign) & kFloatSignMask; + const uint32_t bits = magnitude | sign; + const float result = bitsToFloat(bits); + return result; #else return __builtin_copysign(_value, _sign); #endif // BX_COMPILER_MSVC @@ -177,7 +314,16 @@ namespace bx inline BX_CONSTEXPR_FUNC float trunc(float _a) { - return float(int(_a) ); +#if BX_SIMD_SUPPORTED + if (isConstantEvaluated() ) + { + return truncRef(_a); + } + + return truncSimd(_a); +#else + return truncRef(_a); +#endif // BX_SIMD_SUPPORTED } inline BX_CONSTEXPR_FUNC float fract(float _a) @@ -187,7 +333,10 @@ namespace bx inline BX_CONSTEXPR_FUNC float nms(float _a, float _b, float _c) { - return _c - _a * _b; + const float na = -_a; + const float result = mad(na, _b, _c); + + return result; } inline BX_CONSTEXPR_FUNC float add(float _a, float _b) @@ -252,7 +401,12 @@ namespace bx inline BX_CONSTEXPR_FUNC float mad(float _a, float _b, float _c) { - return add(mul(_a, _b), _c); + const simd32_t aa = simd32_ld(_a); + const simd32_t bb = simd32_ld(_b); + const simd32_t cc = simd32_ld(_c); + const simd32_t result = simd32_f32_madd(aa, bb, cc); + + return bitCast(result); } inline BX_CONSTEXPR_FUNC float rcp(float _a) @@ -275,14 +429,19 @@ namespace bx return mul(_a, rcpSafe(_b) ); } +BX_FP_PRECISE_BEGIN() + inline BX_CONSTEXPR_FUNC float mod(float _a, float _b) { - return _a - _b * floor(div(_a, _b) ); + const float quotient = _a / _b; + const float whole = floor(quotient); + const float result = nms(_b, whole, _a); + return result; } inline BX_CONSTEXPR_FUNC float cos(float _a) { - const float scaled = _a * 2.0f*kInvPi; + const float scaled = _a * (2.0f*kInvPi); const float real = floor(scaled); const float xx = _a - real * kPiHalf; const int32_t bits = int32_t(real) & 3; @@ -412,9 +571,11 @@ namespace bx const float maxaxy = max(ax, ay); const float minaxy = min(ax, ay); + const uint32_t ysign = floatToBits(_y) & kFloatSignMask; + if (maxaxy == 0.0f) { - return _y < 0.0f ? -0.0f : 0.0f; + return bitsToFloat(ysign); } constexpr float kAtan2C0 = -0.013480470f; @@ -424,17 +585,19 @@ namespace bx constexpr float kAtan2C4 = -0.332994597f; constexpr float kAtan2C5 = 0.999995630f; - const float mxy = minaxy / maxaxy; - const float mxysq = square(mxy); - const float tmp0 = mad(kAtan2C0, mxysq, kAtan2C1); - const float tmp1 = mad(tmp0, mxysq, kAtan2C2); - const float tmp2 = mad(tmp1, mxysq, kAtan2C3); - const float tmp3 = mad(tmp2, mxysq, kAtan2C4); - const float tmp4 = mad(tmp3, mxysq, kAtan2C5); - const float tmp5 = tmp4 * mxy; - const float tmp6 = ay > ax ? kPiHalf - tmp5 : tmp5; - const float tmp7 = _x < 0.0f ? kPi - tmp6 : tmp6; - const float result = _y < 0.0f ? -tmp7 : tmp7; + const float mxy = minaxy / maxaxy; + const float mxysq = square(mxy); + const float tmp0 = mad(kAtan2C0, mxysq, kAtan2C1); + const float tmp1 = mad(tmp0, mxysq, kAtan2C2); + const float tmp2 = mad(tmp1, mxysq, kAtan2C3); + const float tmp3 = mad(tmp2, mxysq, kAtan2C4); + const float tmp4 = mad(tmp3, mxysq, kAtan2C5); + const float tmp5 = tmp4 * mxy; + const float tmp6 = ay > ax ? kPiHalf - tmp5 : tmp5; + const float tmp7 = _x < 0.0f ? kPi - tmp6 : tmp6; + + const uint32_t bits = floatToBits(tmp7) | ysign; + const float result = bitsToFloat(bits); return result; } @@ -588,6 +751,8 @@ namespace bx return log(_a) * kInvLogNat2; } +BX_FP_PRECISE_END() + template<> inline BX_CONSTEXPR_FUNC uint8_t countBits(uint32_t _val) { @@ -1199,17 +1364,29 @@ namespace bx inline BX_CONSTEXPR_FUNC Vec3 nms(const Vec3& _a, const Vec3& _b, const Vec3& _c) { - return sub(_c, mul(_a, _b) ); + const float xx = nms(_a.x, _b.x, _c.x); + const float yy = nms(_a.y, _b.y, _c.y); + const float zz = nms(_a.z, _b.z, _c.z); + + return Vec3(xx, yy, zz); } inline BX_CONSTEXPR_FUNC Vec3 mad(const Vec3& _a, const float _b, const Vec3& _c) { - return add(mul(_a, _b), _c); + const float xx = mad(_a.x, _b, _c.x); + const float yy = mad(_a.y, _b, _c.y); + const float zz = mad(_a.z, _b, _c.z); + + return Vec3(xx, yy, zz); } inline BX_CONSTEXPR_FUNC Vec3 mad(const Vec3& _a, const Vec3& _b, const Vec3& _c) { - return add(mul(_a, _b), _c); + const float xx = mad(_a.x, _b.x, _c.x); + const float yy = mad(_a.y, _b.y, _c.y); + const float zz = mad(_a.z, _b.z, _c.z); + + return Vec3(xx, yy, zz); } inline BX_CONSTEXPR_FUNC float dot(const Vec3& _a, const Vec3& _b) diff --git a/include/bx/inline/simd128_neon.inl b/include/bx/inline/simd128_neon.inl index da8f604..168da07 100644 --- a/include/bx/inline/simd128_neon.inl +++ b/include/bx/inline/simd128_neon.inl @@ -334,7 +334,7 @@ namespace bx return vrndnq_f32(_a); #else return simd_f32_round_ni(_a); -#endif +#endif // BX_ARCH_64BIT } template<> @@ -344,7 +344,7 @@ namespace bx return vrndpq_f32(_a); #else return simd_f32_ceil_ni(_a); -#endif +#endif // BX_ARCH_64BIT } template<> @@ -354,7 +354,17 @@ namespace bx return vrndmq_f32(_a); #else return simd_f32_floor_ni(_a); -#endif +#endif // BX_ARCH_64BIT + } + + template<> + BX_SIMD_FORCE_INLINE simd128_neon_t simd128_f32_trunc(simd128_neon_t _a) + { +#if BX_ARCH_64BIT + return vrndq_f32(_a); +#else + return simd_f32_trunc_ni(_a); +#endif // BX_ARCH_64BIT } template<> @@ -382,7 +392,7 @@ namespace bx return vdivq_f32(_a, _b); #else return simd_f32_div_nr_ni(_a, _b); -#endif +#endif // BX_ARCH_64BIT } template<> @@ -621,6 +631,30 @@ namespace bx return vreinterpretq_f32_u32(result); } + template<> + BX_SIMD_FORCE_INLINE simd128_neon_t simd128_i32_div(simd128_neon_t _a, simd128_neon_t _b) + { + return simd_i32_div_ni(_a, _b); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_neon_t simd128_i32_mod(simd128_neon_t _a, simd128_neon_t _b) + { + return simd_i32_mod_ni(_a, _b); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_neon_t simd128_u32_div(simd128_neon_t _a, simd128_neon_t _b) + { + return simd_u32_div_ni(_a, _b); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_neon_t simd128_u32_mod(simd128_neon_t _a, simd128_neon_t _b) + { + return simd_u32_mod_ni(_a, _b); + } + template<> BX_SIMD_FORCE_INLINE simd128_neon_t simd128_u32_cmpgt(simd128_neon_t _a, simd128_neon_t _b) { @@ -912,27 +946,33 @@ namespace bx template<> BX_SIMD_FORCE_INLINE simd128_neon_t simd128_f32_madd(simd128_neon_t _a, simd128_neon_t _b, simd128_neon_t _c) { -#if BX_ARCH_64BIT +#if BX_CONFIG_FMA return vfmaq_f32(_c, _a, _b); #else return simd_f32_madd_ni(_a, _b, _c); -#endif +#endif // BX_CONFIG_FMA } template<> BX_SIMD_FORCE_INLINE simd128_neon_t simd128_f32_msub(simd128_neon_t _a, simd128_neon_t _b, simd128_neon_t _c) { +#if BX_CONFIG_FMA + const simd128_neon_t nmsub = vfmsq_f32(_c, _a, _b); + const simd128_neon_t result = vnegq_f32(nmsub); + return result; +#else return simd_f32_msub_ni(_a, _b, _c); +#endif // BX_CONFIG_FMA } template<> BX_SIMD_FORCE_INLINE simd128_neon_t simd128_f32_nmsub(simd128_neon_t _a, simd128_neon_t _b, simd128_neon_t _c) { -#if BX_ARCH_64BIT +#if BX_CONFIG_FMA return vfmsq_f32(_c, _a, _b); #else return simd_f32_nmsub_ni(_a, _b, _c); -#endif +#endif // BX_CONFIG_FMA } template<> diff --git a/include/bx/inline/simd128_ref.inl b/include/bx/inline/simd128_ref.inl index 917d4a5..35c816a 100644 --- a/include/bx/inline/simd128_ref.inl +++ b/include/bx/inline/simd128_ref.inl @@ -557,6 +557,12 @@ namespace bx return simd_f32_floor_ni(_a); } + template<> + BX_SIMD_FORCE_INLINE simd128_ref_t simd128_f32_trunc(simd128_ref_t _a) + { + return simd_f32_trunc_ni(_a); + } + template<> inline BX_CONSTEXPR_FUNC simd128_ref_t simd128_f32_cmpeq(simd128_ref_t _a, simd128_ref_t _b) { @@ -913,6 +919,30 @@ namespace bx #endif // BX_SIMD_LANGEXT } + template<> + BX_SIMD_FORCE_INLINE simd128_ref_t simd128_i32_div(simd128_ref_t _a, simd128_ref_t _b) + { + return simd_i32_div_ni(_a, _b); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_ref_t simd128_i32_mod(simd128_ref_t _a, simd128_ref_t _b) + { + return simd_i32_mod_ni(_a, _b); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_ref_t simd128_u32_div(simd128_ref_t _a, simd128_ref_t _b) + { + return simd_u32_div_ni(_a, _b); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_ref_t simd128_u32_mod(simd128_ref_t _a, simd128_ref_t _b) + { + return simd_u32_mod_ni(_a, _b); + } + template<> inline BX_CONSTEXPR_FUNC simd128_ref_t simd128_u32_cmplt(simd128_ref_t _a, simd128_ref_t _b) { @@ -1350,19 +1380,42 @@ namespace bx template<> BX_SIMD_FORCE_INLINE simd128_ref_t simd128_f32_madd(simd128_ref_t _a, simd128_ref_t _b, simd128_ref_t _c) { - return simd_f32_madd_ni(_a, _b, _c); + const simd128_f32_ref_t a = bitCast(_a); + const simd128_f32_ref_t b = bitCast(_b); + const simd128_f32_ref_t c = bitCast(_c); + + simd128_f32_ref_t result = {}; + + for (uint32_t ii = 0; ii < 4; ++ii) + { + const simd32_t aa = simd32_ld(a.f32[ii]); + const simd32_t bb = simd32_ld(b.f32[ii]); + const simd32_t cc = simd32_ld(c.f32[ii]); + const simd32_t rr = simd32_f32_madd(aa, bb, cc); + result.f32[ii] = bitCast(rr); + } + + return bitCast(result); } template<> BX_SIMD_FORCE_INLINE simd128_ref_t simd128_f32_msub(simd128_ref_t _a, simd128_ref_t _b, simd128_ref_t _c) { - return simd_f32_msub_ni(_a, _b, _c); + const simd128_f32_ref_t c = bitCast(_c); + const simd128_f32_ref_t nc = { { -c.f32[0], -c.f32[1], -c.f32[2], -c.f32[3] } }; + const simd128_ref_t ncc = bitCast(nc); + const simd128_ref_t result = simd128_f32_madd(_a, _b, ncc); + return result; } template<> BX_SIMD_FORCE_INLINE simd128_ref_t simd128_f32_nmsub(simd128_ref_t _a, simd128_ref_t _b, simd128_ref_t _c) { - return simd_f32_nmsub_ni(_a, _b, _c); + const simd128_f32_ref_t a = bitCast(_a); + const simd128_f32_ref_t na = { { -a.f32[0], -a.f32[1], -a.f32[2], -a.f32[3] } }; + const simd128_ref_t naa = bitCast(na); + const simd128_ref_t result = simd128_f32_madd(naa, _b, _c); + return result; } template<> diff --git a/include/bx/inline/simd128_sse.inl b/include/bx/inline/simd128_sse.inl index 5ae6ab3..25d98a3 100644 --- a/include/bx/inline/simd128_sse.inl +++ b/include/bx/inline/simd128_sse.inl @@ -308,6 +308,12 @@ namespace bx return _mm_floor_ps(_a); } + template<> + BX_SIMD_FORCE_INLINE simd128_sse_t simd128_f32_trunc(simd128_sse_t _a) + { + return _mm_round_ps(_a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + } + template<> BX_SIMD_FORCE_INLINE simd128_sse_t simd128_f32_add(simd128_sse_t _a, simd128_sse_t _b) { @@ -556,6 +562,114 @@ namespace bx return simd_u32_cmplt_ni(_a, _b); } + BX_SIMD_FORCE_INLINE __m128i simd128_sse_i32_quot(__m128i _a, __m128i _b) + { + const __m128i ahi = _mm_unpackhi_epi64(_a, _a); + const __m128i bhi = _mm_unpackhi_epi64(_b, _b); + const __m128d alo_d = _mm_cvtepi32_pd(_a); + const __m128d ahi_d = _mm_cvtepi32_pd(ahi); + const __m128d blo_d = _mm_cvtepi32_pd(_b); + const __m128d bhi_d = _mm_cvtepi32_pd(bhi); + const __m128d qlo = _mm_div_pd(alo_d, blo_d); + const __m128d qhi = _mm_div_pd(ahi_d, bhi_d); + const __m128i ilo = _mm_cvttpd_epi32(qlo); + const __m128i ihi = _mm_cvttpd_epi32(qhi); + const __m128i result = _mm_unpacklo_epi64(ilo, ihi); + return result; + } + + BX_SIMD_FORCE_INLINE __m128i simd128_sse_u32_quot(__m128i _a, __m128i _b) + { + // The signed convert, plus 2^32 where the top bit was set. + const __m128d two32 = _mm_set1_pd(4294967296.0); + const __m128i aneg = _mm_srai_epi32(_a, 31); + const __m128i bneg = _mm_srai_epi32(_b, 31); + const __m128i ahi = _mm_unpackhi_epi64(_a, _a); + const __m128i bhi = _mm_unpackhi_epi64(_b, _b); + const __m128i anhi = _mm_unpackhi_epi64(aneg, aneg); + const __m128i bnhi = _mm_unpackhi_epi64(bneg, bneg); + const __m128d alo_s = _mm_cvtepi32_pd(_a); + const __m128d ahi_s = _mm_cvtepi32_pd(ahi); + const __m128d blo_s = _mm_cvtepi32_pd(_b); + const __m128d bhi_s = _mm_cvtepi32_pd(bhi); + const __m128d anlo_d = _mm_cvtepi32_pd(aneg); + const __m128d anhi_d = _mm_cvtepi32_pd(anhi); + const __m128d bnlo_d = _mm_cvtepi32_pd(bneg); + const __m128d bnhi_d = _mm_cvtepi32_pd(bnhi); + const __m128d alo_f = _mm_mul_pd(anlo_d, two32); + const __m128d ahi_f = _mm_mul_pd(anhi_d, two32); + const __m128d blo_f = _mm_mul_pd(bnlo_d, two32); + const __m128d bhi_f = _mm_mul_pd(bnhi_d, two32); + const __m128d alo_d = _mm_sub_pd(alo_s, alo_f); + const __m128d ahi_d = _mm_sub_pd(ahi_s, ahi_f); + const __m128d blo_d = _mm_sub_pd(blo_s, blo_f); + const __m128d bhi_d = _mm_sub_pd(bhi_s, bhi_f); + const __m128d qlo = _mm_div_pd(alo_d, blo_d); + const __m128d qhi = _mm_div_pd(ahi_d, bhi_d); + const __m128i ilo = _mm_cvttpd_epi32(qlo); + const __m128i ihi = _mm_cvttpd_epi32(qhi); + const __m128i result = _mm_unpacklo_epi64(ilo, ihi); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd128_sse_t simd128_i32_div(simd128_sse_t _a, simd128_sse_t _b) + { + const __m128i a = _mm_castps_si128(_a); + const __m128i b = _mm_castps_si128(_b); + const __m128i zero = _mm_setzero_si128(); + const __m128i quot = simd128_sse_i32_quot(a, b); + const __m128i bzero = _mm_cmpeq_epi32(b, zero); + const __m128i result = _mm_blendv_epi8(quot, a, bzero); + return _mm_castsi128_ps(result); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_sse_t simd128_i32_mod(simd128_sse_t _a, simd128_sse_t _b) + { + const __m128i a = _mm_castps_si128(_a); + const __m128i b = _mm_castps_si128(_b); + const __m128i zero = _mm_setzero_si128(); + const __m128i quot = simd128_sse_i32_quot(a, b); + const __m128i prod = _mm_mullo_epi32(quot, b); + const __m128i rem = _mm_sub_epi32(a, prod); + const __m128i bzero = _mm_cmpeq_epi32(b, zero); + const __m128i result = _mm_andnot_si128(bzero, rem); + return _mm_castsi128_ps(result); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_sse_t simd128_u32_div(simd128_sse_t _a, simd128_sse_t _b) + { + const __m128i a = _mm_castps_si128(_a); + const __m128i b = _mm_castps_si128(_b); + const __m128i zero = _mm_setzero_si128(); + const __m128i one = _mm_set1_epi32(1); + const __m128i quot = simd128_sse_u32_quot(a, b); + const __m128i bzero = _mm_cmpeq_epi32(b, zero); + const __m128i bone = _mm_cmpeq_epi32(b, one); + const __m128i keep = _mm_or_si128(bzero, bone); + const __m128i result = _mm_blendv_epi8(quot, a, keep); + return _mm_castsi128_ps(result); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_sse_t simd128_u32_mod(simd128_sse_t _a, simd128_sse_t _b) + { + const __m128i a = _mm_castps_si128(_a); + const __m128i b = _mm_castps_si128(_b); + const __m128i zero = _mm_setzero_si128(); + const __m128i one = _mm_set1_epi32(1); + const __m128i quot0 = simd128_sse_u32_quot(a, b); + const __m128i bone = _mm_cmpeq_epi32(b, one); + const __m128i quot = _mm_blendv_epi8(quot0, a, bone); + const __m128i prod = _mm_mullo_epi32(quot, b); + const __m128i rem = _mm_sub_epi32(a, prod); + const __m128i bzero = _mm_cmpeq_epi32(b, zero); + const __m128i result = _mm_andnot_si128(bzero, rem); + return _mm_castsi128_ps(result); + } + template<> BX_SIMD_FORCE_INLINE simd128_sse_t simd128_u32_cmpgt(simd128_sse_t _a, simd128_sse_t _b) { @@ -821,23 +935,49 @@ namespace bx return simd128_orx_ni(_a); } +#if BX_CONFIG_FMA template<> BX_SIMD_FORCE_INLINE simd128_sse_t simd128_f32_madd(simd128_sse_t _a, simd128_sse_t _b, simd128_sse_t _c) { - return simd_f32_madd_ni(_a, _b, _c); + return _mm_fmadd_ps(_a, _b, _c); } template<> BX_SIMD_FORCE_INLINE simd128_sse_t simd128_f32_msub(simd128_sse_t _a, simd128_sse_t _b, simd128_sse_t _c) { - return simd_f32_msub_ni(_a, _b, _c); + return _mm_fmsub_ps(_a, _b, _c); } template<> BX_SIMD_FORCE_INLINE simd128_sse_t simd128_f32_nmsub(simd128_sse_t _a, simd128_sse_t _b, simd128_sse_t _c) { - return simd_f32_nmsub_ni(_a, _b, _c); + return _mm_fnmadd_ps(_a, _b, _c); } +#else + template<> + BX_SIMD_FORCE_INLINE simd128_sse_t simd128_f32_madd(simd128_sse_t _a, simd128_sse_t _b, simd128_sse_t _c) + { + const simd128_sse_t prod = _mm_mul_ps(_a, _b); + const simd128_sse_t result = _mm_add_ps(prod, _c); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd128_sse_t simd128_f32_msub(simd128_sse_t _a, simd128_sse_t _b, simd128_sse_t _c) + { + const simd128_sse_t prod = _mm_mul_ps(_a, _b); + const simd128_sse_t result = _mm_sub_ps(prod, _c); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd128_sse_t simd128_f32_nmsub(simd128_sse_t _a, simd128_sse_t _b, simd128_sse_t _c) + { + const simd128_sse_t prod = _mm_mul_ps(_a, _b); + const simd128_sse_t result = _mm_sub_ps(_c, prod); + return result; + } +#endif // BX_CONFIG_FMA template<> BX_SIMD_FORCE_INLINE int simd128_x32_signbitsmask(simd128_sse_t _a) diff --git a/include/bx/inline/simd128_wasm.inl b/include/bx/inline/simd128_wasm.inl index f4059c8..f34e1f1 100644 --- a/include/bx/inline/simd128_wasm.inl +++ b/include/bx/inline/simd128_wasm.inl @@ -299,6 +299,12 @@ namespace bx return wasm_f32x4_floor(_a); } + template<> + BX_SIMD_FORCE_INLINE simd128_wasm_t simd128_f32_trunc(simd128_wasm_t _a) + { + return wasm_f32x4_trunc(_a); + } + template<> BX_SIMD_FORCE_INLINE simd128_wasm_t simd128_f32_add(simd128_wasm_t _a, simd128_wasm_t _b) { @@ -512,6 +518,30 @@ namespace bx return wasm_u32x4_lt(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd128_wasm_t simd128_i32_div(simd128_wasm_t _a, simd128_wasm_t _b) + { + return simd_i32_div_ni(_a, _b); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_wasm_t simd128_i32_mod(simd128_wasm_t _a, simd128_wasm_t _b) + { + return simd_i32_mod_ni(_a, _b); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_wasm_t simd128_u32_div(simd128_wasm_t _a, simd128_wasm_t _b) + { + return simd_u32_div_ni(_a, _b); + } + + template<> + BX_SIMD_FORCE_INLINE simd128_wasm_t simd128_u32_mod(simd128_wasm_t _a, simd128_wasm_t _b) + { + return simd_u32_mod_ni(_a, _b); + } + template<> BX_SIMD_FORCE_INLINE simd128_wasm_t simd128_u32_cmpgt(simd128_wasm_t _a, simd128_wasm_t _b) { diff --git a/include/bx/inline/simd256_avx.inl b/include/bx/inline/simd256_avx.inl index ad904a3..9b009e8 100644 --- a/include/bx/inline/simd256_avx.inl +++ b/include/bx/inline/simd256_avx.inl @@ -178,23 +178,49 @@ namespace bx return _mm256_rsqrt_ps(_a); } +#if BX_CONFIG_FMA template<> BX_SIMD_FORCE_INLINE simd256_avx_t simd256_f32_madd(simd256_avx_t _a, simd256_avx_t _b, simd256_avx_t _c) { - return simd_f32_madd_ni(_a, _b, _c); + return _mm256_fmadd_ps(_a, _b, _c); } template<> BX_SIMD_FORCE_INLINE simd256_avx_t simd256_f32_msub(simd256_avx_t _a, simd256_avx_t _b, simd256_avx_t _c) { - return simd_f32_msub_ni(_a, _b, _c); + return _mm256_fmsub_ps(_a, _b, _c); } template<> BX_SIMD_FORCE_INLINE simd256_avx_t simd256_f32_nmsub(simd256_avx_t _a, simd256_avx_t _b, simd256_avx_t _c) { - return simd_f32_nmsub_ni(_a, _b, _c); + return _mm256_fnmadd_ps(_a, _b, _c); } +#else + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_f32_madd(simd256_avx_t _a, simd256_avx_t _b, simd256_avx_t _c) + { + const simd256_avx_t prod = _mm256_mul_ps(_a, _b); + const simd256_avx_t result = _mm256_add_ps(prod, _c); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_f32_msub(simd256_avx_t _a, simd256_avx_t _b, simd256_avx_t _c) + { + const simd256_avx_t prod = _mm256_mul_ps(_a, _b); + const simd256_avx_t result = _mm256_sub_ps(prod, _c); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_f32_nmsub(simd256_avx_t _a, simd256_avx_t _b, simd256_avx_t _c) + { + const simd256_avx_t prod = _mm256_mul_ps(_a, _b); + const simd256_avx_t result = _mm256_sub_ps(_c, prod); + return result; + } +#endif // BX_CONFIG_FMA template<> BX_SIMD_FORCE_INLINE simd256_avx_t simd256_f32_neg(simd256_avx_t _a) @@ -268,6 +294,12 @@ namespace bx return _mm256_floor_ps(_a); } + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_f32_trunc(simd256_avx_t _a) + { + return _mm256_round_ps(_a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + } + template<> BX_SIMD_FORCE_INLINE simd256_avx_t simd256_f32_cmpeq(simd256_avx_t _a, simd256_avx_t _b) { @@ -520,6 +552,62 @@ namespace bx return simd_u32_cmplt_ni(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_i32_div(simd256_avx_t _a, simd256_avx_t _b) + { + const simd128_sse_t alo = _mm256_castps256_ps128(_a); + const simd128_sse_t blo = _mm256_castps256_ps128(_b); + const simd128_sse_t ahi = _mm256_extractf128_ps(_a, 1); + const simd128_sse_t bhi = _mm256_extractf128_ps(_b, 1); + const simd128_sse_t lo = simd128_i32_div(alo, blo); + const simd128_sse_t hi = simd128_i32_div(ahi, bhi); + const simd256_avx_t lo256 = _mm256_castps128_ps256(lo); + const simd256_avx_t result = _mm256_insertf128_ps(lo256, hi, 1); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_i32_mod(simd256_avx_t _a, simd256_avx_t _b) + { + const simd128_sse_t alo = _mm256_castps256_ps128(_a); + const simd128_sse_t blo = _mm256_castps256_ps128(_b); + const simd128_sse_t ahi = _mm256_extractf128_ps(_a, 1); + const simd128_sse_t bhi = _mm256_extractf128_ps(_b, 1); + const simd128_sse_t lo = simd128_i32_mod(alo, blo); + const simd128_sse_t hi = simd128_i32_mod(ahi, bhi); + const simd256_avx_t lo256 = _mm256_castps128_ps256(lo); + const simd256_avx_t result = _mm256_insertf128_ps(lo256, hi, 1); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_u32_div(simd256_avx_t _a, simd256_avx_t _b) + { + const simd128_sse_t alo = _mm256_castps256_ps128(_a); + const simd128_sse_t blo = _mm256_castps256_ps128(_b); + const simd128_sse_t ahi = _mm256_extractf128_ps(_a, 1); + const simd128_sse_t bhi = _mm256_extractf128_ps(_b, 1); + const simd128_sse_t lo = simd128_u32_div(alo, blo); + const simd128_sse_t hi = simd128_u32_div(ahi, bhi); + const simd256_avx_t lo256 = _mm256_castps128_ps256(lo); + const simd256_avx_t result = _mm256_insertf128_ps(lo256, hi, 1); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_u32_mod(simd256_avx_t _a, simd256_avx_t _b) + { + const simd128_sse_t alo = _mm256_castps256_ps128(_a); + const simd128_sse_t blo = _mm256_castps256_ps128(_b); + const simd128_sse_t ahi = _mm256_extractf128_ps(_a, 1); + const simd128_sse_t bhi = _mm256_extractf128_ps(_b, 1); + const simd128_sse_t lo = simd128_u32_mod(alo, blo); + const simd128_sse_t hi = simd128_u32_mod(ahi, bhi); + const simd256_avx_t lo256 = _mm256_castps128_ps256(lo); + const simd256_avx_t result = _mm256_insertf128_ps(lo256, hi, 1); + return result; + } + template<> BX_SIMD_FORCE_INLINE simd256_avx_t simd256_u32_cmpgt(simd256_avx_t _a, simd256_avx_t _b) { @@ -1025,6 +1113,62 @@ namespace bx return simd_u32_cmplt_ni(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_i32_div(simd256_avx_t _a, simd256_avx_t _b) + { + const simd128_sse_t alo = _mm256_castps256_ps128(_a); + const simd128_sse_t blo = _mm256_castps256_ps128(_b); + const simd128_sse_t ahi = _mm256_extractf128_ps(_a, 1); + const simd128_sse_t bhi = _mm256_extractf128_ps(_b, 1); + const simd128_sse_t lo = simd128_i32_div(alo, blo); + const simd128_sse_t hi = simd128_i32_div(ahi, bhi); + const simd256_avx_t lo256 = _mm256_castps128_ps256(lo); + const simd256_avx_t result = _mm256_insertf128_ps(lo256, hi, 1); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_i32_mod(simd256_avx_t _a, simd256_avx_t _b) + { + const simd128_sse_t alo = _mm256_castps256_ps128(_a); + const simd128_sse_t blo = _mm256_castps256_ps128(_b); + const simd128_sse_t ahi = _mm256_extractf128_ps(_a, 1); + const simd128_sse_t bhi = _mm256_extractf128_ps(_b, 1); + const simd128_sse_t lo = simd128_i32_mod(alo, blo); + const simd128_sse_t hi = simd128_i32_mod(ahi, bhi); + const simd256_avx_t lo256 = _mm256_castps128_ps256(lo); + const simd256_avx_t result = _mm256_insertf128_ps(lo256, hi, 1); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_u32_div(simd256_avx_t _a, simd256_avx_t _b) + { + const simd128_sse_t alo = _mm256_castps256_ps128(_a); + const simd128_sse_t blo = _mm256_castps256_ps128(_b); + const simd128_sse_t ahi = _mm256_extractf128_ps(_a, 1); + const simd128_sse_t bhi = _mm256_extractf128_ps(_b, 1); + const simd128_sse_t lo = simd128_u32_div(alo, blo); + const simd128_sse_t hi = simd128_u32_div(ahi, bhi); + const simd256_avx_t lo256 = _mm256_castps128_ps256(lo); + const simd256_avx_t result = _mm256_insertf128_ps(lo256, hi, 1); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_avx_t simd256_u32_mod(simd256_avx_t _a, simd256_avx_t _b) + { + const simd128_sse_t alo = _mm256_castps256_ps128(_a); + const simd128_sse_t blo = _mm256_castps256_ps128(_b); + const simd128_sse_t ahi = _mm256_extractf128_ps(_a, 1); + const simd128_sse_t bhi = _mm256_extractf128_ps(_b, 1); + const simd128_sse_t lo = simd128_u32_mod(alo, blo); + const simd128_sse_t hi = simd128_u32_mod(ahi, bhi); + const simd256_avx_t lo256 = _mm256_castps128_ps256(lo); + const simd256_avx_t result = _mm256_insertf128_ps(lo256, hi, 1); + return result; + } + template<> BX_SIMD_FORCE_INLINE simd256_avx_t simd256_u32_cmpgt(simd256_avx_t _a, simd256_avx_t _b) { diff --git a/include/bx/inline/simd256_ref.inl b/include/bx/inline/simd256_ref.inl index 9f276e5..8ca60e7 100644 --- a/include/bx/inline/simd256_ref.inl +++ b/include/bx/inline/simd256_ref.inl @@ -260,49 +260,28 @@ namespace bx template<> inline BX_CONST_FUNC simd256_ref_t simd256_f32_madd(simd256_ref_t _a, simd256_ref_t _b, simd256_ref_t _c) { -#if BX_SIMD_LANGEXT - const simd256_f32_langext_t a = bitCast(_a); - const simd256_f32_langext_t b = bitCast(_b); - const simd256_f32_langext_t c = bitCast(_c); - const simd256_f32_langext_t prod = a * b; - const simd256_f32_langext_t sum = prod + c; - const simd256_ref_t result = bitCast(sum); + simd256_ref_t result; + result.lo = simd128_f32_madd(_a.lo, _b.lo, _c.lo); + result.hi = simd128_f32_madd(_a.hi, _b.hi, _c.hi); return result; -#else - return simd_f32_madd_ni(_a, _b, _c); -#endif // BX_SIMD_LANGEXT } template<> inline BX_CONST_FUNC simd256_ref_t simd256_f32_msub(simd256_ref_t _a, simd256_ref_t _b, simd256_ref_t _c) { -#if BX_SIMD_LANGEXT - const simd256_f32_langext_t a = bitCast(_a); - const simd256_f32_langext_t b = bitCast(_b); - const simd256_f32_langext_t c = bitCast(_c); - const simd256_f32_langext_t prod = a * b; - const simd256_f32_langext_t diff = prod - c; - const simd256_ref_t result = bitCast(diff); + simd256_ref_t result; + result.lo = simd128_f32_msub(_a.lo, _b.lo, _c.lo); + result.hi = simd128_f32_msub(_a.hi, _b.hi, _c.hi); return result; -#else - return simd_f32_msub_ni(_a, _b, _c); -#endif // BX_SIMD_LANGEXT } template<> inline BX_CONST_FUNC simd256_ref_t simd256_f32_nmsub(simd256_ref_t _a, simd256_ref_t _b, simd256_ref_t _c) { -#if BX_SIMD_LANGEXT - const simd256_f32_langext_t a = bitCast(_a); - const simd256_f32_langext_t b = bitCast(_b); - const simd256_f32_langext_t c = bitCast(_c); - const simd256_f32_langext_t prod = a * b; - const simd256_f32_langext_t diff = c - prod; - const simd256_ref_t result = bitCast(diff); + simd256_ref_t result; + result.lo = simd128_f32_nmsub(_a.lo, _b.lo, _c.lo); + result.hi = simd128_f32_nmsub(_a.hi, _b.hi, _c.hi); return result; -#else - return simd_f32_nmsub_ni(_a, _b, _c); -#endif // BX_SIMD_LANGEXT } template<> @@ -377,10 +356,10 @@ namespace bx inline BX_CONSTEXPR_FUNC simd256_ref_t simd256_f32_rcp_est(simd256_ref_t _a) { #if BX_SIMD_LANGEXT - const simd256_f32_langext_t one = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; + const simd256_f32_langext_t one = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; const simd256_f32_langext_t a = bitCast(_a); const simd256_f32_langext_t quot = one / a; - const simd256_ref_t result = bitCast(quot); + const simd256_ref_t result = bitCast(quot); return result; #else simd256_ref_t result; @@ -432,6 +411,12 @@ namespace bx return simd_f32_floor_ni(_a); } + template<> + BX_SIMD_FORCE_INLINE simd256_ref_t simd256_f32_trunc(simd256_ref_t _a) + { + return simd_f32_trunc_ni(_a); + } + template<> inline BX_CONSTEXPR_FUNC simd256_ref_t simd256_f32_cmpeq(simd256_ref_t _a, simd256_ref_t _b) { @@ -497,6 +482,40 @@ namespace bx #endif // BX_SIMD_LANGEXT } + template<> + inline BX_CONSTEXPR_FUNC simd256_ref_t simd256_f32_cmple(simd256_ref_t _a, simd256_ref_t _b) + { +#if BX_SIMD_LANGEXT + const simd256_f32_langext_t a = bitCast(_a); + const simd256_f32_langext_t b = bitCast(_b); + const simd256_f32_langext_t cmp = a <= b; + const simd256_ref_t result = bitCast(cmp); + return result; +#else + simd256_ref_t result; + result.lo = simd128_f32_cmple(_a.lo, _b.lo); + result.hi = simd128_f32_cmple(_a.hi, _b.hi); + return result; +#endif // BX_SIMD_LANGEXT + } + + template<> + inline BX_CONSTEXPR_FUNC simd256_ref_t simd256_f32_cmpge(simd256_ref_t _a, simd256_ref_t _b) + { +#if BX_SIMD_LANGEXT + const simd256_f32_langext_t a = bitCast(_a); + const simd256_f32_langext_t b = bitCast(_b); + const simd256_f32_langext_t cmp = a >= b; + const simd256_ref_t result = bitCast(cmp); + return result; +#else + simd256_ref_t result; + result.lo = simd128_f32_cmpge(_a.lo, _b.lo); + result.hi = simd128_f32_cmpge(_a.hi, _b.hi); + return result; +#endif // BX_SIMD_LANGEXT + } + template<> inline BX_CONSTEXPR_FUNC simd256_ref_t simd256_i32_add(simd256_ref_t _a, simd256_ref_t _b) { @@ -721,6 +740,42 @@ namespace bx #endif // BX_SIMD_LANGEXT } + template<> + BX_SIMD_FORCE_INLINE simd256_ref_t simd256_i32_div(simd256_ref_t _a, simd256_ref_t _b) + { + simd256_ref_t result; + result.lo = simd128_i32_div(_a.lo, _b.lo); + result.hi = simd128_i32_div(_a.hi, _b.hi); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_ref_t simd256_i32_mod(simd256_ref_t _a, simd256_ref_t _b) + { + simd256_ref_t result; + result.lo = simd128_i32_mod(_a.lo, _b.lo); + result.hi = simd128_i32_mod(_a.hi, _b.hi); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_ref_t simd256_u32_div(simd256_ref_t _a, simd256_ref_t _b) + { + simd256_ref_t result; + result.lo = simd128_u32_div(_a.lo, _b.lo); + result.hi = simd128_u32_div(_a.hi, _b.hi); + return result; + } + + template<> + BX_SIMD_FORCE_INLINE simd256_ref_t simd256_u32_mod(simd256_ref_t _a, simd256_ref_t _b) + { + simd256_ref_t result; + result.lo = simd128_u32_mod(_a.lo, _b.lo); + result.hi = simd128_u32_mod(_a.hi, _b.hi); + return result; + } + template<> inline BX_CONSTEXPR_FUNC simd256_ref_t simd256_u32_cmplt(simd256_ref_t _a, simd256_ref_t _b) { diff --git a/include/bx/inline/simd32_ref.inl b/include/bx/inline/simd32_ref.inl index 0569438..cd17f00 100644 --- a/include/bx/inline/simd32_ref.inl +++ b/include/bx/inline/simd32_ref.inl @@ -121,31 +121,82 @@ namespace bx return bitCast(result); } +BX_FP_PRECISE_BEGIN() + + inline BX_CONSTEXPR_FUNC simd32_t simd32_f32_madd_ref(simd32_t _a, simd32_t _b, simd32_t _c) + { + const double aa = double(bitCast(_a).f32); + const double bb = double(bitCast(_b).f32); + const double cc = double(bitCast(_c).f32); + const double prod = aa * bb; + const double sum = prod + cc; + + const uint64_t sumBits = bitCast(sum); + + // A non-finite sum is exact; the two-sum below would evaluate Inf - Inf. + if (kDoubleExponentMask == (sumBits & kDoubleExponentMask) ) + { + return simd32_ld(float(sum) ); + } + + const double hi = sum - prod; + const double tmp0 = sum - hi; + const double tmp1 = prod - tmp0; + const double tmp2 = cc - hi; + const double err = tmp1 + tmp2; + + const uint64_t bits = bitCast(sum); + const uint64_t exponent = bits & kDoubleExponentMask; + const bool inexact = 0.0 != err; + const bool even = 0 == (bits & 1); + const bool finite = kDoubleExponentMask != exponent; + const bool up = (0.0 < err) == (0.0 < sum); + const uint64_t nudged = up ? bits + 1 : bits - 1; + const uint64_t odd = (inexact && even && finite) ? nudged : bits; + const double result = bitCast(odd); + + return simd32_ld(float(result) ); + } + +BX_FP_PRECISE_END() + inline BX_CONSTEXPR_FUNC simd32_t simd32_f32_madd(simd32_t _a, simd32_t _b, simd32_t _c) { const simd32_f32_ref_t a = bitCast(_a); const simd32_f32_ref_t b = bitCast(_b); const simd32_f32_ref_t c = bitCast(_c); - const simd32_f32_ref_t result = { .f32 = a.f32 * b.f32 + c.f32 }; - return bitCast(result); + + if (isConstantEvaluated() ) + { + return simd32_f32_madd_ref(_a, _b, _c); + } + +#if BX_CONFIG_FMA +# if BX_COMPILER_MSVC + const float result = fmaf(a.f32, b.f32, c.f32); +# else + const float result = __builtin_fmaf(a.f32, b.f32, c.f32); +# endif // BX_COMPILER_MSVC + return simd32_ld(result); +#else + const float prod = a.f32 * b.f32; + const float result = prod + c.f32; + return simd32_ld(result); +#endif // BX_CONFIG_FMA } inline BX_CONSTEXPR_FUNC simd32_t simd32_f32_msub(simd32_t _a, simd32_t _b, simd32_t _c) { - const simd32_f32_ref_t a = bitCast(_a); - const simd32_f32_ref_t b = bitCast(_b); - const simd32_f32_ref_t c = bitCast(_c); - const simd32_f32_ref_t result = { .f32 = a.f32 * b.f32 - c.f32 }; - return bitCast(result); + const simd32_t nc = simd32_f32_neg(_c); + const simd32_t result = simd32_f32_madd(_a, _b, nc); + return result; } inline BX_CONSTEXPR_FUNC simd32_t simd32_f32_nmsub(simd32_t _a, simd32_t _b, simd32_t _c) { - const simd32_f32_ref_t a = bitCast(_a); - const simd32_f32_ref_t b = bitCast(_b); - const simd32_f32_ref_t c = bitCast(_c); - const simd32_f32_ref_t result = { .f32 = c.f32 - a.f32 * b.f32 }; - return bitCast(result); + const simd32_t na = simd32_f32_neg(_a); + const simd32_t result = simd32_f32_madd(na, _b, _c); + return result; } inline BX_CONSTEXPR_FUNC int simd32_x32_signbitsmask(simd32_t _a) @@ -278,6 +329,13 @@ namespace bx return bitCast(result); } + inline BX_CONSTEXPR_FUNC simd32_t simd32_f32_trunc(simd32_t _a) + { + const simd32_f32_ref_t a = bitCast(_a); + const simd32_f32_ref_t result = { .f32 = trunc(a.f32) }; + return bitCast(result); + } + inline BX_CONSTEXPR_FUNC simd32_t simd32_f32_cmpeq(simd32_t _a, simd32_t _b) { const simd32_f32_ref_t a = bitCast(_a); diff --git a/include/bx/inline/simd64_ref.inl b/include/bx/inline/simd64_ref.inl index b5a1d95..96220e1 100644 --- a/include/bx/inline/simd64_ref.inl +++ b/include/bx/inline/simd64_ref.inl @@ -419,41 +419,36 @@ namespace bx inline BX_CONSTEXPR_FUNC simd64_t simd64_f32_madd(simd64_t _a, simd64_t _b, simd64_t _c) { -#if BX_SIMD_LANGEXT - return bitCast(bitCast(_a) * bitCast(_b) + bitCast(_c)); -#else const simd64_f32_ref_t a = bitCast(_a); const simd64_f32_ref_t b = bitCast(_b); const simd64_f32_ref_t c = bitCast(_c); - const simd64_f32_ref_t result = { { a.f32[0] * b.f32[0] + c.f32[0], a.f32[1] * b.f32[1] + c.f32[1] } }; + + simd64_f32_ref_t result = {}; + + for (uint32_t ii = 0; ii < 2; ++ii) + { + const simd32_t aa = simd32_ld(a.f32[ii]); + const simd32_t bb = simd32_ld(b.f32[ii]); + const simd32_t cc = simd32_ld(c.f32[ii]); + const simd32_t rr = simd32_f32_madd(aa, bb, cc); + result.f32[ii] = bitCast(rr); + } + return bitCast(result); -#endif // BX_SIMD_LANGEXT } inline BX_CONSTEXPR_FUNC simd64_t simd64_f32_msub(simd64_t _a, simd64_t _b, simd64_t _c) { -#if BX_SIMD_LANGEXT - return bitCast(bitCast(_a) * bitCast(_b) - bitCast(_c)); -#else - const simd64_f32_ref_t a = bitCast(_a); - const simd64_f32_ref_t b = bitCast(_b); - const simd64_f32_ref_t c = bitCast(_c); - const simd64_f32_ref_t result = { { a.f32[0] * b.f32[0] - c.f32[0], a.f32[1] * b.f32[1] - c.f32[1] } }; - return bitCast(result); -#endif // BX_SIMD_LANGEXT + const simd64_t nc = simd64_f32_neg(_c); + const simd64_t result = simd64_f32_madd(_a, _b, nc); + return result; } inline BX_CONSTEXPR_FUNC simd64_t simd64_f32_nmsub(simd64_t _a, simd64_t _b, simd64_t _c) { -#if BX_SIMD_LANGEXT - return bitCast(bitCast(_c) - bitCast(_a) * bitCast(_b)); -#else - const simd64_f32_ref_t a = bitCast(_a); - const simd64_f32_ref_t b = bitCast(_b); - const simd64_f32_ref_t c = bitCast(_c); - const simd64_f32_ref_t result = { { c.f32[0] - a.f32[0] * b.f32[0], c.f32[1] - a.f32[1] * b.f32[1] } }; - return bitCast(result); -#endif // BX_SIMD_LANGEXT + const simd64_t na = simd64_f32_neg(_a); + const simd64_t result = simd64_f32_madd(na, _b, _c); + return result; } inline BX_CONSTEXPR_FUNC int simd64_x32_signbitsmask(simd64_t _a) @@ -587,6 +582,13 @@ namespace bx return bitCast(result); } + inline BX_CONSTEXPR_FUNC simd64_t simd64_f32_trunc(simd64_t _a) + { + const simd64_f32_ref_t a = bitCast(_a); + const simd64_f32_ref_t result = { { trunc(a.f32[0]), trunc(a.f32[1]) } }; + return bitCast(result); + } + inline BX_CONSTEXPR_FUNC simd64_t simd64_f32_cmpeq(simd64_t _a, simd64_t _b) { #if BX_SIMD_LANGEXT diff --git a/include/bx/inline/simd_impl.inl b/include/bx/inline/simd_impl.inl index 9cc7f9b..c5a78b0 100644 --- a/include/bx/inline/simd_impl.inl +++ b/include/bx/inline/simd_impl.inl @@ -48,6 +48,7 @@ namespace bx BX_CONSTEXPR_FUNC simd32_t simd32_f32_round(simd32_t _a); BX_CONSTEXPR_FUNC simd32_t simd32_f32_ceil(simd32_t _a); BX_CONSTEXPR_FUNC simd32_t simd32_f32_floor(simd32_t _a); + BX_CONSTEXPR_FUNC simd32_t simd32_f32_trunc(simd32_t _a); BX_CONSTEXPR_FUNC simd32_t simd32_f32_cmpeq(simd32_t _a, simd32_t _b); BX_CONSTEXPR_FUNC simd32_t simd32_f32_cmpneq(simd32_t _a, simd32_t _b); @@ -182,6 +183,7 @@ namespace bx BX_CONSTEXPR_FUNC simd64_t simd64_f32_round(simd64_t _a); BX_CONSTEXPR_FUNC simd64_t simd64_f32_ceil(simd64_t _a); BX_CONSTEXPR_FUNC simd64_t simd64_f32_floor(simd64_t _a); + BX_CONSTEXPR_FUNC simd64_t simd64_f32_trunc(simd64_t _a); BX_CONSTEXPR_FUNC simd64_t simd64_f32_cmpeq(simd64_t _a, simd64_t _b); BX_CONSTEXPR_FUNC simd64_t simd64_f32_cmpneq(simd64_t _a, simd64_t _b); BX_CONSTEXPR_FUNC simd64_t simd64_f32_cmplt(simd64_t _a, simd64_t _b); @@ -483,6 +485,11 @@ namespace bx template Ty simd128_f32_floor(Ty _a); + /// 4xf32 truncate toward zero. + /// + template + Ty simd128_f32_trunc(Ty _a); + /// 4xf32 compare equal (returns mask). /// template @@ -618,6 +625,26 @@ namespace bx template Ty simd128_u32_clamp(Ty _a, Ty _min, Ty _max); + /// 4xi32 division, x / 0 == x and INT32_MIN / -1 == INT32_MIN. + /// + template + Ty simd128_i32_div(Ty _a, Ty _b); + + /// 4xi32 remainder, x % 0 == 0 and INT32_MIN % -1 == 0. + /// + template + Ty simd128_i32_mod(Ty _a, Ty _b); + + /// 4xu32 division, x / 0 == x. + /// + template + Ty simd128_u32_div(Ty _a, Ty _b); + + /// 4xu32 remainder, x % 0 == 0. + /// + template + Ty simd128_u32_mod(Ty _a, Ty _b); + /// 4xu32 compare equal. /// template @@ -996,6 +1023,7 @@ namespace bx template Ty simd256_f32_round(Ty _a); template Ty simd256_f32_ceil(Ty _a); template Ty simd256_f32_floor(Ty _a); + template Ty simd256_f32_trunc(Ty _a); template Ty simd256_f32_cmpeq(Ty _a, Ty _b); template Ty simd256_f32_cmpneq(Ty _a, Ty _b); template Ty simd256_f32_cmplt(Ty _a, Ty _b); @@ -1017,6 +1045,10 @@ namespace bx template Ty simd256_u32_min(Ty _a, Ty _b); template Ty simd256_u32_max(Ty _a, Ty _b); template Ty simd256_u32_clamp(Ty _a, Ty _min, Ty _max); + template Ty simd256_i32_div(Ty _a, Ty _b); + template Ty simd256_i32_mod(Ty _a, Ty _b); + template Ty simd256_u32_div(Ty _a, Ty _b); + template Ty simd256_u32_mod(Ty _a, Ty _b); template Ty simd256_u32_cmpeq(Ty _a, Ty _b); template Ty simd256_u32_cmplt(Ty _a, Ty _b); template Ty simd256_u32_cmpgt(Ty _a, Ty _b); @@ -1220,6 +1252,48 @@ namespace bx template Ty simd_f32_exp_ni(Ty _a); + template + Ty simd_f32_trunc_ni(Ty _a); + + template + Ty simd_f32_fract_ni(Ty _a); + + template + Ty simd_f32_sign_ni(Ty _a); + + template + Ty simd_f32_step_ni(Ty _edge, Ty _a); + + template + Ty simd_f32_smoothstep_ni(Ty _a); + + template + Ty simd_f32_mod_ni(Ty _a, Ty _b); + + template + Ty simd_f32_tan_ni(Ty _a); + + template + Ty simd_f32_acos_ni(Ty _a); + + template + Ty simd_f32_asin_ni(Ty _a); + + template + Ty simd_f32_atan2_ni(Ty _y, Ty _x); + + template + Ty simd_f32_atan_ni(Ty _a); + + template + Ty simd_f32_sinh_ni(Ty _a); + + template + Ty simd_f32_cosh_ni(Ty _a); + + template + Ty simd_f32_tanh_ni(Ty _a); + template Ty simd_f32_cmpneq_ni(Ty _a, Ty _b); @@ -1271,6 +1345,63 @@ namespace bx template Ty simd_u32_cntlz_ni(Ty _a); + template + Ty simd_u32_cntbits_ni(Ty _a); + + template + Ty simd_u32_cnttz_ni(Ty _a); + + template + Ty simd_u32_reversebits_ni(Ty _a); + + template + Ty simd_i32_cmpneq_ni(Ty _a, Ty _b); + + template + Ty simd_i32_cmple_ni(Ty _a, Ty _b); + + template + Ty simd_i32_cmpge_ni(Ty _a, Ty _b); + + template + Ty simd_u32_cmpneq_ni(Ty _a, Ty _b); + + template + Ty simd_u32_cmple_ni(Ty _a, Ty _b); + + template + Ty simd_u32_cmpge_ni(Ty _a, Ty _b); + + template + Ty simd_u32_min_ni(Ty _a, Ty _b); + + template + Ty simd_u32_max_ni(Ty _a, Ty _b); + + template + Ty simd_u32_clamp_ni(Ty _a, Ty _min, Ty _max); + + template + Ty simd_i32_div_ni(Ty _a, Ty _b); + + template + Ty simd_i32_mod_ni(Ty _a, Ty _b); + + template + Ty simd_u32_div_ni(Ty _a, Ty _b); + + template + Ty simd_u32_mod_ni(Ty _a, Ty _b); + + template + Ty simd_f32_ftoi_sat_ni(Ty _a); + + template + Ty simd_f32_ftou_sat_ni(Ty _a); + + template + Ty simd_u32_utof_ni(Ty _a); + template Ty simd_x32_srl_ni(Ty _a, Ty _count); @@ -1294,14 +1425,14 @@ namespace bx } // namespace bx -#if BX_SIMD_AVX -# include "simd256_avx.inl" -#endif // BX_SIMD_AVX - #if BX_SIMD_SSE # include "simd128_sse.inl" #endif // BX_SIMD_SSE +#if BX_SIMD_AVX +# include "simd256_avx.inl" // After SSE: 256-bit fallbacks use the 128-bit ops. +#endif // BX_SIMD_AVX + #if BX_SIMD_NEON # include "simd128_neon.inl" #endif // BX_SIMD_NEON @@ -1315,6 +1446,7 @@ namespace bx BX_CONSTEXPR_FUNC float floor(float _f); BX_CONSTEXPR_FUNC float ceil(float _f); BX_CONSTEXPR_FUNC float round(float _f); + BX_CONSTEXPR_FUNC float trunc(float _f); BX_CONSTEXPR_FUNC float sqrt(float _a); } // namespace bx @@ -1490,10 +1622,10 @@ namespace bx BX_SIMD_FORCE_INLINE simd128_t simd_f32_cmpneq(simd128_t _a, simd128_t _b) { return simd128_f32_cmpneq(_a, _b); } template<> - BX_SIMD_FORCE_INLINE simd128_t simd_f32_cmple(simd128_t _a, simd128_t _b) { return simd128_f32_cmpgt(_b, _a); } + BX_SIMD_FORCE_INLINE simd128_t simd_f32_cmple(simd128_t _a, simd128_t _b) { return simd128_f32_cmple(_a, _b); } template<> - BX_SIMD_FORCE_INLINE simd128_t simd_f32_cmpge(simd128_t _a, simd128_t _b) { return simd128_f32_cmplt(_b, _a); } + BX_SIMD_FORCE_INLINE simd128_t simd_f32_cmpge(simd128_t _a, simd128_t _b) { return simd128_f32_cmpge(_a, _b); } template<> BX_SIMD_FORCE_INLINE simd128_t simd_f32_clamp(simd128_t _a, simd128_t _min, simd128_t _max) { return simd128_f32_clamp(_a, _min, _max); } @@ -1528,6 +1660,48 @@ namespace bx template<> BX_SIMD_FORCE_INLINE simd128_t simd_f32_ldexp(simd128_t _a, simd128_t _b) { return simd_f32_ldexp_ni(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_trunc(simd128_t _a) { return simd128_f32_trunc(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_fract(simd128_t _a) { return simd_f32_fract_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_sign(simd128_t _a) { return simd_f32_sign_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_step(simd128_t _edge, simd128_t _a) { return simd_f32_step_ni(_edge, _a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_smoothstep(simd128_t _a) { return simd_f32_smoothstep_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_mod(simd128_t _a, simd128_t _b) { return simd_f32_mod_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_tan(simd128_t _a) { return simd_f32_tan_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_acos(simd128_t _a) { return simd_f32_acos_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_asin(simd128_t _a) { return simd_f32_asin_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_atan2(simd128_t _y, simd128_t _x) { return simd_f32_atan2_ni(_y, _x); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_atan(simd128_t _a) { return simd_f32_atan_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_sinh(simd128_t _a) { return simd_f32_sinh_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_cosh(simd128_t _a) { return simd_f32_cosh_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_tanh(simd128_t _a) { return simd_f32_tanh_ni(_a); } + template<> BX_SIMD_FORCE_INLINE simd128_t simd_f32_log2(simd128_t _a) { return simd_f32_log2_ni(_a); } @@ -1648,6 +1822,72 @@ namespace bx template<> BX_SIMD_FORCE_INLINE simd128_t simd_u32_cmpgt(simd128_t _a, simd128_t _b) { return simd128_u32_cmpgt(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_i32_cmpneq(simd128_t _a, simd128_t _b) { return simd_i32_cmpneq_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_i32_cmple(simd128_t _a, simd128_t _b) { return simd_i32_cmple_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_i32_cmpge(simd128_t _a, simd128_t _b) { return simd_i32_cmpge_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_cmpneq(simd128_t _a, simd128_t _b) { return simd_u32_cmpneq_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_cmple(simd128_t _a, simd128_t _b) { return simd_u32_cmple_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_cmpge(simd128_t _a, simd128_t _b) { return simd_u32_cmpge_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_min(simd128_t _a, simd128_t _b) { return simd128_u32_min(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_max(simd128_t _a, simd128_t _b) { return simd128_u32_max(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_clamp(simd128_t _a, simd128_t _min, simd128_t _max) { return simd128_u32_clamp(_a, _min, _max); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_i32_div(simd128_t _a, simd128_t _b) { return simd128_i32_div(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_i32_mod(simd128_t _a, simd128_t _b) { return simd128_i32_mod(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_div(simd128_t _a, simd128_t _b) { return simd128_u32_div(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_mod(simd128_t _a, simd128_t _b) { return simd128_u32_mod(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_ftoi_sat(simd128_t _a) { return simd_f32_ftoi_sat_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f32_ftou_sat(simd128_t _a) { return simd_f32_ftou_sat_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_utof(simd128_t _a) { return simd_u32_utof_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_cntlz(simd128_t _a) { return simd_u32_cntlz_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_cnttz(simd128_t _a) { return simd_u32_cnttz_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_cntbits(simd128_t _a) { return simd_u32_cntbits_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_u32_reversebits(simd128_t _a) { return simd_u32_reversebits_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f16_fromf32(simd128_t _a) { return simd_f16_fromf32_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd128_t simd_f16_tof32(simd128_t _a) { return simd_f16_tof32_ni(_a); } + template<> BX_SIMD_FORCE_INLINE simd128_t simd_orc(simd128_t _a, simd128_t _b) { return simd128_orc(_a, _b); } @@ -1663,6 +1903,20 @@ namespace bx template<> BX_SIMD_FORCE_INLINE void simd_stu(void* _ptr, simd128_t _a) { simd128_stu(_ptr, _a); } +#if BX_SIMD_SSE || BX_SIMD_NEON || BX_SIMD_WASM + template<> + BX_SIMD_FORCE_INLINE simd128_ref_t simd_ld(const void* _ptr) { return simd128_ld(_ptr); } + + template<> + BX_SIMD_FORCE_INLINE simd128_ref_t simd_ldu(const void* _ptr) { return simd128_ldu(_ptr); } + + template<> + BX_SIMD_FORCE_INLINE void simd_st(void* _ptr, simd128_ref_t _a) { simd128_st(_ptr, _a); } + + template<> + BX_SIMD_FORCE_INLINE void simd_stu(void* _ptr, simd128_ref_t _a) { simd128_stu(_ptr, _a); } +#endif // BX_SIMD_SSE || BX_SIMD_NEON || BX_SIMD_WASM + template<> BX_SIMD_FORCE_INLINE void simd_x32_st1(void* _ptr, simd128_t _a) { simd128_x32_st1(_ptr, _a); } @@ -1781,10 +2035,10 @@ namespace bx BX_SIMD_FORCE_INLINE simd256_t simd_f32_cmpneq(simd256_t _a, simd256_t _b) { return simd256_f32_cmpneq(_a, _b); } template<> - BX_SIMD_FORCE_INLINE simd256_t simd_f32_cmple(simd256_t _a, simd256_t _b) { return simd256_f32_cmpgt(_b, _a); } + BX_SIMD_FORCE_INLINE simd256_t simd_f32_cmple(simd256_t _a, simd256_t _b) { return simd256_f32_cmple(_a, _b); } template<> - BX_SIMD_FORCE_INLINE simd256_t simd_f32_cmpge(simd256_t _a, simd256_t _b) { return simd256_f32_cmplt(_b, _a); } + BX_SIMD_FORCE_INLINE simd256_t simd_f32_cmpge(simd256_t _a, simd256_t _b) { return simd256_f32_cmpge(_a, _b); } template<> BX_SIMD_FORCE_INLINE simd256_t simd_f32_clamp(simd256_t _a, simd256_t _min, simd256_t _max) { return simd256_f32_clamp(_a, _min, _max); } @@ -1819,6 +2073,48 @@ namespace bx template<> BX_SIMD_FORCE_INLINE simd256_t simd_f32_ldexp(simd256_t _a, simd256_t _b) { return simd_f32_ldexp_ni(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_trunc(simd256_t _a) { return simd256_f32_trunc(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_fract(simd256_t _a) { return simd_f32_fract_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_sign(simd256_t _a) { return simd_f32_sign_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_step(simd256_t _edge, simd256_t _a) { return simd_f32_step_ni(_edge, _a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_smoothstep(simd256_t _a) { return simd_f32_smoothstep_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_mod(simd256_t _a, simd256_t _b) { return simd_f32_mod_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_tan(simd256_t _a) { return simd_f32_tan_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_acos(simd256_t _a) { return simd_f32_acos_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_asin(simd256_t _a) { return simd_f32_asin_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_atan2(simd256_t _y, simd256_t _x) { return simd_f32_atan2_ni(_y, _x); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_atan(simd256_t _a) { return simd_f32_atan_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_sinh(simd256_t _a) { return simd_f32_sinh_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_cosh(simd256_t _a) { return simd_f32_cosh_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_tanh(simd256_t _a) { return simd_f32_tanh_ni(_a); } + template<> BX_SIMD_FORCE_INLINE simd256_t simd_f32_log2(simd256_t _a) { return simd_f32_log2_ni(_a); } @@ -1936,6 +2232,72 @@ namespace bx template<> BX_SIMD_FORCE_INLINE simd256_t simd_u32_cmplt(simd256_t _a, simd256_t _b) { return simd256_u32_cmplt(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_i32_cmpneq(simd256_t _a, simd256_t _b) { return simd_i32_cmpneq_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_i32_cmple(simd256_t _a, simd256_t _b) { return simd_i32_cmple_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_i32_cmpge(simd256_t _a, simd256_t _b) { return simd_i32_cmpge_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_cmpneq(simd256_t _a, simd256_t _b) { return simd_u32_cmpneq_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_cmple(simd256_t _a, simd256_t _b) { return simd_u32_cmple_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_cmpge(simd256_t _a, simd256_t _b) { return simd_u32_cmpge_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_min(simd256_t _a, simd256_t _b) { return simd256_u32_min(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_max(simd256_t _a, simd256_t _b) { return simd256_u32_max(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_clamp(simd256_t _a, simd256_t _min, simd256_t _max) { return simd256_u32_clamp(_a, _min, _max); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_i32_div(simd256_t _a, simd256_t _b) { return simd256_i32_div(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_i32_mod(simd256_t _a, simd256_t _b) { return simd256_i32_mod(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_div(simd256_t _a, simd256_t _b) { return simd256_u32_div(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_mod(simd256_t _a, simd256_t _b) { return simd256_u32_mod(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_ftoi_sat(simd256_t _a) { return simd_f32_ftoi_sat_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f32_ftou_sat(simd256_t _a) { return simd_f32_ftou_sat_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_utof(simd256_t _a) { return simd_u32_utof_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_cntlz(simd256_t _a) { return simd_u32_cntlz_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_cnttz(simd256_t _a) { return simd_u32_cnttz_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_cntbits(simd256_t _a) { return simd_u32_cntbits_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_u32_reversebits(simd256_t _a) { return simd_u32_reversebits_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f16_fromf32(simd256_t _a) { return simd_f16_fromf32_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd256_t simd_f16_tof32(simd256_t _a) { return simd_f16_tof32_ni(_a); } + template<> BX_SIMD_FORCE_INLINE simd256_t simd_u32_cmpgt(simd256_t _a, simd256_t _b) { return simd256_u32_cmpgt(_a, _b); } @@ -1954,6 +2316,20 @@ namespace bx template<> BX_SIMD_FORCE_INLINE void simd_stu(void* _ptr, simd256_t _a) { simd256_stu(_ptr, _a); } +#if BX_SIMD_AVX + template<> + BX_SIMD_FORCE_INLINE simd256_ref_t simd_ld(const void* _ptr) { return simd256_ld(_ptr); } + + template<> + BX_SIMD_FORCE_INLINE simd256_ref_t simd_ldu(const void* _ptr) { return simd256_ldu(_ptr); } + + template<> + BX_SIMD_FORCE_INLINE void simd_st(void* _ptr, simd256_ref_t _a) { simd256_st(_ptr, _a); } + + template<> + BX_SIMD_FORCE_INLINE void simd_stu(void* _ptr, simd256_ref_t _a) { simd256_stu(_ptr, _a); } +#endif // BX_SIMD_AVX + template<> BX_SIMD_FORCE_INLINE void simd_x32_st1(void* _ptr, simd256_t _a) { simd256_x32_st1(_ptr, _a); } @@ -2091,6 +2467,72 @@ namespace bx template<> BX_SIMD_FORCE_INLINE simd32_t simd_u32_cmplt(simd32_t _a, simd32_t _b) { return simd32_u32_cmplt(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_i32_cmpneq(simd32_t _a, simd32_t _b) { return simd_i32_cmpneq_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_i32_cmple(simd32_t _a, simd32_t _b) { return simd_i32_cmple_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_i32_cmpge(simd32_t _a, simd32_t _b) { return simd_i32_cmpge_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_cmpneq(simd32_t _a, simd32_t _b) { return simd_u32_cmpneq_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_cmple(simd32_t _a, simd32_t _b) { return simd_u32_cmple_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_cmpge(simd32_t _a, simd32_t _b) { return simd_u32_cmpge_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_min(simd32_t _a, simd32_t _b) { return simd_u32_min_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_max(simd32_t _a, simd32_t _b) { return simd_u32_max_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_clamp(simd32_t _a, simd32_t _min, simd32_t _max) { return simd_u32_clamp_ni(_a, _min, _max); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_i32_div(simd32_t _a, simd32_t _b) { return simd_i32_div_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_i32_mod(simd32_t _a, simd32_t _b) { return simd_i32_mod_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_div(simd32_t _a, simd32_t _b) { return simd_u32_div_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_mod(simd32_t _a, simd32_t _b) { return simd_u32_mod_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_ftoi_sat(simd32_t _a) { return simd_f32_ftoi_sat_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_ftou_sat(simd32_t _a) { return simd_f32_ftou_sat_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_utof(simd32_t _a) { return simd_u32_utof_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_cntlz(simd32_t _a) { return simd_u32_cntlz_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_cnttz(simd32_t _a) { return simd_u32_cnttz_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_cntbits(simd32_t _a) { return simd_u32_cntbits_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_u32_reversebits(simd32_t _a) { return simd_u32_reversebits_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f16_fromf32(simd32_t _a) { return simd_f16_fromf32_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f16_tof32(simd32_t _a) { return simd_f16_tof32_ni(_a); } + template<> BX_SIMD_FORCE_INLINE simd32_t simd_u32_cmpgt(simd32_t _a, simd32_t _b) { return simd32_u32_cmpgt(_a, _b); } @@ -2226,6 +2668,48 @@ namespace bx template<> BX_SIMD_FORCE_INLINE simd32_t simd_f32_ldexp(simd32_t _a, simd32_t _b) { return simd_f32_ldexp_ni(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_trunc(simd32_t _a) { return simd32_f32_trunc(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_fract(simd32_t _a) { return simd_f32_fract_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_sign(simd32_t _a) { return simd_f32_sign_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_step(simd32_t _edge, simd32_t _a) { return simd_f32_step_ni(_edge, _a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_smoothstep(simd32_t _a) { return simd_f32_smoothstep_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_mod(simd32_t _a, simd32_t _b) { return simd_f32_mod_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_tan(simd32_t _a) { return simd_f32_tan_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_acos(simd32_t _a) { return simd_f32_acos_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_asin(simd32_t _a) { return simd_f32_asin_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_atan2(simd32_t _y, simd32_t _x) { return simd_f32_atan2_ni(_y, _x); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_atan(simd32_t _a) { return simd_f32_atan_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_sinh(simd32_t _a) { return simd_f32_sinh_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_cosh(simd32_t _a) { return simd_f32_cosh_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd32_t simd_f32_tanh(simd32_t _a) { return simd_f32_tanh_ni(_a); } + template<> BX_SIMD_FORCE_INLINE simd32_t simd_f32_ftoi_trunc(simd32_t _a) { return simd32_f32_ftoi_trunc(_a); } @@ -2363,6 +2847,72 @@ namespace bx template<> BX_SIMD_FORCE_INLINE simd64_t simd_u32_cmplt(simd64_t _a, simd64_t _b) { return simd64_u32_cmplt(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_i32_cmpneq(simd64_t _a, simd64_t _b) { return simd_i32_cmpneq_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_i32_cmple(simd64_t _a, simd64_t _b) { return simd_i32_cmple_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_i32_cmpge(simd64_t _a, simd64_t _b) { return simd_i32_cmpge_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_cmpneq(simd64_t _a, simd64_t _b) { return simd_u32_cmpneq_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_cmple(simd64_t _a, simd64_t _b) { return simd_u32_cmple_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_cmpge(simd64_t _a, simd64_t _b) { return simd_u32_cmpge_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_min(simd64_t _a, simd64_t _b) { return simd_u32_min_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_max(simd64_t _a, simd64_t _b) { return simd_u32_max_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_clamp(simd64_t _a, simd64_t _min, simd64_t _max) { return simd_u32_clamp_ni(_a, _min, _max); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_i32_div(simd64_t _a, simd64_t _b) { return simd_i32_div_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_i32_mod(simd64_t _a, simd64_t _b) { return simd_i32_mod_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_div(simd64_t _a, simd64_t _b) { return simd_u32_div_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_mod(simd64_t _a, simd64_t _b) { return simd_u32_mod_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_ftoi_sat(simd64_t _a) { return simd_f32_ftoi_sat_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_ftou_sat(simd64_t _a) { return simd_f32_ftou_sat_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_utof(simd64_t _a) { return simd_u32_utof_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_cntlz(simd64_t _a) { return simd_u32_cntlz_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_cnttz(simd64_t _a) { return simd_u32_cnttz_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_cntbits(simd64_t _a) { return simd_u32_cntbits_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_u32_reversebits(simd64_t _a) { return simd_u32_reversebits_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f16_fromf32(simd64_t _a) { return simd_f16_fromf32_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f16_tof32(simd64_t _a) { return simd_f16_tof32_ni(_a); } + template<> BX_SIMD_FORCE_INLINE simd64_t simd_u32_cmpgt(simd64_t _a, simd64_t _b) { return simd64_u32_cmpgt(_a, _b); } @@ -2498,6 +3048,48 @@ namespace bx template<> BX_SIMD_FORCE_INLINE simd64_t simd_f32_ldexp(simd64_t _a, simd64_t _b) { return simd_f32_ldexp_ni(_a, _b); } + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_trunc(simd64_t _a) { return simd64_f32_trunc(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_fract(simd64_t _a) { return simd_f32_fract_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_sign(simd64_t _a) { return simd_f32_sign_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_step(simd64_t _edge, simd64_t _a) { return simd_f32_step_ni(_edge, _a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_smoothstep(simd64_t _a) { return simd_f32_smoothstep_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_mod(simd64_t _a, simd64_t _b) { return simd_f32_mod_ni(_a, _b); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_tan(simd64_t _a) { return simd_f32_tan_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_acos(simd64_t _a) { return simd_f32_acos_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_asin(simd64_t _a) { return simd_f32_asin_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_atan2(simd64_t _y, simd64_t _x) { return simd_f32_atan2_ni(_y, _x); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_atan(simd64_t _a) { return simd_f32_atan_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_sinh(simd64_t _a) { return simd_f32_sinh_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_cosh(simd64_t _a) { return simd_f32_cosh_ni(_a); } + + template<> + BX_SIMD_FORCE_INLINE simd64_t simd_f32_tanh(simd64_t _a) { return simd_f32_tanh_ni(_a); } + template<> BX_SIMD_FORCE_INLINE simd64_t simd_f32_ftoi_trunc(simd64_t _a) { return simd64_f32_ftoi_trunc(_a); } diff --git a/include/bx/inline/simd_ni.inl b/include/bx/inline/simd_ni.inl index e22fec9..e1c7c09 100644 --- a/include/bx/inline/simd_ni.inl +++ b/include/bx/inline/simd_ni.inl @@ -29,28 +29,29 @@ namespace bx return result; } + // Without an fma instruction these round twice: the fastest fallback. + // simd32_f32_madd_ref is the exact software fma when that is what is needed. template BX_SIMD_INLINE Ty simd_f32_madd_ni(Ty _a, Ty _b, Ty _c) { - const Ty mul = simd_f32_mul(_a, _b); - const Ty result = simd_f32_add(mul, _c); + const Ty prod = simd_f32_mul(_a, _b); + const Ty result = simd_f32_add(prod, _c); return result; } template BX_SIMD_INLINE Ty simd_f32_nmsub_ni(Ty _a, Ty _b, Ty _c) { - const Ty mul = simd_f32_mul(_a, _b); - const Ty result = simd_f32_sub(_c, mul); + const Ty prod = simd_f32_mul(_a, _b); + const Ty result = simd_f32_sub(_c, prod); return result; } template BX_SIMD_INLINE Ty simd_f32_msub_ni(Ty _a, Ty _b, Ty _c) { - const Ty mul = simd_f32_mul(_a, _b); - const Ty result = simd_f32_sub(mul, _c); - + const Ty prod = simd_f32_mul(_a, _b); + const Ty result = simd_f32_sub(prod, _c); return result; } @@ -451,33 +452,64 @@ namespace bx template BX_SIMD_INLINE Ty simd_f32_ceil_ni(Ty _a) { - const Ty tmp = simd_f32_ftoi_trunc(_a); - const Ty tmp0 = simd_i32_itof(tmp); - const Ty mask = simd_f32_cmplt(tmp0, _a); - const Ty one = simd_splat(1.0f); - const Ty tmp1 = simd_and(one, mask); - const Ty result = simd_f32_add(tmp0, tmp1); + const Ty tmp = simd_f32_ftoi_trunc(_a); + const Ty tmp0 = simd_i32_itof(tmp); + const Ty mask = simd_f32_cmplt(tmp0, _a); + const Ty one = simd_splat(1.0f); + const Ty tmp1 = simd_and(one, mask); + const Ty up = simd_f32_add(tmp0, tmp1); + const Ty absmask = simd_splat(uint32_t(~kFloatSignMask) ); + const Ty bigLimit = simd_splat(uint32_t(0x4affffff) ); + const Ty magnitude = simd_and(_a, absmask); + const Ty bigMask = simd_u32_cmpgt(magnitude, bigLimit); + const Ty rounded = simd_selb(bigMask, _a, up); + // float->int->float drops the sign of zero; ceil(-0.5) is -0.0. + const Ty signmask = simd_splat(kFloatSignMask); + const Ty sign = simd_and(_a, signmask); + const Ty result = simd_or(rounded, sign); return result; } template BX_SIMD_INLINE Ty simd_f32_floor_ni(Ty _a) { - const Ty tmp = simd_f32_ftoi_trunc(_a); - const Ty tmp0 = simd_i32_itof(tmp); - const Ty mask = simd_f32_cmpgt(tmp0, _a); - const Ty one = simd_splat(1.0f); - const Ty tmp1 = simd_and(one, mask); - const Ty result = simd_f32_sub(tmp0, tmp1); + const Ty tmp = simd_f32_ftoi_trunc(_a); + const Ty tmp0 = simd_i32_itof(tmp); + const Ty mask = simd_f32_cmpgt(tmp0, _a); + const Ty one = simd_splat(1.0f); + const Ty tmp1 = simd_and(one, mask); + const Ty down = simd_f32_sub(tmp0, tmp1); + const Ty absmask = simd_splat(uint32_t(~kFloatSignMask) ); + const Ty bigLimit = simd_splat(uint32_t(0x4affffff) ); + const Ty magnitude = simd_and(_a, absmask); + const Ty bigMask = simd_u32_cmpgt(magnitude, bigLimit); + const Ty rounded = simd_selb(bigMask, _a, down); + const Ty signmask = simd_splat(kFloatSignMask); + const Ty sign = simd_and(_a, signmask); + const Ty result = simd_or(rounded, sign); return result; } template BX_SIMD_INLINE Ty simd_f32_round_ni(Ty _a) { - const Ty half = simd_splat(0.5f); - const Ty tmp0 = simd_f32_add(_a, half); - const Ty result = simd_f32_floor(tmp0); + const Ty half = simd_splat(0.5f); + const Ty one = simd_splat(1.0f); + const Ty ione = simd_splat(uint32_t(1) ); + const Ty signmask = simd_splat(kFloatSignMask); + const Ty fl = simd_f32_floor(_a); + const Ty fr = simd_f32_sub(_a, fl); + const Ty gtMask = simd_f32_cmpgt(fr, half); + const Ty eqMask = simd_f32_cmpeq(fr, half); + const Ty ifl = simd_f32_ftoi_trunc(fl); + const Ty lsb = simd_and(ifl, ione); + const Ty oddMask = simd_i32_cmpeq(lsb, ione); + const Ty tieUp = simd_and(eqMask, oddMask); + const Ty upMask = simd_or(gtMask, tieUp); + const Ty up = simd_and(upMask, one); + const Ty rd = simd_f32_add(fl, up); + const Ty asign = simd_and(_a, signmask); + const Ty result = simd_or(rd, asign); // rd is 0 or has _a's sign return result; } @@ -496,6 +528,8 @@ namespace bx return result; } +BX_FP_PRECISE_BEGIN() + template BX_SIMD_INLINE Ty simd_f32_cos_ni(Ty _a) { @@ -554,6 +588,197 @@ namespace bx return result; } + template + BX_SIMD_INLINE Ty simd_f32_trunc_ni(Ty _a) + { + const Ty zero = simd_zero(); + const Ty negMask = simd_f32_cmplt(_a, zero); + const Ty up = simd_f32_ceil(_a); + const Ty down = simd_f32_floor(_a); + const Ty result = simd_selb(negMask, up, down); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_fract_ni(Ty _a) + { + const Ty whole = simd_f32_trunc(_a); + const Ty result = simd_f32_sub(_a, whole); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_sign_ni(Ty _a) + { + const Ty zero = simd_zero(); + const Ty one = simd_splat(1.0f); + const Ty posMask = simd_f32_cmpgt(_a, zero); + const Ty negMask = simd_f32_cmplt(_a, zero); + const Ty pos = simd_and(posMask, one); + const Ty neg = simd_and(negMask, one); + const Ty result = simd_f32_sub(pos, neg); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_step_ni(Ty _edge, Ty _a) + { + const Ty one = simd_splat(1.0f); + const Ty ltMask = simd_f32_cmplt(_a, _edge); + const Ty result = simd_andc(one, ltMask); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_smoothstep_ni(Ty _a) + { + const Ty two = simd_splat(2.0f); + const Ty three = simd_splat(3.0f); + const Ty sq = simd_f32_mul(_a, _a); + const Ty twoA = simd_f32_mul(two, _a); + const Ty tmp0 = simd_f32_sub(three, twoA); + const Ty result = simd_f32_mul(sq, tmp0); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_mod_ni(Ty _a, Ty _b) + { + const Ty quotient = simd_f32_div(_a, _b); + const Ty whole = simd_f32_floor(quotient); + const Ty result = simd_f32_nmsub(_b, whole, _a); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_tan_ni(Ty _a) + { + const Ty sine = simd_f32_sin(_a); + const Ty cosine = simd_f32_cos(_a); + const Ty result = simd_f32_div(sine, cosine); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_acos_ni(Ty _a) + { + const Ty acosC0 = simd_splat( 1.5707288f); + const Ty acosC1 = simd_splat(-0.2121144f); + const Ty acosC2 = simd_splat( 0.0742610f); + const Ty acosC3 = simd_splat(-0.0187293f); + const Ty zero = simd_zero(); + const Ty one = simd_splat(1.0f); + const Ty two = simd_splat(2.0f); + const Ty pi = simd_splat(kPi); + const Ty absa = simd_f32_abs(_a); + const Ty tmp0 = simd_f32_madd(acosC3, absa, acosC2); + const Ty tmp1 = simd_f32_madd(tmp0, absa, acosC1); + const Ty tmp2 = simd_f32_madd(tmp1, absa, acosC0); + const Ty rest = simd_f32_sub(one, absa); + const Ty root = simd_f32_sqrt(rest); + const Ty tmp3 = simd_f32_mul(tmp2, root); + const Ty negMask = simd_f32_cmplt(_a, zero); + const Ty negate = simd_and(negMask, one); + const Ty twoNeg = simd_f32_mul(two, negate); + const Ty tmp3n = simd_f32_mul(twoNeg, tmp3); + const Ty tmp4 = simd_f32_sub(tmp3, tmp3n); + const Ty negPi = simd_f32_mul(negate, pi); + const Ty result = simd_f32_add(negPi, tmp4); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_asin_ni(Ty _a) + { + const Ty pi_half = simd_splat(kPiHalf); + const Ty ac = simd_f32_acos(_a); + const Ty result = simd_f32_sub(pi_half, ac); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_atan2_ni(Ty _y, Ty _x) + { + const Ty atan2C0 = simd_splat(-0.013480470f); + const Ty atan2C1 = simd_splat( 0.057477314f); + const Ty atan2C2 = simd_splat(-0.121239071f); + const Ty atan2C3 = simd_splat( 0.195635925f); + const Ty atan2C4 = simd_splat(-0.332994597f); + const Ty atan2C5 = simd_splat( 0.999995630f); + const Ty zero = simd_zero(); + const Ty signmask = simd_splat(kFloatSignMask); + const Ty pi = simd_splat(kPi); + const Ty pi_half = simd_splat(kPiHalf); + const Ty ax = simd_f32_abs(_x); + const Ty ay = simd_f32_abs(_y); + const Ty maxaxy = simd_f32_max(ax, ay); + const Ty minaxy = simd_f32_min(ax, ay); + const Ty zeroMask = simd_f32_cmpeq(maxaxy, zero); + const Ty ysign = simd_and(_y, signmask); + const Ty mxy = simd_f32_div(minaxy, maxaxy); + const Ty mxysq = simd_f32_mul(mxy, mxy); + const Ty tmp0 = simd_f32_madd(atan2C0, mxysq, atan2C1); + const Ty tmp1 = simd_f32_madd(tmp0, mxysq, atan2C2); + const Ty tmp2 = simd_f32_madd(tmp1, mxysq, atan2C3); + const Ty tmp3 = simd_f32_madd(tmp2, mxysq, atan2C4); + const Ty tmp4 = simd_f32_madd(tmp3, mxysq, atan2C5); + const Ty tmp5 = simd_f32_mul(tmp4, mxy); + const Ty gtMask = simd_f32_cmpgt(ay, ax); + const Ty fromHalf = simd_f32_sub(pi_half, tmp5); + const Ty tmp6 = simd_selb(gtMask, fromHalf, tmp5); + const Ty xNegMask = simd_f32_cmplt(_x, zero); + const Ty fromPi = simd_f32_sub(pi, tmp6); + const Ty tmp7 = simd_selb(xNegMask, fromPi, tmp6); + const Ty signed_ = simd_or(tmp7, ysign); + const Ty result = simd_selb(zeroMask, ysign, signed_); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_atan_ni(Ty _a) + { + const Ty one = simd_splat(1.0f); + const Ty result = simd_f32_atan2(_a, one); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_sinh_ni(Ty _a) + { + const Ty half = simd_splat(0.5f); + const Ty na = simd_f32_neg(_a); + const Ty ea = simd_f32_exp(_a); + const Ty ena = simd_f32_exp(na); + const Ty diff = simd_f32_sub(ea, ena); + const Ty result = simd_f32_mul(half, diff); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_cosh_ni(Ty _a) + { + const Ty half = simd_splat(0.5f); + const Ty na = simd_f32_neg(_a); + const Ty ea = simd_f32_exp(_a); + const Ty ena = simd_f32_exp(na); + const Ty sum = simd_f32_add(ea, ena); + const Ty result = simd_f32_mul(half, sum); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_tanh_ni(Ty _a) + { + const Ty one = simd_splat(1.0f); + const Ty two = simd_splat(2.0f); + const Ty twoA = simd_f32_mul(two, _a); + const Ty tmp0 = simd_f32_exp(twoA); + const Ty tmp1 = simd_f32_sub(tmp0, one); + const Ty tmp2 = simd_f32_add(tmp0, one); + const Ty result = simd_f32_div(tmp1, tmp2); + return result; + } + template BX_SIMD_INLINE Ty simd_f32_log_ni(Ty _a) { @@ -703,12 +928,14 @@ namespace bx const Ty result0 = simd_or(pwabs, asign); const Ty bmask = simd_f32_cmplt(absb, smallest); const Ty amask = simd_f32_cmplt(absa, smallest); - const Ty result1 = simd_selb(bmask, one, result0); - const Ty result = simd_selb(amask, zero, result1); + const Ty result1 = simd_selb(amask, zero, result0); + const Ty result = simd_selb(bmask, one, result1); return result; } +BX_FP_PRECISE_END() + template BX_SIMD_INLINE bool simd128_test_any_ni(Ty _a) { @@ -801,6 +1028,287 @@ namespace bx return result; } + template + BX_SIMD_INLINE Ty simd_u32_cntbits_ni(Ty _a) + { + const Ty c55 = simd_splat(uint32_t(0x55555555) ); + const Ty c33 = simd_splat(uint32_t(0x33333333) ); + const Ty c0f = simd_splat(uint32_t(0x0f0f0f0f) ); + const Ty c3f = simd_splat(uint32_t(0x3f) ); + + const Ty p1s = simd_x32_srl(_a, 1); + const Ty p1m = simd_and(p1s, c55); + const Ty p1 = simd_u32_sub(_a, p1m); + + const Ty p2a = simd_and(p1, c33); + const Ty p2s = simd_x32_srl(p1, 2); + const Ty p2m = simd_and(p2s, c33); + const Ty p2 = simd_u32_add(p2a, p2m); + + const Ty p4s = simd_x32_srl(p2, 4); + const Ty p4a = simd_u32_add(p2, p4s); + const Ty p4 = simd_and(p4a, c0f); + + const Ty p8s = simd_x32_srl(p4, 8); + const Ty p8 = simd_u32_add(p4, p8s); + + const Ty p16s = simd_x32_srl(p8, 16); + const Ty p16 = simd_u32_add(p8, p16s); + + const Ty result = simd_and(p16, c3f); + return result; + } + + template + BX_SIMD_INLINE Ty simd_u32_cnttz_ni(Ty _a) + { + const Ty one = simd_splat(uint32_t(1) ); + const Ty na = simd_i32_neg(_a); + const Ty lowest = simd_and(_a, na); + const Ty below = simd_u32_sub(lowest, one); + const Ty result = simd_u32_cntbits_ni(below); + return result; + } + + template + BX_SIMD_INLINE Ty simd_u32_reversebits_ni(Ty _a) + { + const Ty c55 = simd_splat(uint32_t(0x55555555) ); + const Ty c33 = simd_splat(uint32_t(0x33333333) ); + const Ty c0f = simd_splat(uint32_t(0x0f0f0f0f) ); + const Ty cff = simd_splat(uint32_t(0x00ff00ff) ); + + const Ty r1a = simd_x32_srl(_a, 1); + const Ty r1b = simd_and(r1a, c55); + const Ty r1c = simd_and(_a, c55); + const Ty r1d = simd_x32_sll(r1c, 1); + const Ty r1 = simd_or(r1b, r1d); + + const Ty r2a = simd_x32_srl(r1, 2); + const Ty r2b = simd_and(r2a, c33); + const Ty r2c = simd_and(r1, c33); + const Ty r2d = simd_x32_sll(r2c, 2); + const Ty r2 = simd_or(r2b, r2d); + + const Ty r4a = simd_x32_srl(r2, 4); + const Ty r4b = simd_and(r4a, c0f); + const Ty r4c = simd_and(r2, c0f); + const Ty r4d = simd_x32_sll(r4c, 4); + const Ty r4 = simd_or(r4b, r4d); + + const Ty r8a = simd_x32_srl(r4, 8); + const Ty r8b = simd_and(r8a, cff); + const Ty r8c = simd_and(r4, cff); + const Ty r8d = simd_x32_sll(r8c, 8); + const Ty r8 = simd_or(r8b, r8d); + + const Ty r16a = simd_x32_srl(r8, 16); + const Ty r16b = simd_x32_sll(r8, 16); + const Ty result = simd_or(r16a, r16b); + return result; + } + + template + BX_SIMD_INLINE Ty simd_i32_cmpneq_ni(Ty _a, Ty _b) + { + const Ty eq = simd_i32_cmpeq(_a, _b); + const Ty result = simd_not(eq); + return result; + } + + template + BX_SIMD_INLINE Ty simd_i32_cmple_ni(Ty _a, Ty _b) + { + const Ty gt = simd_i32_cmpgt(_a, _b); + const Ty result = simd_not(gt); + return result; + } + + template + BX_SIMD_INLINE Ty simd_i32_cmpge_ni(Ty _a, Ty _b) + { + const Ty lt = simd_i32_cmplt(_a, _b); + const Ty result = simd_not(lt); + return result; + } + + template + BX_SIMD_INLINE Ty simd_u32_cmpneq_ni(Ty _a, Ty _b) + { + const Ty eq = simd_i32_cmpeq(_a, _b); + const Ty result = simd_not(eq); + return result; + } + + template + BX_SIMD_INLINE Ty simd_u32_cmple_ni(Ty _a, Ty _b) + { + const Ty gt = simd_u32_cmpgt(_a, _b); + const Ty result = simd_not(gt); + return result; + } + + template + BX_SIMD_INLINE Ty simd_u32_cmpge_ni(Ty _a, Ty _b) + { + const Ty lt = simd_u32_cmplt(_a, _b); + const Ty result = simd_not(lt); + return result; + } + + template + BX_SIMD_INLINE Ty simd_u32_min_ni(Ty _a, Ty _b) + { + const Ty ltMask = simd_u32_cmplt(_a, _b); + const Ty result = simd_selb(ltMask, _a, _b); + return result; + } + + template + BX_SIMD_INLINE Ty simd_u32_max_ni(Ty _a, Ty _b) + { + const Ty gtMask = simd_u32_cmpgt(_a, _b); + const Ty result = simd_selb(gtMask, _a, _b); + return result; + } + + template + BX_SIMD_INLINE Ty simd_u32_clamp_ni(Ty _a, Ty _min, Ty _max) + { + const Ty lo = simd_u32_max(_a, _min); + const Ty result = simd_u32_min(lo, _max); + return result; + } + + template + BX_SIMD_INLINE Ty simd_i32_div_ni(Ty _a, Ty _b) + { + constexpr uint32_t kNumLanes = sizeof(Ty)/sizeof(int32_t); + + alignas(sizeof(Ty) ) int32_t a[kNumLanes]; + alignas(sizeof(Ty) ) int32_t b[kNumLanes]; + simd_st(a, _a); + simd_st(b, _b); + + for (uint32_t ii = 0; ii < kNumLanes; ++ii) + { + const int32_t aa = a[ii]; + const int32_t bb = b[ii]; + a[ii] = 0 == bb ? aa : (INT32_MIN == aa && -1 == bb) ? aa : aa / bb; + } + + return simd_ld(a); + } + + template + BX_SIMD_INLINE Ty simd_i32_mod_ni(Ty _a, Ty _b) + { + constexpr uint32_t kNumLanes = sizeof(Ty)/sizeof(int32_t); + + alignas(sizeof(Ty) ) int32_t a[kNumLanes]; + alignas(sizeof(Ty) ) int32_t b[kNumLanes]; + simd_st(a, _a); + simd_st(b, _b); + + for (uint32_t ii = 0; ii < kNumLanes; ++ii) + { + const int32_t aa = a[ii]; + const int32_t bb = b[ii]; + a[ii] = 0 == bb ? 0 : (INT32_MIN == aa && -1 == bb) ? 0 : aa % bb; + } + + return simd_ld(a); + } + + template + BX_SIMD_INLINE Ty simd_u32_div_ni(Ty _a, Ty _b) + { + constexpr uint32_t kNumLanes = sizeof(Ty)/sizeof(uint32_t); + + alignas(sizeof(Ty) ) uint32_t a[kNumLanes]; + alignas(sizeof(Ty) ) uint32_t b[kNumLanes]; + simd_st(a, _a); + simd_st(b, _b); + + for (uint32_t ii = 0; ii < kNumLanes; ++ii) + { + const uint32_t aa = a[ii]; + const uint32_t bb = b[ii]; + a[ii] = 0 == bb ? aa : aa / bb; + } + + return simd_ld(a); + } + + template + BX_SIMD_INLINE Ty simd_u32_mod_ni(Ty _a, Ty _b) + { + constexpr uint32_t kNumLanes = sizeof(Ty)/sizeof(uint32_t); + + alignas(sizeof(Ty) ) uint32_t a[kNumLanes]; + alignas(sizeof(Ty) ) uint32_t b[kNumLanes]; + simd_st(a, _a); + simd_st(b, _b); + + for (uint32_t ii = 0; ii < kNumLanes; ++ii) + { + const uint32_t aa = a[ii]; + const uint32_t bb = b[ii]; + a[ii] = 0 == bb ? 0u : aa % bb; + } + + return simd_ld(a); + } + + template + BX_SIMD_INLINE Ty simd_f32_ftoi_sat_ni(Ty _a) + { + const Ty two31 = simd_splat(2147483648.0f); + const Ty intMax = simd_splat(uint32_t(0x7fffffff) ); + const Ty absmask = simd_splat(uint32_t(~kFloatSignMask) ); + const Ty inf = simd_splat(kFloatExponentMask); + const Ty magnitude = simd_and(_a, absmask); + const Ty nanMask = simd_u32_cmpgt(magnitude, inf); + const Ty big = simd_f32_cmpge(_a, two31); + const Ty trunc = simd_f32_ftoi_trunc(_a); + const Ty clamped = simd_selb(big, intMax, trunc); + const Ty result = simd_andc(clamped, nanMask); + return result; + } + + template + BX_SIMD_INLINE Ty simd_f32_ftou_sat_ni(Ty _a) + { + const Ty zero = simd_zero(); + const Ty two31 = simd_splat(2147483648.0f); + const Ty two32 = simd_splat(4294967296.0f); + const Ty topBit = simd_splat(uint32_t(0x80000000) ); + const Ty x = simd_f32_max(_a, zero); + const Ty big = simd_f32_cmpge(x, two32); + const Ty high = simd_f32_cmpge(x, two31); + const Ty shifted = simd_f32_sub(x, two31); + const Ty y = simd_selb(high, shifted, x); + const Ty trunc = simd_f32_ftoi_trunc(y); + const Ty bit = simd_and(high, topBit); + const Ty low = simd_or(trunc, bit); + const Ty result = simd_or(low, big); + return result; + } + + template + BX_SIMD_INLINE Ty simd_u32_utof_ni(Ty _a) + { + const Ty lowMask = simd_splat(uint32_t(0xffff) ); + const Ty two16 = simd_splat(65536.0f); + const Ty hiBits = simd_x32_srl(_a, 16); + const Ty loBits = simd_and(_a, lowMask); + const Ty hi = simd_i32_itof(hiBits); + const Ty lo = simd_i32_itof(loBits); + const Ty hiScaled = simd_f32_mul(hi, two16); + const Ty result = simd_f32_add(hiScaled, lo); + return result; + } + template BX_SIMD_INLINE Ty simd_x32_bit_mask_ni(Ty _count) { @@ -896,8 +1404,7 @@ namespace bx alignas(32) uint8_t oBuf[sizeof(Ty)]; simd_st(aBuf, _a); simd_st(iBuf, _indices); - // Per-16-byte-lane shuffle: matches PSHUFB / vqtbl1q semantics. - // For widths < 16 bytes, the lane is the full register. + constexpr int kLaneBytes = sizeof(Ty) >= 16 ? 16 : int(sizeof(Ty)); constexpr int kLaneMask = kLaneBytes - 1; for (int lane = 0; lane < int(sizeof(Ty)); lane += kLaneBytes) @@ -921,11 +1428,9 @@ namespace bx simd_st(aBuf, _a); simd_st(bBuf, _b); simd_st(iBuf, _indices); - // Two-source per-16-byte-lane shuffle: indices select from concatenated - // [a|b] within the matching 16-byte lane (or full register for < 16 bytes). - // Bit 7 of an index byte zeroes the output byte; bits 0..(log2(2*lane)-1) - // select within the 2*lane concatenation; remaining bits must be 0. + constexpr int kLaneBytes = sizeof(Ty) >= 16 ? 16 : int(sizeof(Ty)); + constexpr int kPairMask = (kLaneBytes * 2) - 1; for (int lane = 0; lane < int(sizeof(Ty)); lane += kLaneBytes) { @@ -943,6 +1448,7 @@ namespace bx } } } + return simd_ld(oBuf); } diff --git a/include/bx/macros.h b/include/bx/macros.h index 9381a2c..70ff2b8 100644 --- a/include/bx/macros.h +++ b/include/bx/macros.h @@ -178,6 +178,29 @@ extern "C" void* __cdecl _alloca(size_t _size); # define BX_PRAGMA_DIAGNOSTIC_IGNORED_CLANG_GCC(_x) #endif // BX_COMPILER_ +/// +#if BX_COMPILER_MSVC +# define BX_FP_CONTRACT_OFF_BEGIN() +# define BX_FP_CONTRACT_OFF_END() +# define BX_FP_PRECISE_BEGIN() __pragma(float_control(push) ) __pragma(float_control(precise, on) ) __pragma(fp_contract(off) ) +# define BX_FP_PRECISE_END() __pragma(float_control(pop) ) +#elif BX_COMPILER_CLANG +# define BX_FP_CONTRACT_OFF_BEGIN() _Pragma("float_control(push)") _Pragma("clang fp contract(off)") +# define BX_FP_CONTRACT_OFF_END() _Pragma("float_control(pop)") +# define BX_FP_PRECISE_BEGIN() _Pragma("float_control(push)") _Pragma("float_control(precise, on)") _Pragma("clang fp contract(off)") +# define BX_FP_PRECISE_END() _Pragma("float_control(pop)") +#elif BX_COMPILER_GCC +# define BX_FP_CONTRACT_OFF_BEGIN() +# define BX_FP_CONTRACT_OFF_END() +# define BX_FP_PRECISE_BEGIN() _Pragma("GCC push_options") _Pragma("GCC optimize(\"no-fast-math\",\"fp-contract=off\")") +# define BX_FP_PRECISE_END() _Pragma("GCC pop_options") +#else +# define BX_FP_CONTRACT_OFF_BEGIN() +# define BX_FP_CONTRACT_OFF_END() +# define BX_FP_PRECISE_BEGIN() +# define BX_FP_PRECISE_END() +#endif // BX_COMPILER_ + /// No default constructor. #define BX_CLASS_NO_DEFAULT_CTOR(_class) \ _class() = delete diff --git a/include/bx/math.h b/include/bx/math.h index bee94e5..acd4393 100644 --- a/include/bx/math.h +++ b/include/bx/math.h @@ -566,7 +566,8 @@ namespace bx /// BX_CONSTEXPR_FUNC float fract(float _a); - /// Returns result of negated multiply-sub operation -(_a * _b - _c) -> _c - _a * _b. + /// Returns result of negated multiply-sub operation -(_a * _b - _c) -> _c - _a * _b, + /// rounded like mad. /// /// @param[in] _a First factor. /// @param[in] _b Second factor. diff --git a/include/bx/simd_t.h b/include/bx/simd_t.h index 4f1c667..d92f7c3 100644 --- a/include/bx/simd_t.h +++ b/include/bx/simd_t.h @@ -72,6 +72,19 @@ # define BX_SIMD_AVX 1 #endif // +#if BX_CONFIG_FMA +# if BX_CPU_X86 +# include // FMA3 +# endif // BX_CPU_X86 +# if BX_COMPILER_MSVC +// With /arch:AVX2 MSVC compiles fmaf to a single vfmadd213ss (no CRT call and, +// unlike _mm_fmadd_ss, no zeroing of the upper lanes); the CRT's own +// declaration is compatible with this one. On ARM64 it compiles to fmadd. +extern "C" float fmaf(float, float, float); +# pragma intrinsic(fmaf) +# endif // BX_COMPILER_MSVC +#endif // BX_CONFIG_FMA + #if BX_CPU_X86 && (defined(__SSE2__) || (BX_COMPILER_MSVC && (BX_ARCH_64BIT || _M_IX86_FP >= 2) ) ) # include # include // SSE4.1 minspec is SSE4.2 so always available @@ -733,6 +746,17 @@ namespace bx template Ty simd_f32_floor(Ty _a); + /// Per-lane f32 truncation toward zero. + /// + /// @param[in] _a Input register. + /// + /// @returns Per-lane integral value not greater in magnitude than `_a`. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_trunc(Ty _a); + /// Per-lane f32 base-2 logarithm: `log2(_a)`. /// /// @param[in] _a Input register. @@ -823,6 +847,97 @@ namespace bx template Ty simd_f32_ldexp(Ty _a, Ty _b); + /// Per-lane f32 fractional part: `_a - trunc(_a)`, as bx::fract. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_fract(Ty _a); + + /// Per-lane f32 sign: -1, 0 or 1, as bx::sign. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_sign(Ty _a); + + /// Per-lane f32 step: 0 where `_a < _edge`, 1 elsewhere, as bx::step. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_step(Ty _edge, Ty _a); + + /// Per-lane f32 Hermite smoothstep: `_a^2 * (3 - 2 * _a)`, as bx::smoothStep. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_smoothstep(Ty _a); + + /// Per-lane f32 remainder: `_a - _b * floor(_a / _b)`, as bx::mod. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_mod(Ty _a, Ty _b); + + /// Per-lane f32 tangent, as bx::tan. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_tan(Ty _a); + + /// Per-lane f32 arc cosine, as bx::acos. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_acos(Ty _a); + + /// Per-lane f32 arc sine, as bx::asin. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_asin(Ty _a); + + /// Per-lane f32 two-argument arc tangent of `_y / _x`, as bx::atan2. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_atan2(Ty _y, Ty _x); + + /// Per-lane f32 arc tangent, as bx::atan. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_atan(Ty _a); + + /// Per-lane f32 hyperbolic sine, as bx::sinh. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_sinh(Ty _a); + + /// Per-lane f32 hyperbolic cosine, as bx::cosh. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_cosh(Ty _a); + + /// Per-lane f32 hyperbolic tangent, as bx::tanh. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_tanh(Ty _a); + /// Per-lane f32 to i32 conversion with truncation toward zero. /// /// @param[in] _a Input f32 register. @@ -1261,6 +1376,109 @@ namespace bx template Ty simd_u32_cmpgt(Ty _a, Ty _b); + /// Per-lane i32 compare not-equal, less-or-equal, greater-or-equal (masks). + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_i32_cmpneq(Ty _a, Ty _b); + + template + Ty simd_i32_cmple(Ty _a, Ty _b); + + template + Ty simd_i32_cmpge(Ty _a, Ty _b); + + /// Per-lane u32 compare not-equal, less-or-equal, greater-or-equal (masks). + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_u32_cmpneq(Ty _a, Ty _b); + + template + Ty simd_u32_cmple(Ty _a, Ty _b); + + template + Ty simd_u32_cmpge(Ty _a, Ty _b); + + /// Per-lane u32 minimum, maximum and clamp. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_u32_min(Ty _a, Ty _b); + + template + Ty simd_u32_max(Ty _a, Ty _b); + + template + Ty simd_u32_clamp(Ty _a, Ty _min, Ty _max); + + /// Per-lane integer division and remainder with the shading-language + /// rules: `x / 0 == x`, `x % 0 == 0`, `INT32_MIN / -1 == INT32_MIN` and + /// `INT32_MIN % -1 == 0`. Never traps. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_i32_div(Ty _a, Ty _b); + + template + Ty simd_i32_mod(Ty _a, Ty _b); + + template + Ty simd_u32_div(Ty _a, Ty _b); + + template + Ty simd_u32_mod(Ty _a, Ty _b); + + /// Per-lane f32 to i32/u32 conversion, saturating: out of range clamps to + /// the type's limits, NaN is 0. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f32_ftoi_sat(Ty _a); + + template + Ty simd_f32_ftou_sat(Ty _a); + + /// Per-lane u32 to f32 conversion, correctly rounded. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_u32_utof(Ty _a); + + /// Per-lane bit counts: leading zeros, trailing zeros (32 for a zero lane) + /// and set bits; and the bit-reversed lane. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_u32_cntlz(Ty _a); + + template + Ty simd_u32_cnttz(Ty _a); + + template + Ty simd_u32_cntbits(Ty _a); + + template + Ty simd_u32_reversebits(Ty _a); + + /// Per-lane f32 to half (in the low 16 bits of the lane) and back, the same + /// bits as bx::halfFromFloat / bx::halfToFloat. + /// + /// @remark Widths: simd32, simd64, simd128, simd256. + /// + template + Ty simd_f16_fromf32(Ty _a); + + template + Ty simd_f16_tof32(Ty _a); + /// Broadcast float value to all lanes. /// /// @param[in] _a Value to broadcast. diff --git a/scripts/toolchain.lua b/scripts/toolchain.lua index feb2b77..1f4db40 100644 --- a/scripts/toolchain.lua +++ b/scripts/toolchain.lua @@ -239,7 +239,6 @@ function toolchain(_buildDir, _libDir) flags { "Cpp20", "ExtraWarnings", - "FloatFast", } if _ACTION == "gmake" or _ACTION == "ninja" then diff --git a/tests/math_test.cpp b/tests/math_test.cpp index bc8c449..eaddf62 100644 --- a/tests/math_test.cpp +++ b/tests/math_test.cpp @@ -6,31 +6,35 @@ #include "test.h" #include #include +#include #include #include // intXX_t #include // UCHAR_* +#include // LDBL_MANT_DIG + +TEST_CASE("fast-math is disabled", "[math][config]") +{ +#if (defined(__FAST_MATH__) && __FAST_MATH__) || defined(_M_FP_FAST) + FAIL("Built with fast math enabled. bx math requires IEEE semantics: drop" + " `-ffast-math` / `/fp:fast` (`FloatFast` in scripts/toolchain.lua)." + ); +#else + SUCCEED("IEEE semantics available."); +#endif // (defined(__FAST_MATH__) && __FAST_MATH__) || defined(_M_FP_FAST) +} TEST_CASE("isFinite, isInfinite, isNan", "[math]") { -#if defined(__FAST_MATH__) && __FAST_MATH__ - SKIP("This unit test fails with fast math is enabled."); -#endif // !defined(__FAST_MATH__) || !__FAST_MATH__ - for (uint64_t ii = 0; ii < UINT32_MAX; ii += rand()%(1<<13)+1) { union { uint32_t ui; float f; } u = { uint32_t(ii) }; BX_UNUSED(u); -#if BX_PLATFORM_OSX || BX_PLATFORM_IOS - REQUIRE(::__isnanf(u.f) == bx::isNan(u.f) ); - REQUIRE(::__isfinitef(u.f) == bx::isFinite(u.f) ); - REQUIRE(::__isinff(u.f) == bx::isInfinite(u.f) ); -#elif BX_COMPILER_MSVC - REQUIRE(!!::isnan(u.f) == bx::isNan(u.f) ); - REQUIRE(!!::isfinite(u.f) == bx::isFinite(u.f) ); - REQUIRE(!!::isinf(u.f) == bx::isInfinite(u.f) ); -#elif BX_PLATFORM_ANDROID +#if BX_PLATFORM_OSX \ + || BX_PLATFORM_IOS \ + || BX_COMPILER_MSVC \ + || BX_PLATFORM_ANDROID REQUIRE(!!::isnan(u.f) == bx::isNan(u.f) ); REQUIRE(!!::isfinite(u.f) == bx::isFinite(u.f) ); REQUIRE(!!::isinf(u.f) == bx::isInfinite(u.f) ); @@ -421,9 +425,7 @@ TEST_CASE("rsqrt", "[math][libm]") } // rsqrtSimd -#if !defined(__FAST_MATH__) || !__FAST_MATH__ REQUIRE(bx::isInfinite(bx::rsqrtSimd(0.0f) ) ); -#endif // !defined(__FAST_MATH__) || !__FAST_MATH__ for (float xx = bx::kNearZero; xx < 100.0f; xx += 0.1f) { @@ -433,10 +435,8 @@ TEST_CASE("rsqrt", "[math][libm]") } // rsqrt -#if !defined(__FAST_MATH__) || !__FAST_MATH__ REQUIRE(bx::isInfinite(1.0f / ::sqrtf(0.0f) ) ); REQUIRE(bx::isInfinite(bx::rsqrt(0.0f) ) ); -#endif // !defined(__FAST_MATH__) || !__FAST_MATH__ for (float xx = bx::kNearZero; xx < 100.0f; xx += 0.1f) { @@ -523,6 +523,99 @@ TEST_CASE("mod", "[math][libm]") STATIC_REQUIRE( 1.0f == bx::mod(1389.0f, 2.0f) ); } +static float madSplit(float _a, float _b, float _c) +{ + volatile float ab = _a * _b; + return ab + _c; +} + +static constexpr float maddRef(float _a, float _b, float _c) +{ + return bx::bitCast( + bx::simd32_f32_madd_ref(bx::simd32_ld(_a), bx::simd32_ld(_b), bx::simd32_ld(_c) ) + ); +} + +TEST_CASE("mad", "[math]") +{ + constexpr float kA = 0x1.000002p+0f; + constexpr float kB = 0x1.fffffcp-1f; + constexpr float kFused = -0x1p-46f; + + STATIC_REQUIRE(kFused == maddRef(kA, kB, -1.0f) ); + + volatile float va = kA; + volatile float vb = kB; + volatile float vc = -1.0f; + REQUIRE(kFused == maddRef(va, vb, vc) ); + + STATIC_REQUIRE( kFused == bx::mad(kA, kB, -1.0f) ); + STATIC_REQUIRE(-kFused == bx::nms(kA, kB, 1.0f) ); + +#if BX_CONFIG_FMA + REQUIRE(kFused == bx::mad(va, vb, vc) ); +#else + REQUIRE(0.0f == bx::mad(va, vb, vc) ); +#endif // BX_CONFIG_FMA + + REQUIRE(0.0f == madSplit(va, vb, vc) ); + + REQUIRE(bx::isNan(bx::mad(bx::kFloatInfinity, 0.0f, 1.0f) ) ); + REQUIRE(bx::isNan(bx::mad(bx::kFloatInfinity, 1.0f, -bx::kFloatInfinity) ) ); + + REQUIRE(bx::kFloatExponentMask == bx::bitCast(bx::mad(3.0e38f, 2.0f, 0.0f) ) ); + +#if BX_CONFIG_FMA + REQUIRE( (bx::kFloatSignMask|bx::kFloatExponentMask) == bx::bitCast(bx::mad(3.0e38f, 2.0f, -bx::kFloatInfinity) ) ); +#else + REQUIRE(bx::isNan(bx::mad(3.0e38f, 2.0f, -bx::kFloatInfinity) ) ); +#endif // BX_CONFIG_FMA + + STATIC_REQUIRE( (bx::kFloatSignMask|bx::kFloatExponentMask) == bx::bitCast(maddRef(3.0e38f, 2.0f, -bx::kFloatInfinity) ) ); + REQUIRE(0 == bx::bitCast(bx::mad( 1.0f, 1.0f, -1.0f) ) ); + + REQUIRE(bx::kFloatSignMask == bx::bitCast(bx::mad(-1.0f, 0.0f, -0.0f) ) ); + + REQUIRE(0 == bx::bitCast(bx::mad(-1.0f, 0.0f, 0.0f) ) ); + + STATIC_REQUIRE(bx::bitCast(0x1p-140f) == bx::bitCast(maddRef(0x1p-100f, 0x1p-40f, 0.0f) ) ); + STATIC_REQUIRE(bx::bitCast(0x1p-149f) == bx::bitCast(maddRef(0x1p-75f, 0x1.000002p-75f, 0.0f) ) ); + STATIC_REQUIRE(bx::bitCast(0.0f) == bx::bitCast(maddRef(0x1p-75f, 0x1p-75f, 0.0f) ) ); + STATIC_REQUIRE(bx::bitCast(0x1p-148f) == bx::bitCast(maddRef(0x1p-75f, 0x1p-75f, 0x1p-149f) ) ); + + STATIC_REQUIRE(0x1p-46f == maddRef(0x1.000002p+0f, 0x1.000002p+0f, -0x1.000004p+0f) ); + STATIC_REQUIRE(1.0f == maddRef(0x1.fffffep+22f, 2.0f, -0x1.fffffcp+23f) ); + + bx::RngMwc rng; + + for (uint32_t ii = 0; ii < 65536; ++ii) + { + auto gen = [&rng]() -> float + { + const uint32_t bits = rng.gen(); + const uint32_t exp = 96 + (bits>>23)%64; + return bx::bitCast( (bits & 0x807fffff) | (exp<<23) ); + }; + + const float aa = gen(); + const float bb = gen(); + const float cc = gen(); + +#if LDBL_MANT_DIG >= 64 + const float ref = float( (long double)(aa) * (long double)(bb) + (long double)(cc) ); +#else + const float ref = ::fmaf(aa, bb, cc); +#endif // LDBL_MANT_DIG >= 64 + const float sw = maddRef(aa, bb, cc); + REQUIRE(bx::bitCast(ref) == bx::bitCast(sw) ); + +#if BX_CONFIG_FMA + const float mad = bx::mad(aa, bb, cc); + REQUIRE(bx::bitCast(ref) == bx::bitCast(mad) ); +#endif // BX_CONFIG_FMA + } +} + typedef float (*MathFloatFn)(float); template @@ -554,6 +647,14 @@ TEST_CASE("round", "[math][libm]") STATIC_REQUIRE( 14.0f == bx::round( 13.89f) ); STATIC_REQUIRE(-14.0f == bx::round( -13.89f) ); + STATIC_REQUIRE( 2.0f == bx::round( 2.5f) ); + STATIC_REQUIRE( 4.0f == bx::round( 3.5f) ); + STATIC_REQUIRE( -2.0f == bx::round( -2.5f) ); + STATIC_REQUIRE( 0.0f == bx::round( 0.49999997f) ); + STATIC_REQUIRE( 0x1p22f == bx::round(0x1p22f + 0.5f) ); + STATIC_REQUIRE( 0x1p22f + 2.0f == bx::round(0x1p22f + 1.5f) ); + STATIC_REQUIRE(bx::kFloatSignMask == bx::bitCast(bx::round(-0.4f) ) ); + testMathFunc1Float( 13.89f); testMathFunc1Float(-13.89f); } @@ -563,6 +664,17 @@ TEST_CASE("trunc", "[math][libm]") STATIC_REQUIRE( 13.0f == bx::trunc( 13.89f) ); STATIC_REQUIRE(-13.0f == bx::trunc(-13.89f) ); + STATIC_REQUIRE( 1.0e10f == bx::trunc( 1.0e10f) ); + STATIC_REQUIRE(-1.0e10f == bx::trunc(-1.0e10f) ); + STATIC_REQUIRE( 1.0e10f == bx::floor( 1.0e10f) ); + STATIC_REQUIRE(-1.0e10f == bx::ceil( -1.0e10f) ); + STATIC_REQUIRE(bx::kFloatSignMask == bx::bitCast(bx::trunc(-0.5f) ) ); + STATIC_REQUIRE(bx::kFloatSignMask == bx::bitCast(bx::ceil( -0.5f) ) ); + STATIC_REQUIRE(bx::kFloatExponentMask == bx::bitCast(bx::floor(bx::kFloatInfinity) ) ); + volatile float nan = bx::bitCast(0x7fc00000u); + REQUIRE(bx::isNan(bx::trunc(nan) ) ); + REQUIRE(bx::isNan(bx::floor(nan) ) ); + testMathFunc1Float( 13.89f); testMathFunc1Float(-13.89f); } @@ -840,6 +952,7 @@ TEST_CASE("signBit", "[math][libm]") { STATIC_REQUIRE( bx::signBit(-0.1389f) ); STATIC_REQUIRE(!bx::signBit( 0.0000f) ); + STATIC_REQUIRE( bx::signBit(-0.0000f) ); STATIC_REQUIRE(!bx::signBit( 0.1389f) ); STATIC_REQUIRE( bx::signBit(-bx::kFloatInfinity) ); diff --git a/tests/simd_test.cpp b/tests/simd_test.cpp index 3bdcebd..f92e604 100644 --- a/tests/simd_test.cpp +++ b/tests/simd_test.cpp @@ -6,6 +6,7 @@ #include "test.h" #include #include +#include #include #if 0 @@ -507,6 +508,524 @@ TEST_CASE("simd128_f32_nmsub", "[simd]") check_f32("f32_nmsub", simd128_f32_nmsub(a, b, c), 4.0f, 8.0f, 10.0f, 10.0f); } +TEST_CASE("simd_f32_madd_fused", "[simd]") +{ + const simd128_t a = simd128_splat(0x1.000002p+0f); + const simd128_t b = simd128_splat(0x1.fffffcp-1f); + const simd128_t c = simd128_splat(1.0f); + const simd128_t nc = simd128_f32_neg(c); + +#if BX_CONFIG_FMA + constexpr uint32_t kNeg = 0xa8800000; // -2^-46 + constexpr uint32_t kPos = 0x28800000; // 2^-46 +#else + constexpr uint32_t kNeg = 0x00000000; + constexpr uint32_t kPos = 0x00000000; +#endif // BX_CONFIG_FMA + + check_u32("f32_madd", simd128_f32_madd(a, b, nc), kNeg, kNeg, kNeg, kNeg); + check_u32("f32_msub", simd128_f32_msub(a, b, c), kNeg, kNeg, kNeg, kNeg); + check_u32("f32_nmsub", simd128_f32_nmsub(a, b, c), kPos, kPos, kPos, kPos); +} + +template +static void testMaddSweep(const char* _name) +{ + BX_UNUSED(_name); + constexpr uint32_t kNumLanes = sizeof(Ty)/sizeof(float); + + bx::RngMwc rng; + + for (uint32_t ii = 0; ii < 4096; ++ii) + { + alignas(sizeof(Ty) ) float a[kNumLanes]; + alignas(sizeof(Ty) ) float b[kNumLanes]; + alignas(sizeof(Ty) ) float c[kNumLanes]; + alignas(sizeof(Ty) ) float madd[kNumLanes]; + alignas(sizeof(Ty) ) float msub[kNumLanes]; + alignas(sizeof(Ty) ) float nmsub[kNumLanes]; + + for (uint32_t jj = 0; jj < kNumLanes; ++jj) + { + auto gen = [&rng]() -> float + { + const uint32_t bits = rng.gen(); + const uint32_t exp = (0 == (bits & 0x1f) ) ? (bits>>23) : 96 + (bits>>23)%64; + return bx::bitCast( (bits & 0x807fffff) | (exp<<23) ); + }; + + a[jj] = gen(); + b[jj] = gen(); + c[jj] = gen(); + } + + simd_st(madd, simd_f32_madd (simd_ld(a), simd_ld(b), simd_ld(c) ) ); + simd_st(msub, simd_f32_msub (simd_ld(a), simd_ld(b), simd_ld(c) ) ); + simd_st(nmsub, simd_f32_nmsub(simd_ld(a), simd_ld(b), simd_ld(c) ) ); + + for (uint32_t jj = 0; jj < kNumLanes; ++jj) + { + const float refMadd = bx::mad( a[jj], b[jj], c[jj]); + const float refMsub = bx::mad( a[jj], b[jj], -c[jj]); + const float refNmsub = bx::mad(-a[jj], b[jj], c[jj]); + + if (bx::isNan(refMadd) ) + { + REQUIRE(bx::isNan(madd[jj]) ); + REQUIRE(bx::isNan(msub[jj]) ); + REQUIRE(bx::isNan(nmsub[jj]) ); + } + else + { + REQUIRE(bx::bitCast(refMadd) == bx::bitCast(madd[jj]) ); + REQUIRE(bx::bitCast(refMsub) == bx::bitCast(msub[jj]) ); + REQUIRE(bx::bitCast(refNmsub) == bx::bitCast(nmsub[jj]) ); + } + } + } +} + +TEST_CASE("simd_f32_madd_sweep", "[simd]") +{ + testMaddSweep("simd32"); + testMaddSweep("simd64"); + testMaddSweep("simd128"); + testMaddSweep("simd256"); +} + +static float parityInput(bx::RngMwc& _rng, float _lo, float _hi, bool _withNan = true) +{ + const uint32_t r = _rng.gen(); + + switch (r & 63) + { + case 0: return 0.0f; + case 1: return -0.0f; + case 2: return bx::kFloatInfinity; + case 3: return -bx::kFloatInfinity; + case 4: return _withNan ? bx::bitCast(0x7fc00000u) : 0.25f; // NaN + case 5: return 1.0f; + case 6: return -1.0f; + case 7: return 0x1p-149f; + case 8: return 0x1p23f; + case 9: return -0x1p23f; + case 10: return 0x1p31f; + case 11: return -0x1p31f; + case 12: return 3.4028235e38f; + case 13: return -3.4028235e38f; + case 14: return 0.5f; + case 15: return -0.5f; + default: break; + } + + const float t = float(_rng.gen() & 0xffffff) * (1.0f/16777216.0f); + return _lo + (_hi - _lo) * t; +} + +static bool sameBits(float _a, float _b) +{ + if (bx::isNan(_a) && bx::isNan(_b) ) + { + return true; + } + + return bx::bitCast(_a) == bx::bitCast(_b); +} + +template +static void parity1(const char* _name, SimdFn _simd, ScalarFn _scalar, float _lo, float _hi, bool _withNan = true) +{ + constexpr uint32_t kNumLanes = sizeof(Ty)/sizeof(float); + bx::RngMwc rng; + uint32_t bad = 0; + float badIn = 0.0f, badScalar = 0.0f, badSimd = 0.0f; + + for (uint32_t ii = 0; ii < 2048; ++ii) + { + alignas(sizeof(Ty) ) float in[kNumLanes]; + alignas(sizeof(Ty) ) float out[kNumLanes]; + + for (uint32_t jj = 0; jj < kNumLanes; ++jj) + { + in[jj] = parityInput(rng, _lo, _hi, _withNan); + } + + simd_st(out, _simd(simd_ld(in) ) ); + + for (uint32_t jj = 0; jj < kNumLanes; ++jj) + { + const float scalar = _scalar(in[jj]); + + if (!sameBits(scalar, out[jj]) ) + { + if (0 == bad++) + { + badIn = in[jj]; badScalar = scalar; badSimd = out[jj]; + } + } + } + } + + INFO(_name << " lanes=" << kNumLanes << ": " << bad << " mismatches, first at " << badIn << " (0x" << std::hex << bx::bitCast(badIn) << ") scalar 0x" << bx::bitCast(badScalar) << " simd 0x" << bx::bitCast(badSimd) << std::dec); + CHECK(0 == bad); +} + +template +static void parity2(const char* _name, SimdFn _simd, ScalarFn _scalar, float _lo0, float _hi0, float _lo1, float _hi1, bool _withNan = true) +{ + constexpr uint32_t kNumLanes = sizeof(Ty)/sizeof(float); + bx::RngMwc rng; + uint32_t bad = 0; + float badIn0 = 0.0f, badIn1 = 0.0f, badScalar = 0.0f, badSimd = 0.0f; + + for (uint32_t ii = 0; ii < 2048; ++ii) + { + alignas(sizeof(Ty) ) float in0[kNumLanes]; + alignas(sizeof(Ty) ) float in1[kNumLanes]; + alignas(sizeof(Ty) ) float out[kNumLanes]; + + for (uint32_t jj = 0; jj < kNumLanes; ++jj) + { + in0[jj] = parityInput(rng, _lo0, _hi0, _withNan); + in1[jj] = parityInput(rng, _lo1, _hi1, _withNan); + } + + simd_st(out, _simd(simd_ld(in0), simd_ld(in1) ) ); + + for (uint32_t jj = 0; jj < kNumLanes; ++jj) + { + const float scalar = _scalar(in0[jj], in1[jj]); + + if (!sameBits(scalar, out[jj]) ) + { + if (0 == bad++) + { + badIn0 = in0[jj]; badIn1 = in1[jj]; badScalar = scalar; badSimd = out[jj]; + } + } + } + } + + INFO(_name << " lanes=" << kNumLanes << ": " << bad << " mismatches, first at " << badIn0 << ", " << badIn1 << " scalar 0x" << std::hex << bx::bitCast(badScalar) << " simd 0x" << bx::bitCast(badSimd) << std::dec); + CHECK(0 == bad); +} + +template +static void parityAll() +{ + parity1("trunc", [](Ty a ) { return simd_f32_trunc(a); }, [](float a ) { return bx::trunc(a); }, -1.0e9f, 1.0e9f); + parity1("floor", [](Ty a ) { return simd_f32_floor(a); }, [](float a ) { return bx::floor(a); }, -1.0e9f, 1.0e9f); + parity1("ceil", [](Ty a ) { return simd_f32_ceil(a); }, [](float a ) { return bx::ceil(a); }, -1.0e9f, 1.0e9f); + parity1("round", [](Ty a ) { return simd_f32_round(a); }, [](float a ) { return bx::round(a); }, -1.0e9f, 1.0e9f); + parity1("round ties", [](Ty a ) { return simd_f32_round(a); }, [](float a ) { return bx::round(a); }, -0x1p23f, 0x1p23f); + parity1("fract", [](Ty a ) { return simd_f32_fract(a); }, [](float a ) { return bx::fract(a); }, -1.0e6f, 1.0e6f); + parity1("sign", [](Ty a ) { return simd_f32_sign(a); }, [](float a ) { return bx::sign(a); }, -10.0f, 10.0f, false); + parity1("smoothstep", [](Ty a ) { return simd_f32_smoothstep(a); }, [](float a ) { return bx::smoothStep(a); }, -2.0f, 2.0f); + parity1("cos", [](Ty a ) { return simd_f32_cos(a); }, [](float a ) { return bx::cos(a); }, -100.0f, 100.0f); + parity1("sin", [](Ty a ) { return simd_f32_sin(a); }, [](float a ) { return bx::sin(a); }, -100.0f, 100.0f); + parity1("tan", [](Ty a ) { return simd_f32_tan(a); }, [](float a ) { return bx::tan(a); }, -100.0f, 100.0f); + parity1("exp", [](Ty a ) { return simd_f32_exp(a); }, [](float a ) { return bx::exp(a); }, -90.0f, 90.0f); + parity1("log", [](Ty a ) { return simd_f32_log(a); }, [](float a ) { return bx::log(a); }, 1.0e-30f, 1.0e30f, false); + parity1("exp2", [](Ty a ) { return simd_f32_exp2(a); }, [](float a ) { return bx::exp2(a); }, -120.0f, 120.0f, false); + parity1("log2", [](Ty a ) { return simd_f32_log2(a); }, [](float a ) { return bx::log2(a); }, 1.0e-30f, 1.0e30f, false); + parity1("acos", [](Ty a ) { return simd_f32_acos(a); }, [](float a ) { return bx::acos(a); }, -1.5f, 1.5f); + parity1("asin", [](Ty a ) { return simd_f32_asin(a); }, [](float a ) { return bx::asin(a); }, -1.5f, 1.5f); + parity1("atan", [](Ty a ) { return simd_f32_atan(a); }, [](float a ) { return bx::atan(a); }, -1000.0f, 1000.0f, false); + parity1("sinh", [](Ty a ) { return simd_f32_sinh(a); }, [](float a ) { return bx::sinh(a); }, -30.0f, 30.0f); + parity1("cosh", [](Ty a ) { return simd_f32_cosh(a); }, [](float a ) { return bx::cosh(a); }, -30.0f, 30.0f); + parity1("tanh", [](Ty a ) { return simd_f32_tanh(a); }, [](float a ) { return bx::tanh(a); }, -30.0f, 30.0f); + parity2("pow", [](Ty a, Ty b) { return simd_f32_pow(a, b); }, [](float a, float b) { return bx::pow(a, b); }, 0.01f, 100.0f, -10.0f, 10.0f, false); + parity2("atan2", [](Ty y, Ty x) { return simd_f32_atan2(y, x); }, [](float y, float x) { return bx::atan2(y, x); }, -100.0f, 100.0f, -100.0f, 100.0f, false); + parity2("step", [](Ty e, Ty a) { return simd_f32_step(e, a); }, [](float e, float a) { return bx::step(e, a); }, -10.0f, 10.0f, -10.0f, 10.0f); + parity2("mod", [](Ty a, Ty b) { return simd_f32_mod(a, b); }, [](float a, float b) { return bx::mod(a, b); }, -100.0f, 100.0f, -10.0f, 10.0f); +} + +TEST_CASE("simd_f32_scalar_parity", "[simd]") +{ + parityAll(); + parityAll(); +} + +static uint32_t intInput(bx::RngMwc& _rng) +{ + const uint32_t r = _rng.gen(); + + switch (r & 31) + { + case 0: return 0; + case 1: return 1; + case 2: return 0xffffffffu; // -1 + case 3: return 0x80000000u; // INT32_MIN + case 4: return 0x7fffffffu; // INT32_MAX + case 5: return 2; + case 6: return 0xfffffffeu; // -2 + case 7: return 0x80000001u; + case 8: return 0x40000000u; + case 9: return 0xc0000000u; + case 10: return r >> 28; // small + case 11: return uint32_t(-int32_t(r >> 28) ); + case 12: return r >> 16; + case 13: return uint32_t(-int32_t(r >> 16) ); + default: break; + } + + return _rng.gen(); +} + +static int32_t refDivI32(int32_t _a, int32_t _b) +{ + return 0 == _b + ? _a : (INT32_MIN == _a && -1 == _b) + ? _a : _a / _b + ; +} + +static int32_t refModI32(int32_t _a, int32_t _b) +{ + return 0 == _b + ? 0 : (INT32_MIN == _a && -1 == _b) + ? 0 : _a % _b + ; +} + +static uint32_t refDivU32(uint32_t _a, uint32_t _b) +{ + return 0 == _b ? _a : _a / _b; +} + +static uint32_t refModU32(uint32_t _a, uint32_t _b) +{ + return 0 == _b ? 0u : _a % _b; +} + +static uint32_t refCntlz(uint32_t _a) +{ + uint32_t n = 0; + for (uint32_t ii = 0; ii < 32; ++ii) + { + if (_a & (0x80000000u >> ii) ) + { + break; + } + + ++n; + } + + return n; +} + +static uint32_t refCnttz(uint32_t _a) +{ + uint32_t n = 0; + for (uint32_t ii = 0; ii < 32; ++ii) + { + if (_a & (1u << ii) ) + { + break; + } + + ++n; + + } + + return n; +} + +static uint32_t refCntbits(uint32_t _a) +{ + uint32_t n = 0; + + for (uint32_t ii = 0; ii < 32; ++ii) + { + n += (_a >> ii) & 1; + } + + return n; +} + +static uint32_t refRevbits(uint32_t _a) +{ + uint32_t r = 0; + + for (uint32_t ii = 0; ii < 32; ++ii) + { + r |= ( (_a >> ii) & 1) << (31 - ii); + } + + return r; +} + +static int32_t refFtoiSat(float _a) +{ + if (bx::isNan(_a) ) + { + return 0; + } + + if (_a >= 2147483648.0f) + { + return INT32_MAX; + } + + if (_a <= -2147483648.0f) + { + return INT32_MIN; + } + + return int32_t(_a); +} + +static uint32_t refFtouSat(float _a) +{ + if (bx::isNan(_a) || _a <= 0.0f) + { + return 0; + } + + if (_a >= 4294967296.0f) + { + return UINT32_MAX; + } + + return uint32_t(int64_t(_a) ); +} + +template +static void intParity2(const char* _name, SimdFn _simd, RefFn _ref, bool _floatInput = false) +{ + constexpr uint32_t kNumLanes = sizeof(Ty)/sizeof(uint32_t); + bx::RngMwc rng; + uint32_t bad = 0; + uint32_t badA = 0, badB = 0, badRef = 0, badSimd = 0; + + for (uint32_t ii = 0; ii < 4096; ++ii) + { + alignas(sizeof(Ty) ) uint32_t a[kNumLanes]; + alignas(sizeof(Ty) ) uint32_t b[kNumLanes]; + alignas(sizeof(Ty) ) uint32_t out[kNumLanes]; + + for (uint32_t jj = 0; jj < kNumLanes; ++jj) + { + a[jj] = _floatInput ? bx::bitCast(parityInput(rng, -1000.0f, 1000.0f, false) ) : intInput(rng); + b[jj] = _floatInput ? bx::bitCast(parityInput(rng, -1000.0f, 1000.0f, false) ) : intInput(rng); + } + + simd_st(out, _simd(simd_ld(a), simd_ld(b) ) ); + + for (uint32_t jj = 0; jj < kNumLanes; ++jj) + { + const uint32_t ref = _ref(a[jj], b[jj]); + + if (ref != out[jj] && 0 == bad++) + { + badA = a[jj]; badB = b[jj]; badRef = ref; badSimd = out[jj]; + } + } + } + + INFO(_name << " lanes=" << kNumLanes << ": " << bad << " mismatches, first at 0x" << std::hex << badA << ", 0x" << badB << " ref 0x" << badRef << " simd 0x" << badSimd << std::dec); + CHECK(0 == bad); +} + +template +static void intParity1(const char* _name, SimdFn _simd, RefFn _ref, bool _floatInput) +{ + constexpr uint32_t kNumLanes = sizeof(Ty)/sizeof(uint32_t); + bx::RngMwc rng; + uint32_t bad = 0; + uint32_t badA = 0, badRef = 0, badSimd = 0; + + for (uint32_t ii = 0; ii < 4096; ++ii) + { + alignas(sizeof(Ty) ) uint32_t a[kNumLanes]; + alignas(sizeof(Ty) ) uint32_t out[kNumLanes]; + + for (uint32_t jj = 0; jj < kNumLanes; ++jj) + { + constexpr bool kWithNan = true; + + a[jj] = _floatInput + ? bx::bitCast(parityInput(rng, -5.0e9f, 5.0e9f, kWithNan) ) + : intInput(rng) + ; + } + + simd_st(out, _simd(simd_ld(a) ) ); + + for (uint32_t jj = 0; jj < kNumLanes; ++jj) + { + const uint32_t ref = _ref(a[jj]); + + if (ref != out[jj] && 0 == bad++) + { + badA = a[jj]; badRef = ref; badSimd = out[jj]; + } + } + } + + INFO(_name << " lanes=" << kNumLanes << ": " << bad << " mismatches, first at 0x" << std::hex << badA << " ref 0x" << badRef << " simd 0x" << badSimd << std::dec); + CHECK(0 == bad); +} + +template +static void intParityAll() +{ + intParity2("i32_div", [](Ty a, Ty b) { return simd_i32_div(a, b); }, [](uint32_t a, uint32_t b) { return uint32_t(refDivI32(int32_t(a), int32_t(b) ) ); }); + intParity2("i32_mod", [](Ty a, Ty b) { return simd_i32_mod(a, b); }, [](uint32_t a, uint32_t b) { return uint32_t(refModI32(int32_t(a), int32_t(b) ) ); }); + intParity2("u32_div", [](Ty a, Ty b) { return simd_u32_div(a, b); }, [](uint32_t a, uint32_t b) { return refDivU32(a, b); }); + intParity2("u32_mod", [](Ty a, Ty b) { return simd_u32_mod(a, b); }, [](uint32_t a, uint32_t b) { return refModU32(a, b); }); + intParity2("u32_min", [](Ty a, Ty b) { return simd_u32_min(a, b); }, [](uint32_t a, uint32_t b) { return a < b ? a : b; }); + intParity2("u32_max", [](Ty a, Ty b) { return simd_u32_max(a, b); }, [](uint32_t a, uint32_t b) { return a > b ? a : b; }); + intParity2("i32_cmpneq", [](Ty a, Ty b) { return simd_i32_cmpneq(a, b); }, [](uint32_t a, uint32_t b) { return a != b ? 0xffffffffu : 0u; }); + intParity2("i32_cmple", [](Ty a, Ty b) { return simd_i32_cmple(a, b); }, [](uint32_t a, uint32_t b) { return int32_t(a) <= int32_t(b) ? 0xffffffffu : 0u; }); + intParity2("i32_cmpge", [](Ty a, Ty b) { return simd_i32_cmpge(a, b); }, [](uint32_t a, uint32_t b) { return int32_t(a) >= int32_t(b) ? 0xffffffffu : 0u; }); + intParity2("u32_cmpneq", [](Ty a, Ty b) { return simd_u32_cmpneq(a, b); }, [](uint32_t a, uint32_t b) { return a != b ? 0xffffffffu : 0u; }); + intParity2("u32_cmple", [](Ty a, Ty b) { return simd_u32_cmple(a, b); }, [](uint32_t a, uint32_t b) { return a <= b ? 0xffffffffu : 0u; }); + intParity2("u32_cmpge", [](Ty a, Ty b) { return simd_u32_cmpge(a, b); }, [](uint32_t a, uint32_t b) { return a >= b ? 0xffffffffu : 0u; }); + intParity1("cntlz", [](Ty a ) { return simd_u32_cntlz(a); }, refCntlz, false); + intParity1("cnttz", [](Ty a ) { return simd_u32_cnttz(a); }, refCnttz, false); + intParity1("cntbits", [](Ty a ) { return simd_u32_cntbits(a); }, refCntbits, false); + intParity1("reversebits",[](Ty a ) { return simd_u32_reversebits(a);}, refRevbits, false); + intParity1("utof", [](Ty a ) { return simd_u32_utof(a); }, [](uint32_t a) { return bx::bitCast(float(a) ); }, false); + intParity1("ftoi_sat", [](Ty a ) { return simd_f32_ftoi_sat(a); }, [](uint32_t a) { return uint32_t(refFtoiSat(bx::bitCast(a) ) ); }, true); + intParity1("ftou_sat", [](Ty a ) { return simd_f32_ftou_sat(a); }, [](uint32_t a) { return refFtouSat(bx::bitCast(a) ); }, true); + intParity1("f16_fromf32",[](Ty a ) { return simd_f16_fromf32(a); }, [](uint32_t a) { return uint32_t(bx::halfFromFloat(bx::bitCast(a) ) ); }, true); + intParity1("f16_tof32", [](Ty a ) { return simd_f16_tof32(a); }, [](uint32_t a) { return bx::bitCast(bx::halfToFloat(uint16_t(a & 0xffff) ) ); }, false); +} + +TEST_CASE("simd_int_scalar_parity", "[simd]") +{ + intParityAll(); + intParityAll(); +} + +static bool cmpNan(uint32_t _a, uint32_t _b) +{ + return bx::isNan(bx::bitCast(_a) ) || bx::isNan(bx::bitCast(_b) ); +} + +template +static void floatCompareParity() +{ + auto f = [](uint32_t _a) { return bx::bitCast(_a); }; + intParity2("f32_cmpeq", [](Ty a, Ty b) { return simd_f32_cmpeq(a, b); }, [f](uint32_t a, uint32_t b) { return !cmpNan(a, b) && f(a) == f(b) ? 0xffffffffu : 0u; }, true); + intParity2("f32_cmpneq", [](Ty a, Ty b) { return simd_f32_cmpneq(a, b); }, [f](uint32_t a, uint32_t b) { return cmpNan(a, b) || f(a) != f(b) ? 0xffffffffu : 0u; }, true); + intParity2("f32_cmplt", [](Ty a, Ty b) { return simd_f32_cmplt(a, b); }, [f](uint32_t a, uint32_t b) { return !cmpNan(a, b) && f(a) < f(b) ? 0xffffffffu : 0u; }, true); + intParity2("f32_cmple", [](Ty a, Ty b) { return simd_f32_cmple(a, b); }, [f](uint32_t a, uint32_t b) { return !cmpNan(a, b) && f(a) <= f(b) ? 0xffffffffu : 0u; }, true); + intParity2("f32_cmpgt", [](Ty a, Ty b) { return simd_f32_cmpgt(a, b); }, [f](uint32_t a, uint32_t b) { return !cmpNan(a, b) && f(a) > f(b) ? 0xffffffffu : 0u; }, true); + intParity2("f32_cmpge", [](Ty a, Ty b) { return simd_f32_cmpge(a, b); }, [f](uint32_t a, uint32_t b) { return !cmpNan(a, b) && f(a) >= f(b) ? 0xffffffffu : 0u; }, true); +} + +TEST_CASE("simd_f32_compare_parity", "[simd]") +{ + floatCompareParity(); + floatCompareParity(); + + const simd128_t a = simd128_splat(2147483648.0f); + check_u32("f32_cmple eq", simd_f32_cmple(a, a), 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu); + check_u32("f32_cmpge eq", simd_f32_cmpge(a, a), 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu); +} + TEST_CASE("simd128_f32_neg", "[simd]") { const simd128_t a = simd128_ld(1.0f, -2.0f, 3.0f, -4.0f); diff --git a/tests/vsnprintf_test.cpp b/tests/vsnprintf_test.cpp index f63987a..0aacde1 100644 --- a/tests/vsnprintf_test.cpp +++ b/tests/vsnprintf_test.cpp @@ -145,11 +145,9 @@ TEST_CASE("Format %f", "[string][printf]") REQUIRE(test(" nan", "%8f", kDoubleNan) ); REQUIRE(test("-NAN ", "%-8F", -kDoubleNan) ); -#if !defined(__FAST_MATH__) || !__FAST_MATH__ REQUIRE(test(" inf", "%8f", bx::kDoubleInfinity) ); REQUIRE(test("inf ", "%-8f", bx::kDoubleInfinity) ); REQUIRE(test(" -INF", "%8F", -bx::kDoubleInfinity) ); -#endif // !defined(__FAST_MATH__) || !__FAST_MATH__ REQUIRE(test(" 1.0", "%4.1f", 1.0) ); REQUIRE(test(" 1.500", "%6.3f", 1.5) );