From fbadfc3abc6026d6c877b3e8847c0962164dfdd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Branimir=20Karad=C5=BEi=C4=87?= Date: Sat, 17 Oct 2026 09:37:35 -0700 Subject: [PATCH] Added SIMD version of BitArray find. (#464) --- include/bx/bitarray.h | 17 + include/bx/inline/bitarray.inl | 113 ++-- include/bx/inline/math.inl | 14 + src/bitarray.cpp | 1096 ++++++++++++++++++++++++-------- tests/bitarray_test.cpp | 427 ++++++++++++- tests/math_test.cpp | 59 ++ 6 files changed, 1426 insertions(+), 300 deletions(-) diff --git a/include/bx/bitarray.h b/include/bx/bitarray.h index 95d0c6b..90141c4 100644 --- a/include/bx/bitarray.h +++ b/include/bx/bitarray.h @@ -153,6 +153,14 @@ namespace bx /// usage asserts in caller's code. uint32_t findRange(uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) const; + /// `findSet` continued after inline check of the first word. `_beginBit` is aligned down to + /// multiple of 64. + uint32_t findSetFrom(uint32_t _beginBit, uint32_t _endBit) const; + + /// `findClear` continued after inline check of the first word. `_beginBit` is aligned down to + /// multiple of 64. + uint32_t findClearFrom(uint32_t _beginBit, uint32_t _endBit) const; + uint64_t* m_ptr; uint32_t m_sizeInBytes; uint32_t m_count; @@ -481,6 +489,15 @@ namespace bx /// void updateMip(uint32_t _beginWord, uint32_t _endWord); + /// Updates the mip bits for words [_beginWord, _endWord) after a range set, where all + /// words except the first and the last one are filled with `_value`. + /// + /// @param[in] _beginWord First word index (inclusive). + /// @param[in] _endWord Last word index (exclusive). + /// @param[in] _value Value the bits were set to. + /// + void updateMipFilled(uint32_t _beginWord, uint32_t _endWord, bool _value); + /// Marks the mip bits past the last word as full, so searches stop there. void setMipTail(); diff --git a/include/bx/inline/bitarray.inl b/include/bx/inline/bitarray.inl index b0cf00a..da10f6a 100644 --- a/include/bx/inline/bitarray.inl +++ b/include/bx/inline/bitarray.inl @@ -93,27 +93,25 @@ namespace bx , m_count ); - uint32_t idx = _beginBit; - - while (idx < _endBit) + if (_beginBit >= _endBit) { - const uint32_t word = idx/64; - const uint64_t bits = m_ptr[word] & (~UINT64_C(0) << (idx & 63) ); - - if (0 != bits) - { - const uint32_t found = word*64 + findFirstSet(bits) - 1; - - return found < _endBit - ? found - : kInvalid - ; - } - - idx = (word + 1)*64; + return kInvalid; } - return kInvalid; + const uint32_t word = _beginBit/64; + const uint64_t bits = m_ptr[word] & (~UINT64_C(0) << (_beginBit & 63) ); + + if (0 != bits) + { + const uint32_t found = word*64 + findFirstSet(bits) - 1; + + return found < _endBit + ? found + : kInvalid + ; + } + + return findSetFrom( (word + 1)*64, _endBit); } inline uint32_t BitArrayViewBase::findClear(uint32_t _beginBit, uint32_t _endBit) const @@ -125,27 +123,25 @@ namespace bx , m_count ); - uint32_t idx = _beginBit; - - while (idx < _endBit) + if (_beginBit >= _endBit) { - const uint32_t word = idx/64; - const uint64_t bits = ~m_ptr[word] & (~UINT64_C(0) << (idx & 63) ); - - if (0 != bits) - { - const uint32_t found = word*64 + findFirstSet(bits) - 1; - - return found < _endBit - ? found - : kInvalid - ; - } - - idx = (word + 1)*64; + return kInvalid; } - return kInvalid; + const uint32_t word = _beginBit/64; + const uint64_t bits = ~m_ptr[word] & (~UINT64_C(0) << (_beginBit & 63) ); + + if (0 != bits) + { + const uint32_t found = word*64 + findFirstSet(bits) - 1; + + return found < _endBit + ? found + : kInvalid + ; + } + + return findClearFrom( (word + 1)*64, _endBit); } inline uint32_t BitArrayViewBase::countBits() const @@ -462,7 +458,19 @@ namespace bx if (_beginBit < _endBit) { MutableBitArrayView(m_ptr, m_sizeInBytes, m_count).set(_beginBit, _endBit, _value); - updateMip(_beginBit/64, alignUp(_endBit, 64)/64); + + const uint32_t beginWord = _beginBit/64; + const uint32_t lastWord = (_endBit - 1)/64; + + if (beginWord == lastWord) + { + const uint64_t mipMask = UINT64_C(1) << (beginWord & 63); + const uint64_t full = ~UINT64_C(0) == m_ptr[beginWord] ? mipMask : 0; + m_mip[beginWord/64] = (m_mip[beginWord/64] & ~mipMask) | full; + return; + } + + updateMipFilled(beginWord, lastWord + 1, _value); } } @@ -489,7 +497,8 @@ namespace bx if (NULL != m_ptr) { MutableBitArrayView(m_ptr, m_sizeInBytes, m_count).clear(); - updateMip(0, m_sizeInBytes/8); + memSet(m_mip, 0, getNumMipWords(m_sizeInBytes/8) * sizeof(uint64_t) ); + setMipTail(); } } @@ -602,18 +611,30 @@ namespace bx inline void MippedBitArrayBase::updateMip(uint32_t _beginWord, uint32_t _endWord) { - for (uint32_t ii = _beginWord; ii < _endWord; ++ii) + if (_beginWord + 1 == _endWord) { - const uint64_t mipMask = UINT64_C(1) << (ii & 63); + const uint64_t mipMask = UINT64_C(1) << (_beginWord & 63); + const uint64_t full = ~UINT64_C(0) == m_ptr[_beginWord] ? mipMask : 0; + m_mip[_beginWord/64] = (m_mip[_beginWord/64] & ~mipMask) | full; + return; + } - if (~UINT64_C(0) == m_ptr[ii]) + for (uint32_t ii = _beginWord; ii < _endWord;) + { + const uint32_t mipWord = ii/64; + const uint32_t end = min(_endWord, (mipWord + 1)*64); + + uint64_t mask = 0; + uint64_t full = 0; + + for (; ii < end; ++ii) { - m_mip[ii/64] |= mipMask; - } - else - { - m_mip[ii/64] &= ~mipMask; + const uint64_t bit = UINT64_C(1) << (ii & 63); + mask |= bit; + full |= ~UINT64_C(0) == m_ptr[ii] ? bit : 0; } + + m_mip[mipWord] = (m_mip[mipWord] & ~mask) | full; } } diff --git a/include/bx/inline/math.inl b/include/bx/inline/math.inl index baf14ec..b3975d1 100644 --- a/include/bx/inline/math.inl +++ b/include/bx/inline/math.inl @@ -2445,6 +2445,13 @@ BX_FP_PRECISE_END() #if BX_COMPILER_GCC || BX_COMPILER_CLANG return __builtin_popcount(_val); #else +# if BX_COMPILER_MSVC && BX_SIMD_SSE + if (!isConstantEvaluated() ) + { + return uint8_t(_mm_popcnt_u32(_val) ); + } +# endif // BX_COMPILER_MSVC && BX_SIMD_SSE + const uint32_t tmp0 = (_val >> 1); const uint32_t tmp1 = (tmp0 & 0x55555555); const uint32_t tmp2 = (_val - tmp1); @@ -2473,6 +2480,13 @@ BX_FP_PRECISE_END() #if BX_COMPILER_GCC || BX_COMPILER_CLANG return __builtin_popcountll(_val); #else +# if BX_COMPILER_MSVC && BX_SIMD_SSE && BX_ARCH_64BIT + if (!isConstantEvaluated() ) + { + return uint8_t(_mm_popcnt_u64(_val) ); + } +# endif // BX_COMPILER_MSVC && BX_SIMD_SSE && BX_ARCH_64BIT + const uint32_t lo = uint32_t(_val&UINT32_MAX); const uint32_t hi = uint32_t(_val>>32); diff --git a/src/bitarray.cpp b/src/bitarray.cpp index 695265b..eb6cd20 100644 --- a/src/bitarray.cpp +++ b/src/bitarray.cpp @@ -9,6 +9,407 @@ namespace bx { + namespace + { +#if BX_SIMD_AVX2 + typedef simd256_t SimdWideT; +#else + typedef simd128_t SimdWideT; +#endif // BX_SIMD_AVX2 + + constexpr uint32_t kNumSimdWords = sizeof(SimdWideT)/sizeof(uint64_t); + constexpr uint32_t kNumSimdDwords = sizeof(SimdWideT)/sizeof(uint32_t); + constexpr uint32_t kSimdDwordMask = (1u << kNumSimdDwords) - 1; + constexpr bool kSimdSupported = 0 != BX_SIMD_SUPPORTED; + + struct Match + { + enum Enum + { + Zero, + NonZero, + NotFull, + }; + }; + + template + BX_FORCE_INLINE bool isMatch(uint64_t _word) + { + if constexpr (Match::Zero == MatchT) + { + return 0 == _word; + } + else if constexpr (Match::NonZero == MatchT) + { + return 0 != _word; + } + else + { + return UINT64_MAX != _word; + } + } + + template + BX_FORCE_INLINE simd32_t matchLanes(SimdWideT _a) + { + const SimdWideT ref = Match::NotFull == MatchT ? simd_splat(UINT32_MAX) : simd_zero(); + const SimdWideT equal = simd_i32_cmpeq(_a, ref); + const simd32_t mask = simd32_ld(uint32_t(simd_x32_signbitsmask(equal) ) ); + const simd32_t maskHi = simd32_x32_srl(mask, 1); + const simd32_t wordLanes = simd32_splat(uint32_t(0x55555555 & kSimdDwordMask) ); + const simd32_t both = simd32_and(mask, maskHi); + const simd32_t equalBoth = simd32_and(both, wordLanes); + + if constexpr (Match::Zero == MatchT) + { + return equalBoth; + } + else + { + return simd32_andc(wordLanes, equalBoth); + } + } + + template + BX_FORCE_INLINE bool anyMatch(SimdWideT _a0, SimdWideT _a1, SimdWideT _a2, SimdWideT _a3) + { + if constexpr (Match::Zero == MatchT) + { + const simd32_t m0 = matchLanes(_a0); + const simd32_t m1 = matchLanes(_a1); + const simd32_t m2 = matchLanes(_a2); + const simd32_t m3 = matchLanes(_a3); + const simd32_t m01 = simd32_or(m0, m1); + const simd32_t m23 = simd32_or(m2, m3); + const simd32_t any = simd32_or(m01, m23); + return 0 != any.u32; + } + else if constexpr (Match::NonZero == MatchT) + { + const SimdWideT a01 = simd_or(_a0, _a1); + const SimdWideT a23 = simd_or(_a2, _a3); + const SimdWideT any = simd_or(a01, a23); + return !simd_test_zero(any, any); + } + else + { + const SimdWideT a01 = simd_and(_a0, _a1); + const SimdWideT a23 = simd_and(_a2, _a3); + const SimdWideT all = simd_and(a01, a23); + const SimdWideT inv = simd_not(all); + return !simd_test_zero(inv, inv); + } + } + + template + inline uint32_t findWordRef(const uint64_t* _ptr, uint32_t _idx, uint32_t _end) + { + for (; _idx < _end; ++_idx) + { + if (isMatch(_ptr[_idx]) ) + { + return _idx; + } + } + + return _end; + } + + template + BX_NO_INLINE uint32_t findWordSimd(const uint64_t* _ptr, uint32_t _idx, uint32_t _end) + { + for (; _idx + 4*kNumSimdWords <= _end; _idx += 4*kNumSimdWords) + { + const SimdWideT a0 = simd_ldu(&_ptr[_idx ]); + const SimdWideT a1 = simd_ldu(&_ptr[_idx + kNumSimdWords]); + const SimdWideT a2 = simd_ldu(&_ptr[_idx + 2*kNumSimdWords]); + const SimdWideT a3 = simd_ldu(&_ptr[_idx + 3*kNumSimdWords]); + + if (anyMatch(a0, a1, a2, a3) ) + { + break; + } + } + + for (; _idx + kNumSimdWords <= _end; _idx += kNumSimdWords) + { + const SimdWideT test = simd_ldu(&_ptr[_idx]); + const simd32_t lanes = matchLanes(test); + + if (0 != lanes.u32) + { + return _idx + countTrailingZeros(lanes.u32)/2; + } + } + + return findWordRef(_ptr, _idx, _end); + } + + template + inline uint32_t findLastWordRef(const uint64_t* _ptr, uint32_t _begin, uint32_t _idx) + { + for (; _idx > _begin; --_idx) + { + if (isMatch(_ptr[_idx-1]) ) + { + return _idx-1; + } + } + + return kInvalid; + } + + template + BX_NO_INLINE uint32_t findLastWordSimd(const uint64_t* _ptr, uint32_t _begin, uint32_t _idx) + { + for (; _idx >= _begin + 4*kNumSimdWords; _idx -= 4*kNumSimdWords) + { + const SimdWideT a0 = simd_ldu(&_ptr[_idx - 4*kNumSimdWords]); + const SimdWideT a1 = simd_ldu(&_ptr[_idx - 3*kNumSimdWords]); + const SimdWideT a2 = simd_ldu(&_ptr[_idx - 2*kNumSimdWords]); + const SimdWideT a3 = simd_ldu(&_ptr[_idx - kNumSimdWords]); + + if (anyMatch(a0, a1, a2, a3) ) + { + break; + } + } + + for (; _idx >= _begin + kNumSimdWords; _idx -= kNumSimdWords) + { + const SimdWideT test = simd_ldu(&_ptr[_idx - kNumSimdWords]); + const simd32_t lanes = matchLanes(test); + + if (0 != lanes.u32) + { + return _idx - kNumSimdWords + (findLastSet(lanes.u32) - 1)/2; + } + } + + return findLastWordRef(_ptr, _begin, _idx); + } + + template + BX_FORCE_INLINE uint32_t findWord(const uint64_t* _ptr, uint32_t _idx, uint32_t _end) + { + if constexpr (SimdT) + { + if (_end - _idx >= 4*kNumSimdWords) + { + return findWordSimd(_ptr, _idx, _end); + } + } + + return findWordRef(_ptr, _idx, _end); + } + + template + BX_FORCE_INLINE uint32_t findLastWord(const uint64_t* _ptr, uint32_t _begin, uint32_t _end) + { + if constexpr (SimdT) + { + if (_end - _begin >= 4*kNumSimdWords) + { + return findLastWordSimd(_ptr, _begin, _end); + } + } + + return findLastWordRef(_ptr, _begin, _end); + } + + constexpr uint32_t kSimdByteMask = 32 == sizeof(SimdWideT) ? UINT32_MAX : (1u << sizeof(SimdWideT) ) - 1; + + template + constexpr uint64_t kElementLowBits = 1 == sizeof(Ty) ? UINT64_C(0x0101010101010101) + : 2 == sizeof(Ty) ? UINT64_C(0x0001000100010001) + : UINT64_C(0x0000000100000001) + ; + + template + constexpr uint32_t kMaskBitsPerWord = 4 == sizeof(Ty) ? 2 : 8; + + template + BX_FORCE_INLINE bool hasZeroElement(uint64_t _word) + { + const uint64_t lowBits = kElementLowBits; + const uint64_t highBits = lowBits << (sizeof(Ty)*8 - 1); + const uint64_t borrow = _word - lowBits; + return 0 != (borrow & ~_word & highBits); + } + + template + BX_FORCE_INLINE simd32_t zeroElementMask(SimdWideT _a) + { + if constexpr (4 == sizeof(Ty) ) + { + const SimdWideT zero = simd_zero(); + const SimdWideT equal = simd_i32_cmpeq(_a, zero); + return simd32_ld(uint32_t(simd_x32_signbitsmask(equal) ) ); + } + else + { + const SimdWideT bias = simd_splat(uint32_t(0x7f7f7f7f) ); + const SimdWideT biased = simd_u8_satadd(_a, bias); + const simd32_t nonZero = simd32_ld(uint32_t(simd_x8_signbitsmask(biased) ) ); + const simd32_t byteMask = simd32_splat(kSimdByteMask); + const simd32_t zero = simd32_andc(byteMask, nonZero); + + if constexpr (1 == sizeof(Ty) ) + { + return zero; + } + else + { + const simd32_t zeroHi = simd32_x32_srl(zero, 1); + const simd32_t pairLanes = simd32_splat(uint32_t(0x55555555) ); + const simd32_t both = simd32_and(zero, zeroHi); + return simd32_and(both, pairLanes); + } + } + } + + template + inline uint32_t findZeroElementWordRef(const uint64_t* _ptr, uint32_t _idx, uint32_t _end) + { + for (; _idx < _end; ++_idx) + { + if (hasZeroElement(_ptr[_idx]) ) + { + return _idx; + } + } + + return _end; + } + + template + BX_NO_INLINE uint32_t findZeroElementWordSimd(const uint64_t* _ptr, uint32_t _idx, uint32_t _end) + { + for (; _idx + 4*kNumSimdWords <= _end; _idx += 4*kNumSimdWords) + { + const SimdWideT a0 = simd_ldu(&_ptr[_idx ]); + const SimdWideT a1 = simd_ldu(&_ptr[_idx + kNumSimdWords]); + const SimdWideT a2 = simd_ldu(&_ptr[_idx + 2*kNumSimdWords]); + const SimdWideT a3 = simd_ldu(&_ptr[_idx + 3*kNumSimdWords]); + const simd32_t m0 = zeroElementMask(a0); + const simd32_t m1 = zeroElementMask(a1); + const simd32_t m2 = zeroElementMask(a2); + const simd32_t m3 = zeroElementMask(a3); + const simd32_t m01 = simd32_or(m0, m1); + const simd32_t m23 = simd32_or(m2, m3); + const simd32_t any = simd32_or(m01, m23); + + if (0 != any.u32) + { + break; + } + } + + for (; _idx + kNumSimdWords <= _end; _idx += kNumSimdWords) + { + const SimdWideT test = simd_ldu(&_ptr[_idx]); + const simd32_t mask = zeroElementMask(test); + + if (0 != mask.u32) + { + return _idx + countTrailingZeros(mask.u32)/kMaskBitsPerWord; + } + } + + return findZeroElementWordRef(_ptr, _idx, _end); + } + + template + BX_FORCE_INLINE uint32_t findZeroElementWord(const uint64_t* _ptr, uint32_t _idx, uint32_t _end) + { + if constexpr (SimdT && 1 < sizeof(Ty) ) + { + return findZeroElementWordSimd(_ptr, _idx, _end); + } + else if constexpr (SimdT) + { + const uint32_t end = min(_end, _idx + 4); + const uint32_t idx = findZeroElementWordRef(_ptr, _idx, end); + + return idx != end || end == _end + ? idx + : findZeroElementWordSimd(_ptr, end, _end) + ; + } + else + { + return findZeroElementWordRef(_ptr, _idx, _end); + } + } + + inline uint32_t countBitsWords(const uint64_t* _ptr, uint32_t _idx, uint32_t _end) + { + const uint64_t* ptr = _ptr + _idx; + const uint64_t* end = _ptr + _end; + + uint32_t count0 = 0; + uint32_t count1 = 0; + uint32_t count2 = 0; + uint32_t count3 = 0; + + for (; end - ptr >= 4; ptr += 4) + { + count0 += countBits(ptr[0]); + count1 += countBits(ptr[1]); + count2 += countBits(ptr[2]); + count3 += countBits(ptr[3]); + } + + for (; ptr < end; ++ptr) + { + count0 += countBits(ptr[0]); + } + + return count0 + count1 + count2 + count3; + } + + } // namespace + + uint32_t bitSetFindWordRef(const uint64_t* _ptr, uint32_t _beginWord, uint32_t _endWord, uint32_t _match) + { + switch (_match) + { + case Match::Zero: return findWordRef(_ptr, _beginWord, _endWord); + case Match::NonZero: return findWordRef(_ptr, _beginWord, _endWord); + default: return findWordRef(_ptr, _beginWord, _endWord); + } + } + + uint32_t bitSetFindWordSimd(const uint64_t* _ptr, uint32_t _beginWord, uint32_t _endWord, uint32_t _match) + { + switch (_match) + { + case Match::Zero: return findWordSimd(_ptr, _beginWord, _endWord); + case Match::NonZero: return findWordSimd(_ptr, _beginWord, _endWord); + default: return findWordSimd(_ptr, _beginWord, _endWord); + } + } + + uint32_t bitSetFindLastWordRef(const uint64_t* _ptr, uint32_t _beginWord, uint32_t _endWord, uint32_t _match) + { + switch (_match) + { + case Match::Zero: return findLastWordRef(_ptr, _beginWord, _endWord); + case Match::NonZero: return findLastWordRef(_ptr, _beginWord, _endWord); + default: return findLastWordRef(_ptr, _beginWord, _endWord); + } + } + + uint32_t bitSetFindLastWordSimd(const uint64_t* _ptr, uint32_t _beginWord, uint32_t _endWord, uint32_t _match) + { + switch (_match) + { + case Match::Zero: return findLastWordSimd(_ptr, _beginWord, _endWord); + case Match::NonZero: return findLastWordSimd(_ptr, _beginWord, _endWord); + default: return findLastWordSimd(_ptr, _beginWord, _endWord); + } + } + inline void bitSetSet(uint64_t* _ptr, uint32_t _idx, bool _value) { const simd32_t mask63 = simd32_splat(uint32_t(63) ); @@ -57,16 +458,12 @@ namespace bx const simd64_t maskE = simd64_x64_srl(allOnes, int( (64-shiftE) & 63) ); const simd64_t valVec = simd64_splat(uint64_t(_value) ); const simd64_t orbits = simd64_u64_sub(zero, valVec); - const simd64_t orbitsB = simd64_and(maskB, orbits); - const simd64_t orbitsE = simd64_and(maskE, orbits); - const simd64_t cmaskB = simd64_not(maskB); - const simd64_t cmaskE = simd64_not(maskE); if (idxB == idxE) { const simd64_t maskBE = simd64_and(maskB, maskE); const simd64_t cmaskBE = simd64_not(maskBE); - const simd64_t orbitsBE = simd64_and(orbitsB, orbitsE); + const simd64_t orbitsBE = simd64_and(orbits, maskBE); const simd64_t prev = simd64_splat(_ptr[idxB]); const simd64_t cleared = simd64_and(prev, cmaskBE); const simd64_t result = simd64_or(cleared, orbitsBE); @@ -75,23 +472,40 @@ namespace bx } { + const simd64_t cmaskB = simd64_not(maskB); + const simd64_t orbitsB = simd64_and(orbits, maskB); const simd64_t prev = simd64_splat(_ptr[idxB]); const simd64_t cleared = simd64_and(prev, cmaskB); const simd64_t result = simd64_or(cleared, orbitsB); _ptr[idxB] = result.u64; } - for (uint32_t idx = idxB+1; idx < idxE; ++idx) - { - _ptr[idx] = orbits.u64; - } - { + const simd64_t cmaskE = simd64_not(maskE); + const simd64_t orbitsE = simd64_and(orbits, maskE); const simd64_t prev = simd64_splat(_ptr[idxE]); const simd64_t cleared = simd64_and(prev, cmaskE); const simd64_t result = simd64_or(cleared, orbitsE); _ptr[idxE] = result.u64; } + + const uint32_t numWords = idxE - idxB - 1; + + if (numWords > 2) + { + bx::memSet(&_ptr[idxB+1], _value ? 0xff : 0, numWords*sizeof(uint64_t) ); + return; + } + + if (numWords > 0) + { + _ptr[idxB+1] = orbits.u64; + } + + if (numWords > 1) + { + _ptr[idxB+2] = orbits.u64; + } } inline bool bitSetGet(const uint64_t* _ptr, uint32_t _idx) @@ -146,10 +560,7 @@ namespace bx const simd64_t maskedB = simd64_and(prevB, maskB); uint32_t count = countBits(maskedB.u64); - for (uint32_t idx = idxB+1; idx < idxE; ++idx) - { - count += countBits(_ptr[idx]); - } + count += countBitsWords(_ptr, idxB+1, idxE); const simd64_t prevE = simd64_splat(_ptr[idxE]); const simd64_t maskedE = simd64_and(prevE, maskE); @@ -158,6 +569,7 @@ namespace bx return count; } + template inline bool bitSetTestAny(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit) { if (_beginBit >= _endBit) @@ -199,12 +611,9 @@ namespace bx } } - for (uint32_t idx = idxB+1; idx < idxE; ++idx) + if (idxE != findWord(_ptr, idxB+1, idxE) ) { - if (0 != _ptr[idx]) - { - return true; - } + return true; } const simd64_t prev = simd64_splat(_ptr[idxE]); @@ -212,6 +621,7 @@ namespace bx return 0 != masked.u64; } + template inline bool bitSetTestAll(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit) { if (_beginBit >= _endBit) @@ -253,12 +663,9 @@ namespace bx } } - for (uint32_t idx = idxB+1; idx < idxE; ++idx) + if (idxE != findWord(_ptr, idxB+1, idxE) ) { - if (UINT64_MAX != _ptr[idx]) - { - return false; - } + return false; } const simd64_t prev = simd64_splat(_ptr[idxE]); @@ -266,6 +673,7 @@ namespace bx return maskE.u64 == masked.u64; } + template inline bool bitSetTestNone(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit) { if (_beginBit >= _endBit) @@ -307,12 +715,9 @@ namespace bx } } - for (uint32_t idx = idxB+1; idx < idxE; ++idx) + if (idxE != findWord(_ptr, idxB+1, idxE) ) { - if (0 != _ptr[idx]) - { - return false; - } + return false; } const simd64_t prev = simd64_splat(_ptr[idxE]); @@ -398,6 +803,89 @@ namespace bx bitSet_setBits(_ptr, _dstBit, bits, _numBits); } + inline void bitSetBlitForward(uint64_t* _ptr, uint32_t _dstBit, uint32_t _srcBit, uint32_t _numBits) + { + const uint32_t numPreBits = (64 - (_dstBit & 63) ) & 63; + + if (0 < numPreBits) + { + copyBits(_ptr, _dstBit, _srcBit, numPreBits); + } + + const uint32_t srcBit = _srcBit + numPreBits; + const uint32_t shift = srcBit & 63; + + uint32_t dst = (_dstBit + numPreBits) >> 6; + uint32_t src = srcBit >> 6; + uint32_t numBits = _numBits - numPreBits; + + if (0 == shift) + { + for (; numBits >= 64; numBits -= 64, ++dst, ++src) + { + _ptr[dst] = _ptr[src]; + } + } + else + { + uint64_t lo = _ptr[src]; + + for (; numBits >= 64; numBits -= 64, ++dst, ++src) + { + const uint64_t hi = _ptr[src+1]; + _ptr[dst] = (lo >> shift) | (hi << (64 - shift) ); + lo = hi; + } + } + + if (0 < numBits) + { + copyBits(_ptr, dst<<6, (src<<6) + shift, numBits); + } + } + + inline void bitSetBlitBackward(uint64_t* _ptr, uint32_t _dstBit, uint32_t _srcBit, uint32_t _numBits) + { + const uint32_t dstEnd = _dstBit + _numBits; + const uint32_t numPostBits = dstEnd & 63; + + if (0 < numPostBits) + { + copyBits(_ptr, dstEnd - numPostBits, _srcBit + _numBits - numPostBits, numPostBits); + } + + const uint32_t srcEnd = _srcBit + _numBits - numPostBits; + const uint32_t shift = srcEnd & 63; + + uint32_t dst = (dstEnd - numPostBits) >> 6; + uint32_t src = srcEnd >> 6; + uint32_t numBits = _numBits - numPostBits; + + if (0 == shift) + { + for (; numBits >= 64; numBits -= 64) + { + _ptr[--dst] = _ptr[--src]; + } + } + else + { + uint64_t hi = _ptr[src]; + + for (; numBits >= 64; numBits -= 64, --src) + { + const uint64_t lo = _ptr[src-1]; + _ptr[--dst] = (lo >> shift) | (hi << (64 - shift) ); + hi = lo; + } + } + + if (0 < numBits) + { + copyBits(_ptr, _dstBit, _srcBit, numBits); + } + } + inline void bitSetBlit(uint64_t* _ptr, uint32_t _dstBit, uint32_t _srcBit, uint32_t _numBits) { if (_dstBit == _srcBit @@ -409,260 +897,311 @@ namespace bx if (_numBits <= 64) { copyBits(_ptr, _dstBit, _srcBit, _numBits); + return; } - else if (_numBits <= 128) + + if ( (_dstBit&7) != (_srcBit&7) ) { - copyBits(_ptr, _dstBit, _srcBit, 64); - - const uint32_t numBits = _numBits - 64; - - if (0 < numBits) + if (_dstBit < _srcBit) { - copyBits(_ptr, _dstBit + 64, _srcBit + 64, numBits); + bitSetBlitForward(_ptr, _dstBit, _srcBit, _numBits); } + else + { + bitSetBlitBackward(_ptr, _dstBit, _srcBit, _numBits); + } + + return; } - else if ( (_dstBit&7) == (_srcBit&7) ) + + const uint32_t numPreBits = (8 - (_dstBit & 7) ) & 7; + const uint32_t dstByte = (_dstBit + numPreBits) >> 3; + const uint32_t srcByte = (_srcBit + numPreBits) >> 3; + const uint32_t rest = _numBits - numPreBits; + const uint32_t numPostBits = rest & 7; + const uint32_t numBytes = rest >> 3; + uint8_t* ptr = (uint8_t*)_ptr; + + if (_dstBit < _srcBit + && 0 < numPreBits) { - const uint32_t dstByte = alignUp(_dstBit, 8)>>3; - const uint32_t srcByte = alignUp(_srcBit, 8)>>3; - const uint32_t numPreBits = (dstByte<<3) - _dstBit; - const uint32_t tmp0 = _numBits - numPreBits; - const uint32_t numPostBits = tmp0&7; - const uint32_t numBytes = tmp0>>3; - uint8_t* ptr = (uint8_t*)_ptr; - - if (0 < numPreBits) - { - copyBits(_ptr, _dstBit, _srcBit, numPreBits); - } - - bx::memMove(ptr + dstByte, ptr + srcByte, numBytes); - - if (0 < numPostBits) - { - copyBits(_ptr, (dstByte + numBytes)<<3, (srcByte + numBytes)<<3, numPostBits); - } + copyBits(_ptr, _dstBit, _srcBit, numPreBits); } - else + + if (_dstBit > _srcBit + && 0 < numPostBits) { - uint32_t idxB = alignUp(_dstBit, 64)>>6; - const uint32_t numPreBits = (idxB<<6) - _dstBit; + copyBits(_ptr, (dstByte + numBytes)<<3, (srcByte + numBytes)<<3, numPostBits); + } - uint32_t srcBit = _srcBit; + bx::memMove(ptr + dstByte, ptr + srcByte, numBytes); - if (0 < numPreBits) + if (_dstBit < _srcBit + && 0 < numPostBits) + { + copyBits(_ptr, (dstByte + numBytes)<<3, (srcByte + numBytes)<<3, numPostBits); + } + + if (_dstBit > _srcBit + && 0 < numPreBits) + { + copyBits(_ptr, _dstBit, _srcBit, numPreBits); + } + } + + template + BX_FORCE_INLINE uint64_t bitSetRunStarts(uint64_t _free, uint32_t _rest) + { + uint64_t runs = _free; + + if constexpr (1 <= NumStepsT) { runs &= runs >> 1; } + if constexpr (2 <= NumStepsT) { runs &= runs >> 2; } + if constexpr (3 <= NumStepsT) { runs &= runs >> 4; } + if constexpr (4 <= NumStepsT) { runs &= runs >> 8; } + if constexpr (5 <= NumStepsT) { runs &= runs >> 16; } + if constexpr (6 <= NumStepsT) { runs &= runs >> 32; } + + const uint64_t tail = runs >> _rest; + + return runs & tail; + } + + template + inline uint32_t bitSetFindShortStepsT(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) + { + constexpr bool kInWord = NumStepsT <= 6; + constexpr bool kSkipToZeroElement = sizeof(ElementT) < sizeof(uint64_t); + + const uint32_t begin = _beginBit/64; + const uint32_t last = (_endBit - 1)/64; + const uint32_t endShift = _endBit & 63; + const uint64_t beginMask = ~(UINT64_MAX << (_beginBit & 63) ); + const uint64_t endMask = 0 == endShift ? 0 : UINT64_MAX << endShift; + const uint32_t rest = kInWord ? _numBits - (1u << NumStepsT) : 0; + + uint32_t carry = 0; + uint32_t ii = begin; + uint64_t occupied = _ptr[ii] | beginMask; + + for (;;) + { + occupied |= ii == last ? endMask : 0; + + const uint32_t pos = ii*64; + const uint32_t tz = countTrailingZeros(occupied); + + if (carry + tz >= _numBits) { - copyBits(_ptr, _dstBit, srcBit, numPreBits); - srcBit += numPreBits; + return pos - carry; } - uint32_t numBits = _numBits - numPreBits; - - for (; numBits >= 64; numBits -= 64, srcBit += 64) + if constexpr (kInWord) { - const uint64_t bits = bitSet_getBits(_ptr, srcBit, 64); - _ptr[idxB++] = bits; + const uint64_t runs = bitSetRunStarts(~occupied, rest); + + if (0 != runs) + { + return pos + countTrailingZeros(runs); + } } - if (0 < numBits) + carry = 0 == occupied ? carry + 64 : countLeadingZeros(occupied); + + if (ii == last) { - copyBits(_ptr, idxB<<6, srcBit, numBits); + return kInvalid; + } + + ++ii; + + if constexpr (kSkipToZeroElement) + { + if (0 == carry + && ii < last + && !hasZeroElement(_ptr[ii]) + && !hasZeroElement(_ptr[ii+1]) ) + { + const uint32_t next = findZeroElementWord(_ptr, ii + 2, last + 1); + + if (next > last) + { + return kInvalid; + } + + ii = max(ii, next - 1); + } + } + + occupied = _ptr[ii]; + + if (UINT64_MAX == occupied + && ii < last) + { + ii = findWord(_ptr, ii + 1, last); + occupied = _ptr[ii]; + carry = 0; } } } - template - inline uint32_t bitSetFindT(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) + template + inline uint32_t bitSetFindShortT(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) { - const Ty* bits = (const Ty*)_ptr; - constexpr uint32_t numBitsPerStep = sizeof(Ty)*8; + // Run of 2*N-1 or more zero bits always contains whole N-bit aligned zero element. + if (_numBits < 4) { return bitSetFindShortStepsT(_ptr, _beginBit, _endBit, _numBits); } + if (_numBits < 8) { return bitSetFindShortStepsT(_ptr, _beginBit, _endBit, _numBits); } + if (_numBits < 15) { return bitSetFindShortStepsT(_ptr, _beginBit, _endBit, _numBits); } + if (_numBits < 16) { return bitSetFindShortStepsT(_ptr, _beginBit, _endBit, _numBits); } + if (_numBits < 31) { return bitSetFindShortStepsT(_ptr, _beginBit, _endBit, _numBits); } + if (_numBits < 32) { return bitSetFindShortStepsT(_ptr, _beginBit, _endBit, _numBits); } + if (_numBits < 63) { return bitSetFindShortStepsT(_ptr, _beginBit, _endBit, _numBits); } + if (_numBits < 64) { return bitSetFindShortStepsT(_ptr, _beginBit, _endBit, _numBits); } + if (_numBits < 65) { return bitSetFindShortStepsT(_ptr, _beginBit, _endBit, _numBits); } - for (uint32_t ii = alignUp(_beginBit, numBitsPerStep)/numBitsPerStep, end = alignUp(_endBit, numBitsPerStep)/numBitsPerStep; ii < end; ++ii) + return bitSetFindShortStepsT(_ptr, _beginBit, _endBit, _numBits); + } + + template + inline uint32_t bitSetFindLongT(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) + { + for (uint32_t ii = alignUp(_beginBit, 64)/64, end = alignUp(_endBit, 64)/64; ii < end; ++ii) { - if (bits[ii] != Ty(0) ) + ii = findWord(_ptr, ii, end); + + if (ii == end) { - continue; + break; } - uint32_t idx = ii*numBitsPerStep; - uint32_t num = min(idx + numBitsPerStep, _endBit) - idx; + uint32_t idx = ii*64; if (idx != _beginBit) { - const Ty test = bits[idx/numBitsPerStep - 1]; - const uint32_t newIdx = max(idx - countLeadingZeros(test), _beginBit); - - num += idx - newIdx; - idx = newIdx; + idx = max(idx - countLeadingZeros(_ptr[ii-1]), _beginBit); } - for (++ii; ii < end; ++ii) - { - if (num >= _numBits) - { - return idx; - } + const uint32_t next = findWord(_ptr, ii+1, end); + const uint32_t tz = next == end ? 0 : countTrailingZeros(_ptr[next]); + const uint32_t runEnd = min(next*64 + tz, _endBit); - const Ty test = bits[ii]; - const uint32_t pos = ii * numBitsPerStep; - num += min(pos + countTrailingZeros(test), _endBit) - pos; - - if (test != Ty(0) ) - { - break; - } - } - - if (num >= _numBits) + if (runEnd - idx >= _numBits) { return idx; } - } - return _endBit; - } - - inline uint32_t bitSetFindUpto14(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) - { - const uint8_t* bits = (const uint8_t*)_ptr; - - const uint8_t beginMask = uint8_t(~(0xff << (_beginBit & 7) ) ); - - uint32_t lz = 0; - - for (uint32_t begin = _beginBit / 8, end = alignUp(_endBit, 8) / 8, ii = begin; ii < end; ++ii) - { - uint8_t test = bits[ii]; - test |= ii == begin ? beginMask : 0; - - if (0xff == test) - { - lz = 0; - continue; - } - - const uint32_t pos = ii * 8; - const uint32_t tz = min(pos + countTrailingZeros(test), _endBit) - pos; - - if (lz + tz >= _numBits) - { - return pos - lz; - } - - lz = countLeadingZeros(test); - - const uint32_t idx = pos + 8; - const uint32_t newIdx = max(idx - lz, _beginBit); - const uint32_t endIdx = min(idx, _endBit); - lz = endIdx > newIdx ? endIdx - newIdx : 0; - - if (lz >= _numBits) - { - return newIdx; - } - } - - return _endBit; - } - - inline uint32_t bitSetFindUpto6(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) - { - const uint8_t* bits = (const uint8_t*)_ptr; - - const uint8_t beginMask = uint8_t(~(0xff << (_beginBit & 7) ) ); - const uint8_t endMask = uint8_t(~(0xff >> ( ( (8-(_endBit & 7) ) & 7) ) ) ); - const uint8_t mask = (1<<_numBits) - 1; - - for (uint32_t begin = _beginBit / 8, end = alignUp(_endBit, 8) / 8, ii = begin; ii < end; ++ii) - { - uint16_t test = bits[ii]; - test |= ii == begin ? beginMask : 0; - test |= ii == end - 1 ? endMask : 0; - test |= 0xff00; - - if (0xffff == test) - { - continue; - } - - for (uint32_t pos = ii*8, posEnd = pos+7; pos < posEnd; ++pos) - { - const uint8_t tz = countTrailingZeros(test^0xff); - test = int16_t(test) >> tz; - - pos += tz; - - if (0 == (test & mask) ) - { - return pos; - } - - test = int16_t(test) >> 1; - } - } - - return _endBit; - } - - inline uint32_t bitSetFindZero(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit) - { - const uint32_t begin = _beginBit/64; - - for (uint32_t ii = begin, end = alignUp(_endBit, 64)/64; ii < end; ++ii) - { - uint64_t zeros = ~_ptr[ii]; - zeros &= ii == begin ? UINT64_MAX << (_beginBit & 63) : UINT64_MAX; - - if (0 != zeros) - { - const uint32_t idx = ii*64 + countTrailingZeros(zeros); - return idx < _endBit ? idx : kInvalid; - } + ii = next; } return kInvalid; } - inline uint32_t bitSetFind(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) + template + BX_NO_INLINE uint32_t bitSetFindRunT(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) { + if (_numBits < 127) + { + return bitSetFindShortT(_ptr, _beginBit, _endBit, _numBits); + } + + return bitSetFindLongT(_ptr, _beginBit, _endBit, _numBits); + } + + template + BX_NO_INLINE uint32_t bitSetFindZeroFromT(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit) + { + const uint32_t endWord = alignUp(_endBit, 64)/64; + const uint32_t ii = findWord(_ptr, _beginBit/64, endWord); + + if (ii == endWord) + { + return kInvalid; + } + + const uint32_t idx = ii*64 + countTrailingZeros(~_ptr[ii]); + return idx < _endBit ? idx : kInvalid; + } + + template + inline uint32_t bitSetFindZeroT(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit) + { + const uint32_t begin = _beginBit/64; + const uint32_t end = alignUp(_endBit, 64)/64; + + if (begin >= end) + { + return kInvalid; + } + + const uint64_t zeros = ~_ptr[begin] & (UINT64_MAX << (_beginBit & 63) ); + + if (0 == zeros) + { + return bitSetFindZeroFromT(_ptr, (begin + 1)*64, _endBit); + } + + const uint32_t idx = begin*64 + countTrailingZeros(zeros); + return idx < _endBit ? idx : kInvalid; + } + + template + inline uint32_t bitSetFindRangeT(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) + { + if (_beginBit >= _endBit + || _endBit - _beginBit < _numBits) + { + return kInvalid; + } + if (1 == _numBits) { - return bitSetFindZero(_ptr, _beginBit, _endBit); + return bitSetFindZeroT(_ptr, _beginBit, _endBit); } - // Run of 2*N-1 or more zero bits always contains whole N-bit aligned zero word. - uint32_t idx; + if (_numBits <= 64) + { + const uint32_t shift = _beginBit & 63; + const uint32_t numFree = min(64 - shift, _endBit - _beginBit); + const uint64_t valid = UINT64_MAX >> (64 - numFree); + const uint64_t avail = (~_ptr[_beginBit/64] >> shift) & valid; + const uint64_t first = UINT64_MAX >> (64 - _numBits); - if (_numBits >= 127) - { - idx = bitSetFindT(_ptr, _beginBit, _endBit, _numBits); - } - else if (_numBits >= 63) - { - idx = bitSetFindT(_ptr, _beginBit, _endBit, _numBits); - } - else if (_numBits >= 31) - { - idx = bitSetFindT(_ptr, _beginBit, _endBit, _numBits); - } - else - { - idx = bitSetFindT(_ptr, _beginBit, _endBit, _numBits); - - if (idx == _endBit - && _numBits < 15) + if (first == (avail & first) ) { - idx = bitSetFindUpto14(_ptr, _beginBit, _endBit, _numBits); + return _beginBit; + } - if (idx == _endBit - && _numBits < 7) - { - idx = bitSetFindUpto6(_ptr, _beginBit, _endBit, _numBits); - } + uint64_t runs = avail; + uint32_t span = 1; + + for (; span*2 <= _numBits; span *= 2) + { + runs &= runs >> span; + } + + runs &= runs >> (_numBits - span); + + if (0 != runs) + { + return _beginBit + countTrailingZeros(runs); } } - return idx != _endBit ? idx : kInvalid; + return bitSetFindRunT(_ptr, _beginBit, _endBit, _numBits); + } + + uint32_t bitSetFindRef(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) + { + return bitSetFindRangeT(_ptr, _beginBit, _endBit, _numBits); + } + + uint32_t bitSetFindSimd(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) + { + return bitSetFindRangeT(_ptr, _beginBit, _endBit, _numBits); + } + + inline uint32_t bitSetFind(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) + { + return bitSetFindRangeT(_ptr, _beginBit, _endBit, _numBits); } uint64_t BitArrayViewBase::get(uint32_t _beginBit, uint32_t _endBit) const @@ -690,35 +1229,74 @@ namespace bx uint32_t BitArrayViewBase::findSet(Element _element) const { - uint32_t from = 0; - uint32_t mask = 0; const uint32_t num = m_sizeInBytes/8; if (kLast == _element) { - from = num; - mask = UINT32_MAX; + const uint32_t idx = findLastWord(m_ptr, 0, num); + + return kInvalid != idx + ? idx*64 + findLastSet(m_ptr[idx]) - 1 + : kInvalid + ; } - for (uint32_t ii = 0; ii < num; ++ii) + const uint32_t idx = findWord(m_ptr, 0, num); + + return idx != num + ? idx*64 + findFirstSet(m_ptr[idx]) - 1 + : kInvalid + ; + } + + uint32_t BitArrayViewBase::findSetFrom(uint32_t _beginBit, uint32_t _endBit) const + { + const uint32_t beginWord = _beginBit/64; + const uint32_t endWord = alignUp(_endBit, 64)/64; + + if (beginWord >= endWord) { - const uint32_t idx = ( (~from + 1) + ii) ^ mask; - const uint64_t word = m_ptr[idx]; - - if (0 == word) - { - continue; - } - - if (kLast == _element) - { - return idx*64 + findLastSet(word) - 1; - } - - return idx*64 + findFirstSet(word) - 1; + return kInvalid; } - return kInvalid; + const uint32_t idx = findWord(m_ptr, beginWord, endWord); + + if (idx == endWord) + { + return kInvalid; + } + + const uint32_t found = idx*64 + findFirstSet(m_ptr[idx]) - 1; + + return found < _endBit + ? found + : kInvalid + ; + } + + uint32_t BitArrayViewBase::findClearFrom(uint32_t _beginBit, uint32_t _endBit) const + { + const uint32_t beginWord = _beginBit/64; + const uint32_t endWord = alignUp(_endBit, 64)/64; + + if (beginWord >= endWord) + { + return kInvalid; + } + + const uint32_t idx = findWord(m_ptr, beginWord, endWord); + + if (idx == endWord) + { + return kInvalid; + } + + const uint32_t found = idx*64 + findFirstSet(~m_ptr[idx]) - 1; + + return found < _endBit + ? found + : kInvalid + ; } uint32_t BitArrayViewBase::countBits(uint32_t _beginBit, uint32_t _endBit) const @@ -754,7 +1332,7 @@ namespace bx , _endBit <= m_count , _beginBit <= _endBit ); - return bitSetTestAny(m_ptr, _beginBit, _endBit); + return bitSetTestAny(m_ptr, _beginBit, _endBit); } bool BitArrayViewBase::testAll(uint32_t _beginBit, uint32_t _endBit) const @@ -772,7 +1350,7 @@ namespace bx , _endBit <= m_count , _beginBit <= _endBit ); - return bitSetTestAll(m_ptr, _beginBit, _endBit); + return bitSetTestAll(m_ptr, _beginBit, _endBit); } bool BitArrayViewBase::testNone(uint32_t _beginBit, uint32_t _endBit) const @@ -790,7 +1368,7 @@ namespace bx , _endBit <= m_count , _beginBit <= _endBit ); - return bitSetTestNone(m_ptr, _beginBit, _endBit); + return bitSetTestNone(m_ptr, _beginBit, _endBit); } void MutableBitArrayView::set(uint32_t _beginBit, uint32_t _endBit, bool _value) @@ -834,4 +1412,16 @@ namespace bx bitSetBlit(m_ptr, _dstBit, _srcBit, _numBits); } + void MippedBitArrayBase::updateMipFilled(uint32_t _beginWord, uint32_t _endWord, bool _value) + { + updateMip(_beginWord, _beginWord + 1); + updateMip(_endWord - 1, _endWord); + + if (_beginWord + 2 < _endWord) + { + const uint32_t numMipWords = getNumMipWords(m_sizeInBytes/8); + MutableBitArrayView(m_mip, numMipWords*sizeof(uint64_t) ).set(_beginWord + 1, _endWord - 1, _value); + } + } + } // namespace bx diff --git a/tests/bitarray_test.cpp b/tests/bitarray_test.cpp index 2bde8d2..b1d4f84 100644 --- a/tests/bitarray_test.cpp +++ b/tests/bitarray_test.cpp @@ -979,6 +979,38 @@ TEST_CASE("BitArray-ranges-ending-on-word-boundary", "[container]") REQUIRE( (UINT64_MAX >> 28) == words[2]); } +namespace bx +{ + uint32_t bitSetFindRef(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits); + uint32_t bitSetFindSimd(const uint64_t* _ptr, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits); + uint32_t bitSetFindWordRef(const uint64_t* _ptr, uint32_t _beginWord, uint32_t _endWord, uint32_t _match); + uint32_t bitSetFindWordSimd(const uint64_t* _ptr, uint32_t _beginWord, uint32_t _endWord, uint32_t _match); + uint32_t bitSetFindLastWordRef(const uint64_t* _ptr, uint32_t _beginWord, uint32_t _endWord, uint32_t _match); + uint32_t bitSetFindLastWordSimd(const uint64_t* _ptr, uint32_t _beginWord, uint32_t _endWord, uint32_t _match); +} + +static bool getBit(const uint64_t* _bits, uint32_t _idx) +{ + return 0 != ( (_bits[_idx/64] >> (_idx & 63) ) & 1); +} + +static uint32_t findFirstFit(const uint64_t* _bits, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) +{ + uint32_t run = 0; + + for (uint32_t ii = _beginBit; ii < _endBit; ++ii) + { + run = getBit(_bits, ii) ? 0 : run + 1; + + if (run == _numBits) + { + return ii + 1 - _numBits; + } + } + + return bx::kInvalid; +} + static uint32_t findRef(const bx::MutableBitArrayView& _bs, uint32_t _beginBit, uint32_t _endBit, uint32_t _numBits) { for (uint32_t ii = _beginBit; ii + _numBits <= _endBit; ++ii) @@ -1073,7 +1105,7 @@ TEST_CASE("BitArray-find-bench", "[.][!benchmark]") char name[64]; - for (uint32_t numBits : { 4u, 12u, 40u, 200u, 300u }) + for (uint32_t numBits : { 1u, 2u, 4u, 6u, 7u, 12u, 14u, 15u, 24u, 31u, 40u, 63u, 64u, 127u, 200u, 300u }) { uint32_t idx = 0; @@ -1084,6 +1116,399 @@ TEST_CASE("BitArray-find-bench", "[.][!benchmark]") return idx; }; + bx::snprintf(name, sizeof(name), "find ref %u", numBits); + BENCHMARK(name) + { + return bx::bitSetFindRef(bits, 0, kNumBits, numBits); + }; + REQUIRE( (bx::kInvalid == idx) == (bx::kInvalid == findRef(bs, 0, kNumBits, numBits) ) ); } } + +TEST_CASE("BitArray-find-simd-matches-ref", "[container]") +{ + constexpr uint32_t kMaxWords = 80; + + uint64_t bits[kMaxWords]; + + bx::RngMwc rng; + + for (uint32_t iter = 0; iter < 50000; ++iter) + { + const uint32_t numWords = 1 + rng.gen() % kMaxWords; + const uint32_t numBits = numWords*64; + bx::MutableBitArrayView bs(bits, numWords*sizeof(uint64_t) ); + + const uint32_t density = rng.gen() % 6; + + bs.clear(); + + for (uint32_t ii = 0, num = rng.gen() % (4 + numWords*4); ii < num; ++ii) + { + const uint32_t begin = rng.gen() % numBits; + const uint32_t end = bx::min(numBits, begin + 1 + rng.gen() % (4 << density) ); + bs.set(begin, end, true); + } + + if (0 == (rng.gen() & 3) ) + { + bs.set(0, numBits, true); + + for (uint32_t ii = 0, num = rng.gen() % 4; ii < num; ++ii) + { + const uint32_t begin = rng.gen() % numBits; + const uint32_t end = bx::min(numBits, begin + 1 + rng.gen() % 320); + bs.set(begin, end, false); + } + } + + const uint32_t runBits = 0 == (rng.gen() & 1) ? 1 + rng.gen() % 8 : 1 + rng.gen() % 300; + const uint32_t beginBit = rng.gen() % numBits; + const uint32_t endBit = beginBit + 1 + rng.gen() % (numBits - beginBit); + + const uint32_t ref = bx::bitSetFindRef (bits, beginBit, endBit, runBits); + const uint32_t simd = bx::bitSetFindSimd(bits, beginBit, endBit, runBits); + + INFO("Iteration " << iter << ", words " << numWords << ", range [" << beginBit << ", " << endBit << "), num bits " << runBits << ", simd " << simd << ", ref " << ref); + + REQUIRE(ref == simd); + REQUIRE(findFirstFit(bits, beginBit, endBit, runBits) == simd); + } +} + +TEST_CASE("BitArray-ops-bench", "[.][!benchmark]") +{ + constexpr uint32_t kNumBits = 64<<10; + + alignas(64) static uint64_t bits[kNumBits/64]; + bx::MutableBitArrayView bs(bits, sizeof(bits) ); + + char name[64]; + + for (uint32_t numBits : { 7u, 64u, 300u, 4096u }) + { + bs.clear(); + + bx::snprintf(name, sizeof(name), "set %u", numBits); + BENCHMARK(name) + { + bs.set(13, 13 + numBits, true); + bs.set(13, 13 + numBits, false); + return bits[0]; + }; + } + + bs.clear(); + + BENCHMARK("countBits 64K") + { + return bs.countBits(1, kNumBits - 1); + }; + + BENCHMARK("testNone 64K") + { + return bs.testNone(1, kNumBits - 1); + }; + + BENCHMARK("testAny 64K") + { + return bs.testAny(1, kNumBits - 1); + }; + + BENCHMARK("findSet first 64K") + { + return bs.findSet(bx::kFirst); + }; + + BENCHMARK("findSet range 64K") + { + return bs.findSet(1, kNumBits - 1); + }; + + bs.set(0, kNumBits, true); + + BENCHMARK("testAll 64K") + { + return bs.testAll(1, kNumBits - 1); + }; + + BENCHMARK("findClear range 64K") + { + return bs.findClear(1, kNumBits - 1); + }; + + BENCHMARK("find 1 64K") + { + return bs.find(1); + }; + + BENCHMARK("find 12 64K") + { + return bs.find(12); + }; + + bx::RngMwc rng; + + for (uint32_t ii = 0; ii < kNumBits; ++ii) + { + bits[ii/64] = (uint64_t(rng.gen() ) << 32) | rng.gen(); + } + + BENCHMARK("countBits random 64K") + { + return bs.countBits(1, kNumBits - 1); + }; + + for (uint32_t numBits : { 300u, 4096u }) + { + bx::snprintf(name, sizeof(name), "blit %u", numBits); + BENCHMARK(name) + { + bs.blit(1000, 3, numBits); + return bits[20]; + }; + + bx::snprintf(name, sizeof(name), "blit aligned %u", numBits); + BENCHMARK(name) + { + bs.blit(1024, 128, numBits); + return bits[20]; + }; + } +} + +static void fillRandom(bx::RngMwc& _rng, uint64_t* _bits, uint32_t _numWords) +{ + const uint32_t mode = _rng.gen() % 4; + + for (uint32_t ii = 0; ii < _numWords; ++ii) + { + const uint64_t word = (uint64_t(_rng.gen() ) << 32) | _rng.gen(); + const uint32_t pick = _rng.gen() % 8; + + switch (mode) + { + case 0: _bits[ii] = word; break; + case 1: _bits[ii] = 0 == pick ? word : 0; break; + case 2: _bits[ii] = 0 == pick ? word : UINT64_MAX; break; + default: _bits[ii] = 0 == pick ? word : 1 == pick ? 0 : UINT64_MAX; break; + } + } +} + +TEST_CASE("BitArray-find-word-simd-matches-ref", "[container]") +{ + constexpr uint32_t kMaxWords = 80; + + uint64_t bits[kMaxWords]; + + bx::RngMwc rng; + + for (uint32_t iter = 0; iter < 50000; ++iter) + { + const uint32_t numWords = 1 + rng.gen() % kMaxWords; + fillRandom(rng, bits, numWords); + + const uint32_t begin = rng.gen() % (numWords + 1); + const uint32_t end = begin + rng.gen() % (numWords - begin + 1); + const uint32_t match = rng.gen() % 3; + + INFO("Iteration " << iter << ", words " << numWords << ", range [" << begin << ", " << end << "), match " << match); + + REQUIRE(bx::bitSetFindWordRef(bits, begin, end, match) == bx::bitSetFindWordSimd(bits, begin, end, match) ); + REQUIRE(bx::bitSetFindLastWordRef(bits, begin, end, match) == bx::bitSetFindLastWordSimd(bits, begin, end, match) ); + } +} + +TEST_CASE("BitArray-range-ops-random", "[container]") +{ + constexpr uint32_t kMaxWords = 80; + + uint64_t bits[kMaxWords]; + + bx::RngMwc rng; + + for (uint32_t iter = 0; iter < 50000; ++iter) + { + const uint32_t numWords = 1 + rng.gen() % kMaxWords; + const uint32_t numBits = numWords*64; + fillRandom(rng, bits, numWords); + + bx::MutableBitArrayView bs(bits, numWords*sizeof(uint64_t) ); + + const uint32_t beginBit = rng.gen() % (numBits + 1); + const uint32_t endBit = beginBit + rng.gen() % (numBits - beginBit + 1); + + uint32_t count = 0; + uint32_t firstSet = bx::kInvalid; + uint32_t firstClr = bx::kInvalid; + + for (uint32_t ii = beginBit; ii < endBit; ++ii) + { + const bool bit = getBit(bits, ii); + count += bit; + firstSet = bit && bx::kInvalid == firstSet ? ii : firstSet; + firstClr = !bit && bx::kInvalid == firstClr ? ii : firstClr; + } + + const uint32_t numRange = endBit - beginBit; + + INFO("Iteration " << iter << ", words " << numWords << ", range [" << beginBit << ", " << endBit << ")"); + + REQUIRE(count == bs.countBits(beginBit, endBit) ); + REQUIRE( (0 != count) == bs.testAny(beginBit, endBit) ); + REQUIRE( (0 == count) == bs.testNone(beginBit, endBit) ); + REQUIRE( (numRange == count) == bs.testAll(beginBit, endBit) ); + REQUIRE(firstSet == bs.findSet(beginBit, endBit) ); + REQUIRE(firstClr == bs.findClear(beginBit, endBit) ); + + uint32_t first = bx::kInvalid; + uint32_t last = bx::kInvalid; + + for (uint32_t ii = 0; ii < numBits; ++ii) + { + if (getBit(bits, ii) ) + { + first = bx::kInvalid == first ? ii : first; + last = ii; + } + } + + REQUIRE(first == bs.findSet(bx::kFirst) ); + REQUIRE(last == bs.findSet(bx::kLast) ); + } +} + +TEST_CASE("BitArray-blit-random", "[container]") +{ + constexpr uint32_t kNumWords = 40; + constexpr uint32_t kNumBits = kNumWords*64; + + uint64_t bits[kNumWords]; + uint64_t expected[kNumWords]; + + bx::RngMwc rng; + + for (uint32_t iter = 0; iter < 50000; ++iter) + { + fillRandom(rng, bits, kNumWords); + bx::memCopy(expected, bits, sizeof(bits) ); + + const uint32_t numBits = 1 + rng.gen() % (0 == (rng.gen() & 1) ? 160 : kNumBits/2); + const uint32_t srcBit = rng.gen() % (kNumBits - numBits + 1); + const uint32_t dstBit = 0 == (rng.gen() & 3) + ? bx::min(kNumBits - numBits, (srcBit & ~7u) + (rng.gen() % 64)*8) + : rng.gen() % (kNumBits - numBits + 1) + ; + + bool tmp[kNumBits]; + + for (uint32_t ii = 0; ii < numBits; ++ii) + { + tmp[ii] = getBit(expected, srcBit + ii); + } + + for (uint32_t ii = 0; ii < numBits; ++ii) + { + const uint32_t idx = dstBit + ii; + expected[idx/64] = (expected[idx/64] & ~(UINT64_C(1) << (idx & 63) ) ) | (uint64_t(tmp[ii]) << (idx & 63) ); + } + + bx::MutableBitArrayView bs(bits, sizeof(bits) ); + bs.blit(dstBit, srcBit, numBits); + + INFO("Iteration " << iter << ", dst " << dstBit << ", src " << srcBit << ", num bits " << numBits); + + REQUIRE(0 == bx::memCmp(bits, expected, sizeof(bits) ) ); + } +} + +TEST_CASE("BitArray-set-random", "[container]") +{ + constexpr uint32_t kNumWords = 40; + constexpr uint32_t kNumBits = kNumWords*64; + + uint64_t bits[kNumWords]; + uint64_t expected[kNumWords]; + + bx::RngMwc rng; + + for (uint32_t iter = 0; iter < 50000; ++iter) + { + fillRandom(rng, bits, kNumWords); + bx::memCopy(expected, bits, sizeof(bits) ); + + const uint32_t beginBit = rng.gen() % (kNumBits + 1); + const uint32_t endBit = beginBit + rng.gen() % (kNumBits - beginBit + 1); + const bool value = 0 != (rng.gen() & 1); + + for (uint32_t idx = beginBit; idx < endBit; ++idx) + { + expected[idx/64] = (expected[idx/64] & ~(UINT64_C(1) << (idx & 63) ) ) | (uint64_t(value) << (idx & 63) ); + } + + bx::MutableBitArrayView bs(bits, sizeof(bits) ); + bs.set(beginBit, endBit, value); + + INFO("Iteration " << iter << ", range [" << beginBit << ", " << endBit << "), value " << value); + + REQUIRE(0 == bx::memCmp(bits, expected, sizeof(bits) ) ); + } +} + +TEST_CASE("MippedBitArray-random", "[container]") +{ + constexpr uint32_t kNumBits = 64*130; + constexpr uint32_t kNumWords = kNumBits/64; + + bx::FixedMippedBitArrayT array; + array.setCount(kNumBits); + + bx::RngMwc rng; + + for (uint32_t iter = 0; iter < 20000; ++iter) + { + const uint32_t op = rng.gen() % 6; + const uint32_t beginBit = rng.gen() % kNumBits; + const uint32_t numBits = 1 + rng.gen() % (0 == (rng.gen() & 3) ? kNumBits - beginBit : bx::min(kNumBits - beginBit, 300) ); + const uint32_t endBit = beginBit + numBits; + + switch (op) + { + case 0: array.set(beginBit, endBit, true); break; + case 1: array.set(beginBit, endBit, false); break; + case 2: array.set(beginBit, 0 != (rng.gen() & 1) ); break; + case 3: array.set(beginBit, bx::min(endBit, beginBit + 64), (uint64_t(rng.gen() ) << 32) | rng.gen() ); break; + case 4: array.blit(rng.gen() % (kNumBits - numBits + 1), beginBit, numBits); break; + default: if (0 == rng.gen() % 64) { array.clear(); } break; + } + + const uint64_t* words = (const uint64_t*)array.getPtr(); + const uint64_t* mip = array.getMip(); + + INFO("Iteration " << iter << ", op " << op << ", range [" << beginBit << ", " << endBit << ")"); + + for (uint32_t ii = 0; ii < kNumWords; ++ii) + { + const bool full = UINT64_MAX == words[ii]; + const bool mipFull = 0 != ( (mip[ii/64] >> (ii & 63) ) & 1); + REQUIRE(full == mipFull); + } + + for (uint32_t ii = kNumWords; ii < bx::alignUp(kNumWords, 64u); ++ii) + { + REQUIRE(0 != ( (mip[ii/64] >> (ii & 63) ) & 1) ); + } + + const uint32_t findBegin = rng.gen() % kNumBits; + uint32_t expected = bx::kInvalid; + + for (uint32_t ii = findBegin; ii < kNumBits && bx::kInvalid == expected; ++ii) + { + expected = getBit(words, ii) ? bx::kInvalid : ii; + } + + REQUIRE(expected == array.findClear(findBegin, kNumBits) ); + } +} diff --git a/tests/math_test.cpp b/tests/math_test.cpp index 847fa74..41f172c 100644 --- a/tests/math_test.cpp +++ b/tests/math_test.cpp @@ -205,6 +205,65 @@ TEST_CASE("countTrailingZeros", "[math]") STATIC_REQUIRE(64 == bx::countTrailingZeros(0) ); } +template +static uint8_t countLeadingZerosRef(Ty _val) +{ + uint8_t count = 0; + + for (int32_t ii = sizeof(Ty)*8 - 1; ii >= 0 && 0 == ( (_val >> ii) & 1); --ii) + { + ++count; + } + + return count; +} + +template +static uint8_t countTrailingZerosRef(Ty _val) +{ + uint8_t count = 0; + + for (uint32_t ii = 0; ii < sizeof(Ty)*8 && 0 == ( (_val >> ii) & 1); ++ii) + { + ++count; + } + + return count; +} + +template +static void testCountZeros(Ty _val) +{ + volatile Ty val = _val; + REQUIRE(countLeadingZerosRef(_val) == bx::countLeadingZeros(val) ); + REQUIRE(countTrailingZerosRef(_val) == bx::countTrailingZeros(val) ); +} + +TEST_CASE("countLeadingZeros/countTrailingZeros runtime", "[math]") +{ + testCountZeros(0); + testCountZeros(0); + + for (uint32_t ii = 0; ii < 64; ++ii) + { + testCountZeros(uint32_t(1) << (ii & 31) ); + testCountZeros(uint64_t(1) << ii); + testCountZeros(UINT64_MAX << ii); + testCountZeros(UINT64_MAX >> ii); + } + + bx::RngMwc rng; + + for (uint32_t ii = 0; ii < 10000; ++ii) + { + const uint64_t val = (uint64_t(rng.gen() ) << 32) | rng.gen(); + const uint32_t shift = rng.gen() % 64; + testCountZeros(uint32_t(val) >> (shift & 31) ); + testCountZeros(val >> shift); + testCountZeros(val << shift); + } +} + TEST_CASE("countLeadingZeros", "[math]") { STATIC_REQUIRE( 7 == bx::countLeadingZeros(1) );