Added atomicLoad, atomicStore, atomicExchange, cpuRelax, getStackAddress, getHardwareThreads, timerResolutionBegin/End, memoryAllocationGranularity, and sanitizer.h. Fixed memoryPageSize returning allocation granularity on Windows. (#435)

This commit is contained in:
Branimir Karadžić
2026-09-29 19:16:07 -07:00
committed by GitHub
parent e98d9f4b35
commit 3b8366af83
12 changed files with 436 additions and 26 deletions

View File

@@ -61,6 +61,29 @@ namespace bx
///
void* atomicExchangePtr(void** _ptr, void* _new);
/// Sequentially consistent atomic load of 32-bit, or 64-bit value.
template<typename Ty>
Ty atomicLoad(const volatile Ty* _ptr);
/// Sequentially consistent atomic store of 32-bit, or 64-bit value.
template<typename Ty>
void atomicStore(volatile Ty* _ptr, Ty _value);
/// Atomic store without ordering, for values that don't need to be immediately
/// visible to other threads.
template<typename Ty>
void atomicStoreRelaxed(volatile Ty* _ptr, Ty _value);
/// Atomic exchange of 32-bit, or 64-bit value. Returns previous value.
template<typename Ty>
Ty atomicExchange(volatile Ty* _ptr, Ty _new);
/// Atomic compare and swap of pointer sized integer. Returns previous value.
uintptr_t atomicCompareAndSwapPtr(volatile uintptr_t* _ptr, uintptr_t _old, uintptr_t _new);
/// Hint to CPU that caller is spin waiting.
void cpuRelax();
} // namespace bx
#include "inline/cpu.inl"

View File

@@ -6,9 +6,20 @@
#ifndef BX_DEBUG_H_HEADER_GUARD
#define BX_DEBUG_H_HEADER_GUARD
#include "platform.h"
#include <stdint.h> // uint32_t
#include <stdarg.h> // va_list
/// Address of current stack frame.
#if BX_COMPILER_MSVC
extern "C" void* _AddressOfReturnAddress();
# pragma intrinsic(_AddressOfReturnAddress)
# define BX_STACK_FRAME_ADDRESS() uintptr_t(_AddressOfReturnAddress() )
#else
# define BX_STACK_FRAME_ADDRESS() uintptr_t(__builtin_frame_address(0) )
#endif // BX_COMPILER_MSVC
namespace bx
{
class Error;

View File

@@ -58,6 +58,15 @@ extern "C" void* _InterlockedExchangePointer(void* volatile* _ptr, void* _value)
#endif
extern "C" long _InterlockedExchange(long volatile* _ptr, long _value);
# pragma intrinsic(_InterlockedExchange)
extern "C" int64_t _InterlockedExchange64(int64_t volatile* _ptr, int64_t _value);
# pragma intrinsic(_InterlockedExchange64)
extern "C" void* _InterlockedCompareExchangePointer(void* volatile* _ptr, void* _exchange, void* _comparand);
# pragma intrinsic(_InterlockedCompareExchangePointer)
# if BX_PLATFORM_WINRT
# define _InterlockedExchangeAdd64 InterlockedExchangeAdd64
# endif // BX_PLATFORM_WINRT
@@ -290,7 +299,7 @@ namespace bx
inline Ty atomicFetchTestAndAdd(volatile Ty* _ptr, Ty _test, Ty _value)
{
Ty oldVal;
Ty newVal = *_ptr;
Ty newVal = atomicLoad(_ptr);
do
{
oldVal = newVal;
@@ -305,7 +314,7 @@ namespace bx
inline Ty atomicFetchTestAndSub(volatile Ty* _ptr, Ty _test, Ty _value)
{
Ty oldVal;
Ty newVal = *_ptr;
Ty newVal = atomicLoad(_ptr);
do
{
oldVal = newVal;
@@ -320,7 +329,7 @@ namespace bx
Ty atomicFetchAndAddsat(volatile Ty* _ptr, Ty _value, Ty _max)
{
Ty oldVal;
Ty newVal = *_ptr;
Ty newVal = atomicLoad(_ptr);
do
{
oldVal = newVal;
@@ -335,7 +344,7 @@ namespace bx
Ty atomicFetchAndSubsat(volatile Ty* _ptr, Ty _value, Ty _min)
{
Ty oldVal;
Ty newVal = *_ptr;
Ty newVal = atomicLoad(_ptr);
do
{
oldVal = newVal;
@@ -355,4 +364,85 @@ namespace bx
#endif // BX_COMPILER_*
}
template<typename Ty>
inline Ty atomicLoad(const volatile Ty* _ptr)
{
static_assert(4 == sizeof(Ty) || 8 == sizeof(Ty), "Only 32-bit, and 64-bit types are supported.");
#if BX_COMPILER_MSVC
return *_ptr;
#else
return __atomic_load_n(_ptr, __ATOMIC_SEQ_CST);
#endif // BX_COMPILER_*
}
template<typename Ty>
inline void atomicStore(volatile Ty* _ptr, Ty _value)
{
static_assert(4 == sizeof(Ty) || 8 == sizeof(Ty), "Only 32-bit, and 64-bit types are supported.");
#if BX_COMPILER_MSVC
// Exchange, like clang, and GCC do. Store followed by `mfence` is 8x slower on
// AMD Zen 4.
atomicExchange(_ptr, _value);
#else
__atomic_store_n(_ptr, _value, __ATOMIC_SEQ_CST);
#endif // BX_COMPILER_*
}
template<typename Ty>
inline void atomicStoreRelaxed(volatile Ty* _ptr, Ty _value)
{
static_assert(4 == sizeof(Ty) || 8 == sizeof(Ty), "Only 32-bit, and 64-bit types are supported.");
#if BX_COMPILER_MSVC
*_ptr = _value;
#else
__atomic_store_n(_ptr, _value, __ATOMIC_RELAXED);
#endif // BX_COMPILER_*
}
template<typename Ty>
inline Ty atomicExchange(volatile Ty* _ptr, Ty _new)
{
static_assert(4 == sizeof(Ty) || 8 == sizeof(Ty), "Only 32-bit, and 64-bit types are supported.");
#if BX_COMPILER_MSVC
if constexpr (8 == sizeof(Ty) )
{
return Ty(_InterlockedExchange64( (volatile int64_t*)_ptr, int64_t(_new) ) );
}
else
{
return Ty(_InterlockedExchange( (volatile long*)_ptr, long(_new) ) );
}
#else
return __atomic_exchange_n(_ptr, _new, __ATOMIC_SEQ_CST);
#endif // BX_COMPILER_*
}
inline uintptr_t atomicCompareAndSwapPtr(volatile uintptr_t* _ptr, uintptr_t _old, uintptr_t _new)
{
#if BX_COMPILER_MSVC
return uintptr_t(_InterlockedCompareExchangePointer( (void* volatile*)_ptr, (void*)_new, (void*)_old) );
#else
return __sync_val_compare_and_swap(_ptr, _old, _new);
#endif // BX_COMPILER_*
}
inline void cpuRelax()
{
#if BX_COMPILER_MSVC
# if BX_CPU_X86
_mm_pause();
# else
YieldProcessor();
# endif // BX_CPU_X86
#elif BX_CPU_X86
__builtin_ia32_pause();
#elif BX_CPU_ARM && BX_ARCH_64BIT
__asm__ volatile("yield");
#endif // BX_COMPILER_*
}
} // namespace bx

View File

@@ -35,7 +35,7 @@ namespace bx
{
m_last->m_next = BX_NEW(m_allocator, Node)(_ptr);
atomicExchangePtr( (void**)&m_last, m_last->m_next);
while (m_first != m_divider)
while (m_first != atomicLoad(&m_divider) )
{
Node* node = m_first;
m_first = m_first->m_next;
@@ -45,7 +45,7 @@ namespace bx
inline void* SpScUnboundedQueue::peek()
{
if (m_divider != m_last)
if (m_divider != atomicLoad(&m_last) )
{
return m_divider->m_next->m_ptr;
}
@@ -55,7 +55,7 @@ namespace bx
inline void* SpScUnboundedQueue::pop()
{
if (m_divider != m_last)
if (m_divider != atomicLoad(&m_last) )
{
void* ptr = m_divider->m_next->m_ptr;
atomicExchangePtr( (void**)&m_divider, m_divider->m_next);

View File

@@ -328,6 +328,69 @@ extern "C" void* __cdecl _alloca(size_t _size);
} \
BX_MACRO_BLOCK_END
#if BX_CLANG_HAS_FEATURE(thread_sanitizer) || defined(__SANITIZE_THREAD__)
# define BX_SANITIZER_THREAD 1
#else
# define BX_SANITIZER_THREAD 0
#endif // BX_CLANG_HAS_FEATURE(thread_sanitizer)
#if BX_CLANG_HAS_FEATURE(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
# define BX_SANITIZER_ADDRESS 1
#else
# define BX_SANITIZER_ADDRESS 0
#endif // BX_CLANG_HAS_FEATURE(address_sanitizer)
#if BX_SANITIZER_THREAD
extern "C"
{
void __tsan_mutex_pre_lock(void* _addr, unsigned _flags);
void __tsan_mutex_post_lock(void* _addr, unsigned _flags, int _recursion);
int __tsan_mutex_pre_unlock(void* _addr, unsigned _flags);
void __tsan_mutex_post_unlock(void* _addr, unsigned _flags);
void* __tsan_get_current_fiber();
void* __tsan_create_fiber(unsigned _flags);
void __tsan_destroy_fiber(void* _fiber);
void __tsan_switch_to_fiber(void* _fiber, unsigned _flags);
}
/// Annotations of custom mutex for ThreadSanitizer.
# define BX_TSAN_MUTEX_PRE_LOCK(_ptr) __tsan_mutex_pre_lock( (void*)(_ptr), 0)
# define BX_TSAN_MUTEX_POST_LOCK(_ptr) __tsan_mutex_post_lock( (void*)(_ptr), 0, 0)
# define BX_TSAN_MUTEX_PRE_UNLOCK(_ptr) __tsan_mutex_pre_unlock( (void*)(_ptr), 0)
# define BX_TSAN_MUTEX_POST_UNLOCK(_ptr) __tsan_mutex_post_unlock( (void*)(_ptr), 0)
/// Annotations of fiber switch for ThreadSanitizer.
# define BX_TSAN_FIBER_CURRENT() __tsan_get_current_fiber()
# define BX_TSAN_FIBER_CREATE() __tsan_create_fiber(0)
# define BX_TSAN_FIBER_DESTROY(_fiber) __tsan_destroy_fiber(_fiber)
# define BX_TSAN_FIBER_SWITCH(_fiber) __tsan_switch_to_fiber(_fiber, 0)
#else
# define BX_TSAN_MUTEX_PRE_LOCK(_ptr) BX_NOOP(_ptr)
# define BX_TSAN_MUTEX_POST_LOCK(_ptr) BX_NOOP(_ptr)
# define BX_TSAN_MUTEX_PRE_UNLOCK(_ptr) BX_NOOP(_ptr)
# define BX_TSAN_MUTEX_POST_UNLOCK(_ptr) BX_NOOP(_ptr)
# define BX_TSAN_FIBER_CURRENT() NULL
# define BX_TSAN_FIBER_CREATE() NULL
# define BX_TSAN_FIBER_DESTROY(_fiber) BX_NOOP(_fiber)
# define BX_TSAN_FIBER_SWITCH(_fiber) BX_NOOP(_fiber)
#endif // BX_SANITIZER_THREAD
#if BX_SANITIZER_ADDRESS
extern "C"
{
void __sanitizer_start_switch_fiber(void** _fakeStackSave, const void* _bottom, size_t _size);
void __sanitizer_finish_switch_fiber(void* _fakeStackSave, const void** _bottomOld, size_t* _sizeOld);
}
/// Annotations of fiber switch for AddressSanitizer.
# define BX_ASAN_FIBER_SWITCH_START(_fakeStackSave, _bottom, _size) __sanitizer_start_switch_fiber(_fakeStackSave, _bottom, _size)
# define BX_ASAN_FIBER_SWITCH_FINISH(_fakeStack, _outBottom, _outSize) __sanitizer_finish_switch_fiber(_fakeStack, _outBottom, _outSize)
#else
# define BX_ASAN_FIBER_SWITCH_START(_fakeStackSave, _bottom, _size) BX_NOOP(_fakeStackSave, _bottom, _size)
# define BX_ASAN_FIBER_SWITCH_FINISH(_fakeStack, _outBottom, _outSize) BX_NOOP(_fakeStack, _outBottom, _outSize)
#endif // BX_SANITIZER_ADDRESS
// static_assert sometimes causes unused-local-typedef...
BX_PRAGMA_DIAGNOSTIC_IGNORED_CLANG("-Wunused-local-typedef")

View File

@@ -67,6 +67,25 @@ namespace bx
/// Yield remainder of the current thread's time slice to another runnable thread.
void yield();
/// Request OS timer resolution, which affects precision of `sleep` and wait timeouts.
/// Every call must be matched with `timerResolutionEnd` with the same value.
///
/// @param[in] _ms Timer resolution in milliseconds, 0 = no request.
///
void timerResolutionBegin(uint32_t _ms);
/// End request made with `timerResolutionBegin`.
///
/// @param[in] _ms Value passed to `timerResolutionBegin`.
///
void timerResolutionEnd(uint32_t _ms);
/// Get number of hardware threads available to the process.
///
/// @returns Number of hardware threads, at least 1.
///
uint32_t getHardwareThreads();
/// Get the OS-level identifier of the calling thread.
///
/// @returns Platform-specific thread id.
@@ -184,6 +203,13 @@ namespace bx
///
size_t memoryPageSize(uint32_t _flags = 0);
/// Query granularity of base address of fresh reservation made by `memoryMap`. It's
/// larger than page size on Windows (64 KiB), and equal to page size elsewhere.
///
/// @returns Allocation granularity in bytes.
///
size_t memoryAllocationGranularity();
} // namespace bx
#include "inline/os.inl"

View File

@@ -430,11 +430,7 @@ namespace bx
{
if (BX_ENABLED( (BX_PLATFORM_LINUX || BX_PLATFORM_OSX) && BX_ARCH_64BIT) )
{
#if BX_COMPILER_GCC || BX_COMPILER_CLANG
const uintptr_t* stackFrame = (const uintptr_t*)__builtin_frame_address(0);
#else
const uintptr_t* stackFrame = NULL;
#endif // BX_COMPILER_...
const uintptr_t* stackFrame = (const uintptr_t*)BX_STACK_FRAME_ADDRESS();
uint32_t num = 0;
@@ -620,7 +616,7 @@ namespace bx
return num;
}
else if (BX_ENABLED(BX_PLATFORM_WINDOWS && BX_CPU_X86 && BX_ARCH_64BIT) )
else if (BX_ENABLED(BX_PLATFORM_WINDOWS && BX_ARCH_64BIT) )
{
return getCallStackWinRtl(_skip + 1 /* skip self */, _max, _outStack);
}

View File

@@ -47,29 +47,47 @@ namespace bx
{
}
// Reference(s):
// - Futexes Are Tricky, Ulrich Drepper, mutex2
// https://www.akkadia.org/drepper/futex.pdf
//
void Mutex::lock()
{
uint32_t* futex = (uint32_t*)m_internal;
if (State::Unlocked == atomicCompareAndSwap<uint32_t>(futex, State::Unlocked, State::Locked) )
BX_TSAN_MUTEX_PRE_LOCK(futex);
uint32_t state = atomicCompareAndSwap<uint32_t>(futex, State::Unlocked, State::Locked);
if (State::Unlocked != state)
{
return;
if (State::Contested != state)
{
state = atomicExchange<uint32_t>(futex, State::Contested);
}
while (State::Unlocked != state)
{
crt0::futexWait(futex, State::Contested);
state = atomicExchange<uint32_t>(futex, State::Contested);
}
}
while (State::Unlocked != atomicCompareAndSwap<uint32_t>(futex, State::Locked, State::Contested) )
{
crt0::futexWait(futex, State::Contested);
}
BX_TSAN_MUTEX_POST_LOCK(futex);
}
void Mutex::unlock()
{
uint32_t* futex = (uint32_t*)m_internal;
if (State::Contested == atomicCompareAndSwap<uint32_t>(futex, State::Locked, State::Unlocked) )
BX_TSAN_MUTEX_PRE_UNLOCK(futex);
if (State::Contested == atomicExchange<uint32_t>(futex, State::Unlocked) )
{
crt0::futexWake(futex, State::Locked);
crt0::futexWake(futex, 1);
}
BX_TSAN_MUTEX_POST_UNLOCK(futex);
}
#else

View File

@@ -77,6 +77,68 @@ namespace bx
#endif // BX_PLATFORM_
}
#if BX_PLATFORM_WINDOWS
typedef uint32_t (__stdcall* TimePeriodFn)(uint32_t _ms);
static void timePeriod(const StringView& _name, uint32_t _ms)
{
if (0 == _ms)
{
return;
}
// Loaded on demand, so that users of bx don't have to link with winmm.
static void* s_winmm = dlopen("winmm.dll");
if (NULL != s_winmm)
{
TimePeriodFn fn = dlsym<TimePeriodFn>(s_winmm, _name);
if (NULL != fn)
{
fn(_ms);
}
}
}
void timerResolutionBegin(uint32_t _ms)
{
timePeriod("timeBeginPeriod", _ms);
}
void timerResolutionEnd(uint32_t _ms)
{
timePeriod("timeEndPeriod", _ms);
}
#else
void timerResolutionBegin(uint32_t _ms)
{
BX_UNUSED(_ms);
}
void timerResolutionEnd(uint32_t _ms)
{
BX_UNUSED(_ms);
}
#endif // BX_PLATFORM_WINDOWS
uint32_t getHardwareThreads()
{
#if BX_CRT_NONE
return 1;
#elif BX_PLATFORM_WINDOWS || BX_PLATFORM_WINRT
SYSTEM_INFO si;
memSet(&si, 0, sizeof(si) );
::GetNativeSystemInfo(&si);
return max<uint32_t>(1, si.dwNumberOfProcessors);
#elif BX_PLATFORM_POSIX && defined(_SC_NPROCESSORS_ONLN)
const long num = ::sysconf(_SC_NPROCESSORS_ONLN);
return uint32_t(max<long>(1, num) );
#else
return 1;
#endif // BX_PLATFORM_
}
uint32_t getTid()
{
#if BX_PLATFORM_WINDOWS
@@ -439,9 +501,9 @@ namespace bx
const uint32_t advise = _flags & Memory::AdviseMask;
const uint32_t page = _flags & Memory::PageMask;
const size_t pageSize = memoryPageSize();
BX_ASSERT(_alignment <= pageSize, "Alignments greater than the page size are not implemented (requested %zu, page %zu).", _alignment, pageSize);
BX_UNUSED(_alignment, pageSize);
const size_t granularity = memoryAllocationGranularity();
BX_ASSERT(_alignment <= granularity, "Alignments greater than the allocation granularity are not implemented (requested %zu, granularity %zu).", _alignment, granularity);
BX_UNUSED(_alignment, granularity);
#if BX_PLATFORM_LINUX || BX_PLATFORM_OSX
if (NULL == _address)
@@ -608,7 +670,7 @@ namespace bx
BX_UNUSED(advise);
return _address;
#else
BX_UNUSED(_address, _size, _alignment, _flags, state, protect, advise, page, pageSize);
BX_UNUSED(_address, _size, _alignment, _flags, state, protect, advise, page, granularity);
BX_ERROR_SET(_err, kErrorMemoryMapFailed, "memoryMap: Not implemented!");
return NULL;
#endif // BX_PLATFORM_*
@@ -667,11 +729,23 @@ namespace bx
SYSTEM_INFO si;
memSet(&si, 0, sizeof(si) );
::GetSystemInfo(&si);
return si.dwAllocationGranularity;
return si.dwPageSize;
#else
BX_UNUSED(page);
return 16<<10;
#endif // BX_PLATFORM_WINDOWS
}
size_t memoryAllocationGranularity()
{
#if BX_PLATFORM_WINDOWS
SYSTEM_INFO si;
memSet(&si, 0, sizeof(si) );
::GetSystemInfo(&si);
return si.dwAllocationGranularity;
#else
return memoryPageSize();
#endif // BX_PLATFORM_WINDOWS
}
} // namespace bx

View File

@@ -44,3 +44,36 @@ TEST_CASE("atomic", "")
REQUIRE(test == 1337);
}
TEST_CASE("atomic load, store, exchange", "")
{
uint32_t value32 = 1337;
REQUIRE(1337 == bx::atomicLoad(&value32) );
bx::atomicStore(&value32, 1389u);
REQUIRE(1389 == value32);
bx::atomicStoreRelaxed(&value32, 1453u);
REQUIRE(1453 == value32);
REQUIRE(1453 == bx::atomicExchange(&value32, 42u) );
REQUIRE(42 == value32);
uint64_t value64 = UINT64_C(0x123456789abcdef0);
REQUIRE(UINT64_C(0x123456789abcdef0) == bx::atomicLoad(&value64) );
bx::atomicStore(&value64, UINT64_C(0xfedcba9876543210) );
REQUIRE(UINT64_C(0xfedcba9876543210) == value64);
REQUIRE(UINT64_C(0xfedcba9876543210) == bx::atomicExchange(&value64, UINT64_C(1) ) );
REQUIRE(1 == value64);
uintptr_t ptr = 16;
REQUIRE(16 == bx::atomicCompareAndSwapPtr(&ptr, 8, 32) );
REQUIRE(16 == ptr);
REQUIRE(16 == bx::atomicCompareAndSwapPtr(&ptr, 16, 32) );
REQUIRE(32 == ptr);
bx::cpuRelax();
}

39
tests/debug_test.cpp Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright 2010-2026 Branimir Karadzic. All rights reserved.
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
*/
#include "test.h"
#include <bx/debug.h>
static BX_NO_INLINE uintptr_t getCalleeStackAddress()
{
return BX_STACK_FRAME_ADDRESS();
}
TEST_CASE("BX_STACK_FRAME_ADDRESS", "[debug]")
{
const uintptr_t local = uintptr_t(&local);
const uintptr_t addr = BX_STACK_FRAME_ADDRESS();
// Stack grows down on all supported platforms, callee frame is below caller's.
REQUIRE(getCalleeStackAddress() < addr);
// AddressSanitizer keeps locals on its fake stack.
if (!BX_SANITIZER_ADDRESS)
{
REQUIRE( (addr > local ? addr - local : local - addr) < 64<<10);
}
}
TEST_CASE("getCallStackFast", "[debug]")
{
uintptr_t stack[32];
const uint32_t num = bx::getCallStackFast(0, BX_COUNTOF(stack), stack);
// Linux walks frame pointers, and compilers there omit them when optimizing.
if (!BX_ENABLED(BX_PLATFORM_LINUX) )
{
REQUIRE(0 < num);
}
}

View File

@@ -131,3 +131,40 @@ TEST_CASE("memoryMap-protection-change", "[os]")
bx::memoryUnmap(addr, size, &err);
REQUIRE(err.isOk() );
}
TEST_CASE("memoryAllocationGranularity", "[os]")
{
const size_t pageSize = bx::memoryPageSize();
const size_t granularity = bx::memoryAllocationGranularity();
REQUIRE(granularity >= pageSize);
REQUIRE(0 == (granularity & (granularity - 1) ) );
if (BX_ENABLED(BX_PLATFORM_WINDOWS) )
{
REQUIRE(4<<10 == pageSize);
REQUIRE(64<<10 == granularity);
}
bx::Error err;
void* addr = bx::memoryMap(NULL, pageSize, granularity, bx::Memory::ReadWrite, &err);
REQUIRE(err.isOk() );
REQUIRE(bx::isAligned(uintptr_t(addr), granularity) );
bx::memoryUnmap(addr, pageSize, &err);
REQUIRE(err.isOk() );
}
TEST_CASE("getHardwareThreads", "[os]")
{
REQUIRE(1 <= bx::getHardwareThreads() );
}
TEST_CASE("timerResolution", "[os]")
{
bx::timerResolutionBegin(0);
bx::timerResolutionEnd(0);
bx::timerResolutionBegin(1);
bx::sleep(1);
bx::timerResolutionEnd(1);
}