From b81fe9db72877ff7d85b97554d09b7dbd6873fca Mon Sep 17 00:00:00 2001 From: Bartosz Taudul Date: Sat, 19 Sep 2026 14:06:08 +0200 Subject: [PATCH] Rewrite server-side lock tracking as an append-only event engine. Capture has serialized lock events since d6f32a083 and 3e3aa80fa, so the out-of-order reconstruction machinery has nothing left to repair. Mark now targets the recorded Wait/Obtain event index; the old reverse scan could walk off the start of the timeline for a thread that never waited. --- cmake/server.cmake | 1 + server/TracyEvent.hpp | 66 ------- server/TracyLocks.cpp | 359 ++++++++++++++++++++++++++++++++++++ server/TracyLocks.hpp | 169 +++++++++++++++++ server/TracyWorker.cpp | 403 +++++++++-------------------------------- server/TracyWorker.hpp | 4 +- 6 files changed, 616 insertions(+), 386 deletions(-) create mode 100644 server/TracyLocks.cpp create mode 100644 server/TracyLocks.hpp diff --git a/cmake/server.cmake b/cmake/server.cmake index 8298d219..dfe6b47a 100644 --- a/cmake/server.cmake +++ b/cmake/server.cmake @@ -16,6 +16,7 @@ set(TRACY_SERVER_DIR ${CMAKE_CURRENT_LIST_DIR}/../server) set(TRACY_SERVER_SOURCES TracyBroadcast.cpp TracyMemory.cpp + TracyLocks.cpp TracyMmap.cpp TracyPrint.cpp TracySysUtil.cpp diff --git a/server/TracyEvent.hpp b/server/TracyEvent.hpp index d3854a75..b19e9c99 100644 --- a/server/TracyEvent.hpp +++ b/server/TracyEvent.hpp @@ -307,72 +307,6 @@ struct HwSampleData }; -struct LockEvent -{ - enum class Type : uint8_t - { - Wait, - Obtain, - Release, - WaitShared, - ObtainShared, - ReleaseShared - }; - - tracy_force_inline int64_t Time() const { return int64_t( _time_srcloc ) >> 16; } - tracy_force_inline void SetTime( int64_t time ) { assert( time < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_time_srcloc)+2, &time, 4 ); memcpy( ((char*)&_time_srcloc)+6, ((char*)&time)+4, 2 ); } - tracy_force_inline int16_t SrcLoc() const { return int16_t( _time_srcloc & 0xFFFF ); } - tracy_force_inline void SetSrcLoc( int16_t srcloc ) { memcpy( &_time_srcloc, &srcloc, 2 ); } - - uint64_t _time_srcloc; - uint8_t thread; - Type type; -}; - -struct LockEventShared : public LockEvent -{ - uint64_t waitShared; - uint64_t sharedList; -}; - -struct LockEventPtr -{ - short_ptr ptr; - uint8_t lockingThread; - uint8_t lockCount; - uint64_t waitList; -}; - -constexpr size_t MaxLockThreads = sizeof( LockEventPtr::waitList ) * 8; -static_assert( std::numeric_limits::max() >= MaxLockThreads, "Not enough space for lock count." ); - - -enum class LockType : uint8_t; - -struct LockMap -{ - struct TimeRange - { - int64_t start = std::numeric_limits::max(); - int64_t end = std::numeric_limits::min(); - }; - - StringIdx customName; - int16_t srcloc; - Vector timeline; - unordered_flat_map threadMap; - std::vector threadList; - LockType type; - int64_t timeAnnounce; - int64_t timeTerminate; - bool valid; - bool isContended; - uint64_t lockingThread; - - TimeRange range[64]; -}; - - struct GpuEvent { tracy_force_inline int64_t CpuStart() const { return int64_t( _cpuStart_srcloc ) >> 16; } diff --git a/server/TracyLocks.cpp b/server/TracyLocks.cpp new file mode 100644 index 00000000..5cdeddf0 --- /dev/null +++ b/server/TracyLocks.cpp @@ -0,0 +1,359 @@ +#include + +#include "TracyLocks.hpp" + +namespace tracy +{ + +void InitLockMap( LockMap& map, int16_t srcloc, LockType type, int64_t announce ) +{ + map.srcloc = srcloc; + map.type = type; + map.timeAnnounce = announce; + map.timeTerminate = 0; + map.valid = true; + map.isContended = false; +} + +void ReserveLockSlots( LockMap& map, const uint64_t* threadIds, size_t count ) +{ + const size_t base = map.threads.size(); + assert( base + count <= 0xFFFF ); + map.threadMap.reserve( base + count ); + map.threads.reserve( base + count ); + for( size_t i=0; isecond; + if( map.threads.size() >= LockEvent::NoThread ) return LockEvent::NoThread; + const auto slot = ( uint16_t )map.threads.size(); + map.threadMap.emplace( thread, slot ); + LockThreadInfo ti; + ti.thread = thread; + map.threads.push_back( std::move( ti ) ); + return slot; +} + +static tracy_force_inline void EraseSlot( Vector& vec, uint16_t slot ) +{ + for( size_t i=0; i 0 && map.curLockingThread == slot; + const bool exclOthers = ( map.curWaitCount - ( waiting ? 1 : 0 ) ) > 0; + const bool sharedOthers = ( map.curWaitSharedCount - ( sharedWaiting ? 1 : 0 ) ) > 0; + + if( map.type == LockType::Lockable ) + { + if( holding ) return exclOthers ? LockEventState::HasBlockingLock : LockEventState::HasLock; + if( map.curLockCount > 0 && waiting ) return LockEventState::WaitLock; + return LockEventState::Nothing; + } + else + { + if( holding ) return ( exclOthers || sharedOthers ) ? LockEventState::HasBlockingLock : LockEventState::HasLock; + if( map.curLockCount > 0 && ( waiting || sharedWaiting ) ) return LockEventState::WaitLock; + // a shared holder's own exclusive wait is an upgrade request - a deadlock - so the check is self-inclusive + if( map.curLockCount == 0 && sharedHolding ) return map.curWaitCount > 0 ? LockEventState::HasBlockingLock : LockEventState::HasLock; + if( map.curLockCount == 0 && map.curSharedCount > 0 && waiting ) return LockEventState::WaitLock; + return LockEventState::Nothing; + } +} + +static void SegmentPass( LockMap& map, uint16_t self, LockEvent::Type type, uint32_t idx ) +{ + const bool acquire = type == LockEvent::Type::Obtain || type == LockEvent::Type::ObtainShared; + + map.passScratch.clear(); + map.passScratch.push_back( self ); + for( auto s : map.activeSlots ) + { + if( s != self ) map.passScratch.push_back( s ); + } + if( acquire || ( type == LockEvent::Type::Release && map.curLockCount == 0 ) ) + { + for( auto s : map.pendingStarts ) + { + if( s != self ) map.passScratch.push_back( s ); + } + } + + for( auto T : map.passScratch ) + { + auto& ti = map.threads[T]; + const bool open = ti.curState != LockEventState::Nothing; + auto desired = ResolveLockState( map, T ); + if( open && PinnedWait( ti.curState, ti.flags ) ) desired = LockEventState::WaitLock; + + const uint8_t snapState = ( uint8_t )desired; + const uint8_t snapFlags = ti.flags; + + if( !open ) + { + if( desired != LockEventState::Nothing ) + { + LockSegment seg; + seg.evStart = idx; + seg.nextEv = LockEvent::NoEvent; + seg.state = snapState; + seg.flags = snapFlags; + ti.segments.push_back( seg ); + CacheSeverity( ti, ( uint32_t )ti.segments.size() - 1, snapState ); + ti.curState = snapState; + if( !PinnedWait( snapState, snapFlags ) ) map.activeSlots.push_back( T ); + } + } + else + { + auto& seg = ti.segments.back(); + if( seg.state != snapState || seg.flags != snapFlags ) + { + seg.nextEv = idx; + ti.curState = LockEventState::Nothing; + EraseSlot( map.activeSlots, T ); + if( desired != LockEventState::Nothing ) + { + LockSegment ns; + ns.evStart = idx; + ns.nextEv = LockEvent::NoEvent; + ns.state = snapState; + ns.flags = snapFlags; + ti.segments.push_back( ns ); + CacheSeverity( ti, ( uint32_t )ti.segments.size() - 1, snapState ); + ti.curState = snapState; + if( !PinnedWait( snapState, snapFlags ) ) map.activeSlots.push_back( T ); + } + } + } + + // inversion handling (traces from clients that emit the release event after the unlock): + // an ObtainShared arriving before the exclusive Release resolves to Nothing until + // the exclusive drains - pending so the drain event opens the hold. With the release + // event ordered before the unlock, SharedHolding under an exclusive holder is unreachable. + const bool need = ti.curState == LockEventState::Nothing && ( ( ti.flags & ( LockEventFlags::Waiting | LockEventFlags::SharedWaiting ) ) != 0 || + ( map.legacyInversions && map.type == LockType::SharedLockable && ( ti.flags & LockEventFlags::SharedHolding ) != 0 && map.curLockCount > 0 ) ); + if( need ) + { + if( !ti.inPending ) + { + ti.inPending = true; + map.pendingStarts.push_back( T ); + } + } + else if( ti.inPending ) + { + ti.inPending = false; + EraseSlot( map.pendingStarts, T ); + } + } +} + +void AppendLockEvent( LockMap& map, int64_t time, uint16_t slot, LockEvent::Type type, int16_t srcloc ) +{ + if( slot == LockEvent::NoThread ) return; + auto& ti = map.threads[slot]; + + switch( type ) + { + case LockEvent::Type::Wait: + if( !( ti.flags & LockEventFlags::Waiting ) ) + { + ti.flags |= LockEventFlags::Waiting; + map.curWaitCount++; + } + break; + case LockEvent::Type::WaitShared: + if( !( ti.flags & LockEventFlags::SharedWaiting ) ) + { + ti.flags |= LockEventFlags::SharedWaiting; + map.curWaitSharedCount++; + } + break; + case LockEvent::Type::Obtain: + assert( map.curLockCount < UINT16_MAX ); + if( ti.flags & LockEventFlags::Waiting ) + { + ti.flags &= ~LockEventFlags::Waiting; + map.curWaitCount--; + } + if( map.curLockingThread != slot ) + { + assert( map.legacyInversions || map.curLockCount == 0 ); + if( map.curLockCount > 0 ) map.threads[map.curLockingThread].flags &= ~LockEventFlags::LockHolding; + } + ti.flags |= LockEventFlags::LockHolding; + map.curLockingThread = slot; + map.curLockCount++; + break; + case LockEvent::Type::Release: + if( map.curLockCount != 0 ) + { + map.curLockCount--; + if( map.curLockCount == 0 ) map.threads[map.curLockingThread].flags &= ~LockEventFlags::LockHolding; + } + break; + case LockEvent::Type::ObtainShared: + if( ti.flags & LockEventFlags::SharedWaiting ) + { + ti.flags &= ~LockEventFlags::SharedWaiting; + map.curWaitSharedCount--; + } + if( !( ti.flags & LockEventFlags::SharedHolding ) ) + { + ti.flags |= LockEventFlags::SharedHolding; + map.curSharedCount++; + } + break; + case LockEvent::Type::ReleaseShared: + if( ti.flags & LockEventFlags::SharedHolding ) + { + ti.flags &= ~LockEventFlags::SharedHolding; + map.curSharedCount--; + } + break; + default: + break; + } + + assert( map.timeline.empty() || map.timeline.back().Time() <= time ); + assert( map.timeline.size() < LockEvent::NoEvent ); + const uint32_t idx = ( uint32_t )map.timeline.size(); + + LockEvent ev; + ev.SetTime( time ); + ev.SetSrcLoc( srcloc ); + ev.thread = slot; + ev.type = ( uint8_t )type; + map.timeline.push_back( ev ); + if( type == LockEvent::Type::Obtain || type == LockEvent::Type::Release ) + map.holderChanges.push_back( { idx, map.curLockingThread, map.curLockCount } ); + if( srcloc != 0 && ( ti.marks.empty() || ti.marks.back() != idx ) ) ti.marks.push_back( idx ); + + if( ti.firstTime > time ) ti.firstTime = time; + if( ti.lastTime < time ) ti.lastTime = time; + + switch( type ) + { + case LockEvent::Type::Wait: + case LockEvent::Type::WaitShared: + ti.openWaitStart = time; + ti.hasOpenWait = true; + break; + case LockEvent::Type::Obtain: + case LockEvent::Type::ObtainShared: + if( ti.hasOpenWait ) + { + ti.waitTotal += time - ti.openWaitStart; + ti.waitCount++; + ti.hasOpenWait = false; + } + break; + default: + break; + } + + switch( type ) + { + case LockEvent::Type::Wait: + case LockEvent::Type::Obtain: + case LockEvent::Type::WaitShared: + case LockEvent::Type::ObtainShared: + ti.lastWaitObtain = idx; + break; + default: + break; + } + + if( map.curLockCount != 0 ) + { + if( !map.holdOpen ) + { + map.holdOpen = true; + map.openHoldStart = time; + } + } + else if( map.holdOpen ) + { + map.holdTotal += time - map.openHoldStart; + map.holdOpen = false; + } + + if( map.curWaitCount != 0 ) + { + if( !map.waitAggOpen ) + { + map.waitAggOpen = true; + map.openWaitAggStart = time; + } + if( map.curWaitCount > map.maxWaiting ) map.maxWaiting = map.curWaitCount; + } + else if( map.waitAggOpen ) + { + map.waitTotalAgg += time - map.openWaitAggStart; + map.waitAggOpen = false; + } + + if( !map.isContended ) + { + if( map.type == LockType::Lockable ) + { + map.isContended = map.curLockCount != 0 && map.curWaitCount != 0; + } + else + { + map.isContended = ( map.curLockCount != 0 && ( map.curWaitCount != 0 || map.curWaitSharedCount != 0 ) ) || ( map.curSharedCount != 0 && map.curWaitCount != 0 ); + } + } + + SegmentPass( map, slot, type, idx ); +} + +bool ApplyLockMark( LockMap& map, uint16_t slot, int16_t srcloc ) +{ + if( slot == LockEvent::NoThread ) return false; + auto& ti = map.threads[slot]; + const auto idx = ti.lastWaitObtain; + if( idx == LockEvent::NoEvent ) return false; + map.timeline[idx].SetSrcLoc( srcloc ); + if( ti.marks.empty() || ti.marks.back() != idx ) ti.marks.push_back( idx ); + return true; +} + +} diff --git a/server/TracyLocks.hpp b/server/TracyLocks.hpp new file mode 100644 index 00000000..971f0e05 --- /dev/null +++ b/server/TracyLocks.hpp @@ -0,0 +1,169 @@ +#ifndef __TRACYLOCKS_HPP__ +#define __TRACYLOCKS_HPP__ + +#include +#include +#include +#include +#include + +#include "TracyEvent.hpp" +#include "TracyVector.hpp" +#include "tracy_robin_hood.h" +#include "../public/common/TracyForceInline.hpp" +#include "../public/common/TracyQueue.hpp" + +namespace tracy +{ + +#pragma pack( push, 1 ) + +struct LockEvent +{ + enum class Type : uint8_t + { + Wait, + Obtain, + Release, + WaitShared, + ObtainShared, + ReleaseShared + }; + + static constexpr uint32_t NoEvent = 0xFFFFFFFF; + static constexpr uint16_t NoThread = 0xFFFF; + + tracy_force_inline int64_t Time() const { return int64_t( _time_srcloc ) >> 16; } + tracy_force_inline void SetTime( int64_t time ) { assert( time < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_time_srcloc)+2, &time, 4 ); memcpy( ((char*)&_time_srcloc)+6, ((char*)&time)+4, 2 ); } + tracy_force_inline int16_t SrcLoc() const { return int16_t( _time_srcloc & 0xFFFF ); } + tracy_force_inline void SetSrcLoc( int16_t srcloc ) { memcpy( &_time_srcloc, &srcloc, 2 ); } + + uint64_t _time_srcloc; + uint16_t thread; + uint8_t type; +}; + +namespace LockEventState +{ +enum Type : uint8_t +{ + Nothing = 1 << 0, + HasLock = 1 << 1, + HasBlockingLock = 1 << 2, + WaitLock = 1 << 3 +}; +} + +namespace LockEventFlags +{ +enum : uint8_t +{ + Waiting = 1 << 0, + SharedWaiting = 1 << 1, + SharedHolding = 1 << 2, + LockHolding = 1 << 3 +}; +} + +struct LockSegment +{ + uint32_t evStart; + uint32_t nextEv; // event that closed it; NoEvent = open + uint8_t state; + uint8_t flags; // LockEventFlags of the segment thread at evStart +}; + +struct LockHolderChange +{ + uint32_t idx; // event index at which the exclusive state changed + uint16_t holder; // curLockingThread after the event + uint16_t count; // curLockCount after the event +}; +#pragma pack( pop ) + +struct LockThreadInfo +{ + uint64_t thread; + int64_t firstTime = std::numeric_limits::max(); + int64_t lastTime = std::numeric_limits::min(); + int64_t waitTotal = 0; // sum of Wait/WaitShared -> Obtain/ObtainShared durations + uint64_t waitCount = 0; // completed wait pairs + int64_t openWaitStart = 0; + bool hasOpenWait = false; + uint32_t lastWaitObtain = LockEvent::NoEvent; + uint8_t flags = 0; + uint8_t curState = LockEventState::Nothing; + bool inPending = false; // membership of LockMap::pendingStarts + Vector segments; // sorted by evStart; at most one open (last) + Vector marks; // own srcloc != 0 event indices, ascending + Vector yellowSegs; // indices into segments, ascending; segment states are final at creation, so appends keep this searchable + Vector redSegs; +}; + +struct LockMap +{ + StringIdx customName; + int16_t srcloc; + LockType type; + int64_t timeAnnounce; + int64_t timeTerminate; + bool valid; + bool isContended; + bool legacyInversions = false; // traces from clients predating the release-ordering fix can invert handoffs; see SegmentPass + + Vector timeline; // append-only, sorted by time + Vector threads; // slot-indexed + unordered_flat_map threadMap; + + // The waiter counters always equal the respective flag's popcount over all + // slots; decrements are flag-gated, so they cannot underflow. + uint16_t curLockingThread = 0; + uint16_t curLockCount = 0; + uint16_t curWaitCount = 0, curWaitSharedCount = 0, curSharedCount = 0; + Vector pendingStarts; // slots without an open segment whose derived state can change on an acquire or on the exclusive draining to zero: waiting flags, and shared holds recorded under an exclusive holder + Vector activeSlots; // slots with an open segment, excluding pinned WaitLock slots (see PinnedWait) + Vector holderChanges; // exclusive holder/count after each Obtain|Release, ascending idx + + int64_t holdTotal = 0, waitTotalAgg = 0; + uint32_t maxWaiting = 0; + int64_t openHoldStart = 0, openWaitAggStart = 0; + bool holdOpen = false, waitAggOpen = false; + + Vector passScratch; // candidate set of the per-event segment pass + + ~LockMap() + { + for( auto& ti : threads ) + { + ti.segments.~Vector(); + ti.marks.~Vector(); + ti.yellowSegs.~Vector(); + ti.redSegs.~Vector(); + } + } +}; + +void InitLockMap( LockMap& map, int16_t srcloc, LockType type, int64_t announce ); +void ReserveLockSlots( LockMap& map, const uint64_t* threadIds, size_t count ); +uint16_t GetLockSlot( LockMap& map, uint64_t thread ); +void AppendLockEvent( LockMap& map, int64_t time, uint16_t slot, LockEvent::Type type, int16_t srcloc = 0 ); +bool ApplyLockMark( LockMap& map, uint16_t slot, int16_t srcloc ); +LockEventState::Type ResolveLockState( const LockMap& map, uint16_t slot ); + +struct LockHolderInfo +{ + uint16_t holder; + uint16_t count; +}; +tracy_force_inline LockHolderInfo HolderAt( const LockMap& map, uint32_t at ) +{ + const auto& lc = map.holderChanges; + auto it = std::upper_bound( lc.begin(), lc.end(), at, []( uint32_t v, const LockHolderChange& e ) { return v < e.idx; } ); + if( it == lc.begin() ) return { 0, 0 }; + --it; + return { it->holder, it->count }; +} + +} + +#endif diff --git a/server/TracyWorker.cpp b/server/TracyWorker.cpp index d91cf22b..2bba2679 100644 --- a/server/TracyWorker.cpp +++ b/server/TracyWorker.cpp @@ -10,6 +10,7 @@ # include #endif +#include #include #include #include @@ -59,154 +60,6 @@ static const int CurrentVersion = FileVersion( Version::Major, Version::Minor, V static const int MinSupportedVersion = FileVersion( 0, 11, 0 ); -static void UpdateLockCountLockable( LockMap& lockmap, size_t pos ) -{ - auto& timeline = lockmap.timeline; - bool isContended = lockmap.isContended; - uint8_t lockingThread; - uint8_t lockCount; - uint64_t waitList; - - if( pos == 0 ) - { - lockingThread = 0; - lockCount = 0; - waitList = 0; - } - else - { - const auto& tl = timeline[pos-1]; - lockingThread = tl.lockingThread; - lockCount = tl.lockCount; - waitList = tl.waitList; - } - const auto end = timeline.size(); - - while( pos != end ) - { - auto& tl = timeline[pos]; - const auto tbit = uint64_t( 1 ) << tl.ptr->thread; - switch( (LockEvent::Type)tl.ptr->type ) - { - case LockEvent::Type::Wait: - waitList |= tbit; - break; - case LockEvent::Type::Obtain: - assert( lockCount < std::numeric_limits::max() ); - assert( ( waitList & tbit ) != 0 ); - waitList &= ~tbit; - lockingThread = tl.ptr->thread; - lockCount++; - break; - case LockEvent::Type::Release: - assert( lockCount > 0 ); - lockCount--; - break; - default: - break; - } - tl.lockingThread = lockingThread; - tl.waitList = waitList; - tl.lockCount = lockCount; - if( !isContended ) isContended = lockCount != 0 && waitList != 0; - pos++; - } - - lockmap.isContended = isContended; -} - -static void UpdateLockCountSharedLockable( LockMap& lockmap, size_t pos ) -{ - auto& timeline = lockmap.timeline; - bool isContended = lockmap.isContended; - uint8_t lockingThread; - uint8_t lockCount; - uint64_t waitShared; - uint64_t waitList; - uint64_t sharedList; - - if( pos == 0 ) - { - lockingThread = 0; - lockCount = 0; - waitShared = 0; - waitList = 0; - sharedList = 0; - } - else - { - const auto& tl = timeline[pos-1]; - const auto tlp = (const LockEventShared*)(const LockEvent*)tl.ptr; - lockingThread = tl.lockingThread; - lockCount = tl.lockCount; - waitShared = tlp->waitShared; - waitList = tl.waitList; - sharedList = tlp->sharedList; - } - const auto end = timeline.size(); - - // ObtainShared and ReleaseShared should assert on lockCount == 0, but - // due to the async retrieval of data from threads that's not possible. - while( pos != end ) - { - auto& tl = timeline[pos]; - const auto tlp = (LockEventShared*)(LockEvent*)tl.ptr; - const auto tbit = uint64_t( 1 ) << tlp->thread; - switch( (LockEvent::Type)tlp->type ) - { - case LockEvent::Type::Wait: - waitList |= tbit; - break; - case LockEvent::Type::WaitShared: - waitShared |= tbit; - break; - case LockEvent::Type::Obtain: - assert( lockCount < std::numeric_limits::max() ); - assert( ( waitList & tbit ) != 0 ); - waitList &= ~tbit; - lockingThread = tlp->thread; - lockCount++; - break; - case LockEvent::Type::Release: - assert( lockCount > 0 ); - lockCount--; - break; - case LockEvent::Type::ObtainShared: - assert( ( waitShared & tbit ) != 0 ); - assert( ( sharedList & tbit ) == 0 ); - waitShared &= ~tbit; - sharedList |= tbit; - break; - case LockEvent::Type::ReleaseShared: - assert( ( sharedList & tbit ) != 0 ); - sharedList &= ~tbit; - break; - default: - break; - } - tl.lockingThread = lockingThread; - tlp->waitShared = waitShared; - tl.waitList = waitList; - tlp->sharedList = sharedList; - tl.lockCount = lockCount; - if( !isContended ) isContended = ( lockCount != 0 && ( waitList != 0 || waitShared != 0 ) ) || ( sharedList != 0 && waitList != 0 ); - pos++; - } - - lockmap.isContended = isContended; -} - -static inline void UpdateLockCount( LockMap& lockmap, size_t pos ) -{ - if( lockmap.type == LockType::Lockable ) - { - UpdateLockCountLockable( lockmap, pos ); - } - else - { - UpdateLockCountSharedLockable( lockmap, pos ); - } -} static tracy_force_inline void WriteTimeOffset( FileWrite& f, int64_t& refTime, int64_t time ) { @@ -223,12 +76,6 @@ static tracy_force_inline int64_t ReadTimeOffset( FileRead& f, int64_t& refTime return refTime; } -static tracy_force_inline void UpdateLockRange( LockMap& lockmap, const LockEvent& ev, int64_t lt ) -{ - auto& range = lockmap.range[ev.thread]; - if( range.start > lt ) range.start = lt; - if( range.end < lt ) range.end = lt; -} template static uint64_t ReadHwSampleVec( FileRead& f, SortedVector& vec, Slab& slab ) @@ -923,24 +770,33 @@ Worker::Worker( FileRead& f, EventType::Type eventMask, bool bgTasks, bool allow uint64_t tsz; f.Read8( id, lockmap.customName, lockmap.srcloc, lockmap.type, lockmap.valid, lockmap.timeAnnounce, lockmap.timeTerminate, tsz ); lockmap.isContended = false; - lockmap.threadMap.reserve( tsz ); - lockmap.threadList.reserve( tsz ); - for( uint64_t i=0; i= FileVersion( 0, 14, 2 ) ) { - uint64_t t; - f.Read( t ); - lockmap.threadMap.emplace( t, i ); - lockmap.threadList.emplace_back( t ); + uint8_t inversionFlag; + f.Read( inversionFlag ); + lockmap.legacyInversions = inversionFlag; + } + else + { + lockmap.legacyInversions = true; + } + uint64_t threadCnt = tsz; + if( threadCnt > 0xFFFF ) throw FileReadError(); + uint64_t threadIds[ 256 ]; + while( threadCnt != 0 ) + { + const size_t chunk = threadCnt > 256 ? 256 : ( size_t )threadCnt; + f.Read( threadIds, sizeof( uint64_t ) * chunk ); + ReserveLockSlots( lockmap, threadIds, chunk ); + threadCnt -= chunk; } f.Read( tsz ); - lockmap.timeline.reserve_exact( tsz, m_slab ); - auto ptr = lockmap.timeline.data(); + lockmap.timeline.reserve( ( size_t )tsz ); int64_t refTime = lockmap.timeAnnounce; if( fileVer >= FileVersion( 0, 14, 2 ) ) { for( uint64_t i=0; i() : m_slab.Alloc(); const int64_t lt = ReadTimeOffset( f, refTime ); int16_t srcloc; f.Read( srcloc ); @@ -948,38 +804,41 @@ Worker::Worker( FileRead& f, EventType::Type eventMask, bool bgTasks, bool allow f.Read( thread ); uint8_t type; f.Read( type ); - assert( thread < MaxLockThreads ); - lev->SetTime( lt ); - lev->SetSrcLoc( srcloc ); - lev->thread = ( uint8_t )thread; - lev->type = ( LockEvent::Type )type; - *ptr++ = { lev }; - UpdateLockRange( lockmap, *lev, lt ); + if( thread >= lockmap.threads.size() ) throw FileReadError(); + AppendLockEvent( lockmap, lt, thread, ( LockEvent::Type )type, srcloc ); } } else { + struct LegacyEvent + { + int64_t time; + int16_t srcloc; + uint16_t thread; + uint8_t type; + }; + // pre-0.14.2 files can hold events out of time order and the views + // binary-search the timeline, so restore order before replay + Vector legacy; + legacy.reserve( ( size_t )tsz ); for( uint64_t i=0; i() : m_slab.Alloc(); - const int64_t lt = ReadTimeOffset( f, refTime ); - int16_t srcloc; - f.Read( srcloc ); + LegacyEvent ev; + ev.time = ReadTimeOffset( f, refTime ); + f.Read( ev.srcloc ); uint8_t t8; f.Read( t8 ); - const uint16_t thread = t8; - uint8_t type; - f.Read( type ); - assert( thread < MaxLockThreads ); - lev->SetTime( lt ); - lev->SetSrcLoc( srcloc ); - lev->thread = ( uint8_t )thread; - lev->type = ( LockEvent::Type )type; - *ptr++ = { lev }; - UpdateLockRange( lockmap, *lev, lt ); + ev.thread = t8; + f.Read( ev.type ); + legacy.push_back( ev ); + } + std::stable_sort( legacy.begin(), legacy.end(), [] ( const LegacyEvent& lhs, const LegacyEvent& rhs ) { return lhs.time < rhs.time; } ); + for( const auto& ev : legacy ) + { + if( ev.thread >= lockmap.threads.size() ) throw FileReadError(); + AppendLockEvent( lockmap, ev.time, ev.thread, ( LockEvent::Type )ev.type, ev.srcloc ); } } - UpdateLockCount( lockmap, 0 ); m_data.lockMap.emplace( id, lockmapPtr ); } } @@ -993,6 +852,7 @@ Worker::Worker( FileRead& f, EventType::Type eventMask, bool bgTasks, bool allow f.Read( type ); f.Skip( sizeof( LockMap::valid ) + sizeof( LockMap::timeAnnounce ) + sizeof( LockMap::timeTerminate ) ); f.Read( tsz ); + if( fileVer >= FileVersion( 0, 14, 2 ) ) f.Skip( sizeof( uint8_t ) ); f.Skip( tsz * sizeof( uint64_t ) ); f.Read( tsz ); if( fileVer >= FileVersion( 0, 14, 2 ) ) @@ -1001,7 +861,7 @@ Worker::Worker( FileRead& f, EventType::Type eventMask, bool bgTasks, bool allow } else { - f.Skip( tsz * ( sizeof( int64_t ) + sizeof( int16_t ) + sizeof( LockEvent::thread ) + sizeof( LockEvent::type ) ) ); + f.Skip( tsz * ( sizeof( int64_t ) + sizeof( int16_t ) + sizeof( uint8_t ) + sizeof( uint8_t ) ) ); } } } @@ -3775,41 +3635,10 @@ void Worker::NewZone( ZoneEvent* zone ) #endif } -void Worker::InsertLockEvent( LockMap& lockmap, LockEvent* lev, uint64_t thread, int64_t time ) +void Worker::AppendLock( LockMap& lock, int64_t time, uint16_t slot, LockEvent::Type type ) { if( m_data.lastTime < time ) m_data.lastTime = time; - - NoticeThread( thread ); - - auto it = lockmap.threadMap.find( thread ); - if( it == lockmap.threadMap.end() ) - { - if( lockmap.threadList.size() >= MaxLockThreads ) - { - LockThreadOverflowFailure(); - return; - } - it = lockmap.threadMap.emplace( thread, lockmap.threadList.size() ).first; - lockmap.threadList.emplace_back( thread ); - } - lev->thread = it->second; - assert( lev->thread == it->second ); - auto& timeline = lockmap.timeline; - if( timeline.empty() ) - { - timeline.push_back( { lev } ); - UpdateLockCount( lockmap, timeline.size() - 1 ); - } - else - { - assert( timeline.back().ptr->Time() <= time ); - timeline.push_back_non_empty( { lev } ); - UpdateLockCount( lockmap, timeline.size() - 1 ); - } - - auto& range = lockmap.range[it->second]; - if( range.start > time ) range.start = time; - if( range.end < time ) range.end = time; + AppendLockEvent( lock, time, slot, type ); } bool Worker::CheckString( uint64_t ptr ) @@ -5684,13 +5513,7 @@ void Worker::ProcessLockAnnounce( const QueueLockAnnounce& ev ) auto it = m_data.lockMap.find( ev.id ); assert( it == m_data.lockMap.end() ); auto lm = m_slab.AllocInit(); - lm->srcloc = ShrinkSourceLocation( ev.lckloc ); - lm->type = ev.type; - lm->timeAnnounce = TscTime( ev.time ); - lm->timeTerminate = 0; - lm->valid = true; - lm->isContended = false; - lm->lockingThread = 0; + InitLockMap( *lm, ShrinkSourceLocation( ev.lckloc ), ev.type, TscTime( ev.time ) ); m_data.lockMap.emplace( ev.id, lm ); CheckSourceLocation( ev.lckloc ); } @@ -5702,35 +5525,32 @@ void Worker::ProcessLockTerminate( const QueueLockTerminate& ev ) it->second->timeTerminate = TscTime( ev.time ); } -void Worker::ProcessLockWait( const QueueLockWait& ev ) +void Worker::ProcessLockThreadEvent( uint64_t id, int64_t time, uint64_t thread, LockEvent::Type type ) { - auto it = m_data.lockMap.find( ev.id ); + auto it = m_data.lockMap.find( id ); assert( it != m_data.lockMap.end() ); auto& lock = *it->second; + assert( type < LockEvent::Type::WaitShared || lock.type == LockType::SharedLockable ); - auto lev = lock.type == LockType::Lockable ? m_slab.Alloc() : m_slab.Alloc(); - const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); - lev->SetTime( time ); - lev->SetSrcLoc( 0 ); - lev->type = LockEvent::Type::Wait; + const auto lt = TscTime( RefTime( m_refTimeSerial, time ) ); + NoticeThread( thread ); + const auto slot = GetLockSlot( lock, thread ); + if( slot == LockEvent::NoThread ) + { + LockThreadOverflowFailure(); + return; + } + AppendLock( lock, lt, slot, type ); +} - InsertLockEvent( lock, lev, ev.thread, time ); +void Worker::ProcessLockWait( const QueueLockWait& ev ) +{ + ProcessLockThreadEvent( ev.id, ev.time, ev.thread, LockEvent::Type::Wait ); } void Worker::ProcessLockObtain( const QueueLockObtain& ev ) { - auto it = m_data.lockMap.find( ev.id ); - assert( it != m_data.lockMap.end() ); - auto& lock = *it->second; - - auto lev = lock.type == LockType::Lockable ? m_slab.Alloc() : m_slab.Alloc(); - const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); - lev->SetTime( time ); - lev->SetSrcLoc( 0 ); - lev->type = LockEvent::Type::Obtain; - - InsertLockEvent( lock, lev, ev.thread, time ); - lock.lockingThread = ev.thread; + ProcessLockThreadEvent( ev.id, ev.time, ev.thread, LockEvent::Type::Obtain ); } void Worker::ProcessLockRelease( const QueueLockRelease& ev ) @@ -5739,61 +5559,24 @@ void Worker::ProcessLockRelease( const QueueLockRelease& ev ) assert( it != m_data.lockMap.end() ); auto& lock = *it->second; - auto lev = lock.type == LockType::Lockable ? m_slab.Alloc() : m_slab.Alloc(); const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); - lev->SetTime( time ); - lev->SetSrcLoc( 0 ); - lev->type = LockEvent::Type::Release; - - InsertLockEvent( lock, lev, lock.lockingThread, time ); + if( lock.curLockCount == 0 ) return; + AppendLock( lock, time, lock.curLockingThread, LockEvent::Type::Release ); } void Worker::ProcessLockSharedWait( const QueueLockWait& ev ) { - auto it = m_data.lockMap.find( ev.id ); - assert( it != m_data.lockMap.end() ); - auto& lock = *it->second; - - assert( lock.type == LockType::SharedLockable ); - auto lev = m_slab.Alloc(); - const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); - lev->SetTime( time ); - lev->SetSrcLoc( 0 ); - lev->type = LockEvent::Type::WaitShared; - - InsertLockEvent( lock, lev, ev.thread, time ); + ProcessLockThreadEvent( ev.id, ev.time, ev.thread, LockEvent::Type::WaitShared ); } void Worker::ProcessLockSharedObtain( const QueueLockObtain& ev ) { - auto it = m_data.lockMap.find( ev.id ); - assert( it != m_data.lockMap.end() ); - auto& lock = *it->second; - - assert( lock.type == LockType::SharedLockable ); - auto lev = m_slab.Alloc(); - const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); - lev->SetTime( time ); - lev->SetSrcLoc( 0 ); - lev->type = LockEvent::Type::ObtainShared; - - InsertLockEvent( lock, lev, ev.thread, time ); + ProcessLockThreadEvent( ev.id, ev.time, ev.thread, LockEvent::Type::ObtainShared ); } void Worker::ProcessLockSharedRelease( const QueueLockReleaseShared& ev ) { - auto it = m_data.lockMap.find( ev.id ); - assert( it != m_data.lockMap.end() ); - auto& lock = *it->second; - - assert( lock.type == LockType::SharedLockable ); - auto lev = m_slab.Alloc(); - const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); - lev->SetTime( time ); - lev->SetSrcLoc( 0 ); - lev->type = LockEvent::Type::ReleaseShared; - - InsertLockEvent( lock, lev, ev.thread, time ); + ProcessLockThreadEvent( ev.id, ev.time, ev.thread, LockEvent::Type::ReleaseShared ); } void Worker::ProcessLockMark( const QueueLockMark& ev ) @@ -5803,27 +5586,8 @@ void Worker::ProcessLockMark( const QueueLockMark& ev ) assert( lit != m_data.lockMap.end() ); auto& lockmap = *lit->second; auto tid = lockmap.threadMap.find( ev.thread ); - assert( tid != lockmap.threadMap.end() ); - const auto thread = tid->second; - auto it = lockmap.timeline.end(); - for(;;) - { - --it; - if( it->ptr->thread == thread ) - { - switch( it->ptr->type ) - { - case LockEvent::Type::Obtain: - case LockEvent::Type::ObtainShared: - case LockEvent::Type::Wait: - case LockEvent::Type::WaitShared: - it->ptr->SetSrcLoc( ShrinkSourceLocation( ev.srcloc ) ); - return; - default: - break; - } - } - } + if( tid == lockmap.threadMap.end() ) return; + ApplyLockMark( lockmap, tid->second, ShrinkSourceLocation( ev.srcloc ) ); } void Worker::ProcessLockName( const QueueLockName& ev ) @@ -8542,23 +8306,24 @@ void Worker::Write( FileWrite& f, bool fiDict ) f.Write( &v.second->valid, sizeof( v.second->valid ) ); f.Write( &v.second->timeAnnounce, sizeof( v.second->timeAnnounce ) ); f.Write( &v.second->timeTerminate, sizeof( v.second->timeTerminate ) ); - sz = v.second->threadList.size(); + sz = v.second->threads.size(); f.Write( &sz, sizeof( sz ) ); - for( auto& t : v.second->threadList ) + const uint8_t inversionFlag = v.second->legacyInversions; + f.Write( &inversionFlag, sizeof( inversionFlag ) ); + for( auto& t : v.second->threads ) { - f.Write( &t, sizeof( t ) ); + f.Write( &t.thread, sizeof( t.thread ) ); } int64_t refTime = v.second->timeAnnounce; sz = v.second->timeline.size(); f.Write( &sz, sizeof( sz ) ); for( auto& lev : v.second->timeline ) { - WriteTimeOffset( f, refTime, lev.ptr->Time() ); - const int16_t srcloc = lev.ptr->SrcLoc(); + WriteTimeOffset( f, refTime, lev.Time() ); + const int16_t srcloc = lev.SrcLoc(); f.Write( &srcloc, sizeof( srcloc ) ); - const uint16_t thread = lev.ptr->thread; - f.Write( &thread, sizeof( thread ) ); - f.Write( &lev.ptr->type, sizeof( lev.ptr->type ) ); + f.Write( &lev.thread, sizeof( lev.thread ) ); + f.Write( &lev.type, sizeof( lev.type ) ); } } @@ -9069,7 +8834,7 @@ static const char* s_failureReasons[] = { "Multiple frame images were sent for a single frame.", "Fiber execution stopped on a thread which is not executing a fiber.", "Too many source locations. You cannot have more than 32K static or dynamic source locations.", - "Too many threads. You cannot have more than 64 distinct threads waiting on a single lock.", + "Too many threads. You cannot have more than 65K distinct threads interacting with a single lock.", }; static_assert( sizeof( s_failureReasons ) / sizeof( *s_failureReasons ) == (int)Worker::Failure::NUM_FAILURES, "Missing failure reason description." ); diff --git a/server/TracyWorker.hpp b/server/TracyWorker.hpp index c2fcf549..5499e43e 100644 --- a/server/TracyWorker.hpp +++ b/server/TracyWorker.hpp @@ -19,6 +19,7 @@ #include "../public/common/TracySocket.hpp" #include "tracy_robin_hood.h" #include "TracyEvent.hpp" +#include "TracyLocks.hpp" #include "TracyShortPtr.hpp" #include "TracySlab.hpp" #include "TracyStringDiscovery.hpp" @@ -970,7 +971,8 @@ private: tracy_force_inline void NewZone( ZoneEvent* zone ); - void InsertLockEvent( LockMap& lockmap, LockEvent* lev, uint64_t thread, int64_t time ); + void AppendLock( LockMap& lock, int64_t time, uint16_t slot, LockEvent::Type type ); + void ProcessLockThreadEvent( uint64_t id, int64_t time, uint64_t thread, LockEvent::Type type ); bool CheckString( uint64_t ptr ); void CheckThreadString( uint64_t id );