Track deadlocks in the worker from the lock event state.

The live per-frame scan visits only the set of locks with nonzero wait
counts, updated at each lock event: every wait-for edge originates at a
waiter, so uncontented locks cannot contribute anything. File loads replay
events without touching the set and are covered by the one-shot full-map
scan at end of load.
This commit is contained in:
Bartosz Taudul
2026-09-22 02:21:31 +02:00
parent 5808e7ed1a
commit eee89de2d9
4 changed files with 128 additions and 0 deletions

View File

@@ -583,4 +583,62 @@ void DetectLockDeadlocks( const unordered_flat_map<uint32_t, LockMap*>& lockMap,
DetectLockDeadlocksImpl( lockMap, &candidates, groups, members );
}
static bool ThreadSetSubset( const Vector<DeadlockMember>& sup, uint32_t sf, uint32_t sc,
const Vector<DeadlockMember>& sub, uint32_t bf, uint32_t bc )
{
if( bc > sc ) return false;
uint32_t j = 0;
for( uint32_t i=0; i<bc; i++ )
{
const auto t = sub[bf+i].thread;
while( j < sc && sup[sf+j].thread < t ) j++;
if( j == sc || sup[sf+j].thread != t ) return false;
j++;
}
return true;
}
void MergeDetectedDeadlocks( Vector<DeadlockGroup>& groups, Vector<DeadlockMember>& members,
const Vector<DeadlockGroup>& found, const Vector<DeadlockMember>& foundMembers )
{
for( const auto& g : found )
{
Vector<uint32_t> drop;
bool duplicate = false;
for( uint32_t si=0; si<groups.size(); si++ )
{
const auto& s = groups[si];
if( !ThreadSetSubset( foundMembers, g.first, g.cnt, members, s.first, s.cnt ) ) continue;
if( s.cnt == g.cnt )
{
duplicate = true;
break;
}
drop.push_back( si );
}
if( duplicate ) continue;
if( !drop.empty() )
{
std::sort( drop.begin(), drop.end() );
Vector<DeadlockGroup> keptGroups;
Vector<DeadlockMember> keptMembers;
for( uint32_t si=0; si<groups.size(); si++ )
{
if( std::binary_search( drop.begin(), drop.end(), si ) ) continue;
const auto& s = groups[si];
const uint32_t first = (uint32_t)keptMembers.size();
for( uint32_t k=0; k<s.cnt; k++ ) keptMembers.push_back( members[s.first+k] );
keptGroups.push_back( { s.time, first, s.cnt } );
}
groups = std::move( keptGroups );
members = std::move( keptMembers );
}
const uint32_t first = (uint32_t)members.size();
for( uint32_t i=0; i<g.cnt; i++ ) members.push_back( foundMembers[g.first+i] );
groups.push_back( { g.time, first, g.cnt } );
}
}
}

View File

@@ -277,6 +277,14 @@ void DetectLockDeadlocks( const unordered_flat_map<uint32_t, LockMap*>& lockMap,
void DetectLockDeadlocks( const unordered_flat_map<uint32_t, LockMap*>& lockMap,
const unordered_flat_set<uint32_t>& candidates,
Vector<DeadlockGroup>& groups, Vector<DeadlockMember>& members );
// Reconcile a detection pass against previously reported groups. A found group equal
// to a stored one is a repeat of the still-formed cycle and is skipped. A found group
// containing stored ones absorbs them: cycle members are frozen and can never
// fragment, so a later pass can only regroup reported threads into larger sets.
void MergeDetectedDeadlocks( Vector<DeadlockGroup>& groups, Vector<DeadlockMember>& members,
const Vector<DeadlockGroup>& found, const Vector<DeadlockMember>& foundMembers );
}
#endif

View File

@@ -1623,6 +1623,8 @@ Worker::Worker( FileRead& f, EventType::Type eventMask, bool bgTasks, bool allow
}
}
DetectDeadlocks();
s_loadProgress.total.store( 0, std::memory_order_relaxed );
m_loadTime = std::chrono::duration_cast<std::chrono::nanoseconds>( std::chrono::high_resolution_clock::now() - loadStart ).count();
@@ -3653,6 +3655,40 @@ void Worker::AppendLock( LockMap& lock, int64_t time, uint16_t slot, LockEvent::
AppendLockEvent( lock, time, slot, type );
}
bool Worker::IsDeadlockedThread( uint64_t thread ) const
{
for( auto& m : m_data.deadlockMembers )
{
if( m.thread == thread ) return true;
}
return false;
}
bool Worker::IsDeadlockedPair( uint64_t thread, uint32_t lock ) const
{
for( auto& m : m_data.deadlockMembers )
{
if( m.thread == thread && m.lock == lock ) return true;
}
return false;
}
void Worker::DetectDeadlocks()
{
Vector<DeadlockGroup> found;
Vector<DeadlockMember> foundMembers;
DetectLockDeadlocks( m_data.lockMap, found, foundMembers );
MergeDetectedDeadlocks( m_data.deadlockGroups, m_data.deadlockMembers, found, foundMembers );
}
void Worker::DetectDeadlocksLive()
{
Vector<DeadlockGroup> found;
Vector<DeadlockMember> foundMembers;
DetectLockDeadlocks( m_data.lockMap, m_data.waitingLocks, found, foundMembers );
MergeDetectedDeadlocks( m_data.deadlockGroups, m_data.deadlockMembers, found, foundMembers );
}
bool Worker::CheckString( uint64_t ptr )
{
if( ptr == 0 ) return true;
@@ -5554,6 +5590,13 @@ void Worker::ProcessLockThreadEvent( uint64_t id, int64_t time, uint64_t thread,
}
td->inLocks = 1;
AppendLock( lock, lt, slot, type );
if( lock.curWaitCount + lock.curWaitSharedCount != 0 ) m_data.waitingLocks.insert( ( uint32_t )id );
else m_data.waitingLocks.erase( ( uint32_t )id );
if( type == LockEvent::Type::Wait || type == LockEvent::Type::WaitShared ||
type == LockEvent::Type::Obtain || type == LockEvent::Type::ObtainShared )
{
m_detectDeadlocks = true;
}
}
void Worker::ProcessLockWait( const QueueLockWait& ev )

View File

@@ -386,6 +386,10 @@ private:
CrashEvent crashEvent;
Vector<DeadlockGroup> deadlockGroups;
Vector<DeadlockMember> deadlockMembers;
unordered_flat_set<uint32_t> waitingLocks;
unordered_flat_map<uint64_t, ContextSwitch*> ctxSwitch;
CpuData cpuData[256];
@@ -520,6 +524,13 @@ public:
};
MainThreadDataLockGuard ObtainLockForMainThread() { return { m_data }; }
void UpdateDeadlocks()
{
if( !m_detectDeadlocks ) return;
m_detectDeadlocks = false;
if( !m_data.waitingLocks.empty() ) DetectDeadlocksLive();
}
size_t GetFrameCount( const FrameData& fd ) const { return fd.frames.size(); }
size_t GetFullFrameCount( const FrameData& fd ) const;
bool AreFramesUsed() const;
@@ -618,6 +629,10 @@ public:
#endif
const CrashEvent& GetCrashEvent() const { return m_data.crashEvent; }
const Vector<DeadlockGroup>& GetDeadlockGroups() const { return m_data.deadlockGroups; }
const Vector<DeadlockMember>& GetDeadlockMembers() const { return m_data.deadlockMembers; }
bool IsDeadlockedThread( uint64_t thread ) const;
bool IsDeadlockedPair( uint64_t thread, uint32_t lock ) const;
// Some zones may have incomplete timing data (only start time is available, end hasn't arrived yet).
// GetZoneEnd() will try to infer the end time by looking at child zones (parent zone can't end
@@ -973,6 +988,8 @@ private:
void AppendLock( LockMap& lock, int64_t time, uint16_t slot, LockEvent::Type type );
void ProcessLockThreadEvent( uint64_t id, int64_t time, uint64_t thread, LockEvent::Type type );
void DetectDeadlocks();
void DetectDeadlocksLive();
bool CheckString( uint64_t ptr );
void CheckThreadString( uint64_t id );
@@ -1100,6 +1117,8 @@ private:
bool m_terminate = false;
bool m_crashed = false;
bool m_disconnect = false;
bool m_detectDeadlocks = false;
void* m_stream; // LZ4_streamDecode_t*
char* m_buffer;
int m_bufferOffset;