Scarfski/tracy
1
0
Fork 0
mirror of https://github.com/wolfpld/tracy.git synced 2026-10-24 01:30:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add an option to collect CUDA kernel activity as CUPTI_ACTIVITY_KIND_KERNEL.

Browse Source 
With CONCURRENT_KERNEL, MEMCPY or MEMSET activity enabled, CUPTI 13.0
faults the device with an illegal memory access once the nodes inside CUDA
graph conditional (IF/WHILE) bodies have executed about 699,050 times in a
context. TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY enables KERNEL activity
instead, which avoids the fault at the cost of serializing kernels. In that
mode CUPTI reports conditional-body kernels once per graph launch without
timestamps, so those records are skipped.
This commit is contained in:
Juan Jose Casafranca
2026-10-22 19:45:43 +00:00
parent e15c78fdac
commit 52ebe6c258
1 changed files with 24 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
public/tracy/TracyCUDA.hpp
View File

@@ -58,6 +58,17 @@ using CUDACtx = std::nullptr_t;
#define TRACY_CUDA_ENABLE_CUDA_CALL_STATS (0)
#endif//TRACY_CUDA_ENABLE_CUDA_CALL_STATS
// Collect kernel activity as CUPTI_ACTIVITY_KIND_KERNEL instead of
// CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL. CUPTI then serializes kernel
// execution, but this avoids a CUPTI fault in applications that use CUDA graph
// conditional nodes (IF/WHILE): with CONCURRENT_KERNEL, MEMCPY or MEMSET
// activity enabled, the device hits an illegal memory access once the nodes
// inside conditional bodies have executed about 699,050 times in a context
// (observed with CUPTI 13.0). Enabling KERNEL activity avoids it.
#ifndef TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY
#define TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY (0)
#endif//TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY
namespace {
// TODO(marcos): wrap these in structs for better type safety
@@ -671,6 +682,7 @@ namespace tracy
// or 0 for records that are not graph-launched or don't have the field.
static GraphID getGraphIdFromRecord(const CUpti_Activity* record) {
switch (record->kind) {
case CUPTI_ACTIVITY_KIND_KERNEL:
case CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL:
return reinterpret_cast<const CUpti_ActivityKernel9*>(record)->graphId;
case CUPTI_ACTIVITY_KIND_MEMCPY:
@@ -1102,10 +1114,17 @@ namespace tracy
switch (record->kind)
{
case CUPTI_ACTIVITY_KIND_KERNEL:
case CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL:
{
ZoneNamedN(kernel, "tracy::CUDACtx::DoProcessDeviceEvent[kernel]", instrument);
CUpti_ActivityKernel9* kernel9 = (CUpti_ActivityKernel9*) record;
// With CUPTI_ACTIVITY_KIND_KERNEL, kernels inside CUDA graph
// conditional bodies are reported once per graph launch and
// without timestamps.
if (kernel9->start == 0) {
break;
}
APICallInfo apiCall;
if (!matchGraphActivityToAPICall(kernel9->correlationId, kernel9->graphId, apiCall)) {
return matchError(kernel9->correlationId, "KERNEL");
@@ -1247,8 +1266,12 @@ namespace tracy
};
static constexpr CUpti_ActivityKind activities[] = {
//CUPTI_ACTIVITY_KIND_KERNEL, // mutually exclusive with CONCURRENT_KERNEL
// KERNEL and CONCURRENT_KERNEL are mutually exclusive
#if TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY
CUPTI_ACTIVITY_KIND_KERNEL,
#else
CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL,
#endif
CUPTI_ACTIVITY_KIND_MEMCPY,
CUPTI_ACTIVITY_KIND_MEMSET,
CUPTI_ACTIVITY_KIND_SYNCHRONIZATION,