From 52ebe6c2587de4893df7745ca2de50bbeefdc52f Mon Sep 17 00:00:00 2001 From: Juan Jose Casafranca Date: Thu, 22 Oct 2026 19:45:43 +0000 Subject: [PATCH 1/2] Add an option to collect CUDA kernel activity as CUPTI_ACTIVITY_KIND_KERNEL. With CONCURRENT_KERNEL, MEMCPY or MEMSET activity enabled, CUPTI 13.0 faults the device with an illegal memory access once the nodes inside CUDA graph conditional (IF/WHILE) bodies have executed about 699,050 times in a context. TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY enables KERNEL activity instead, which avoids the fault at the cost of serializing kernels. In that mode CUPTI reports conditional-body kernels once per graph launch without timestamps, so those records are skipped. --- public/tracy/TracyCUDA.hpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/public/tracy/TracyCUDA.hpp b/public/tracy/TracyCUDA.hpp index 4de959b7..b9cffbba 100644 --- a/public/tracy/TracyCUDA.hpp +++ b/public/tracy/TracyCUDA.hpp @@ -58,6 +58,17 @@ using CUDACtx = std::nullptr_t; #define TRACY_CUDA_ENABLE_CUDA_CALL_STATS (0) #endif//TRACY_CUDA_ENABLE_CUDA_CALL_STATS +// Collect kernel activity as CUPTI_ACTIVITY_KIND_KERNEL instead of +// CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL. CUPTI then serializes kernel +// execution, but this avoids a CUPTI fault in applications that use CUDA graph +// conditional nodes (IF/WHILE): with CONCURRENT_KERNEL, MEMCPY or MEMSET +// activity enabled, the device hits an illegal memory access once the nodes +// inside conditional bodies have executed about 699,050 times in a context +// (observed with CUPTI 13.0). Enabling KERNEL activity avoids it. +#ifndef TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY +#define TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY (0) +#endif//TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY + namespace { // TODO(marcos): wrap these in structs for better type safety @@ -671,6 +682,7 @@ namespace tracy // or 0 for records that are not graph-launched or don't have the field. static GraphID getGraphIdFromRecord(const CUpti_Activity* record) { switch (record->kind) { + case CUPTI_ACTIVITY_KIND_KERNEL: case CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL: return reinterpret_cast(record)->graphId; case CUPTI_ACTIVITY_KIND_MEMCPY: @@ -1102,10 +1114,17 @@ namespace tracy switch (record->kind) { + case CUPTI_ACTIVITY_KIND_KERNEL: case CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL: { ZoneNamedN(kernel, "tracy::CUDACtx::DoProcessDeviceEvent[kernel]", instrument); CUpti_ActivityKernel9* kernel9 = (CUpti_ActivityKernel9*) record; + // With CUPTI_ACTIVITY_KIND_KERNEL, kernels inside CUDA graph + // conditional bodies are reported once per graph launch and + // without timestamps. + if (kernel9->start == 0) { + break; + } APICallInfo apiCall; if (!matchGraphActivityToAPICall(kernel9->correlationId, kernel9->graphId, apiCall)) { return matchError(kernel9->correlationId, "KERNEL"); @@ -1247,8 +1266,12 @@ namespace tracy }; static constexpr CUpti_ActivityKind activities[] = { - //CUPTI_ACTIVITY_KIND_KERNEL, // mutually exclusive with CONCURRENT_KERNEL + // KERNEL and CONCURRENT_KERNEL are mutually exclusive + #if TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY + CUPTI_ACTIVITY_KIND_KERNEL, + #else CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL, + #endif CUPTI_ACTIVITY_KIND_MEMCPY, CUPTI_ACTIVITY_KIND_MEMSET, CUPTI_ACTIVITY_KIND_SYNCHRONIZATION, From d0ddff87da3fd37003de206365c31bca8f1a2693 Mon Sep 17 00:00:00 2001 From: Juan Jose Casafranca Date: Thu, 22 Oct 2026 21:30:24 +0000 Subject: [PATCH 2/2] Rename TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY to TRACY_CUDA_FORCE_KERNEL_SERIAL_EXECUTION. --- public/tracy/TracyCUDA.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/tracy/TracyCUDA.hpp b/public/tracy/TracyCUDA.hpp index b9cffbba..b5f7c99d 100644 --- a/public/tracy/TracyCUDA.hpp +++ b/public/tracy/TracyCUDA.hpp @@ -65,9 +65,9 @@ using CUDACtx = std::nullptr_t; // activity enabled, the device hits an illegal memory access once the nodes // inside conditional bodies have executed about 699,050 times in a context // (observed with CUPTI 13.0). Enabling KERNEL activity avoids it. -#ifndef TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY -#define TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY (0) -#endif//TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY +#ifndef TRACY_CUDA_FORCE_KERNEL_SERIAL_EXECUTION +#define TRACY_CUDA_FORCE_KERNEL_SERIAL_EXECUTION (0) +#endif//TRACY_CUDA_FORCE_KERNEL_SERIAL_EXECUTION namespace { @@ -1267,7 +1267,7 @@ namespace tracy static constexpr CUpti_ActivityKind activities[] = { // KERNEL and CONCURRENT_KERNEL are mutually exclusive - #if TRACY_CUDA_SERIALIZED_KERNEL_ACTIVITY + #if TRACY_CUDA_FORCE_KERNEL_SERIAL_EXECUTION CUPTI_ACTIVITY_KIND_KERNEL, #else CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL,