From aa657d543889ff1845893fe30c65a6fef377d086 Mon Sep 17 00:00:00 2001 From: Basil Milanich Date: Tue, 24 Mar 2026 09:24:23 -0500 Subject: [PATCH] Fix missing GPU zones for CUDA Graph-launched kernels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When kernels are launched via CUDA Graphs (cuGraphLaunch), CUPTI delivers CONCURRENT_KERNEL, MEMCPY, and MEMSET activity records but no corresponding API callback fires for the individual operations. This means matchActivityToAPICall() always fails, and every GPU activity record is silently dropped by matchError(). Fix this by falling back to a synthetic APICallInfo using the GPU timestamps from the activity record when no API correlation exists. This produces correct GPU zones with kernel names and timing — just without the CPU-to-GPU launch correlation arrow. Tested on NVIDIA H100 with CUDA 13.1: before this fix, 0 GPU zones appeared for CUDA Graph workloads; after, all kernel and memcpy zones are visible in the Tracy timeline. --- public/tracy/TracyCUDA.hpp | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/public/tracy/TracyCUDA.hpp b/public/tracy/TracyCUDA.hpp index 5a64c8d9..b941e676 100644 --- a/public/tracy/TracyCUDA.hpp +++ b/public/tracy/TracyCUDA.hpp @@ -998,7 +998,12 @@ namespace tracy CUpti_ActivityKernel9* kernel9 = (CUpti_ActivityKernel9*) record; APICallInfo apiCall; if (!matchActivityToAPICall(kernel9->correlationId, apiCall)) { - return matchError(kernel9->correlationId, "KERNEL"); + // Fallback for CUDA Graph-launched kernels: create GPU zone + // using kernel timestamps when no API callback correlation exists. + auto* host = PersistentState::Get().profilerHost; + if (!host) return; + TracyTimestamp cpuTime = tracyFromCUpti(kernel9->start); + apiCall = APICallInfo{ cpuTime, cpuTime, kernel9->start, host }; } apiCall.host->EmitGpuZone(apiCall.start, apiCall.end, kernel9->start, kernel9->end, getKernelSourceLocation(kernel9->name), kernel9->contextId, kernel9->streamId); auto latency_ms = (kernel9->start - apiCall.cupti) / 1'000'000.0; @@ -1012,7 +1017,12 @@ namespace tracy CUpti_ActivityMemcpy5* memcpy5 = (CUpti_ActivityMemcpy5*) record; APICallInfo apiCall; if (!matchActivityToAPICall(memcpy5->correlationId, apiCall)) { - return matchError(memcpy5->correlationId, "MEMCPY"); + // Fallback for CUDA Graph memcpy: create GPU zone using + // activity timestamps when no API callback correlation exists. + auto* host = PersistentState::Get().profilerHost; + if (!host) return; + TracyTimestamp cpuTime = tracyFromCUpti(memcpy5->start); + apiCall = APICallInfo{ cpuTime, cpuTime, memcpy5->start, host }; } static constexpr tracy::SourceLocationData TracyCUPTISrcLocDeviceMemcpy { "CUDA::memcpy", TracyFunction, TracyFile, (uint32_t)TracyLine, tracy::Color::Blue }; apiCall.host->EmitGpuZone(apiCall.start, apiCall.end, memcpy5->start, memcpy5->end, &TracyCUPTISrcLocDeviceMemcpy, memcpy5->contextId, memcpy5->streamId); @@ -1028,7 +1038,12 @@ namespace tracy CUpti_ActivityMemset4* memset4 = (CUpti_ActivityMemset4*) record; APICallInfo apiCall; if (!matchActivityToAPICall(memset4->correlationId, apiCall)) { - return matchError(memset4->correlationId, "MEMSET"); + // Fallback for CUDA Graph memset: create GPU zone using + // activity timestamps when no API callback correlation exists. + auto* host = PersistentState::Get().profilerHost; + if (!host) return; + TracyTimestamp cpuTime = tracyFromCUpti(memset4->start); + apiCall = APICallInfo{ cpuTime, cpuTime, memset4->start, host }; } static constexpr tracy::SourceLocationData TracyCUPTISrcLocDeviceMemset { "CUDA::memset", TracyFunction, TracyFile, (uint32_t)TracyLine, tracy::Color::Blue }; apiCall.host->EmitGpuZone(apiCall.start, apiCall.end, memset4->start, memset4->end, &TracyCUPTISrcLocDeviceMemset, memset4->contextId, memset4->streamId);