From 39cd16a92bdb507e388b7285d81d221d89a49719 Mon Sep 17 00:00:00 2001 From: Bartosz Taudul Date: Thu, 30 Apr 2026 02:41:48 +0200 Subject: [PATCH] Move specialized LLM skills out of the system prompt. --- profiler/CMakeLists.txt | 2 ++ profiler/src/llm/skill.callstack.md | 6 ++++++ profiler/src/llm/skill.optimization.md | 8 ++++++++ profiler/src/llm/system.prompt.md | 18 +++--------------- profiler/src/llm/tools.json | 17 +++++++++++++++++ profiler/src/profiler/TracyLlm.cpp | 19 +++++++++++++++++-- profiler/src/profiler/TracyLlm.hpp | 10 ++++++++++ profiler/src/profiler/TracyLlmChat.cpp | 14 ++++++++++++-- profiler/src/profiler/TracyLlmChat.hpp | 8 +++++++- profiler/src/profiler/TracyLlmTools.cpp | 15 ++++++++++++++- profiler/src/profiler/TracyLlmTools.hpp | 5 ++++- 11 files changed, 100 insertions(+), 22 deletions(-) create mode 100644 profiler/src/llm/skill.callstack.md create mode 100644 profiler/src/llm/skill.optimization.md diff --git a/profiler/CMakeLists.txt b/profiler/CMakeLists.txt index bde089d4..3cef85f9 100644 --- a/profiler/CMakeLists.txt +++ b/profiler/CMakeLists.txt @@ -144,6 +144,8 @@ set(PROFILER_FILES ) Embed(PROFILER_FILES SystemPrompt src/llm/system.prompt.md) +Embed(PROFILER_FILES SkillCallstack src/llm/skill.callstack.md) +Embed(PROFILER_FILES SkillOptimization src/llm/skill.optimization.md) Embed(PROFILER_FILES ToolsJson src/llm/tools.json) Embed(PROFILER_FILES FontFixed src/font/FiraCode-Retina.ttf) Embed(PROFILER_FILES FontIcons src/font/Font\ Awesome\ 6\ Free-Solid-900.otf) diff --git a/profiler/src/llm/skill.callstack.md b/profiler/src/llm/skill.callstack.md new file mode 100644 index 00000000..a6902390 --- /dev/null +++ b/profiler/src/llm/skill.callstack.md @@ -0,0 +1,6 @@ +### Inspecting callstacks + +1. Focus on user's code. Ignore standard library boilerplate. +2. Retrieve source code to verify callstack validity. Source locations in callstacks are return locations, and the call site may actually be near the reported source line. +3. Top of the callstack is the most interesting, as it shows what the program is doing *now*. The bottom of the callstack shows what the program did to do what it's doing. +4. If the callstack contains Tracy's crash handler, the profiled program has crashed. In this case, ignore the crash handler and any functions it may be calling. The crash happened *before* the handler intercepted it. diff --git a/profiler/src/llm/skill.optimization.md b/profiler/src/llm/skill.optimization.md new file mode 100644 index 00000000..0f305562 --- /dev/null +++ b/profiler/src/llm/skill.optimization.md @@ -0,0 +1,8 @@ +### Program optimization + +1. Start by mapping the assembly instructions to the source code. All the reasoning should be performed with source code first. The assembly can only be used as a supplementary source. +2. Analyze the available data, looking where the majority of the run time is spent. Always look at the code as a whole. Do not stop after finding a bunch of interesting spots. +3. Figure out what algorithms are in use, how the data is structured and how it flows, reason about trade-offs taken. +4. Reason if the code can be made to perform better. Note that some code will already be optimal, despite having hot spots. +5. Formulate the optimization strategies and present them to the user. +6. Do not provide concrete speed up percentages. It is only possible to know how faster the code is by measuring it after the changes. You can't do that. diff --git a/profiler/src/llm/system.prompt.md b/profiler/src/llm/system.prompt.md index af2c2434..001db24a 100644 --- a/profiler/src/llm/system.prompt.md +++ b/profiler/src/llm/system.prompt.md @@ -53,20 +53,8 @@ Insert links inline in the text, for example: "Function xyz() is located at [lin ## Case specific operation -In certain situations you must use a specialized workflow. +Specialized instructions and workflows for specific tasks are provided with the `skill` tool. If the task description matches the skill description, you must load the skill in question to gather the required abilities, *before* doing anything else. -### Program optimization +Available skills: -1. Start by mapping the assembly instructions to the source code. All the reasoning should be performed with source code first. The assembly can only be used as a supplementary source. -2. Analyze the available data, looking where the majority of the run time is spent. Always look at the code as a whole. Do not stop after finding a bunch of interesting spots. -3. Figure out what algorithms are in use, how the data is structured and how it flows, reason about trade-offs taken. -4. Reason if the code can be made to perform better. Note that some code will already be optimal, despite having hot spots. -5. Formulate the optimization strategies and present them to the user. -6. Do not provide concrete speed up percentages. It is only possible to know how faster the code is by measuring it after the changes. You can't do that. - -### Inspecting callstacks - -1. Focus on user's code. Ignore standard library boilerplate. -2. Retrieve source code to verify callstack validity. Source locations in callstacks are return locations, and the call site may actually be near the reported source line. -3. Top of the callstack is the most interesting, as it shows what the program is doing *now*. The bottom of the callstack shows what the program did to do what it's doing. -4. If the callstack contains Tracy's crash handler, the profiled program has crashed. In this case, ignore the crash handler and any functions it may be calling. The crash happened *before* the handler intercepted it. +%SKILLS% diff --git a/profiler/src/llm/tools.json b/profiler/src/llm/tools.json index 65772db5..7df6ebf1 100644 --- a/profiler/src/llm/tools.json +++ b/profiler/src/llm/tools.json @@ -1,4 +1,21 @@ [ +{ + "type": "function", + "function": { + "name": "skill", + "description": "Learn a specialized skill for the task needing specific knowledge.", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the skill, chosen from the list of available skills." + } + }, + "required": ["name"] + } + } +}, { "type": "function", "function": { diff --git a/profiler/src/profiler/TracyLlm.cpp b/profiler/src/profiler/TracyLlm.cpp index ccdf6ba9..a9b0643a 100644 --- a/profiler/src/profiler/TracyLlm.cpp +++ b/profiler/src/profiler/TracyLlm.cpp @@ -17,6 +17,8 @@ #include "../public/common/TracySystem.hpp" #include "data/SystemPrompt.hpp" +#include "data/SkillCallstack.hpp" +#include "data/SkillOptimization.hpp" #include "data/ToolsJson.hpp" namespace tracy @@ -42,6 +44,9 @@ TracyLlm::TracyLlm( Worker& worker, View& view, const TracyManualData& manual ) atexit( curl_global_cleanup ); } + AddSkill( "callstack", "Analyze content of a call stack or crash trace", Unembed( SkillCallstack ) ); + AddSkill( "optimization", "General code optimization workflow", Unembed( SkillOptimization ) ); + m_systemPrompt = Unembed( SystemPrompt ); auto toolsJson = Unembed( ToolsJson ); m_toolsJson = nlohmann::json::parse( toolsJson->data(), toolsJson->data() + toolsJson->size() ); @@ -51,8 +56,8 @@ TracyLlm::TracyLlm( Worker& worker, View& view, const TracyManualData& manual ) ResetChat(); m_api = std::make_unique(); - m_chatUi = std::make_unique( view, worker ); - m_tools = std::make_unique( worker, manual ); + m_chatUi = std::make_unique( view, worker, m_skills ); + m_tools = std::make_unique( worker, manual, m_skills ); m_busy = true; QueueConnect(); @@ -872,15 +877,20 @@ void TracyLlm::UpdateSystemPrompt() static constexpr std::string_view UserToken = "%USER%"; static constexpr std::string_view TimeToken = "%TIME%"; static constexpr std::string_view ProgramNameToken = "%PROGRAMNAME%"; + static constexpr std::string_view SkillsToken = "%SKILLS%"; auto userName = GetUserFullName(); if( !userName ) userName = GetUserLogin(); + std::string skills; + for( auto& skill : m_skills ) skills += skill.name + ": " + skill.description + "\n"; + auto systemPrompt = std::string( m_systemPrompt->data(), m_systemPrompt->size() ); Replace( systemPrompt, UserToken, userName ); Replace( systemPrompt, TimeToken, m_tools->GetCurrentTime() ); Replace( systemPrompt, ProgramNameToken, m_worker.GetCaptureProgram() ); + Replace( systemPrompt, SkillsToken, skills ); if( !m_api ) { @@ -1304,4 +1314,9 @@ bool TracyLlm::OnResponse( const nlohmann::json& json ) return true; } +void TracyLlm::AddSkill( std::string&& name, std::string&& description, const std::shared_ptr& content ) +{ + m_skills.emplace_back( std::move( name ), std::move( description ), std::string( content->data(), content->size() ) ); +} + } diff --git a/profiler/src/profiler/TracyLlm.hpp b/profiler/src/profiler/TracyLlm.hpp index ad0684f3..70904c11 100644 --- a/profiler/src/profiler/TracyLlm.hpp +++ b/profiler/src/profiler/TracyLlm.hpp @@ -23,6 +23,13 @@ class TracyManualData; class View; class Worker; +struct LlmSkill +{ + std::string name; + std::string description; + std::string content; +}; + class TracyLlm { enum class Task @@ -81,6 +88,8 @@ private: void AppendResponse( const char* name, const nlohmann::json& delta ); bool OnResponse( const nlohmann::json& json ); + void AddSkill( std::string&& name, std::string&& description, const std::shared_ptr& content ); + std::unique_ptr m_api; std::unique_ptr m_chatUi; std::unique_ptr m_tools; @@ -111,6 +120,7 @@ private: std::vector m_chat; std::string m_summary; + std::vector m_skills; std::shared_ptr m_systemPrompt; nlohmann::json m_toolsJson; diff --git a/profiler/src/profiler/TracyLlmChat.cpp b/profiler/src/profiler/TracyLlmChat.cpp index 36574085..7f7a34f8 100644 --- a/profiler/src/profiler/TracyLlmChat.cpp +++ b/profiler/src/profiler/TracyLlmChat.cpp @@ -5,6 +5,7 @@ #include #include "TracyImGui.hpp" +#include "TracyLlm.hpp" #include "TracyLlmChat.hpp" #include "TracyMouse.hpp" #include "TracyPrint.hpp" @@ -68,7 +69,7 @@ static const char* utfendl( const char* str, int len ) } -static std::string ToolCallDescription( const nlohmann::json& json ) +std::string TracyLlmChat::ToolCallDescription( const nlohmann::json& json ) const { if( !json.contains( "arguments" ) ) return ""; nlohmann::json args; @@ -127,13 +128,22 @@ static std::string ToolCallDescription( const nlohmann::json& json ) if( args.contains( "path" ) ) path = ", path: " + args["path"].get_ref(); return "Source search: " + args["query"].get_ref() + caseInsensitive + path; } + else if( name == "skill" ) + { + if( !args.contains( "name" ) ) return ""; + auto skill = args["name"].get_ref(); + auto it = std::ranges::find_if( m_skills, [&skill]( const auto& s ) { return s.name == skill; } ); + if( it == m_skills.end() ) return ""; + return "Learn skill: " + it->description; + } return ""; } -TracyLlmChat::TracyLlmChat( View& view, Worker& worker ) +TracyLlmChat::TracyLlmChat( View& view, Worker& worker, const std::vector& skills ) : m_width( new float[NumRoles] ) , m_markdown( &view, &worker ) + , m_skills( skills ) { } diff --git a/profiler/src/profiler/TracyLlmChat.hpp b/profiler/src/profiler/TracyLlmChat.hpp index f0597f79..9c3674c6 100644 --- a/profiler/src/profiler/TracyLlmChat.hpp +++ b/profiler/src/profiler/TracyLlmChat.hpp @@ -2,12 +2,14 @@ #define __TRACYLLMCHAT_HPP__ #include +#include #include "TracyMarkdown.hpp" namespace tracy { +struct LlmSkill; class View; class Worker; @@ -35,7 +37,7 @@ public: ToolCall }; - TracyLlmChat( View& view, Worker& worker ); + TracyLlmChat( View& view, Worker& worker, const std::vector& skills ); ~TracyLlmChat(); void Begin(); @@ -50,6 +52,8 @@ private: void PrintThink( const char* str, size_t size ); + [[nodiscard]] std::string ToolCallDescription( const nlohmann::json& json ) const; + float* m_width; float m_maxWidth; @@ -61,6 +65,8 @@ private: Markdown m_markdown; std::string m_label; + + const std::vector& m_skills; }; } diff --git a/profiler/src/profiler/TracyLlmTools.cpp b/profiler/src/profiler/TracyLlmTools.cpp index 7921f11d..71bb0bdc 100644 --- a/profiler/src/profiler/TracyLlmTools.cpp +++ b/profiler/src/profiler/TracyLlmTools.cpp @@ -10,6 +10,7 @@ #include #include "TracyConfig.hpp" +#include "TracyLlm.hpp" #include "TracyLlmApi.hpp" #include "TracyLlmTools.hpp" #include "TracyManualData.hpp" @@ -78,9 +79,10 @@ static std::unique_ptr ParseHtml( const std::string& html ) return doc; } -TracyLlmTools::TracyLlmTools( Worker& worker, const TracyManualData& manual ) +TracyLlmTools::TracyLlmTools( Worker& worker, const TracyManualData& manual, const std::vector& skills ) : m_worker( worker ) , m_manual( manual ) + , m_skills( skills ) { int idx = 0; for( auto& chunk : m_manual.GetChunks() ) @@ -182,6 +184,10 @@ std::string TracyLlmTools::HandleToolCalls( const std::string& tool, const nlohm std::string empty; return SourceSearch( Param( "query" ), ParamOptBool( "case_insensitive", false ), ParamOptString( "path", empty ) ); } + else if( tool == "skill" ) + { + return GetSkill( Param( "name" ) ); + } return "Unknown tool call: " + tool; } catch( const std::exception& e ) @@ -1008,4 +1014,11 @@ std::string TracyLlmTools::SourceSearch( std::string query, bool caseInsensitive return ret; } +std::string TracyLlmTools::GetSkill( const std::string& name ) const +{ + auto it = std::ranges::find_if( m_skills, [&name]( const auto& skill ) { return skill.name == name; } ); + if( it == m_skills.end() ) return "No such skill."; + return it->content; +} + } diff --git a/profiler/src/profiler/TracyLlmTools.hpp b/profiler/src/profiler/TracyLlmTools.hpp index 6d1666e0..531edf5f 100644 --- a/profiler/src/profiler/TracyLlmTools.hpp +++ b/profiler/src/profiler/TracyLlmTools.hpp @@ -16,6 +16,7 @@ class EmbedData; namespace tracy { +struct LlmSkill; class TracyLlmApi; class TracyManualData; class Worker; @@ -31,7 +32,7 @@ public: float progress = 0; }; - TracyLlmTools( Worker& worker, const TracyManualData& manual ); + TracyLlmTools( Worker& worker, const TracyManualData& manual, const std::vector& skills ); ~TracyLlmTools(); std::string HandleToolCalls( const std::string& tool, const nlohmann::json& json, TracyLlmApi& api, int contextSize, bool hasEmbeddingsModel ); @@ -62,6 +63,7 @@ private: std::string SearchManual( const std::string& query, TracyLlmApi& api, bool hasEmbeddingsModel ); std::string SourceFile( const std::string& file, uint32_t line, uint32_t context, uint32_t contextBack ) const; std::string SourceSearch( std::string query, bool caseInsensitive, const std::string& path ) const; + std::string GetSkill( const std::string& name ) const; void ManualEmbeddingsWorker( TracyLlmApi& api ); @@ -79,6 +81,7 @@ private: Worker& m_worker; const TracyManualData& m_manual; + const std::vector& m_skills; }; }