From 845933040d7161a741fd86f47333beea32c83610 Mon Sep 17 00:00:00 2001 From: Bartosz Taudul Date: Mon, 9 Jun 2025 21:25:26 +0200 Subject: [PATCH] Perform tool calls with json. --- profiler/src/llm/system.prompt | 9 ++-- profiler/src/profiler/TracyLlm.cpp | 51 ++++++++---------- profiler/src/profiler/TracyLlmTools.cpp | 70 ++++++++++++------------- profiler/src/profiler/TracyLlmTools.hpp | 3 +- 4 files changed, 60 insertions(+), 73 deletions(-) diff --git a/profiler/src/llm/system.prompt b/profiler/src/llm/system.prompt index 315e3519..f75d59f1 100644 --- a/profiler/src/llm/system.prompt +++ b/profiler/src/llm/system.prompt @@ -26,16 +26,13 @@ The user is asking about the weather in San Francisco. I need to use the weather 2. *Tool Usage (``):* - If, in the `` block, you decide you need to use a tool, the next block generated *MUST* be a `` block. - The tag name MUST be exactly `tool`. - - The first line inside the `` block *MUST* contain the *exact function name* from the list of available tools. - - If the function requires parameters, provide them on subsequent lines, *one parameter per line*. Provide parameters in plain form, *without quotation marks*. - *After generating a `` block, you MUST END YOUR RESPONSE FOR THIS TURN.* Do not generate any other text or tags after the `` block. The system will process this tool call and provide you with the result in the next step. - - Example: + - The tool name and its parameters (if applicable) must be passed as a json data. For example: -The user is asking about the weather in San Francisco. I need to use the weather checking tool. The function name is 'check_weather', the parameter is the city name. +The user is asking about the weather in San Francisco. I need to use the weather checking tool. The tool name is 'check_weather', the parameter is the city name. -check_weather -San Francisco +{"tool": "check_weather", "city": "San Francisco"} 3. *Tool Output (``):* diff --git a/profiler/src/profiler/TracyLlm.cpp b/profiler/src/profiler/TracyLlm.cpp index 77f824b6..514b41ec 100644 --- a/profiler/src/profiler/TracyLlm.cpp +++ b/profiler/src/profiler/TracyLlm.cpp @@ -733,20 +733,6 @@ void TracyLlm::SendMessage( std::unique_lock& lock ) } } -static std::vector SplitLines( const std::string& str ) -{ - std::vector lines; - auto pos = 0; - while( pos < str.size() ) - { - auto next = str.find( '\n', pos ); - if( next == std::string::npos ) next = str.size(); - if( pos != next ) lines.emplace_back( str.substr( pos, next - pos ) ); - pos = next + 1; - } - return lines; -} - bool TracyLlm::OnResponse( const nlohmann::json& json ) { std::unique_lock lock( m_lock ); @@ -804,24 +790,29 @@ bool TracyLlm::OnResponse( const nlohmann::json& json ) if( end != std::string::npos ) { while( end > pos && str[end-1] == '\n' ) end--; - auto data = str.substr( pos, end - pos ); - auto lines = SplitLines( data ); - if( !lines.empty() ) + const auto tool = str.substr( pos, end - pos ); + lock.unlock(); + + TracyLlmTools::ToolReply reply; + try { - isTool = true; - auto tool = lines[0]; - lines.erase( lines.begin() ); - lock.unlock(); - const auto reply = m_tools->HandleToolCalls( tool, lines, *m_api, m_api->GetModels()[m_modelIdx].contextSize, m_embedIdx >= 0 ); - auto output = "\n" + reply.reply; - lock.lock(); - AddMessage( std::move( output ), "user" ); - m_jobs.emplace_back( WorkItem { - .task = Task::SendMessage, - .callback = nullptr - } ); - m_cv.notify_all(); + auto json = nlohmann::json::parse( tool ); + reply = m_tools->HandleToolCalls( json, *m_api, m_api->GetModels()[m_modelIdx].contextSize, m_embedIdx >= 0 ); } + catch( const nlohmann::json::exception& e ) + { + reply.reply = e.what(); + } + + isTool = true; + auto output = "\n" + reply.reply; + lock.lock(); + AddMessage( std::move( output ), "user" ); + m_jobs.emplace_back( WorkItem { + .task = Task::SendMessage, + .callback = nullptr + } ); + m_cv.notify_all(); } } } diff --git a/profiler/src/profiler/TracyLlmTools.cpp b/profiler/src/profiler/TracyLlmTools.cpp index 8a9b39c8..ebc52b28 100644 --- a/profiler/src/profiler/TracyLlmTools.cpp +++ b/profiler/src/profiler/TracyLlmTools.cpp @@ -181,51 +181,49 @@ TracyLlmTools::~TracyLlmTools() CancelManualEmbeddings(); } -TracyLlmTools::ToolReply TracyLlmTools::HandleToolCalls( const std::string& name, const std::vector& args, TracyLlmApi& api, int contextSize, bool hasEmbeddingsModel ) +#define Param(name) json[name].get_ref() + +TracyLlmTools::ToolReply TracyLlmTools::HandleToolCalls( const nlohmann::json& json, TracyLlmApi& api, int contextSize, bool hasEmbeddingsModel ) { m_ctxSize = contextSize; - if( name == "fetch_web_page" ) + try { - if( args.empty() ) return { .reply = "Missing URL argument" }; - return { .reply = FetchWebPage( args[0] ) }; + auto name = json["tool"].get_ref(); + if( name == "search_wikipedia" ) + { + return SearchWikipedia( Param( "query" ), Param( "language" ) ); + } + else if( name == "get_wikipedia" ) + { + return { .reply = GetWikipedia( Param( "page" ), Param( "language" ) ) }; + } + else if( name == "get_dictionary" ) + { + return { .reply = GetDictionary( Param( "word" ), Param( "language" ) ) }; + } + else if( name == "search_web" ) + { + return { .reply = SearchWeb( Param( "query" ) ) }; + } + else if( name == "get_webpage" ) + { + return { .reply = GetWebpage( Param( "url" ) ) }; + } + else if( name == "user_manual" ) + { + return { .reply = SearchManual( Param( "query" ), api, hasEmbeddingsModel ) }; + } + return { .reply = "Unknown tool call: " + name }; } - if( name == "search_wikipedia" ) + catch( const nlohmann::json::exception& e ) { - if( args.empty() ) return { .reply = "Missing search term argument" }; - if( args.size() < 2 ) return { .reply = "Missing language argument" }; - return SearchWikipedia( args[0], args[1] ); + return { .reply = e.what() }; } - if( name == "get_wikipedia" ) - { - if( args.empty() ) return { .reply = "Missing page name argument" }; - if( args.size() < 2 ) return { .reply = "Missing language argument" }; - return { .reply = GetWikipedia( args[0], args[1] ) }; - } - if( name == "get_dictionary" ) - { - if( args.empty() ) return { .reply = "Missing word argument" }; - if( args.size() < 2 ) return { .reply = "Missing language argument" }; - return { .reply = GetDictionary( args[0], args[1] ) }; - } - if( name == "search_web" ) - { - if( args.empty() ) return { .reply = "Missing search term argument" }; - return { .reply = SearchWeb( args[0] ) }; - } - if( name == "get_webpage" ) - { - if( args.empty() ) return { .reply = "Missing URL argument" }; - return { .reply = GetWebpage( args[0] ) }; - } - if( name == "user_manual" ) - { - if( args.empty() ) return { .reply = "Missing search term argument" }; - return { .reply = SearchManual( args[0], api, hasEmbeddingsModel ) }; - } - return { .reply = "Unknown tool call: " + name }; } +#undef Param + std::string TracyLlmTools::GetCurrentTime() const { auto t = time( nullptr ); diff --git a/profiler/src/profiler/TracyLlmTools.hpp b/profiler/src/profiler/TracyLlmTools.hpp index cd365913..8f34a909 100644 --- a/profiler/src/profiler/TracyLlmTools.hpp +++ b/profiler/src/profiler/TracyLlmTools.hpp @@ -1,6 +1,7 @@ #ifndef __TRACYLLMTOOLS_HPP__ #define __TRACYLLMTOOLS_HPP__ +#include #include #include #include @@ -37,7 +38,7 @@ public: TracyLlmTools(); ~TracyLlmTools(); - ToolReply HandleToolCalls( const std::string& name, const std::vector& args, TracyLlmApi& api, int contextSize, bool hasEmbeddingsModel ); + ToolReply HandleToolCalls( const nlohmann::json& json, TracyLlmApi& api, int contextSize, bool hasEmbeddingsModel ); std::string GetCurrentTime() const; [[nodiscard]] EmbeddingState GetManualEmbeddingsState() const;