diff --git a/profiler/src/profiler/TracyLlmTools.cpp b/profiler/src/profiler/TracyLlmTools.cpp index 395d2714..3652279e 100644 --- a/profiler/src/profiler/TracyLlmTools.cpp +++ b/profiler/src/profiler/TracyLlmTools.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -6,8 +7,12 @@ #include #include +#include "TracyEmbed.hpp" +#include "TracyLlmApi.hpp" #include "TracyLlmTools.hpp" +#include "data/Manual.hpp" + constexpr const char* NoNetworkAccess = "Internet access is disabled by the user. You may inform the user that he can enable it in the settings, so that you can use the tools to gather information."; #define NetworkCheckString if( !m_netAccess ) return NoNetworkAccess @@ -42,6 +47,11 @@ static std::string UrlEncode( const std::string& str ) return out; } +TracyLlmTools::~TracyLlmTools() +{ + CancelManualEmbeddings(); +} + TracyLlmTools::ToolReply TracyLlmTools::HandleToolCalls( const std::string& name, const std::vector& args, int contextSize ) { m_ctxSize = contextSize; @@ -88,6 +98,81 @@ std::string TracyLlmTools::GetCurrentTime() return buffer; } +TracyLlmTools::EmbeddingState TracyLlmTools::GetManualEmbeddingsState() const +{ + std::lock_guard lock( m_lock ); + return m_manualEmbeddingState; +} + +void TracyLlmTools::BuildManualEmbeddings( const std::string& model, TracyLlmApi& api ) +{ + std::unique_lock lock( m_lock ); + assert( !m_manualEmbeddingState.inProgress ); + if( m_manualEmbeddingState.done && m_manualEmbeddingState.model == model ) return; + + lock.unlock(); + if( m_thread.joinable() ) m_thread.join(); + + assert( !m_cancel ); + m_manualEmbeddingState = { .model = model, .inProgress = true }; + m_thread = std::thread( [this, &api] { ManualEmbeddingsWorker( api ); } ); +} + +void TracyLlmTools::ManualEmbeddingsWorker( TracyLlmApi& api ) +{ + constexpr auto Chunk = 1000; + constexpr auto Overlap = 200; + + auto manual = Unembed( Manual ); + + const auto sz = (int)manual->size(); + const auto chunks = ( sz + Chunk - 1 ) / Chunk; + for( int i=0; idata() + start, end - start ); + req["model"] = m_manualEmbeddingState.model; + + nlohmann::json response; + api.Embeddings( req, response ); + + std::vector embeddings; + for( auto& item : response["data"][0]["embedding"] ) + { + embeddings.emplace_back( item.get() ); + } + } + + std::lock_guard lock( m_lock ); + m_manualEmbeddingState.inProgress = false; + m_manualEmbeddingState.done = true; +} + +void TracyLlmTools::CancelManualEmbeddings() +{ + if( m_thread.joinable() ) + { + m_lock.lock(); + m_cancel = true; + m_lock.unlock(); + m_thread.join(); + m_cancel = false; + } +} + int TracyLlmTools::CalcMaxSize() const { if( m_ctxSize <= 0 ) return 32*1024; diff --git a/profiler/src/profiler/TracyLlmTools.hpp b/profiler/src/profiler/TracyLlmTools.hpp index e626df01..5616de88 100644 --- a/profiler/src/profiler/TracyLlmTools.hpp +++ b/profiler/src/profiler/TracyLlmTools.hpp @@ -1,7 +1,9 @@ #ifndef __TRACYLLMTOOLS_HPP__ #define __TRACYLLMTOOLS_HPP__ +#include #include +#include #include #include "tracy_robin_hood.h" @@ -9,6 +11,8 @@ namespace tracy { +class TracyLlmApi; + class TracyLlmTools { public: @@ -18,9 +22,23 @@ public: std::string image; }; + struct EmbeddingState + { + std::string model; + bool done = false; + bool inProgress = false; + float progress = 0; + }; + + ~TracyLlmTools(); + ToolReply HandleToolCalls( const std::string& name, const std::vector& args, int contextSize ); std::string GetCurrentTime(); + [[nodiscard]] EmbeddingState GetManualEmbeddingsState() const; + void BuildManualEmbeddings( const std::string& model, TracyLlmApi& api ); + void CancelManualEmbeddings(); + bool m_netAccess = true; private: @@ -33,9 +51,16 @@ private: std::string GetDictionary( std::string word, const std::string& lang ); std::string SearchWeb( std::string query ); + void ManualEmbeddingsWorker( TracyLlmApi& api ); + unordered_flat_map m_webCache; int m_ctxSize; + + mutable std::mutex m_lock; + std::thread m_thread; + bool m_cancel = false; + EmbeddingState m_manualEmbeddingState; }; }