diff --git a/profiler/src/main.cpp b/profiler/src/main.cpp index a8dadd66..71daa519 100644 --- a/profiler/src/main.cpp +++ b/profiler/src/main.cpp @@ -125,6 +125,7 @@ tracy::Config s_config; tracy::AchievementsMgr* s_achievements; static const tracy::data::AchievementItem* s_achievementItem = nullptr; static bool s_switchAchievementCategory = false; +static std::unique_ptr llmProbe; static float smoothstep( float x ) { @@ -444,6 +445,7 @@ int main( int argc, char** argv ) if( updateThread.joinable() ) updateThread.join(); if( updateNotesThread.joinable() ) updateNotesThread.join(); view.reset(); + llmProbe.reset(); tracy::FreeTexture( zigzagTex, RunOnMainThread ); tracy::FreeTexture( iconTex, RunOnMainThread ); @@ -875,6 +877,7 @@ static void DrawContents() llmModels.clear(); s_config.llmAddress = "http://localhost:11434"; SaveConfig(); + llmProbe.reset(); } ImGui::SameLine(); char llmAddress[1024]; @@ -886,6 +889,7 @@ static void DrawContents() llmModels.clear(); s_config.llmAddress = llmAddress; SaveConfig(); + llmProbe.reset(); } if( llmstatus == 1 ) ImGui::BeginDisabled(); @@ -894,14 +898,25 @@ static void DrawContents() if( doCheck ) { llmstatus = 1; - // TODO put into a thread? - tracy::TracyLlm llm; - const bool valid = llm.IsValid(); - llmstatus = valid ? 2 : 3; - if( valid ) + llmProbe = std::make_unique(); + activeFrames = 3; + } + else if( llmProbe ) + { + activeFrames = 3; + if( llmProbe->IsValid() ) { - llmVersion = llm.GetVersion(); - llmModels = llm.GetModels(); + if( !llmProbe->IsBusy() ) + { + llmstatus = 2; + llmVersion = llmProbe->GetVersion(); + llmModels = llmProbe->GetModels(); + } + } + else + { + llmstatus = 3; + llmProbe.reset(); } } @@ -1377,6 +1392,7 @@ static void DrawContents() clients.clear(); } if( loadThread.joinable() ) loadThread.join(); + llmProbe.reset(); view->NotifyRootWindowSize( display_w, display_h ); if( !view->Draw() ) { diff --git a/profiler/src/profiler/TracyLlm.cpp b/profiler/src/profiler/TracyLlm.cpp index 583960dc..b688a3fb 100644 --- a/profiler/src/profiler/TracyLlm.cpp +++ b/profiler/src/profiler/TracyLlm.cpp @@ -11,6 +11,7 @@ namespace tracy { TracyLlm::TracyLlm() + : m_exit( false ) { if( !s_config.llm ) return; @@ -21,29 +22,34 @@ TracyLlm::TracyLlm() return; } - try - { - LoadModels(); - } - catch( const std::exception& e ) - { - m_ollama.reset(); - return; - } - - auto it = std::ranges::find_if( m_models, []( const auto& model ) { return model.name == s_config.llmModel; } ); - if( it == m_models.end() ) - { - m_modelIdx = 0; - } - else - { - m_modelIdx = std::distance( m_models.begin(), it ); - } + m_jobs.emplace_back( WorkItem { + .task = Task::LoadModels, + .callback = [this] { + auto it = std::ranges::find_if( m_models, []( const auto& model ) { return model.name == s_config.llmModel; } ); + if( it == m_models.end() ) + { + m_modelIdx = 0; + } + else + { + m_modelIdx = std::distance( m_models.begin(), it ); + } + } + } ); + m_thread = std::thread( [this] { Worker(); } ); } TracyLlm::~TracyLlm() { + if( m_thread.joinable() ) + { + { + std::lock_guard lock( m_lock ); + m_exit.store( true, std::memory_order_release ); + m_cv.notify_all(); + } + m_thread.join(); + } } std::string TracyLlm::GetVersion() const @@ -60,9 +66,40 @@ void TracyLlm::Draw() ImGui::End(); } +void TracyLlm::Worker() +{ + std::unique_lock lock( m_lock ); + while( !m_exit.load( std::memory_order_acquire ) ) + { + m_cv.wait( lock, [this] { return !m_jobs.empty() || m_exit.load( std::memory_order_acquire ); } ); + if( m_exit.load( std::memory_order_acquire ) ) break; + + auto job = m_jobs.back(); + m_jobs.pop_back(); + m_busy = true; + lock.unlock(); + + switch( job.task ) + { + case Task::LoadModels: + LoadModels(); + break; + default: + assert( false ); + break; + } + + job.callback(); + + lock.lock(); + m_busy = false; + } +}; + void TracyLlm::LoadModels() { - m_models.clear(); + std::vector m; + const auto models = m_ollama->list_models(); for( const auto& model : models ) { @@ -70,8 +107,12 @@ void TracyLlm::LoadModels() const auto& modelInfo = info["model_info"]; const auto architecture = modelInfo["general.architecture"].get(); const auto& ctx = modelInfo[architecture + ".context_length"]; - m_models.emplace_back( LlmModel { .name = model, .ctxSize = ctx.get() } ); + m.emplace_back( LlmModel { .name = model, .ctxSize = ctx.get() } ); } + + m_modelsLock.lock(); + std::swap( m_models, m ); + m_modelsLock.unlock(); } } diff --git a/profiler/src/profiler/TracyLlm.hpp b/profiler/src/profiler/TracyLlm.hpp index a5acb8bd..25fc3f6f 100644 --- a/profiler/src/profiler/TracyLlm.hpp +++ b/profiler/src/profiler/TracyLlm.hpp @@ -1,8 +1,13 @@ #ifndef __TRACYLLM_HPP__ #define __TRACYLLM_HPP__ +#include +#include +#include #include +#include #include +#include #include class Ollama; @@ -12,6 +17,17 @@ namespace tracy class TracyLlm { + enum class Task + { + LoadModels + }; + + struct WorkItem + { + Task task; + std::function callback; + }; + public: struct LlmModel { @@ -23,21 +39,35 @@ public: ~TracyLlm(); [[nodiscard]] bool IsValid() const { return (bool)m_ollama; } + [[nodiscard]] bool IsBusy() const { std::lock_guard lock( m_lock); return m_busy; } + [[nodiscard]] std::string GetVersion() const; - [[nodiscard]] std::vector GetModels() const { return m_models; } + [[nodiscard]] std::vector GetModels() const { std::lock_guard lock( m_modelsLock ); return m_models; } void Draw(); bool m_show = false; private: + void Worker(); + void LoadModels(); std::unique_ptr m_ollama; + + mutable std::mutex m_modelsLock; std::vector m_models; size_t m_modelIdx; size_t m_ctxSize; + + std::atomic m_exit; + std::condition_variable m_cv; + std::thread m_thread; + + mutable std::mutex m_lock; + std::vector m_jobs; + bool m_busy; }; }