Load LLM models in a thread.

profiler/src/main.cpp

@@ -125,6 +125,7 @@ tracy::Config s_config;
 tracy::AchievementsMgr* s_achievements;
 static const tracy::data::AchievementItem* s_achievementItem = nullptr;
 static bool s_switchAchievementCategory = false;
+static std::unique_ptr<tracy::TracyLlm> llmProbe;
 
 static float smoothstep( float x )
 {
@@ -444,6 +445,7 @@ int main( int argc, char** argv )
     if( updateThread.joinable() ) updateThread.join();
     if( updateNotesThread.joinable() ) updateNotesThread.join();
     view.reset();
+    llmProbe.reset();
 
     tracy::FreeTexture( zigzagTex, RunOnMainThread );
     tracy::FreeTexture( iconTex, RunOnMainThread );
@@ -875,6 +877,7 @@ static void DrawContents()
                         llmModels.clear();
                         s_config.llmAddress = "http://localhost:11434";
                         SaveConfig();
+                        llmProbe.reset();
                     }
                     ImGui::SameLine();
                     char llmAddress[1024];
@@ -886,6 +889,7 @@ static void DrawContents()
                         llmModels.clear();
                         s_config.llmAddress = llmAddress;
                         SaveConfig();
+                        llmProbe.reset();
                     }
 
                     if( llmstatus == 1 ) ImGui::BeginDisabled();
@@ -894,14 +898,25 @@ static void DrawContents()
                     if( doCheck )
                     {
                         llmstatus = 1;
-                        // TODO put into a thread?
-                        tracy::TracyLlm llm;
-                        const bool valid = llm.IsValid();
-                        llmstatus = valid ? 2 : 3;
-                        if( valid )
+                        llmProbe = std::make_unique<tracy::TracyLlm>();
+                        activeFrames = 3;
+                    }
+                    else if( llmProbe )
+                    {
+                        activeFrames = 3;
+                        if( llmProbe->IsValid() )
                         {
-                            llmVersion = llm.GetVersion();
-                            llmModels = llm.GetModels();
+                            if( !llmProbe->IsBusy() )
+                            {
+                                llmstatus = 2;
+                                llmVersion = llmProbe->GetVersion();
+                                llmModels = llmProbe->GetModels();
+                            }
+                        }
+                        else
+                        {
+                            llmstatus = 3;
+                            llmProbe.reset();
                         }
                     }
 
@@ -1377,6 +1392,7 @@ static void DrawContents()
             clients.clear();
         }
         if( loadThread.joinable() ) loadThread.join();
+        llmProbe.reset();
         view->NotifyRootWindowSize( display_w, display_h );
         if( !view->Draw() )
         {

profiler/src/profiler/TracyLlm.cpp

@@ -11,6 +11,7 @@ namespace tracy
 {
 
 TracyLlm::TracyLlm()
+    : m_exit( false )
 {
     if( !s_config.llm ) return;
 
@@ -21,29 +22,34 @@ TracyLlm::TracyLlm()
         return;
     }
 
-    try
-    {
-        LoadModels();
-    }
-    catch( const std::exception& e )
-    {
-        m_ollama.reset();
-        return;
-    }
-
-    auto it = std::ranges::find_if( m_models, []( const auto& model ) { return model.name == s_config.llmModel; } );
-    if( it == m_models.end() )
-    {
-        m_modelIdx = 0;
-    }
-    else
-    {
-        m_modelIdx = std::distance( m_models.begin(), it );
-    }
+    m_jobs.emplace_back( WorkItem {
+        .task = Task::LoadModels,
+        .callback = [this] {
+            auto it = std::ranges::find_if( m_models, []( const auto& model ) { return model.name == s_config.llmModel; } );
+            if( it == m_models.end() )
+            {
+                m_modelIdx = 0;
+            }
+            else
+            {
+                m_modelIdx = std::distance( m_models.begin(), it );
+            }
+        }
+    } );
+    m_thread = std::thread( [this] { Worker(); } );
 }
 
 TracyLlm::~TracyLlm()
 {
+    if( m_thread.joinable() )
+    {
+        {
+            std::lock_guard lock( m_lock );
+            m_exit.store( true, std::memory_order_release );
+            m_cv.notify_all();
+        }
+        m_thread.join();
+    }
 }
 
 std::string TracyLlm::GetVersion() const
@@ -60,9 +66,40 @@ void TracyLlm::Draw()
     ImGui::End();
 }
 
+void TracyLlm::Worker()
+{
+    std::unique_lock lock( m_lock );
+    while( !m_exit.load( std::memory_order_acquire ) )
+    {
+        m_cv.wait( lock, [this] { return !m_jobs.empty() || m_exit.load( std::memory_order_acquire ); } );
+        if( m_exit.load( std::memory_order_acquire ) ) break;
+
+        auto job = m_jobs.back();
+        m_jobs.pop_back();
+        m_busy = true;
+        lock.unlock();
+
+        switch( job.task )
+        {
+        case Task::LoadModels:
+            LoadModels();
+            break;
+        default:
+            assert( false );
+            break;
+        }
+
+        job.callback();
+
+        lock.lock();
+        m_busy = false;
+    }
+};
+
 void TracyLlm::LoadModels()
 {
-    m_models.clear();
+    std::vector<LlmModel> m;
+
     const auto models = m_ollama->list_models();
     for( const auto& model : models )
     {
@@ -70,8 +107,12 @@ void TracyLlm::LoadModels()
         const auto& modelInfo = info["model_info"];
         const auto architecture = modelInfo["general.architecture"].get<std::string>();
         const auto& ctx = modelInfo[architecture + ".context_length"];
-        m_models.emplace_back( LlmModel { .name = model, .ctxSize = ctx.get<size_t>() } );
+        m.emplace_back( LlmModel { .name = model, .ctxSize = ctx.get<size_t>() } );
     }
+
+    m_modelsLock.lock();
+    std::swap( m_models, m );
+    m_modelsLock.unlock();
 }
 
 }

profiler/src/profiler/TracyLlm.hpp

@@ -1,8 +1,13 @@
 #ifndef __TRACYLLM_HPP__
 #define __TRACYLLM_HPP__
 
+#include <atomic>
+#include <condition_variable>
+#include <functional>
 #include <memory>
+#include <mutex>
 #include <string>
+#include <thread>
 #include <vector>
 
 class Ollama;
@@ -12,6 +17,17 @@ namespace tracy
 
 class TracyLlm
 {
+    enum class Task
+    {
+        LoadModels
+    };
+
+    struct WorkItem
+    {
+        Task task;
+        std::function<void()> callback;
+    };
+
 public:
     struct LlmModel
     {
@@ -23,21 +39,35 @@ public:
     ~TracyLlm();
 
     [[nodiscard]] bool IsValid() const { return (bool)m_ollama; }
+    [[nodiscard]] bool IsBusy() const { std::lock_guard lock( m_lock); return m_busy; }
+
     [[nodiscard]] std::string GetVersion() const;
-    [[nodiscard]] std::vector<LlmModel> GetModels() const { return m_models; }
+    [[nodiscard]] std::vector<LlmModel> GetModels() const { std::lock_guard lock( m_modelsLock ); return m_models; }
 
     void Draw();
 
     bool m_show = false;
 
 private:
+    void Worker();
+
     void LoadModels();
 
     std::unique_ptr<Ollama> m_ollama;
+
+    mutable std::mutex m_modelsLock;
     std::vector<LlmModel> m_models;
 
     size_t m_modelIdx;
     size_t m_ctxSize;
+
+    std::atomic<bool> m_exit;
+    std::condition_variable m_cv;
+    std::thread m_thread;
+
+    mutable std::mutex m_lock;
+    std::vector<WorkItem> m_jobs;
+    bool m_busy;
 };
 
 }