diff --git a/profiler/src/main.cpp b/profiler/src/main.cpp index fa3a3458..d7003bfa 100644 --- a/profiler/src/main.cpp +++ b/profiler/src/main.cpp @@ -239,6 +239,7 @@ static void LoadConfig() if( ini_sget( ini, "llm", "enabled", "%d", &v ) ) s_config.llm = v; if( v2 = ini_get( ini, "llm", "address" ); v2 ) s_config.llmAddress = v2; if( v2 = ini_get( ini, "llm", "model" ); v2 ) s_config.llmModel = v2; + if( ini_sget( ini, "llm", "context", "%d", &v ) ) s_config.llmContext = v; ini_free( ini ); } @@ -277,6 +278,7 @@ static bool SaveConfig() fprintf( f, "enabled = %i\n", (int)s_config.llm ); fprintf( f, "address = %s\n", s_config.llmAddress.c_str() ); fprintf( f, "model = %s\n", s_config.llmModel.c_str() ); + fprintf( f, "context = %i\n", s_config.llmContext ); fclose( f ); return true; @@ -973,7 +975,7 @@ static void DrawContents() } if( isSelected ) ImGui::SetItemDefaultFocus(); ImGui::SameLine(); - ImGui::TextDisabled( "(ctx: %s)", tracy::RealToString( llmModels[i].ctxSize ) ); + ImGui::TextDisabled( "(max context: %s)", tracy::RealToString( llmModels[i].ctxSize ) ); } ImGui::EndCombo(); } @@ -984,6 +986,52 @@ static void DrawContents() } } + ImGui::TextUnformatted( "Context size" ); + ImGui::SameLine(); + ImGui::SetNextItemWidth( 120 * dpiScale ); + if( ImGui::InputInt( "##contextsize", &s_config.llmContext, 1024, 8192 ) ) + { + s_config.llmContext = std::clamp( s_config.llmContext, 2048, 10240 * 1024 ); + SaveConfig(); + } + ImGui::Indent(); + if( ImGui::Button( "4K" ) ) + { + s_config.llmContext = 4 * 1024; + SaveConfig(); + } + ImGui::SameLine(); + if( ImGui::Button( "8K" ) ) + { + s_config.llmContext = 8 * 1024; + SaveConfig(); + } + ImGui::SameLine(); + if( ImGui::Button( "16K" ) ) + { + s_config.llmContext = 16 * 1024; + SaveConfig(); + } + ImGui::SameLine(); + if( ImGui::Button( "32K" ) ) + { + s_config.llmContext = 32 * 1024; + SaveConfig(); + } + ImGui::SameLine(); + if( ImGui::Button( "64K" ) ) + { + s_config.llmContext = 64 * 1024; + SaveConfig(); + } + ImGui::SameLine(); + if( ImGui::Button( "128K" ) ) + { + s_config.llmContext = 128 * 1024; + SaveConfig(); + } + ImGui::Unindent(); + ImGui::Unindent(); } diff --git a/profiler/src/profiler/TracyConfig.hpp b/profiler/src/profiler/TracyConfig.hpp index a6a7be0a..b6f564d0 100644 --- a/profiler/src/profiler/TracyConfig.hpp +++ b/profiler/src/profiler/TracyConfig.hpp @@ -27,6 +27,7 @@ struct Config bool llm = true; std::string llmAddress = "http://localhost:11434"; std::string llmModel; + int llmContext = 32*1024; }; } diff --git a/profiler/src/profiler/TracyLlm.cpp b/profiler/src/profiler/TracyLlm.cpp index f47b5eb9..034d8488 100644 --- a/profiler/src/profiler/TracyLlm.cpp +++ b/profiler/src/profiler/TracyLlm.cpp @@ -143,20 +143,32 @@ void TracyLlm::Draw() } if( m_modelIdx == i ) ImGui::SetItemDefaultFocus(); ImGui::SameLine(); - ImGui::TextDisabled( "(ctx: %s)", tracy::RealToString( m_models[i].ctxSize ) ); + ImGui::TextDisabled( "(max context: %s)", tracy::RealToString( m_models[i].ctxSize ) ); } ImGui::EndCombo(); } ImGui::AlignTextToFramePadding(); - TextDisabledUnformatted( "Context size:" ); + ImGui::TextUnformatted( "Context size:" ); ImGui::SameLine(); - ImGui::SetNextItemWidth( 80 * scale ); - if( ImGui::InputInt( "##contextSize", &m_ctxPercent ) ) { m_ctxPercent = std::clamp( m_ctxPercent, 1, 100 ); } + ImGui::SetNextItemWidth( 120 * scale ); + if( ImGui::InputInt( "##contextsize", &s_config.llmContext, 1024, 8192 ) ) + { + s_config.llmContext = std::clamp( s_config.llmContext, 2048, 10240 * 1024 ); + } + ImGui::Indent(); + if( ImGui::Button( "4K" ) ) s_config.llmContext = 4 * 1024; ImGui::SameLine(); - ImGui::TextUnformatted( "%" ); + if( ImGui::Button( "8K" ) ) s_config.llmContext = 8 * 1024; ImGui::SameLine(); - ImGui::TextDisabled( "(%s)", RealToString( m_models[m_modelIdx].ctxSize * m_ctxPercent / 100 ) ); + if( ImGui::Button( "16K" ) ) s_config.llmContext = 16 * 1024; + ImGui::SameLine(); + if( ImGui::Button( "32K" ) ) s_config.llmContext = 32 * 1024; + ImGui::SameLine(); + if( ImGui::Button( "64K" ) ) s_config.llmContext = 64 * 1024; + ImGui::SameLine(); + if( ImGui::Button( "128K" ) ) s_config.llmContext = 128 * 1024; + ImGui::Unindent(); ImGui::TreePop(); } @@ -429,7 +441,7 @@ void TracyLlm::LoadModels() const auto& modelInfo = info["model_info"]; const auto& architecture = modelInfo["general.architecture"].get_ref(); const auto& ctx = modelInfo[architecture + ".context_length"]; - m.emplace_back( LlmModel { .name = model, .ctxSize = ctx.get() } ); + m.emplace_back( LlmModel { .name = model, .ctxSize = ctx.get() } ); } m_modelsLock.lock(); @@ -459,7 +471,7 @@ void TracyLlm::ResetChat() void TracyLlm::SendMessage( ollama::messages&& messages ) { ollama::options options; - options["num_ctx"] = m_models[m_modelIdx].ctxSize * m_ctxPercent / 100; + options["num_ctx"] = std::min( m_models[m_modelIdx].ctxSize, s_config.llmContext ); // The chat() call will fire a callback right away, so the assistant message needs to be there already m_chat->emplace_back( ollama::message( "assistant", "" ) ); diff --git a/profiler/src/profiler/TracyLlm.hpp b/profiler/src/profiler/TracyLlm.hpp index 137229ff..e4e473cc 100644 --- a/profiler/src/profiler/TracyLlm.hpp +++ b/profiler/src/profiler/TracyLlm.hpp @@ -56,7 +56,7 @@ public: struct LlmModel { std::string name; - size_t ctxSize; + int ctxSize; }; TracyLlm(); @@ -96,7 +96,6 @@ private: std::vector m_models; size_t m_modelIdx; - int m_ctxPercent = 100; std::atomic m_exit; std::condition_variable m_cv;