Scarfski/tracy
1
0
Fork 0
mirror of https://github.com/wolfpld/tracy.git synced 2026-10-17 14:25:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework context size handling.

Browse Source 
The user now specifies max context size to be used for all models.
This commit is contained in:
Bartosz Taudul
2025-05-13 19:24:16 +02:00
parent 74b6e481bb
commit 8fd1530c8f
4 changed files with 71 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
profiler/src/main.cpp
View File

@@ -239,6 +239,7 @@ static void LoadConfig()
if( ini_sget( ini, "llm", "enabled", "%d", &v ) ) s_config.llm = v;
if( v2 = ini_get( ini, "llm", "address" ); v2 ) s_config.llmAddress = v2;
if( v2 = ini_get( ini, "llm", "model" ); v2 ) s_config.llmModel = v2;
if( ini_sget( ini, "llm", "context", "%d", &v ) ) s_config.llmContext = v;
ini_free( ini );
}
@@ -277,6 +278,7 @@ static bool SaveConfig()
fprintf( f, "enabled = %i\n", (int)s_config.llm );
fprintf( f, "address = %s\n", s_config.llmAddress.c_str() );
fprintf( f, "model = %s\n", s_config.llmModel.c_str() );
fprintf( f, "context = %i\n", s_config.llmContext );
fclose( f );
return true;
@@ -973,7 +975,7 @@ static void DrawContents()
}
if( isSelected ) ImGui::SetItemDefaultFocus();
ImGui::SameLine();
ImGui::TextDisabled( "(ctx: %s)", tracy::RealToString( llmModels[i].ctxSize ) );
ImGui::TextDisabled( "(max context: %s)", tracy::RealToString( llmModels[i].ctxSize ) );
}
ImGui::EndCombo();
}
@@ -984,6 +986,52 @@ static void DrawContents()
}
}
ImGui::TextUnformatted( "Context size" );
ImGui::SameLine();
ImGui::SetNextItemWidth( 120 * dpiScale );
if( ImGui::InputInt( "##contextsize", &s_config.llmContext, 1024, 8192 ) )
{
s_config.llmContext = std::clamp( s_config.llmContext, 2048, 10240 * 1024 );
SaveConfig();
}
ImGui::Indent();
if( ImGui::Button( "4K" ) )
{
s_config.llmContext = 4 * 1024;
SaveConfig();
}
ImGui::SameLine();
if( ImGui::Button( "8K" ) )
{
s_config.llmContext = 8 * 1024;
SaveConfig();
}
ImGui::SameLine();
if( ImGui::Button( "16K" ) )
{
s_config.llmContext = 16 * 1024;
SaveConfig();
}
ImGui::SameLine();
if( ImGui::Button( "32K" ) )
{
s_config.llmContext = 32 * 1024;
SaveConfig();
}
ImGui::SameLine();
if( ImGui::Button( "64K" ) )
{
s_config.llmContext = 64 * 1024;
SaveConfig();
}
ImGui::SameLine();
if( ImGui::Button( "128K" ) )
{
s_config.llmContext = 128 * 1024;
SaveConfig();
}
ImGui::Unindent();
ImGui::Unindent();
}

1
profiler/src/profiler/TracyConfig.hpp
View File

@@ -27,6 +27,7 @@ struct Config
bool llm = true;
std::string llmAddress = "http://localhost:11434";
std::string llmModel;
int llmContext = 32*1024;
};
}

28
profiler/src/profiler/TracyLlm.cpp
View File

@@ -143,20 +143,32 @@ void TracyLlm::Draw()
}
if( m_modelIdx == i ) ImGui::SetItemDefaultFocus();
ImGui::SameLine();
ImGui::TextDisabled( "(ctx: %s)", tracy::RealToString( m_models[i].ctxSize ) );
ImGui::TextDisabled( "(max context: %s)", tracy::RealToString( m_models[i].ctxSize ) );
}
ImGui::EndCombo();
}
ImGui::AlignTextToFramePadding();
TextDisabledUnformatted( "Context size:" );
ImGui::TextUnformatted( "Context size:" );
ImGui::SameLine();
ImGui::SetNextItemWidth( 80 * scale );
if( ImGui::InputInt( "##contextSize", &m_ctxPercent ) ) { m_ctxPercent = std::clamp( m_ctxPercent, 1, 100 ); }
ImGui::SetNextItemWidth( 120 * scale );
if( ImGui::InputInt( "##contextsize", &s_config.llmContext, 1024, 8192 ) )
{
s_config.llmContext = std::clamp( s_config.llmContext, 2048, 10240 * 1024 );
}
ImGui::Indent();
if( ImGui::Button( "4K" ) ) s_config.llmContext = 4 * 1024;
ImGui::SameLine();
ImGui::TextUnformatted( "%" );
if( ImGui::Button( "8K" ) ) s_config.llmContext = 8 * 1024;
ImGui::SameLine();
ImGui::TextDisabled( "(%s)", RealToString( m_models[m_modelIdx].ctxSize * m_ctxPercent / 100 ) );
if( ImGui::Button( "16K" ) ) s_config.llmContext = 16 * 1024;
ImGui::SameLine();
if( ImGui::Button( "32K" ) ) s_config.llmContext = 32 * 1024;
ImGui::SameLine();
if( ImGui::Button( "64K" ) ) s_config.llmContext = 64 * 1024;
ImGui::SameLine();
if( ImGui::Button( "128K" ) ) s_config.llmContext = 128 * 1024;
ImGui::Unindent();
ImGui::TreePop();
}
@@ -429,7 +441,7 @@ void TracyLlm::LoadModels()
const auto& modelInfo = info["model_info"];
const auto& architecture = modelInfo["general.architecture"].get_ref<const std::string&>();
const auto& ctx = modelInfo[architecture + ".context_length"];
m.emplace_back( LlmModel { .name = model, .ctxSize = ctx.get<size_t>() } );
m.emplace_back( LlmModel { .name = model, .ctxSize = ctx.get<int>() } );
}
m_modelsLock.lock();
@@ -459,7 +471,7 @@ void TracyLlm::ResetChat()
void TracyLlm::SendMessage( ollama::messages&& messages )
{
ollama::options options;
options["num_ctx"] = m_models[m_modelIdx].ctxSize * m_ctxPercent / 100;
options["num_ctx"] = std::min( m_models[m_modelIdx].ctxSize, s_config.llmContext );
// The chat() call will fire a callback right away, so the assistant message needs to be there already
m_chat->emplace_back( ollama::message( "assistant", "" ) );

3
profiler/src/profiler/TracyLlm.hpp
View File

@@ -56,7 +56,7 @@ public:
struct LlmModel
{
std::string name;
size_t ctxSize;
int ctxSize;
};
TracyLlm();
@@ -96,7 +96,6 @@ private:
std::vector<LlmModel> m_models;
size_t m_modelIdx;
int m_ctxPercent = 100;
std::atomic<bool> m_exit;
std::condition_variable m_cv;