Scarfski/tracy
1
0
Fork 0
mirror of https://github.com/wolfpld/tracy.git synced 2026-10-13 12:25:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

When managing context don't stop after first removal if quota is not met.

Browse Source
This commit is contained in:
Bartosz Taudul
2025-06-01 21:49:05 +02:00
parent 30c2f45c2c
commit 31cd82f78d
2 changed files with 28 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
profiler/src/profiler/TracyLlm.cpp
View File

@@ -781,37 +781,47 @@ void TracyLlm::AddMessage( std::string&& str, const char* role )
m_chat.emplace_back( std::move( msg ) );
}
void TracyLlm::SendMessage( std::unique_lock<std::mutex>& lock )
void TracyLlm::ManageContext()
{
const auto& models = m_api->GetModels();
const auto ctxSize = models[m_modelIdx].contextSize;
if( ctxSize > 0 && (float)m_usedCtx / ctxSize > 0.7f )
if( ctxSize <= 0 ) return;
const auto quota = int( ctxSize * 0.7f );
if( m_usedCtx < quota ) return;
size_t idx = 0;
std::vector<std::pair<size_t, size_t>> toolOutputs;
for( auto& msg : m_chat )
{
size_t idx = 0;
std::vector<std::pair<size_t, size_t>> toolOutputs;
for( auto& msg : m_chat )
if( msg["role"].get_ref<const std::string&>() == "user" )
{
if( msg["role"].get_ref<const std::string&>() == "user" )
auto& content = msg["content"];
const auto& str = content.get_ref<const std::string&>();
if( str.starts_with( "<tool_output>\n" ) )
{
auto& content = msg["content"];
const auto& str = content.get_ref<const std::string&>();
if( str.starts_with( "<tool_output>\n" ) )
{
toolOutputs.emplace_back( str.size(), idx );
}
toolOutputs.emplace_back( str.size(), idx );
}
idx++;
}
if( toolOutputs.size() > 1 )
idx++;
}
if( toolOutputs.size() > 1 )
{
toolOutputs.pop_back(); // keep the last tool output
std::ranges::stable_sort( toolOutputs, []( const auto& a, const auto& b ) { return a.first > b.first; } );
for( auto& v : toolOutputs )
{
toolOutputs.pop_back(); // keep the last tool output
std::ranges::stable_sort( toolOutputs, []( const auto& a, const auto& b ) { return a.first > b.first; } );
auto& v = toolOutputs[0];
m_usedCtx -= v.first / 4;
m_chat[v.second]["content"] = ForgetMsg;
m_usedCtx += strlen( ForgetMsg ) / 4;
if( m_usedCtx < quota ) break;
}
}
}
void TracyLlm::SendMessage( std::unique_lock<std::mutex>& lock )
{
ManageContext();
bool debug = false;
#ifndef NDEBUG

1
profiler/src/profiler/TracyLlm.hpp
View File

@@ -62,6 +62,7 @@ private:
void ResetChat();
void AddMessage( std::string&& str, const char* role );
void ManageContext();
void SendMessage( std::unique_lock<std::mutex>& lock );
bool OnResponse( const nlohmann::json& json );