Scarfski/tracy
1
0
Fork 0
mirror of https://github.com/wolfpld/tracy.git synced 2026-10-24 01:30:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add hard limit on context usage.

Browse Source 
Big MoE models + big context size = slow prompt processing. Limit tool
reply size to a reasonable amount.
This commit is contained in:
Bartosz Taudul
2026-01-07 16:30:34 +01:00
parent f00694fae0
commit 14f0ed1cba
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
profiler/src/profiler/TracyLlmTools.cpp
View File

@@ -324,12 +324,13 @@ void TracyLlmTools::CancelManualEmbeddings()
int TracyLlmTools::CalcMaxSize() const
{
if( m_ctxSize <= 0 ) return 32*1024;
constexpr int limit = 32*1024;
if( m_ctxSize <= 0 ) return limit;
// Limit the size of the response to avoid exceeding the context size
// Assume average token size is 4 bytes. Make space for 3 articles to be retrieved.
const auto maxSize = ( m_ctxSize * 4 ) / 3;
return maxSize;
const int maxSize = ( m_ctxSize * 4 ) / 3;
return std::min( maxSize, limit );
}
std::string TracyLlmTools::TrimString( std::string&& str ) const