From 14f0ed1cba2f0e1fbc60680fdddd464f0ebc9312 Mon Sep 17 00:00:00 2001 From: Bartosz Taudul Date: Wed, 7 Jan 2026 16:30:34 +0100 Subject: [PATCH] Add hard limit on context usage. Big MoE models + big context size = slow prompt processing. Limit tool reply size to a reasonable amount. --- profiler/src/profiler/TracyLlmTools.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/profiler/src/profiler/TracyLlmTools.cpp b/profiler/src/profiler/TracyLlmTools.cpp index 22b13ef5..30f0a28d 100644 --- a/profiler/src/profiler/TracyLlmTools.cpp +++ b/profiler/src/profiler/TracyLlmTools.cpp @@ -324,12 +324,13 @@ void TracyLlmTools::CancelManualEmbeddings() int TracyLlmTools::CalcMaxSize() const { - if( m_ctxSize <= 0 ) return 32*1024; + constexpr int limit = 32*1024; + if( m_ctxSize <= 0 ) return limit; // Limit the size of the response to avoid exceeding the context size // Assume average token size is 4 bytes. Make space for 3 articles to be retrieved. - const auto maxSize = ( m_ctxSize * 4 ) / 3; - return maxSize; + const int maxSize = ( m_ctxSize * 4 ) / 3; + return std::min( maxSize, limit ); } std::string TracyLlmTools::TrimString( std::string&& str ) const