From 931bc977877638d2a124df90b096ca074e4f374a Mon Sep 17 00:00:00 2001 From: Bartosz Taudul Date: Thu, 22 May 2025 22:13:51 +0200 Subject: [PATCH] Trimming tool response now properly handles UTF-8. This is important to have, as otherwise json parser will fail. --- profiler/src/profiler/TracyLlmTools.cpp | 25 +++++++++++++++++++------ profiler/src/profiler/TracyLlmTools.hpp | 1 + 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/profiler/src/profiler/TracyLlmTools.cpp b/profiler/src/profiler/TracyLlmTools.cpp index f4253b32..395d2714 100644 --- a/profiler/src/profiler/TracyLlmTools.cpp +++ b/profiler/src/profiler/TracyLlmTools.cpp @@ -98,6 +98,23 @@ int TracyLlmTools::CalcMaxSize() const return maxSize; } +std::string TracyLlmTools::TrimString( std::string&& str ) const +{ + auto maxSize = CalcMaxSize(); + if( str.size() < maxSize ) return str; + + // Check if UTF-8 continuation byte will be removed, meaning an UTF-8 character is split in the middle + if( ( str[maxSize] & 0xC0 ) == 0xC0 ) + { + // Remove the current UTF-8 character + while( maxSize > 0 && ( str[maxSize-1] & 0xC0 ) == 0xC0 ) maxSize--; + // Finally, remove the first byte of a UTF-8 multi-byte sequence + //assert( ( str[maxSize-1] & 0xC0 ) == 0x80 ); + if( maxSize > 0 ) maxSize--; + } + return str.substr( 0, maxSize ); +} + static size_t WriteFn( void* _data, size_t size, size_t num, void* ptr ) { const auto data = (unsigned char*)_data; @@ -203,9 +220,7 @@ std::string TracyLlmTools::GetWikipedia( std::string page, const std::string& la std::ranges::replace( page, ' ', '_' ); auto res = FetchWebPage( "https://" + lang + ".wikipedia.org/w/rest.php/v1/page/" + page ); - const auto maxSize = CalcMaxSize(); - if( res.size() > maxSize ) res = res.substr( 0, maxSize ); - return res; + return TrimString( std::move( res ) ); } std::string TracyLlmTools::GetDictionary( std::string word, const std::string& lang ) @@ -227,9 +242,7 @@ std::string TracyLlmTools::GetDictionary( std::string word, const std::string& l const auto key = page0["key"].get_ref(); auto res = FetchWebPage( "https://" + lang + ".wiktionary.org/w/rest.php/v1/page/" + key ); - const auto maxSize = CalcMaxSize(); - if( res.size() > maxSize ) res = res.substr( 0, maxSize ); - return res; + return TrimString( std::move( res ) ); } static std::string RemoveNewline( std::string str ) diff --git a/profiler/src/profiler/TracyLlmTools.hpp b/profiler/src/profiler/TracyLlmTools.hpp index 5f819b1d..e626df01 100644 --- a/profiler/src/profiler/TracyLlmTools.hpp +++ b/profiler/src/profiler/TracyLlmTools.hpp @@ -25,6 +25,7 @@ public: private: [[nodiscard]] int CalcMaxSize() const; + [[nodiscard]] std::string TrimString( std::string&& str ) const; std::string FetchWebPage( const std::string& url ); ToolReply SearchWikipedia( std::string query, const std::string& lang );