Scarfski/tracy
1
0
Fork 0
mirror of https://github.com/wolfpld/tracy.git synced 2026-10-17 14:25:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement llama-swap tokenization request.

Browse Source
This commit is contained in:
Bartosz Taudul
2025-06-29 13:42:56 +02:00
parent 4969be344e
commit b36b6a0aa2
2 changed files with 24 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
profiler/src/profiler/TracyLlmApi.cpp
View File

@@ -239,6 +239,29 @@ bool TracyLlmApi::Embeddings( const nlohmann::json& req, nlohmann::json& respons
return true;
}
int TracyLlmApi::Tokenize( const std::string& text, int modelIdx )
{
if( m_type == Type::LlamaSwap )
{
std::string buf;
nlohmann::json req = { { "content", text } };
auto res = PostRequest( m_url + "/upstream/" + m_models[modelIdx].name + "/tokenize", req.dump( -1, ' ', false, nlohmann::json::error_handler_t::replace ), buf, true );
if( res != 200 ) return -1;
try
{
auto json = nlohmann::json::parse( buf );
return json["tokens"].size();
}
catch( const std::exception& )
{
return -1;
}
}
return -1;
}
int64_t TracyLlmApi::GetRequest( const std::string& url, std::string& response )
{
assert( m_curl );

1
profiler/src/profiler/TracyLlmApi.hpp
View File

@@ -35,6 +35,7 @@ public:
bool Connect( const char* url );
bool ChatCompletion( const nlohmann::json& req, const std::function<bool(const nlohmann::json&)>& callback, int modelIdx );
bool Embeddings( const nlohmann::json& req, nlohmann::json& response, bool separateConnection = false );
[[nodiscard]] int Tokenize( const std::string& text, int modelIdx );
[[nodiscard]] bool IsConnected() const { return m_curl != nullptr; }
[[nodiscard]] const std::vector<LlmModel>& GetModels() const { return m_models; }