Scarfski/tracy
1
0
Fork 0
mirror of https://github.com/wolfpld/tracy.git synced 2026-09-06 10:28:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Embeddings calls may need to be done on a separate connection.

Browse Source 
The completion reply may be unfinished when the embeddings call is executed.
This commit is contained in:
Bartosz Taudul
2025-05-23 18:58:50 +02:00
parent 75e9e589b8
commit 96e03b079e
2 changed files with 27 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
profiler/src/profiler/TracyLlmApi.cpp
View File

@@ -169,12 +169,12 @@ bool TracyLlmApi::ChatCompletion( const nlohmann::json& req, const std::function
return false;
}
bool TracyLlmApi::Embeddings( const nlohmann::json& req, nlohmann::json& response )
bool TracyLlmApi::Embeddings( const nlohmann::json& req, nlohmann::json& response, bool separateConnection )
{
assert( m_curl );
std::string buf;
auto res = PostRequest( m_url + "/v1/embeddings", req.dump(), buf );
auto res = PostRequest( m_url + "/v1/embeddings", req.dump(), buf, separateConnection );
if( res != 200 ) return false;
response = nlohmann::json::parse( buf );
@@ -204,7 +204,7 @@ int64_t TracyLlmApi::GetRequest( const std::string& url, std::string& response )
return http_code;
}
int64_t TracyLlmApi::PostRequest( const std::string& url, const std::string& data, std::string& response )
int64_t TracyLlmApi::PostRequest( const std::string& url, const std::string& data, std::string& response, bool separateConnection )
{
assert( m_curl );
response.clear();
@@ -213,19 +213,32 @@ int64_t TracyLlmApi::PostRequest( const std::string& url, const std::string& dat
hdr = curl_slist_append( hdr, "Accept: application/json" );
hdr = curl_slist_append( hdr, "Content-Type: application/json" );
curl_easy_setopt( m_curl, CURLOPT_URL, url.c_str() );
curl_easy_setopt( m_curl, CURLOPT_HTTPHEADER, hdr );
curl_easy_setopt( m_curl, CURLOPT_POSTFIELDS, data.c_str() );
curl_easy_setopt( m_curl, CURLOPT_POSTFIELDSIZE, data.size() );
curl_easy_setopt( m_curl, CURLOPT_WRITEDATA, &response );
curl_easy_setopt( m_curl, CURLOPT_WRITEFUNCTION, WriteFn );
auto curl = m_curl;
if( separateConnection )
{
curl = curl_easy_init();
if( !curl ) return -1;
SetupCurl( curl );
}
auto res = curl_easy_perform( m_curl );
curl_easy_setopt( curl, CURLOPT_URL, url.c_str() );
curl_easy_setopt( curl, CURLOPT_HTTPHEADER, hdr );
curl_easy_setopt( curl, CURLOPT_POSTFIELDS, data.c_str() );
curl_easy_setopt( curl, CURLOPT_POSTFIELDSIZE, data.size() );
curl_easy_setopt( curl, CURLOPT_WRITEDATA, &response );
curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, WriteFn );
auto res = curl_easy_perform( curl );
curl_slist_free_all( hdr );
if( res != CURLE_OK ) return -1;
if( res != CURLE_OK )
{
if( separateConnection ) curl_easy_cleanup( curl );
return -1;
}
int64_t http_code = 0;
curl_easy_getinfo( m_curl, CURLINFO_RESPONSE_CODE, &http_code );
curl_easy_getinfo( curl, CURLINFO_RESPONSE_CODE, &http_code );
if( separateConnection ) curl_easy_cleanup( curl );
return http_code;
}

4
profiler/src/profiler/TracyLlmApi.hpp
View File

@@ -32,7 +32,7 @@ public:
bool Connect( const char* url );
bool ChatCompletion( const nlohmann::json& req, const std::function<bool(const nlohmann::json&)>& callback, int modelIdx );
bool Embeddings( const nlohmann::json& req, nlohmann::json& response );
bool Embeddings( const nlohmann::json& req, nlohmann::json& response, bool separateConnection = false );
[[nodiscard]] bool IsConnected() const { return m_curl != nullptr; }
[[nodiscard]] const std::vector<LlmModel>& GetModels() const { return m_models; }
@@ -41,7 +41,7 @@ private:
void SetupCurl( void* curl );
int64_t GetRequest( const std::string& url, std::string& response );
int64_t PostRequest( const std::string& url, const std::string& data, std::string& response );
int64_t PostRequest( const std::string& url, const std::string& data, std::string& response, bool separateConnection = false );
void* m_curl = nullptr;
std::string m_url;