Scarfski/tracy
1
0
Fork 0
mirror of https://github.com/wolfpld/tracy.git synced 2026-09-11 04:48:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Get context size with llama-swap.

Browse Source
This commit is contained in:
Bartosz Taudul
2025-06-08 23:05:18 +02:00
parent e2ba66cf04
commit 14cc545efd
1 changed files with 25 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
profiler/src/profiler/TracyLlmApi.cpp
View File

@@ -162,15 +162,33 @@ bool TracyLlmApi::ChatCompletion( const nlohmann::json& req, const std::function
curl_easy_getinfo( m_curl, CURLINFO_RESPONSE_CODE, &http_code );
if( http_code == 200 )
{
if( m_type == Type::LmStudio && m_models[modelIdx].contextSize <= 0 )
if( m_models[modelIdx].contextSize <= 0 )
{
curl_easy_reset( m_curl );
SetupCurl( m_curl );
std::string buf;
if( GetRequest( m_url + "/api/v0/models/" + m_models[modelIdx].name, buf ) == 200 )
if( m_type == Type::LlamaSwap )
{
auto json = nlohmann::json::parse( buf );
if( json.contains( "loaded_context_length" ) ) m_models[modelIdx].contextSize = json["loaded_context_length"].get<int>();
curl_easy_reset( m_curl );
SetupCurl( m_curl );
std::string buf;
if( GetRequest( m_url + "/upstream/" + m_models[modelIdx].name + "/props", buf ) == 200 )
{
auto json = nlohmann::json::parse( buf );
if( json.contains( "default_generation_settings" ) )
{
auto& settings = json["default_generation_settings"];
if( settings.contains( "n_ctx" ) ) m_models[modelIdx].contextSize = settings["n_ctx"].get<int>();
}
}
}
else if( m_type == Type::LmStudio )
{
curl_easy_reset( m_curl );
SetupCurl( m_curl );
std::string buf;
if( GetRequest( m_url + "/api/v0/models/" + m_models[modelIdx].name, buf ) == 200 )
{
auto json = nlohmann::json::parse( buf );
if( json.contains( "loaded_context_length" ) ) m_models[modelIdx].contextSize = json["loaded_context_length"].get<int>();
}
}
}
return true;