Scarfski/tracy
1
0
Fork 0
mirror of https://github.com/wolfpld/tracy.git synced 2026-10-17 14:25:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove readability.js support.

Browse Source 
The current method of web retrieval works well enough.
This commit is contained in:
Bartosz Taudul
2025-06-28 15:05:22 +02:00
parent 154c055fcb
commit 047fa3b98f
4 changed files with 67 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
profiler/src/profiler/TracyConfig.cpp
View File

@@ -38,7 +38,6 @@ void LoadConfig()
if( v2 = ini_get( ini, "llm", "address" ); v2 ) s_config.llmAddress = v2;
if( v2 = ini_get( ini, "llm", "model" ); v2 ) s_config.llmModel = v2;
if( v2 = ini_get( ini, "llm", "embeddings" ); v2 ) s_config.llmEmbeddingsModel = v2;
if( v2 = ini_get( ini, "llm", "readability" ); v2 ) s_config.llmReadability = v2;
if( v2 = ini_get( ini, "llm", "useragent" ); v2 ) s_config.llmUserAgent = v2;
if( v2 = ini_get( ini, "llm", "searchIdentifier" ); v2 ) s_config.llmSearchIdentifier = v2;
if( v2 = ini_get( ini, "llm", "searchApiKey" ); v2 ) s_config.llmSearchApiKey = v2;
@@ -81,7 +80,6 @@ bool SaveConfig()
fprintf( f, "address = %s\n", s_config.llmAddress.c_str() );
fprintf( f, "model = %s\n", s_config.llmModel.c_str() );
fprintf( f, "embeddings = %s\n", s_config.llmEmbeddingsModel.c_str() );
fprintf( f, "readability = %s\n", s_config.llmReadability.c_str() );
fprintf( f, "useragent = %s\n", s_config.llmUserAgent.c_str() );
fprintf( f, "searchIdentifier = %s\n", s_config.llmSearchIdentifier.c_str() );
fprintf( f, "searchApiKey = %s\n", s_config.llmSearchApiKey.c_str() );

1
profiler/src/profiler/TracyConfig.hpp
View File

@@ -28,7 +28,6 @@ struct Config
std::string llmAddress = "http://localhost:11434";
std::string llmModel;
std::string llmEmbeddingsModel;
std::string llmReadability;
std::string llmUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36";
std::string llmSearchIdentifier;
std::string llmSearchApiKey;

12
profiler/src/profiler/TracyLlm.cpp
View File

@@ -280,18 +280,6 @@ void TracyLlm::Draw()
{
char buf[1024];
ImGui::AlignTextToFramePadding();
ImGui::TextUnformatted( "Readability.js:" );
ImGui::SameLine();
snprintf( buf, sizeof( buf ), "%s", s_config.llmReadability.c_str() );
if( ImGui::InputTextWithHint( "##readability", "http://127.0.0.1:3000", buf, sizeof( buf ) ) )
{
s_config.llmReadability = buf;
SaveConfig();
}
ImGui::SameLine();
if( ImGui::Button( ICON_FA_HOUSE "##readability" ) ) OpenWebpage( "https://github.com/phpdocker-io/readability-js-server" );
ImGui::AlignTextToFramePadding();
ImGui::TextUnformatted( "User agent:" );
ImGui::SameLine();

179
profiler/src/profiler/TracyLlmTools.cpp
View File

@@ -704,123 +704,78 @@ std::string TracyLlmTools::GetWebpage( const std::string& url )
{
NetworkCheckString;
auto data = FetchWebPage( url, false );
auto doc = ParseHtml( data );
if( !doc ) return "Error: Failed to parse HTML";
auto body = doc->select_node( "/html/body" );
if( !body ) return "Error: Failed to parse HTML";
auto node = body.node();
RemoveTag( node, "//script" );
RemoveTag( node, "//style" );
RemoveTag( node, "//link" );
RemoveTag( node, "//meta" );
RemoveTag( node, "//svg" );
RemoveTag( node, "//template" );
RemoveTag( node, "//ins" );
RemoveAttributes( node, "//body" );
RemoveAttributes( node, "//div" );
RemoveAttributes( node, "//p" );
RemoveAttributes( node, "//a", { "href", "title" } );
RemoveAttributes( node, "//img", { "src", "alt" } );
RemoveAttributes( node, "//li" );
RemoveAttributes( node, "//ul", { "role" } );
RemoveAttributes( node, "//td", { "colspan" } );
RemoveAttributes( node, "//tr" );
RemoveAttributes( node, "//hr" );
RemoveAttributes( node, "//th", { "colspan", "rowspan" } );
RemoveAttributes( node, "//table", { "role" } );
RemoveAttributes( node, "//col" );
RemoveAttributes( node, "//span" );
RemoveAttributes( node, "//pre" );
RemoveAttributes( node, "//button" );
RemoveAttributes( node, "//label", { "title" } );
RemoveAttributes( node, "//input", { "type", "placeholder" } );
RemoveAttributes( node, "//form", { "action", "method" } );
RemoveAttributes( node, "//textarea", { "placeholder" } );
RemoveAttributes( node, "//dialog" );
RemoveAttributes( node, "//header" );
RemoveAttributes( node, "//footer" );
RemoveAttributes( node, "//section" );
RemoveAttributes( node, "//article" );
RemoveAttributes( node, "//aside" );
RemoveAttributes( node, "//figure" );
RemoveAttributes( node, "//main" );
RemoveAttributes( node, "//summary" );
RemoveAttributes( node, "//details" );
RemoveAttributes( node, "//nav" );
RemoveAttributes( node, "//bdi" );
RemoveAttributes( node, "//time", { "datetime" } );
RemoveAttributes( node, "//h1" );
RemoveAttributes( node, "//h2" );
RemoveAttributes( node, "//h3" );
RemoveAttributes( node, "//h4" );
RemoveAttributes( node, "//h5" );
RemoveAttributes( node, "//h6" );
RemoveAttributes( node, "//strong" );
RemoveAttributes( node, "//em" );
RemoveAttributes( node, "//i" );
RemoveAttributes( node, "//b" );
RemoveAttributes( node, "//u" );
RemoveEmptyTags( node );
std::string response;
xml_writer writer( response );
body.node().print( writer, nullptr, pugi::format_raw | pugi::format_no_declaration | pugi::format_no_escapes );
if( !s_config.llmReadability.empty() )
{
auto curl = curl_easy_init();
if( !curl ) return "Error: Failed to initialize cURL";
RemoveNewline( response );
auto it = std::ranges::unique( response, []( char a, char b ) { return ( a == ' ' || a == '\t' ) && ( b == ' ' || b == '\t' ); } );
response.erase( it.begin(), it.end() );
nlohmann::json post;
post["url"] = url;
auto postStr = post.dump( -1, ' ', false, nlohmann::json::error_handler_t::replace );
std::string buf;
curl_slist *hdr = nullptr;
hdr = curl_slist_append( hdr, "Accept: application/json" );
hdr = curl_slist_append( hdr, "Content-Type: application/json" );
curl_easy_setopt( curl, CURLOPT_NOSIGNAL, 1L );
curl_easy_setopt( curl, CURLOPT_URL, "http://localhost:3000/" );
curl_easy_setopt( curl, CURLOPT_HTTPHEADER, hdr );
curl_easy_setopt( curl, CURLOPT_CA_CACHE_TIMEOUT, 604800L );
curl_easy_setopt( curl, CURLOPT_FOLLOWLOCATION, 1L );
curl_easy_setopt( curl, CURLOPT_TIMEOUT, 120 );
curl_easy_setopt( curl, CURLOPT_POSTFIELDS, postStr.c_str() );
curl_easy_setopt( curl, CURLOPT_POSTFIELDSIZE, postStr.size() );
curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, WriteFn );
curl_easy_setopt( curl, CURLOPT_WRITEDATA, &buf );
curl_easy_setopt( curl, CURLOPT_USERAGENT, s_config.llmUserAgent.c_str() );
auto res = curl_easy_perform( curl );
curl_slist_free_all( hdr );
if( res == CURLE_OK )
{
try
{
auto json = nlohmann::json::parse( buf );
if( json.contains( "content" ) ) response = json["content"].get_ref<const std::string&>();
}
catch( const nlohmann::json::exception& e ) {}
}
curl_easy_cleanup( curl );
}
if( response.empty() )
{
auto data = FetchWebPage( url, false );
auto doc = ParseHtml( data );
if( !doc ) return "Error: Failed to parse HTML";
auto body = doc->select_node( "/html/body" );
if( !body ) return "Error: Failed to parse HTML";
auto node = body.node();
RemoveTag( node, "//script" );
RemoveTag( node, "//style" );
RemoveTag( node, "//link" );
RemoveTag( node, "//meta" );
RemoveTag( node, "//svg" );
RemoveTag( node, "//template" );
RemoveTag( node, "//ins" );
RemoveAttributes( node, "//body" );
RemoveAttributes( node, "//div" );
RemoveAttributes( node, "//p" );
RemoveAttributes( node, "//a", { "href", "title" } );
RemoveAttributes( node, "//img", { "src", "alt" } );
RemoveAttributes( node, "//li" );
RemoveAttributes( node, "//ul", { "role" } );
RemoveAttributes( node, "//td", { "colspan" } );
RemoveAttributes( node, "//tr" );
RemoveAttributes( node, "//hr" );
RemoveAttributes( node, "//th", { "colspan", "rowspan" } );
RemoveAttributes( node, "//table", { "role" } );
RemoveAttributes( node, "//col" );
RemoveAttributes( node, "//span" );
RemoveAttributes( node, "//pre" );
RemoveAttributes( node, "//button" );
RemoveAttributes( node, "//label", { "title" } );
RemoveAttributes( node, "//input", { "type", "placeholder" } );
RemoveAttributes( node, "//form", { "action", "method" } );
RemoveAttributes( node, "//textarea", { "placeholder" } );
RemoveAttributes( node, "//dialog" );
RemoveAttributes( node, "//header" );
RemoveAttributes( node, "//footer" );
RemoveAttributes( node, "//section" );
RemoveAttributes( node, "//article" );
RemoveAttributes( node, "//aside" );
RemoveAttributes( node, "//figure" );
RemoveAttributes( node, "//main" );
RemoveAttributes( node, "//summary" );
RemoveAttributes( node, "//details" );
RemoveAttributes( node, "//nav" );
RemoveAttributes( node, "//bdi" );
RemoveAttributes( node, "//time", { "datetime" } );
RemoveAttributes( node, "//h1" );
RemoveAttributes( node, "//h2" );
RemoveAttributes( node, "//h3" );
RemoveAttributes( node, "//h4" );
RemoveAttributes( node, "//h5" );
RemoveAttributes( node, "//h6" );
RemoveAttributes( node, "//strong" );
RemoveAttributes( node, "//em" );
RemoveAttributes( node, "//i" );
RemoveAttributes( node, "//b" );
RemoveAttributes( node, "//u" );
RemoveEmptyTags( node );
xml_writer writer( response );
body.node().print( writer, nullptr, pugi::format_raw | pugi::format_no_declaration | pugi::format_no_escapes );
RemoveNewline( response );
auto it = std::ranges::unique( response, []( char a, char b ) { return ( a == ' ' || a == '\t' ) && ( b == ' ' || b == '\t' ); } );
response.erase( it.begin(), it.end() );
}
response = TrimString( std::move( response ) );
m_webCache.emplace( url, response );
return response;
}