diff --git a/profiler/src/profiler/TracyConfig.cpp b/profiler/src/profiler/TracyConfig.cpp index 3d1c8f3d..aaeaf308 100644 --- a/profiler/src/profiler/TracyConfig.cpp +++ b/profiler/src/profiler/TracyConfig.cpp @@ -38,7 +38,6 @@ void LoadConfig() if( v2 = ini_get( ini, "llm", "address" ); v2 ) s_config.llmAddress = v2; if( v2 = ini_get( ini, "llm", "model" ); v2 ) s_config.llmModel = v2; if( v2 = ini_get( ini, "llm", "embeddings" ); v2 ) s_config.llmEmbeddingsModel = v2; - if( v2 = ini_get( ini, "llm", "readability" ); v2 ) s_config.llmReadability = v2; if( v2 = ini_get( ini, "llm", "useragent" ); v2 ) s_config.llmUserAgent = v2; if( v2 = ini_get( ini, "llm", "searchIdentifier" ); v2 ) s_config.llmSearchIdentifier = v2; if( v2 = ini_get( ini, "llm", "searchApiKey" ); v2 ) s_config.llmSearchApiKey = v2; @@ -81,7 +80,6 @@ bool SaveConfig() fprintf( f, "address = %s\n", s_config.llmAddress.c_str() ); fprintf( f, "model = %s\n", s_config.llmModel.c_str() ); fprintf( f, "embeddings = %s\n", s_config.llmEmbeddingsModel.c_str() ); - fprintf( f, "readability = %s\n", s_config.llmReadability.c_str() ); fprintf( f, "useragent = %s\n", s_config.llmUserAgent.c_str() ); fprintf( f, "searchIdentifier = %s\n", s_config.llmSearchIdentifier.c_str() ); fprintf( f, "searchApiKey = %s\n", s_config.llmSearchApiKey.c_str() ); diff --git a/profiler/src/profiler/TracyConfig.hpp b/profiler/src/profiler/TracyConfig.hpp index f1e8ae08..1146f7c7 100644 --- a/profiler/src/profiler/TracyConfig.hpp +++ b/profiler/src/profiler/TracyConfig.hpp @@ -28,7 +28,6 @@ struct Config std::string llmAddress = "http://localhost:11434"; std::string llmModel; std::string llmEmbeddingsModel; - std::string llmReadability; std::string llmUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"; std::string llmSearchIdentifier; std::string llmSearchApiKey; diff --git a/profiler/src/profiler/TracyLlm.cpp b/profiler/src/profiler/TracyLlm.cpp index e86c8074..ddb89866 100644 --- a/profiler/src/profiler/TracyLlm.cpp +++ b/profiler/src/profiler/TracyLlm.cpp @@ -280,18 +280,6 @@ void TracyLlm::Draw() { char buf[1024]; - ImGui::AlignTextToFramePadding(); - ImGui::TextUnformatted( "Readability.js:" ); - ImGui::SameLine(); - snprintf( buf, sizeof( buf ), "%s", s_config.llmReadability.c_str() ); - if( ImGui::InputTextWithHint( "##readability", "http://127.0.0.1:3000", buf, sizeof( buf ) ) ) - { - s_config.llmReadability = buf; - SaveConfig(); - } - ImGui::SameLine(); - if( ImGui::Button( ICON_FA_HOUSE "##readability" ) ) OpenWebpage( "https://github.com/phpdocker-io/readability-js-server" ); - ImGui::AlignTextToFramePadding(); ImGui::TextUnformatted( "User agent:" ); ImGui::SameLine(); diff --git a/profiler/src/profiler/TracyLlmTools.cpp b/profiler/src/profiler/TracyLlmTools.cpp index 2246513e..6d824fcc 100644 --- a/profiler/src/profiler/TracyLlmTools.cpp +++ b/profiler/src/profiler/TracyLlmTools.cpp @@ -704,123 +704,78 @@ std::string TracyLlmTools::GetWebpage( const std::string& url ) { NetworkCheckString; + auto data = FetchWebPage( url, false ); + auto doc = ParseHtml( data ); + if( !doc ) return "Error: Failed to parse HTML"; + + auto body = doc->select_node( "/html/body" ); + if( !body ) return "Error: Failed to parse HTML"; + + auto node = body.node(); + RemoveTag( node, "//script" ); + RemoveTag( node, "//style" ); + RemoveTag( node, "//link" ); + RemoveTag( node, "//meta" ); + RemoveTag( node, "//svg" ); + RemoveTag( node, "//template" ); + RemoveTag( node, "//ins" ); + RemoveAttributes( node, "//body" ); + RemoveAttributes( node, "//div" ); + RemoveAttributes( node, "//p" ); + RemoveAttributes( node, "//a", { "href", "title" } ); + RemoveAttributes( node, "//img", { "src", "alt" } ); + RemoveAttributes( node, "//li" ); + RemoveAttributes( node, "//ul", { "role" } ); + RemoveAttributes( node, "//td", { "colspan" } ); + RemoveAttributes( node, "//tr" ); + RemoveAttributes( node, "//hr" ); + RemoveAttributes( node, "//th", { "colspan", "rowspan" } ); + RemoveAttributes( node, "//table", { "role" } ); + RemoveAttributes( node, "//col" ); + RemoveAttributes( node, "//span" ); + RemoveAttributes( node, "//pre" ); + RemoveAttributes( node, "//button" ); + RemoveAttributes( node, "//label", { "title" } ); + RemoveAttributes( node, "//input", { "type", "placeholder" } ); + RemoveAttributes( node, "//form", { "action", "method" } ); + RemoveAttributes( node, "//textarea", { "placeholder" } ); + RemoveAttributes( node, "//dialog" ); + RemoveAttributes( node, "//header" ); + RemoveAttributes( node, "//footer" ); + RemoveAttributes( node, "//section" ); + RemoveAttributes( node, "//article" ); + RemoveAttributes( node, "//aside" ); + RemoveAttributes( node, "//figure" ); + RemoveAttributes( node, "//main" ); + RemoveAttributes( node, "//summary" ); + RemoveAttributes( node, "//details" ); + RemoveAttributes( node, "//nav" ); + RemoveAttributes( node, "//bdi" ); + RemoveAttributes( node, "//time", { "datetime" } ); + RemoveAttributes( node, "//h1" ); + RemoveAttributes( node, "//h2" ); + RemoveAttributes( node, "//h3" ); + RemoveAttributes( node, "//h4" ); + RemoveAttributes( node, "//h5" ); + RemoveAttributes( node, "//h6" ); + RemoveAttributes( node, "//strong" ); + RemoveAttributes( node, "//em" ); + RemoveAttributes( node, "//i" ); + RemoveAttributes( node, "//b" ); + RemoveAttributes( node, "//u" ); + RemoveEmptyTags( node ); + std::string response; + xml_writer writer( response ); + body.node().print( writer, nullptr, pugi::format_raw | pugi::format_no_declaration | pugi::format_no_escapes ); - if( !s_config.llmReadability.empty() ) - { - auto curl = curl_easy_init(); - if( !curl ) return "Error: Failed to initialize cURL"; + RemoveNewline( response ); + auto it = std::ranges::unique( response, []( char a, char b ) { return ( a == ' ' || a == '\t' ) && ( b == ' ' || b == '\t' ); } ); + response.erase( it.begin(), it.end() ); - nlohmann::json post; - post["url"] = url; - auto postStr = post.dump( -1, ' ', false, nlohmann::json::error_handler_t::replace ); - - std::string buf; - - curl_slist *hdr = nullptr; - hdr = curl_slist_append( hdr, "Accept: application/json" ); - hdr = curl_slist_append( hdr, "Content-Type: application/json" ); - - curl_easy_setopt( curl, CURLOPT_NOSIGNAL, 1L ); - curl_easy_setopt( curl, CURLOPT_URL, "http://localhost:3000/" ); - curl_easy_setopt( curl, CURLOPT_HTTPHEADER, hdr ); - curl_easy_setopt( curl, CURLOPT_CA_CACHE_TIMEOUT, 604800L ); - curl_easy_setopt( curl, CURLOPT_FOLLOWLOCATION, 1L ); - curl_easy_setopt( curl, CURLOPT_TIMEOUT, 120 ); - curl_easy_setopt( curl, CURLOPT_POSTFIELDS, postStr.c_str() ); - curl_easy_setopt( curl, CURLOPT_POSTFIELDSIZE, postStr.size() ); - curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, WriteFn ); - curl_easy_setopt( curl, CURLOPT_WRITEDATA, &buf ); - curl_easy_setopt( curl, CURLOPT_USERAGENT, s_config.llmUserAgent.c_str() ); - - auto res = curl_easy_perform( curl ); - curl_slist_free_all( hdr ); - - if( res == CURLE_OK ) - { - try - { - auto json = nlohmann::json::parse( buf ); - if( json.contains( "content" ) ) response = json["content"].get_ref(); - } - catch( const nlohmann::json::exception& e ) {} - } - curl_easy_cleanup( curl ); - } - - if( response.empty() ) - { - auto data = FetchWebPage( url, false ); - - auto doc = ParseHtml( data ); - if( !doc ) return "Error: Failed to parse HTML"; - - auto body = doc->select_node( "/html/body" ); - if( !body ) return "Error: Failed to parse HTML"; - - auto node = body.node(); - RemoveTag( node, "//script" ); - RemoveTag( node, "//style" ); - RemoveTag( node, "//link" ); - RemoveTag( node, "//meta" ); - RemoveTag( node, "//svg" ); - RemoveTag( node, "//template" ); - RemoveTag( node, "//ins" ); - RemoveAttributes( node, "//body" ); - RemoveAttributes( node, "//div" ); - RemoveAttributes( node, "//p" ); - RemoveAttributes( node, "//a", { "href", "title" } ); - RemoveAttributes( node, "//img", { "src", "alt" } ); - RemoveAttributes( node, "//li" ); - RemoveAttributes( node, "//ul", { "role" } ); - RemoveAttributes( node, "//td", { "colspan" } ); - RemoveAttributes( node, "//tr" ); - RemoveAttributes( node, "//hr" ); - RemoveAttributes( node, "//th", { "colspan", "rowspan" } ); - RemoveAttributes( node, "//table", { "role" } ); - RemoveAttributes( node, "//col" ); - RemoveAttributes( node, "//span" ); - RemoveAttributes( node, "//pre" ); - RemoveAttributes( node, "//button" ); - RemoveAttributes( node, "//label", { "title" } ); - RemoveAttributes( node, "//input", { "type", "placeholder" } ); - RemoveAttributes( node, "//form", { "action", "method" } ); - RemoveAttributes( node, "//textarea", { "placeholder" } ); - RemoveAttributes( node, "//dialog" ); - RemoveAttributes( node, "//header" ); - RemoveAttributes( node, "//footer" ); - RemoveAttributes( node, "//section" ); - RemoveAttributes( node, "//article" ); - RemoveAttributes( node, "//aside" ); - RemoveAttributes( node, "//figure" ); - RemoveAttributes( node, "//main" ); - RemoveAttributes( node, "//summary" ); - RemoveAttributes( node, "//details" ); - RemoveAttributes( node, "//nav" ); - RemoveAttributes( node, "//bdi" ); - RemoveAttributes( node, "//time", { "datetime" } ); - RemoveAttributes( node, "//h1" ); - RemoveAttributes( node, "//h2" ); - RemoveAttributes( node, "//h3" ); - RemoveAttributes( node, "//h4" ); - RemoveAttributes( node, "//h5" ); - RemoveAttributes( node, "//h6" ); - RemoveAttributes( node, "//strong" ); - RemoveAttributes( node, "//em" ); - RemoveAttributes( node, "//i" ); - RemoveAttributes( node, "//b" ); - RemoveAttributes( node, "//u" ); - RemoveEmptyTags( node ); - - xml_writer writer( response ); - body.node().print( writer, nullptr, pugi::format_raw | pugi::format_no_declaration | pugi::format_no_escapes ); - - RemoveNewline( response ); - auto it = std::ranges::unique( response, []( char a, char b ) { return ( a == ' ' || a == '\t' ) && ( b == ' ' || b == '\t' ); } ); - response.erase( it.begin(), it.end() ); - } response = TrimString( std::move( response ) ); m_webCache.emplace( url, response ); + return response; }