peachyontop

C++ Web Scraper [ More Efficient ]

May 28th, 2025
837
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
C++ 1.75 KB | None | 0 0
  1. #include <iostream>
  2. #include <string>
  3. #include <curl/curl.h>
  4. #include <gumbo.h>
  5.  
  6. // Callback function for libcurl
  7. size_t WriteCallback(void* contents, size_t size, size_t nmemb, std::string* output) {
  8.     size_t totalSize = size * nmemb;
  9.     output->append((char*)contents, totalSize);
  10.     return totalSize;
  11. }
  12.  
  13. // Function to fetch webpage content
  14. std::string fetchHTML(const std::string& url) {
  15.     CURL* curl = curl_easy_init();
  16.     std::string htmlContent;
  17.  
  18.     if (curl) {
  19.         curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
  20.         curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
  21.         curl_easy_setopt(curl, CURLOPT_WRITEDATA, &htmlContent);
  22.         curl_easy_perform(curl);
  23.         curl_easy_cleanup(curl);
  24.     }
  25.  
  26.     return htmlContent;
  27. }
  28.  
  29. // Function to parse HTML and extract `<title>` tag
  30. void extractTitle(const std::string& html) {
  31.     GumboOutput* output = gumbo_parse(html.c_str());
  32.     GumboNode* root = output->root;
  33.  
  34.     if (root->type == GUMBO_NODE_ELEMENT) {
  35.         GumboVector children = root->v.element.children;
  36.  
  37.         for (size_t i = 0; i < children.length; ++i) {
  38.             GumboNode* child = (GumboNode*)children.data[i];
  39.             if (child->type == GUMBO_NODE_ELEMENT && child->v.element.tag == GUMBO_TAG_TITLE) {
  40.                 std::cout << "Page Title: " << child->v.element.children.data[0] << std::endl;
  41.                 break;
  42.             }
  43.         }
  44.     }
  45.  
  46.     gumbo_destroy_output(&kGumboDefaultOptions, output);
  47. }
  48.  
  49. int main() {
  50.     std::string url = "https://example.com";
  51.     std::string html = fetchHTML(url);
  52.    
  53.     if (!html.empty()) {
  54.         extractTitle(html);
  55.     } else {
  56.         std::cout << "Failed to fetch the webpage!" << std::endl;
  57.     }
  58.  
  59.     return 0;
  60. }
Advertisement
Add Comment
Please, Sign In to add comment