Advertisement
Guest User

Untitled

a guest
Dec 12th, 2019
98
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
PHP 1.02 KB | None | 0 0
  1. <?php
  2. require __DIR__ . '/vendor/autoload.php';
  3.  
  4. class CrawlerSupport{
  5.  
  6.     private $use_proxy;
  7.  
  8.     function __construct($use_proxy = false){
  9.         $this->use_proxy = $use_proxy;
  10.     }
  11.  
  12.  
  13.     function loadUrl($url){
  14.         if($html = $this->curl($url)){
  15.             return new Rct567\DomQuery\DomQuery($html);
  16.         }
  17.         return false;
  18.     }
  19.  
  20.     private function curl($url, $try = 1){
  21.         $ch = curl_init();
  22.         curl_setopt($ch, CURLOPT_URL, $url);
  23.         curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
  24.         curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
  25.  
  26.         if($this->use_proxy){
  27.             //TODO::Set curl to use proxy
  28.         }
  29.  
  30.         $response_content =  curl_exec($ch);
  31.         $response_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
  32.  
  33.         if($response_code !== 200 && $try <= 4){
  34.             return $this->curl($url, $try++);
  35.         }elseif($response_code !== 200){
  36.             return false;
  37.         }
  38.         curl_close($ch);
  39.  
  40.         return $response_content;
  41.     }
  42. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement