View difference between Paste ID: NMZsvTP2 and e0wY8crW

 <?php
 error_reporting(-1);
 ini_set('display_errors', true);
 stream_context_set_default(
     array(
         'http' => array(
             'method' => 'HEAD'
         )
     )
 );
 
 $links = [];
 $checked = [];
-loadRecursive(getMainUrl(), $links, $checked);
+loadRecursive(getMainUrl(), $links);
 
 var_dump($links);
 function getMainUrl()
 {
     return 'http://theory.phphtml.net';
 }
 
-function loadRecursive1($url, array &$found, array &$checked)
+function loadRecursive1($url, array &$found)
 {
 
     echo "Parsing " . $url . " <br/>";
-    $checked[] = $url;
+
     $html = loadHTML($url);
     $links = findLinks($html);
     $counter = 0;
     foreach ($links as $link) {
 
         $link = getMainUrl() . $link;
         $linkKey = md5($link);
         if ($link === $url) {
             continue;
         }
 
         $found[$linkKey] = $link;
         $counter++;
 
-        //  loadRecursive1($link, $found, $checked);
+
     }
     echo "Found " . $counter . " links on " . $url . "<hr/>";
 }
 
 
 function loadRecursive($url, array &$found)
-function loadRecursive($url, array &$found, array &$checked)
+
 
     echo "Parsing " . $url . " <br/>";
 
-    $checked[] = $url;
+
     $html = loadHTML($url);
     $links = findLinks($html);
     $counter = 0;
     foreach ($links as $link) {
 
         $link = getMainUrl() . $link;
         $linkKey = md5($link);
         if ($link === $url) {
             continue;
         }
 
         $found[$linkKey] = $link;
         $counter++;
     }
     foreach ($found as $link) {
         loadRecursive1($link, $found);
-        loadRecursive1($link, $found, $checked);
+
 
     echo "Found " . $counter . " links on " . $url . "<hr/>";
 }
 
 function urlIsOk($url)
 {
 
     $headers = @get_headers($url, 1);
     if (!$headers) {
         return false;
     }
     if (!preg_match("~(\\d{3})~", $headers[0], $matches)) {
         return false;
     }
     $statusCode = (int)$matches[0];
     if ($statusCode < 200 || $statusCode > 400) {
         return false;
     }
     if ($headers['Content-Type'] !== 'text/html') {
         return false;
     }
 
     return true;
 }
 
 function loadHTML($url)
 {
     if (!urlIsOk($url)) {
         return '';
     }
     $curl = curl_init();
     curl_setopt($curl, CURLOPT_URL, $url);
     curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
     curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
     curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0);
     curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, 0);
     $result = curl_exec($curl);
     curl_close($curl);
     if ($result === false) {
         //echo 'broken url';
         return '';
     }
 
     return (string)$result;
 }
 
 function findLinks($html)
 {
     $domDocument = new DOMDocument();
     @$domDocument->loadHTML($html);
 
     $xpath = new DOMXPath($domDocument);
 
     /**
      * @var DOMNodeList $elements
      */
     $elements = $xpath->query('//a[@href]'); //найдти все елементы которыи имеют href аттрибут
     $links = [];
 
     foreach ($elements as $element) {
         $href = $element->getAttribute('href');
         if ($href === '') {
             continue;
         }
         $isAbsolute = strpos($href, 'http') !== false;
         $isAnchor = strpos($href, '#') !== false;
         if ($isAbsolute || $isAnchor) {
             continue;
         }
         $links[] = $href;
     }
 
     return array_unique($links);
 }