View difference between Paste ID: 55Mf5AuB and q2D4bN33
SHOW: | | - or go back to the newest paste.
1
<?php
2
/*************************************************************
3
 Simple site crawler to create a search engine XML Sitemap.
4
 Version 1.0
5
 Free to use, without any warranty.
6
 Written by Elmar Hanlhofer https://www.plop.at 01/Feb/2012.
7
*************************************************************/
8
    // Set the output file name.
9
    $file = "sitemap.xml";
10
    // Set the tart URL. Here is https used, use http:// for 
11
    // non SSL websites.
12
    $url = "https://www.artcyclopedia.com";       
13
    // Set true or false to define how the script is used.
14
    // true:  As CLI script.
15
    // false: As Website script.
16
    define (CLI, true); 
17
    define (VERSION, "1.0") ;                                            
18
    define (NL, CLI ? "\n" : "<br>")  ;
19
    // Define here the URLs to skip. All URLs that start with 
20
    // the defined URL will be skipped too.
21
    // Example: "https://www.plop.at/print" will also skip
22
    // https://www.plop.at/print/bootmanager.html
23-
 $skip = array (  "replytocom" , "whatsapp:" ,   "account"  , "register" , "?display=" , "?download"  , "movies_actors"     )   ;  
23+
 $skip = array (  "replytocom" , "whatsapp:" ,   "account"  , "register" , "?display=" , "?download"      )   ;  
24
  //    Define what file types should be scanned  .
25
  $extension = array (  ".html",   ".php",  ); // never use it anymore 
26
  $caturl = [  "cats" , "catgores" ,   "tag" , "tags"          ]  ; 
27
    // Scan frequency
28
    $freq = "daily";
29
    // Page priority
30
    $priority = "0.5";
31
    // Init end ==========================    
32
33
function Path ($p)
34
{
35
    $a   = explode ("/", $p);
36
    $len = strlen ($a[count ($a) - 1]);
37
    return (substr ($p, 0, strlen ($p) - $len));
38
}
39
40
function GetUrl ($url)
41
{
42
    $agent = "Mozilla/5.0 (compatible; Plop PHP XML Sitemap Generator/" . VERSION . ")";
43
    $ch = curl_init();
44
    curl_setopt ($ch, CURLOPT_URL, $url);
45
    curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1);
46
    curl_setopt ($ch, CURLOPT_USERAGENT, $agent);
47
    $data = curl_exec($ch);
48
    curl_close($ch);
49
    return $data;
50
}
51
52
function GetQuotedUrl ($str)
53
{
54
    if ($str[0] != '"') return $str; // Only process a string 
55
                                     // starting with double quote
56
    $ret = "";    
57
    $len = strlen ($str);    
58
    for ($i = 1; $i < $len; $i++) // Start with 1 to skip first quote
59
    {
60
        if ($str[$i] == '"') break; // End quote reached
61
        $ret .= $str[$i];
62
    }    
63
    return $ret;
64
}
65
66
function Scan ($url)
67
{
68
    global $scanned, $pf, $extension, $skip, $freq, $priority;    
69
    echo $url . NL;
70
    array_push ($scanned, $url);
71
    $html = GetUrl ($url);
72
    $a1   = explode ("<a", $html);
73
echo ' Number of URLs are  ' .count (  $a1   ) ." \n\r " ; 
74
    foreach ($a1 as $val)
75
    {
76
        $anker_parts = explode (">", $val);
77
        $a = $anker_parts[0];        
78
        $href_split  = explode ("href=", $a);
79
        $href_string = $href_split[1];        
80
        if ($href_string[0] == '"')
81
        {
82
            $next_url = GetQuotedUrl ($href_string);
83
        }
84
        else
85
        {
86
            $spaces_split = explode (" ", $href_string);
87
            $next_url     = str_replace ("\"", "", $spaces_split[0]);
88
        }
89
        $fragment_split = explode ("#", $next_url);
90
        $next_url       = $fragment_split[0] ;        
91
        if ((substr ($next_url, 0, 7) != "http://")  && 
92
            (substr ($next_url, 0, 8) != "https://"))
93
        {
94
            if ($next_url[0] == '/')
95
            {
96
                $next_url = "$scanned[0]$next_url";
97
            }
98
            else
99
            {
100
                $next_url = Path ($url) . $next_url;
101
            }
102
        }        
103
   if (  substr (    $next_url  , 0  , strlen (  $scanned[0]  )  ) ==   $scanned[0]  )
104
      {   $ignore = false ;
105
 if (  isset (     $skip     )     )                 //  filter bad urls     
106
  {     
107
     $last=  end (  explode (  $url   ,    $next_url  )   )      ;            print_r(  $last  )    ;       //    
108
    str_replace(  $skip  , '' ,  $last   , $counter  );
109
if (  $counter  !==   0  ) 
110
     {   $ignore   =   true   ;
111
          echo   'ignore'  ; 
112
                         }        
113
                         } 
114
            if (!$ignore && !in_array ($next_url, $scanned))
115
            {
116
                foreach ($extension as $ext)
117
                {
118
                    if (   strpos (   $next_url  ,   $ext    )   > 0)
119
                        {      echo "  $ext " ;              } //   
120
                        fwrite ($pf, "  <url>\n" .
121
                                     "    <loc>" . htmlentities ($next_url) ."</loc>\n" .
122
                                     "    <changefreq>$freq</changefreq>\n" .
123
                                     "    <priority>$priority</priority>\n" .
124
                                     "  </url>\n");
125
                        Scan ($next_url);
126
                }
127
            }
128
        }
129
    }
130
}
131
132
    $pf = fopen ($file, "w");
133
    if (!$pf)
134
    {
135
        echo "Cannot create $file!" . NL;
136
        return;
137
    }
138
 fwrite ($pf, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" .
139
                 "<!-- Created with Plop PHP XML Sitemap Generator " . VERSION . " https://www.plop.at -->\n" .
140
                 "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"\n" .
141
                 "        xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" .
142
                 "        xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9\n" .
143
                 "        http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">\n" .
144
                 "  <url>\n" .
145
                 "    <loc>$url/</loc>\n" . 
146
                 "  </url>\n");
147
    $scanned = array();
148
    Scan ($url);    
149
    fwrite ($pf, "</urlset>\n");
150
    fclose ($pf);
151
    echo "Done." . NL;
152
    echo "$file created." . NL;
153
?>