SHOW:
|
|
- or go back to the newest paste.
| 1 | <?php | |
| 2 | /************************************************************* | |
| 3 | Simple site crawler to create a search engine XML Sitemap. | |
| 4 | Version 1.0 | |
| 5 | Free to use, without any warranty. | |
| 6 | Written by Elmar Hanlhofer https://www.plop.at 01/Feb/2012. | |
| 7 | *************************************************************/ | |
| 8 | // Set the output file name. | |
| 9 | $file = "sitemap.xml"; | |
| 10 | // Set the tart URL. Here is https used, use http:// for | |
| 11 | // non SSL websites. | |
| 12 | $url = "https://www.artcyclopedia.com"; | |
| 13 | // Set true or false to define how the script is used. | |
| 14 | // true: As CLI script. | |
| 15 | // false: As Website script. | |
| 16 | define (CLI, true); | |
| 17 | define (VERSION, "1.0") ; | |
| 18 | define (NL, CLI ? "\n" : "<br>") ; | |
| 19 | // Define here the URLs to skip. All URLs that start with | |
| 20 | // the defined URL will be skipped too. | |
| 21 | // Example: "https://www.plop.at/print" will also skip | |
| 22 | // https://www.plop.at/print/bootmanager.html | |
| 23 | - | $skip = array ( "replytocom" , "whatsapp:" , "account" , "register" , "?display=" , "?download" , "movies_actors" ) ; |
| 23 | + | $skip = array ( "replytocom" , "whatsapp:" , "account" , "register" , "?display=" , "?download" ) ; |
| 24 | // Define what file types should be scanned . | |
| 25 | $extension = array ( ".html", ".php", ); // never use it anymore | |
| 26 | $caturl = [ "cats" , "catgores" , "tag" , "tags" ] ; | |
| 27 | // Scan frequency | |
| 28 | $freq = "daily"; | |
| 29 | // Page priority | |
| 30 | $priority = "0.5"; | |
| 31 | // Init end ========================== | |
| 32 | ||
| 33 | function Path ($p) | |
| 34 | {
| |
| 35 | $a = explode ("/", $p);
| |
| 36 | $len = strlen ($a[count ($a) - 1]); | |
| 37 | return (substr ($p, 0, strlen ($p) - $len)); | |
| 38 | } | |
| 39 | ||
| 40 | function GetUrl ($url) | |
| 41 | {
| |
| 42 | $agent = "Mozilla/5.0 (compatible; Plop PHP XML Sitemap Generator/" . VERSION . ")"; | |
| 43 | $ch = curl_init(); | |
| 44 | curl_setopt ($ch, CURLOPT_URL, $url); | |
| 45 | curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1); | |
| 46 | curl_setopt ($ch, CURLOPT_USERAGENT, $agent); | |
| 47 | $data = curl_exec($ch); | |
| 48 | curl_close($ch); | |
| 49 | return $data; | |
| 50 | } | |
| 51 | ||
| 52 | function GetQuotedUrl ($str) | |
| 53 | {
| |
| 54 | if ($str[0] != '"') return $str; // Only process a string | |
| 55 | // starting with double quote | |
| 56 | $ret = ""; | |
| 57 | $len = strlen ($str); | |
| 58 | for ($i = 1; $i < $len; $i++) // Start with 1 to skip first quote | |
| 59 | {
| |
| 60 | if ($str[$i] == '"') break; // End quote reached | |
| 61 | $ret .= $str[$i]; | |
| 62 | } | |
| 63 | return $ret; | |
| 64 | } | |
| 65 | ||
| 66 | function Scan ($url) | |
| 67 | {
| |
| 68 | global $scanned, $pf, $extension, $skip, $freq, $priority; | |
| 69 | echo $url . NL; | |
| 70 | array_push ($scanned, $url); | |
| 71 | $html = GetUrl ($url); | |
| 72 | $a1 = explode ("<a", $html);
| |
| 73 | echo ' Number of URLs are ' .count ( $a1 ) ." \n\r " ; | |
| 74 | foreach ($a1 as $val) | |
| 75 | {
| |
| 76 | $anker_parts = explode (">", $val);
| |
| 77 | $a = $anker_parts[0]; | |
| 78 | $href_split = explode ("href=", $a);
| |
| 79 | $href_string = $href_split[1]; | |
| 80 | if ($href_string[0] == '"') | |
| 81 | {
| |
| 82 | $next_url = GetQuotedUrl ($href_string); | |
| 83 | } | |
| 84 | else | |
| 85 | {
| |
| 86 | $spaces_split = explode (" ", $href_string);
| |
| 87 | $next_url = str_replace ("\"", "", $spaces_split[0]);
| |
| 88 | } | |
| 89 | $fragment_split = explode ("#", $next_url);
| |
| 90 | $next_url = $fragment_split[0] ; | |
| 91 | if ((substr ($next_url, 0, 7) != "http://") && | |
| 92 | (substr ($next_url, 0, 8) != "https://")) | |
| 93 | {
| |
| 94 | if ($next_url[0] == '/') | |
| 95 | {
| |
| 96 | $next_url = "$scanned[0]$next_url"; | |
| 97 | } | |
| 98 | else | |
| 99 | {
| |
| 100 | $next_url = Path ($url) . $next_url; | |
| 101 | } | |
| 102 | } | |
| 103 | if ( substr ( $next_url , 0 , strlen ( $scanned[0] ) ) == $scanned[0] ) | |
| 104 | { $ignore = false ;
| |
| 105 | if ( isset ( $skip ) ) // filter bad urls | |
| 106 | {
| |
| 107 | $last= end ( explode ( $url , $next_url ) ) ; print_r( $last ) ; // | |
| 108 | str_replace( $skip , '' , $last , $counter ); | |
| 109 | if ( $counter !== 0 ) | |
| 110 | { $ignore = true ;
| |
| 111 | echo 'ignore' ; | |
| 112 | } | |
| 113 | } | |
| 114 | if (!$ignore && !in_array ($next_url, $scanned)) | |
| 115 | {
| |
| 116 | foreach ($extension as $ext) | |
| 117 | {
| |
| 118 | if ( strpos ( $next_url , $ext ) > 0) | |
| 119 | { echo " $ext " ; } //
| |
| 120 | fwrite ($pf, " <url>\n" . | |
| 121 | " <loc>" . htmlentities ($next_url) ."</loc>\n" . | |
| 122 | " <changefreq>$freq</changefreq>\n" . | |
| 123 | " <priority>$priority</priority>\n" . | |
| 124 | " </url>\n"); | |
| 125 | Scan ($next_url); | |
| 126 | } | |
| 127 | } | |
| 128 | } | |
| 129 | } | |
| 130 | } | |
| 131 | ||
| 132 | $pf = fopen ($file, "w"); | |
| 133 | if (!$pf) | |
| 134 | {
| |
| 135 | echo "Cannot create $file!" . NL; | |
| 136 | return; | |
| 137 | } | |
| 138 | fwrite ($pf, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" . | |
| 139 | "<!-- Created with Plop PHP XML Sitemap Generator " . VERSION . " https://www.plop.at -->\n" . | |
| 140 | "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"\n" . | |
| 141 | " xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" . | |
| 142 | " xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9\n" . | |
| 143 | " http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">\n" . | |
| 144 | " <url>\n" . | |
| 145 | " <loc>$url/</loc>\n" . | |
| 146 | " </url>\n"); | |
| 147 | $scanned = array(); | |
| 148 | Scan ($url); | |
| 149 | fwrite ($pf, "</urlset>\n"); | |
| 150 | fclose ($pf); | |
| 151 | echo "Done." . NL; | |
| 152 | echo "$file created." . NL; | |
| 153 | ?> |