x2Fusion

Freenode Scraper [botbot.me].php

May 20th, 2016
132
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
PHP 1.24 KB | None | 0 0
  1. <?php
  2.  
  3. echo PHP_EOL . 'START' . PHP_EOL;
  4.  
  5. $networks = file_get_contents('https://botbot.me/freenode/');
  6. preg_match_all('/<a href=".*">\#(.*)<\/a>/', $networks, $channels);
  7.  
  8. foreach($channels[1] as $channel)
  9. {
  10.     $channelData = file_get_contents('https://botbot.me/freenode/' . $channel . '/');
  11.     preg_match_all('/<li class="month"><a class="jump-date" data-date=".*" href="\/freenode\/.*\/(.*)\/"><span>.*<\/span><\/a><\/li>/', $channelData, $dates);
  12.    
  13.     mkdir("output/$channel/", 755, true);
  14.    
  15.     foreach($dates[1] as $date)
  16.     {
  17.         mkdir("output/$channel/$date/", 755, true);
  18.        
  19.         $channelData = file_get_contents('https://botbot.me/freenode/' . $channel . '/' . $date . '/');
  20.         preg_match_all('/<div class="message">(.*)<\/div>/', $channelData, $channelChatter);
  21.        
  22.         if(count($channelChatter[1]) > 0)
  23.         {
  24.             foreach($channelChatter[1] as $channelChat)
  25.             {
  26.                 $channelChat = htmlspecialchars_decode(preg_replace_callback("/(&#[0-9]+;)/", function($m) { return mb_convert_encoding($m[1], "UTF-8", "HTML-ENTITIES"); }, $channelChat));
  27.                 echo $channelChat . PHP_EOL . '####' . PHP_EOL;
  28.                 file_put_contents("output/$channel/$date/output.txt", $channelChat . "~##########~##########~", FILE_APPEND);
  29.             }
  30.         }
  31.     }
  32. }
  33.  
  34. echo PHP_EOL . 'END' . PHP_EOL;
Add Comment
Please, Sign In to add comment