1cloud.pl

use 5.010;
use strict;
use warnings;
use utf8;

use LWP;
use File::Slurp qw/read_file write_file/;

#--------------------------------------------------------

my $pages   = 100;
my $outfile = "1db.txt";

#--------------------------------------------------------

my $lwp = new LWP::UserAgent;

sub get_board_page {
   my($page) = @_;
   my $res = $lwp->get("http://1chan.ru/news/all/$page/");
   die $res->status_line unless $res->is_success;
   return $res->content;
}

sub get_thread_page {
   my($thread) = @_;
   my $res = $lwp->get("http://1chan.ru/news/res/$thread/");
   die $res->status_line unless $res->is_success;
   return $res->content;
}

sub parse_threads {
   my($html) = @_;
   my @threads = $html =~ m~<span class="b-blog-entry_b-info_b-link">\s*?<a href="http://1chan.ru/news/res/(\d+)/">~gs;
   return @threads;
}

sub parse_comments {
   my($html) = @_;
   my @comments = $html =~ m~<div class="b-comment_b-body g-clearfix">\s*<p>(.*?)</p>\s*?</div>?~gs;
   return @comments;
}

sub prepare_comments {
   my @comments = @_;
   return map {
      utf8::decode($_);
      $_ = lc $_;
      tr/ё/е/;
      s~[^а-я\s]~ ~g;
      s~\s+~ ~g;
      s~^\s+|\s+$~~g;
      utf8::encode($_);
      $_;
   } @comments;
}

my %used_threads;
my @comments;
for my $page (0 .. $pages) {
   say "Get page '$page'";
   for my $thread (parse_threads get_board_page $page) {
      next if $used_threads{$thread}++;
      say "Get thread '$thread'";
      push @comments, prepare_comments parse_comments get_thread_page $thread;
   }
}

write_file($outfile, join " ", @comments);