Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- my $max_conn = 100;
- my $query = "SELECT ip FROM intranet_ip";
- my $res = $db->query($query);
- my @urls = Mojo::URL->new();
- my @allUrls;
- while ( my $row = $res->hash ) {
- my $pgsql_ip = $row->{ip};
- push @allUrls, $pgsql_ip;
- }
- my $ua = Mojo::UserAgent->new( max_redirects => 5 );
- my $active = 0;
- Mojo::IOLoop->recurring(
- 0 => sub {
- for ( $active + 1 .. $max_conn ) {
- return ( $active or Mojo::IOLoop->stop )
- unless my $url = shift @urls;
- ++$active;
- $ua->get( $url => \&get_callback );
- }
- }
- );
- sub get_callback {
- my ( undef, $tx ) = @_;
- --$active;
- my $url = $tx->req->url;
- if ( ( !$tx->res->is_status_class(200) )
- or ( $tx->res->headers->content_type !~ m{^text/html\b}ix ) )
- {
- say "url getting problem: $url";
- return;
- }
- #say $url;
- return;
- }
- for my $href (@allUrls) {
- my $origUrl = $href;
- @urls = ();
- push( @urls, Mojo::URL->new($origUrl) );
- $active = 0;
- say "Processing: $href";
- ### Start event loop if necessary
- Mojo::IOLoop->start unless Mojo::IOLoop->is_running;
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement