Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/perl
- use strict;
- use warnings;
- use Data::Dumper;
- use Devel::Timer;
- use Furl;
- use EV;
- use AnyEvent;
- use AnyEvent::Util 'fork_call';
- my @urls = (
- 'http://www.perlmonks.org/?',
- 'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://dklab.ru/chicken/nablas/21.html',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://www.perlmonks.org/?',
- 'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
- 'http://perldoc.perl.org/functions/localtime.html',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://www.perlmonks.org/?node_id=1018717',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://perldoc.perl.org/functions/localtime.html',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://perldoc.perl.org/functions/localtime.html',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://www.perlmonks.org/?',
- 'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://www.perlmonks.org/?node_id=1018717',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://perldoc.perl.org/functions/localtime.html',
- 'http://www.perlmonks.org/?',
- 'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://perldoc.perl.org/functions/localtime.html',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://www.perlmonks.org/?node_id=1018717',
- 'http://www.perlmonks.org/?',
- 'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://dklab.ru/chicken/nablas/21.html',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://perldoc.perl.org/functions/localtime.html',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://www.perlmonks.org/?node_id=1018717',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://dklab.ru/chicken/nablas/21.html',
- 'http://perldoc.perl.org/functions/localtime.html',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://dklab.ru/chicken/nablas/21.html',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://www.perlmonks.org/?node_id=1018717',
- 'http://www.perlmonks.org/?',
- 'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
- 'http://perldoc.perl.org/functions/localtime.html',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://dklab.ru/chicken/nablas/21.html',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://www.perlmonks.org/?node_id=1018717',
- 'http://perldoc.perl.org/functions/localtime.html',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
- 'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
- 'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
- 'http://dklab.ru/chicken/nablas/21.html',
- );
- sub fetch {
- my $url = shift;
- my $furl = Furl::HTTP->new(agent => 'Furl/0.31',timeout => 3);
- print "start $url\n";
- my ($ver, $code, $msg, $headers, $body) = $furl->get($url);
- my $size = length $body;
- print "finished $url, $size bytes\n";
- return ($code, $msg, $headers, $body);
- }
- my %resps;
- my $timer = Devel::Timer->new();
- $timer->mark('foreach');
- $AnyEvent::Util::MAX_FORKS = 20;
- my $cv = AE::cv;
- foreach my $url (@urls) {
- $timer->mark('next foreach');
- $cv->begin;
- fork_call {
- print "getting $url... ";
- my ($code, $msg, $headers, $body)=fetch($url);
- print "[$code]\n";
- return ($url, $code, $msg, $headers, $body);
- }
- sub {
- print "adding 2 %resps\n";
- my ($url, $code, $msg, $headers, $body)=@_;
- $resps{$url}->{'code'}=$code;
- $resps{$url}->{'msg'}=$msg;
- $resps{$url}->{'headers'}=$headers;
- $resps{$url}->{'body'}=$body;
- $cv->end;
- };
- }
- $cv->recv;
- $timer->mark('end');
- print "\nall data is ready, press <ENTER>:";
- <STDIN>;
- print Dumper(%resps);
- print "\n<PRESS ENTER>to print timer report\n";
- <STDIN>;
- $timer->report();
- sleep(3);
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement