Advertisement
Guest User

Perl REAL concurrent HTTP using AnyEvent and Furl

a guest
Apr 2nd, 2017
278
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Perl 5.97 KB | None | 0 0
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use Data::Dumper;
  5. use Devel::Timer;
  6. use Furl;
  7. use EV;
  8. use AnyEvent;
  9. use AnyEvent::Util 'fork_call';
  10.  
  11. my @urls = (
  12.     'http://www.perlmonks.org/?',
  13.     'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
  14.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  15.     'http://dklab.ru/chicken/nablas/21.html',
  16.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  17.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  18.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  19.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  20.     'http://www.perlmonks.org/?',
  21.     'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
  22.     'http://perldoc.perl.org/functions/localtime.html',
  23.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  24.     'http://www.perlmonks.org/?node_id=1018717',
  25.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  26.     'http://perldoc.perl.org/functions/localtime.html',
  27.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  28.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  29.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  30.     'http://perldoc.perl.org/functions/localtime.html',
  31.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  32.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  33.     'http://www.perlmonks.org/?',
  34.     'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
  35.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  36.     'http://www.perlmonks.org/?node_id=1018717',
  37.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  38.     'http://perldoc.perl.org/functions/localtime.html',
  39.     'http://www.perlmonks.org/?',
  40.     'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
  41.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  42.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  43.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  44.     'http://perldoc.perl.org/functions/localtime.html',
  45.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  46.     'http://www.perlmonks.org/?node_id=1018717',
  47.     'http://www.perlmonks.org/?',
  48.     'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
  49.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  50.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  51.     'http://dklab.ru/chicken/nablas/21.html',
  52.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  53.     'http://perldoc.perl.org/functions/localtime.html',
  54.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  55.     'http://www.perlmonks.org/?node_id=1018717',
  56.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  57.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  58.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  59.     'http://dklab.ru/chicken/nablas/21.html',
  60.     'http://perldoc.perl.org/functions/localtime.html',
  61.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  62.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  63.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  64.     'http://dklab.ru/chicken/nablas/21.html',
  65.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  66.     'http://www.perlmonks.org/?node_id=1018717',
  67.     'http://www.perlmonks.org/?',
  68.     'http://stackoverflow.com/questions/43024722/perl-too-slow-concurrent-download-with-both-httpasync-netasynchttp',
  69.     'http://perldoc.perl.org/functions/localtime.html',
  70.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  71.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  72.     'http://dklab.ru/chicken/nablas/21.html',
  73.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  74.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  75.     'http://www.perlmonks.org/?node_id=1018717',
  76.     'http://perldoc.perl.org/functions/localtime.html',
  77.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  78.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Headers.pm',
  79.     'http://search.cpan.org/~ether/HTTP-Message-6.11/lib/HTTP/Response.pm',
  80.     'http://search.cpan.org/dist/HTTP-Async/lib/HTTP/Async.pm',
  81.     'http://dklab.ru/chicken/nablas/21.html',
  82. );
  83.  
  84.  
  85. sub fetch {
  86.     my $url = shift;
  87.     my $furl = Furl::HTTP->new(agent => 'Furl/0.31',timeout => 3);
  88.     print "start $url\n";
  89.     my ($ver, $code, $msg, $headers, $body) = $furl->get($url);
  90.     my $size = length $body;
  91.     print "finished $url, $size bytes\n";
  92.     return ($code, $msg, $headers, $body);
  93. }
  94.  
  95. my %resps;
  96.  
  97. my $timer = Devel::Timer->new();
  98. $timer->mark('foreach');
  99. $AnyEvent::Util::MAX_FORKS = 20;
  100. my $cv = AE::cv;
  101. foreach my $url (@urls) {
  102.     $timer->mark('next foreach');
  103.     $cv->begin;
  104.     fork_call {
  105.         print "getting $url... ";
  106.         my ($code, $msg, $headers, $body)=fetch($url);
  107.         print "[$code]\n";
  108.         return ($url, $code, $msg, $headers, $body);
  109.         }
  110.         sub {
  111.             print "adding 2 %resps\n";
  112.             my ($url, $code, $msg, $headers, $body)=@_;
  113.             $resps{$url}->{'code'}=$code;
  114.             $resps{$url}->{'msg'}=$msg;
  115.             $resps{$url}->{'headers'}=$headers;
  116.             $resps{$url}->{'body'}=$body;
  117.             $cv->end;
  118.         };
  119. }
  120. $cv->recv;
  121. $timer->mark('end');
  122.  
  123. print "\nall data is ready, press <ENTER>:";
  124. <STDIN>;
  125. print Dumper(%resps);
  126. print "\n<PRESS ENTER>to print timer report\n";
  127. <STDIN>;
  128. $timer->report();
  129. sleep(3);
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement