Guest User

Danbooru downloader with pools

a guest
Feb 14th, 2012
445
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Perl 3.54 KB | None | 0 0
  1. #!/usr/bin/perl
  2.  
  3. use strict;
  4. use LWP::UserAgent;
  5. use HTTP::Response;
  6. use threads;
  7. use threads::shared;
  8. use File::Basename;
  9. use Digest::SHA1 qw(sha1_hex);
  10. use URI::Escape;
  11. use Cwd;
  12.  
  13. my $stop = 0;
  14. $SIG{'INT'} = 'SIGINT_handler';
  15.  
  16. $| = 1; #flush stdout immediately
  17.  
  18. my $user;
  19. my $pass;
  20. my $directory :shared;
  21. $directory = 'images';
  22. my @tags;
  23. my $tags;
  24. my $pool;
  25. my $url_tags = 'http://danbooru.donmai.us/post/index.xml';
  26. my $url_pool = 'http://danbooru.donmai.us/pool/show.xml';
  27. my $limit = 1000;
  28. my $threads = 5;
  29.  
  30. $_ = join ' ', @ARGV;
  31. if (/--help|-h/i)
  32. {
  33.     show_help();
  34.     exit;
  35. }
  36.  
  37. if (s/-(-user|u) (\S+)//)
  38. {
  39.     $user = $2;
  40. }
  41.  
  42. if (s/-(-pass|p) (\S+)//)
  43. {
  44.     $pass = $2;
  45. }
  46.  
  47. if (s/-(-limit|l) (\d+)//)
  48. {
  49.     $limit = $2;
  50. }
  51.  
  52. if (s/-(-threads|t) (\d+)//)
  53. {
  54.     $threads = $2;
  55. }
  56.  
  57. if (s/-(-pool|p) (\d+)//)
  58. {
  59.     $pool = $2;
  60. }
  61.  
  62. if (s/-(-dir|d) (\S+)//)
  63. {
  64.     $directory = $2;
  65. }
  66.  
  67. @tags = split / /, $_;
  68.  
  69. map { s/[^A-Za-z0-9_]//g; chomp; } @tags;
  70. $tags = join '+', @tags;
  71. my @tags;
  72.  
  73. if (($tags eq '' and $pool eq '') or ($user eq '' or $pass eq ''))
  74. {
  75.     show_help();
  76.     exit;
  77. }
  78.  
  79. my $url;
  80. if ($pool eq '') { $url = $url_tags; }
  81. else             { $url = $url_pool; }
  82.  
  83. my $hashed_pass = sha1_hex("choujin-steiner--$pass--");
  84. $hashed_pass = uri_escape($hashed_pass);
  85. $user = uri_escape($user);
  86. $url .= "?login=$user&password_hash=$hashed_pass";
  87.  
  88. $directory =~ s/\/|\\$//;
  89.  
  90. if (!-d $directory)
  91. {
  92.     mkdir $directory;
  93. }
  94.  
  95. chdir $directory;
  96.  
  97. fetch_site(1) =~ /<posts count="(\d+)"/;
  98. if ($1)
  99. {
  100.     print "Total count match: $1, per page: $limit\n";
  101. }
  102. else
  103. {
  104.     print "ERROR: can't get total count match\n";
  105.     exit;
  106. }
  107.  
  108. my @files :shared;
  109. my $max = int $1/$limit + 1;
  110.  
  111. for (1..$max)
  112. {
  113.     if ($stop)
  114.     {
  115.         print "Interruped by SIGINT\n";
  116.         exit(0);
  117.     }
  118.    
  119.     my $data = fetch_site($_);
  120.     while ($data =~ /file_url="([^"]+)"/g)
  121.     {
  122.         push @files, $1;
  123.     }
  124. }
  125.  
  126. my @thr;
  127. my $file;
  128. if ($#files+1 < $threads) { $threads = $#files+1; };
  129.  
  130. print "THREADS: $threads, FILES ".($#files+1)."\n";
  131.  
  132. for (1..$threads)
  133. {
  134.     if ($file = shift @files)
  135.     {
  136.         $thr[$_] = threads->create(\&save_file, $file);
  137.     }
  138. }
  139.  
  140. while (sleep 1)
  141. {
  142.     for (1..$threads)
  143.     {
  144.         if ($thr[$_]->is_joinable)
  145.         {
  146.             $thr[$_]->join;
  147.             if ($file = shift @files)
  148.             {
  149.                 $thr[$_] = threads->create(\&save_file, $file);
  150.             }
  151.         }
  152.     }
  153.     last if ($#files == -1 && threads->list == 0);
  154. }
  155.  
  156. sub fetch_site
  157. {
  158.     my $page = shift;
  159.     my $ua = LWP::UserAgent->new;  
  160.     print "Getting [$url] page $page... ";
  161.     my $local_url = $url;
  162.     if ($pool eq '') { $local_url .= "&tags=$tags&page=$page&limit=$limit"; }
  163.     else             { $local_url .= "&id=$pool&page=$page&limit=$limit"; }
  164.     my $responce = $ua->get($local_url);   
  165.     if ($responce->is_success)
  166.     {
  167.         print "OK\n";
  168.         return $responce->content;
  169.     }
  170.     else
  171.     {
  172.         print 'ERROR ' . $responce->code . ' ' . $responce->message . "\n";
  173.         return undef;
  174.     }
  175. }
  176.  
  177. sub save_file
  178. {
  179.     my $name = shift;
  180.     my $ua = LWP::UserAgent->new;
  181.     my $filename;
  182.     #print cwd()."\n";
  183.     $name =~ /([^\/\\]+?)$/;
  184.     $filename = $1;
  185.     print ''.($#files+1)." files left, saving $filename\n";
  186.     $ua->get($name, ':content_file' => $filename); 
  187.     threads->exit;
  188. }
  189.  
  190. sub show_help
  191. {
  192.     print "Danbooru download script.
  193. Usage: ".basename($0)." -u <username> -p <pass> <options> {--pool <pool_name>|space_separated_tag_list}
  194. Options:
  195.     --limit(-l)     files PER PAGE (defaut $limit)
  196.     --threads(-t)       number of threads (default $threads)
  197.     --dir(-d)       dir to save images (default `$directory')";
  198. }
  199.  
  200. sub SIGINT_handler
  201. {
  202.     @files = ();
  203.     $stop = 1;
  204. }
Add Comment
Please, Sign In to add comment