Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/perl
- use strict;
- use LWP::UserAgent;
- use HTTP::Response;
- use threads;
- use threads::shared;
- use File::Basename;
- use Digest::SHA1 qw(sha1_hex);
- use URI::Escape;
- use Cwd;
- my $stop = 0;
- $SIG{'INT'} = 'SIGINT_handler';
- $| = 1; #flush stdout immediately
- my $user;
- my $pass;
- my $directory :shared;
- $directory = 'images';
- my @tags;
- my $tags;
- my $pool;
- my $url_tags = 'http://danbooru.donmai.us/post/index.xml';
- my $url_pool = 'http://danbooru.donmai.us/pool/show.xml';
- my $limit = 1000;
- my $threads = 5;
- $_ = join ' ', @ARGV;
- if (/--help|-h/i)
- {
- show_help();
- exit;
- }
- if (s/-(-user|u) (\S+)//)
- {
- $user = $2;
- }
- if (s/-(-pass|p) (\S+)//)
- {
- $pass = $2;
- }
- if (s/-(-limit|l) (\d+)//)
- {
- $limit = $2;
- }
- if (s/-(-threads|t) (\d+)//)
- {
- $threads = $2;
- }
- if (s/-(-pool|p) (\d+)//)
- {
- $pool = $2;
- }
- if (s/-(-dir|d) (\S+)//)
- {
- $directory = $2;
- }
- @tags = split / /, $_;
- map { s/[^A-Za-z0-9_]//g; chomp; } @tags;
- $tags = join '+', @tags;
- my @tags;
- if (($tags eq '' and $pool eq '') or ($user eq '' or $pass eq ''))
- {
- show_help();
- exit;
- }
- my $url;
- if ($pool eq '') { $url = $url_tags; }
- else { $url = $url_pool; }
- my $hashed_pass = sha1_hex("choujin-steiner--$pass--");
- $hashed_pass = uri_escape($hashed_pass);
- $user = uri_escape($user);
- $url .= "?login=$user&password_hash=$hashed_pass";
- $directory =~ s/\/|\\$//;
- if (!-d $directory)
- {
- mkdir $directory;
- }
- chdir $directory;
- fetch_site(1) =~ /<posts count="(\d+)"/;
- if ($1)
- {
- print "Total count match: $1, per page: $limit\n";
- }
- else
- {
- print "ERROR: can't get total count match\n";
- exit;
- }
- my @files :shared;
- my $max = int $1/$limit + 1;
- for (1..$max)
- {
- if ($stop)
- {
- print "Interruped by SIGINT\n";
- exit(0);
- }
- my $data = fetch_site($_);
- while ($data =~ /file_url="([^"]+)"/g)
- {
- push @files, $1;
- }
- }
- my @thr;
- my $file;
- if ($#files+1 < $threads) { $threads = $#files+1; };
- print "THREADS: $threads, FILES ".($#files+1)."\n";
- for (1..$threads)
- {
- if ($file = shift @files)
- {
- $thr[$_] = threads->create(\&save_file, $file);
- }
- }
- while (sleep 1)
- {
- for (1..$threads)
- {
- if ($thr[$_]->is_joinable)
- {
- $thr[$_]->join;
- if ($file = shift @files)
- {
- $thr[$_] = threads->create(\&save_file, $file);
- }
- }
- }
- last if ($#files == -1 && threads->list == 0);
- }
- sub fetch_site
- {
- my $page = shift;
- my $ua = LWP::UserAgent->new;
- print "Getting [$url] page $page... ";
- my $local_url = $url;
- if ($pool eq '') { $local_url .= "&tags=$tags&page=$page&limit=$limit"; }
- else { $local_url .= "&id=$pool&page=$page&limit=$limit"; }
- my $responce = $ua->get($local_url);
- if ($responce->is_success)
- {
- print "OK\n";
- return $responce->content;
- }
- else
- {
- print 'ERROR ' . $responce->code . ' ' . $responce->message . "\n";
- return undef;
- }
- }
- sub save_file
- {
- my $name = shift;
- my $ua = LWP::UserAgent->new;
- my $filename;
- #print cwd()."\n";
- $name =~ /([^\/\\]+?)$/;
- $filename = $1;
- print ''.($#files+1)." files left, saving $filename\n";
- $ua->get($name, ':content_file' => $filename);
- threads->exit;
- }
- sub show_help
- {
- print "Danbooru download script.
- Usage: ".basename($0)." -u <username> -p <pass> <options> {--pool <pool_name>|space_separated_tag_list}
- Options:
- --limit(-l) files PER PAGE (defaut $limit)
- --threads(-t) number of threads (default $threads)
- --dir(-d) dir to save images (default `$directory')";
- }
- sub SIGINT_handler
- {
- @files = ();
- $stop = 1;
- }
Add Comment
Please, Sign In to add comment