API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Jun 3rd, 2017
68
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Diff 7.70 KB | None | 0 0
raw download clone embed print report
  1. diff --git a/kadr.pl b/kadr.pl
  2. index e419af5..d0ec76b 100644
  3. --- a/kadr.pl
  4. +++ b/kadr.pl
  5. @@ -17,6 +17,10 @@
  6.  
  7.  use v5.10;
  8.  use common::sense;
  9. +use AnyEvent;
  10. +use Coro;
  11. +use Coro::Channel;
  12. +use lib qw(Parse-TitleSyntax/lib Term-StatusLine/lib);
  13.  use Config::YAML;
  14.  use Data::Types qw(:int);
  15.  use DBI::SpeedySimple;
  16. @@ -88,6 +92,8 @@ EOF
  17.  $conf->read("$appdir/config");
  18.  $conf->write;
  19.  
  20. +our $offline_mode = @ARGV ~~ /--offline-mode/;
  21. +
  22.  my $parsets = Parse::TitleSyntax->new($conf->{file_naming_scheme});
  23.  $parsets->AddFunctions(Parse::TitleSyntax::Functions::Regexp->new($parsets));
  24.  
  25. @@ -134,17 +140,85 @@ my $a = AniDB::UDPClient->new({
  26.     port => 3700,
  27.  });
  28.  
  29. -my @files = sort(find_files(@{$conf->{dirs}->{to_scan}}));
  30. +my $totalfiles;
  31. +my $sl = Term::StatusLine::XofX->new(total_item_count => sub { $totalfiles; });
  32. +my %channel;
  33. +my @files;
  34. +for (('files','hashing','hashed','done')) {
  35. +   $channel{$_} = new Coro::Channel;
  36. +}
  37. +
  38. +async { # Finds files and queues them for hashing
  39. +   my(@paths) = @{$conf->{dirs}->{to_scan}};
  40. +   my @dirs;
  41. +   for(@paths) {
  42. +       if(!is_dir($_)) {
  43. +           if(is_file($_)) {
  44. +               $totalfiles++;
  45. +               @files = sort(@files, $_);
  46. +           }
  47. +           else {
  48. +               say "Warning: Not a directory: $_";
  49. +           }
  50. +       }
  51. +       else {
  52. +           push @dirs, $_;
  53. +       }
  54. +   }
  55. +   cede;
  56. +   for my $dir (@dirs) {
  57. +       opendir my $dh, $dir;
  58. +       for (readdir $dh) {
  59. +           if(!($_ eq '.' or $_ eq '..')) {
  60. +               $_ = "$dir/$_";
  61. +               if(is_dir($_)) {
  62. +                   push @dirs, $_;
  63. +               }
  64. +               else {
  65. +                   $totalfiles++;
  66. +                   @files = sort(@files, decode_utf8($_));
  67. +               }
  68. +           }
  69. +       }
  70. +       close($dh);
  71. +       cede;
  72. +   }
  73. +   push @files, undef;
  74. +};
  75. +async { # queues the progressively sorted files for hashing
  76. +   while(@files == 0 || defined $files[0]) {
  77. +       $channel{files}->put(shift @files) if defined $files[0];
  78. +       cede;
  79. +   }
  80. +   $channel{files}->shutdown;
  81. +};
  82. +
  83.  my @ed2k_of_processed_files;
  84. -my $sl = Term::StatusLine::XofX->new(total_item_count => scalar(@files));
  85. -for(@files) {
  86. -   $sl->update('++', $_);
  87. -   next if !valid_file($_);
  88. -   if(my $ed2k = process_file($_)) {
  89. -       push @ed2k_of_processed_files, $ed2k;
  90. +async { # requests hashing and process the file as soon as done
  91. +   while(($_=$channel{files}->get)) {
  92. +       $sl->update('++', $_);
  93. +       next unless valid_file($_);
  94. +       $channel{hashing}->put($_);
  95. +       async_pool {
  96. +           if(($_=$channel{hashed}->get)) {
  97. +               push @ed2k_of_processed_files, $_->[0] if process_file(@$_);
  98. +           } else {
  99. +               $channel{done}->shutdown;
  100. +           }
  101. +       };
  102.     }
  103. -}
  104. -$sl->finalize;
  105. +    $channel{hashing}->shutdown;
  106. +   $sl->finalize;
  107. +};
  108. +
  109. +async {
  110. +   while(($_=$channel{hashing}->get)) {
  111. +       $channel{hashed}->put([ed2k_hash($_),$_]);
  112. +   }
  113. +   $channel{hashed}->shutdown;
  114. +};
  115. +
  116. +$channel{done}->get;
  117.  
  118.  if($conf->{anidb}->{update_records_for_deleted_files}) {
  119.     my @missing_files = @{$db->{dbh}->selectall_arrayref('SELECT ed2k, size, filename FROM known_files WHERE ed2k NOT IN (' . join(',', map { "'$_'" } @ed2k_of_processed_files) . ') ORDER BY filename')};
  120. @@ -180,48 +254,10 @@ sub valid_file {
  121.     return 1;
  122.  }
  123.  
  124. -sub find_files {
  125. -   my(@paths) = @_;
  126. -   my(@dirs, @files);
  127. -   for(@paths) {
  128. -       if(!is_dir($_)) {
  129. -           if(is_file($_)) {
  130. -               push @files, $_;
  131. -           }
  132. -           else {
  133. -               say "Warning: Not a directory: $_";
  134. -           }
  135. -       }
  136. -       else {
  137. -           push @dirs, $_;
  138. -       }
  139. -   }
  140. -   my $sl = Term::StatusLine::XofX->new(label => 'Scanning Directory', total_item_count => sub { scalar(@dirs) });
  141. -   for my $dir (@dirs) {
  142. -       $sl->update('++', $dir);
  143. -       opendir(my $dh, $dir);
  144. -       for(readdir($dh)) {
  145. -           if(!($_ eq '.' or $_ eq '..')) {
  146. -               $_ = "$dir/$_";
  147. -               if(is_dir($_)) {
  148. -                   push @dirs, $_;
  149. -               }
  150. -               else {
  151. -                   push @files, decode_utf8($_);
  152. -               }
  153. -           }
  154. -       }
  155. -       close($dh);
  156. -   }
  157. -   $sl->finalize;
  158. -   return @files;
  159. -}
  160. -
  161.  sub process_file {
  162. -   my $file = shift;
  163. -   my $ed2k = ed2k_hash($file);
  164. +   my ($ed2k, $file) = @_;
  165.     my $fileinfo = $a->file_query({ed2k => $ed2k, size => -s $file});
  166. -   my $proc_sl = Term::StatusLine::Freeform->new(parent => $sl);
  167. +   my $proc_sl = Term::StatusLine::Freeform->new();
  168.  
  169.     if(!defined $fileinfo) {
  170.         $proc_sl->finalize_and_log('Ignored');
  171. @@ -273,6 +309,9 @@ sub process_file {
  172.     }
  173.  
  174.     for(keys %$fileinfo) {
  175. +       delete $fileinfo->{$_} && next if $fileinfo->{$_} eq '';
  176. +       $fileinfo->{$_} =~ s/H264\/AVC/H.264/g;
  177. +       $fileinfo->{$_} =~ s/Vorbis \(Ogg Vorbis\)/Vorbis/g;
  178.         $fileinfo->{$_} =~ s/\//∕/g;
  179.         $fileinfo->{$_} =~ s/\`/'/g;
  180.     }
  181. @@ -345,6 +384,7 @@ sub avdump {
  182.         [qr/H\s+(\d+).(\d{2})/, sub {
  183.             my @m = @{shift->matchlist};
  184.             $avsl->update(int(int($m[0]) + int($m[1]) / 100));
  185. +           cede;
  186.             exp_continue;
  187.         }],
  188.         [qr/P\s+(\d+).(\d{2})/, sub {
  189. @@ -352,6 +392,7 @@ sub avdump {
  190.             if($avsl->label eq 'AvHashing') {
  191.                 $avsl->label('AvParsing');
  192.             }
  193. +           cede;
  194.             $avsl->update(int(int($m[0]) + int($m[1]) / 100));
  195.             exp_continue;
  196.         }],
  197. @@ -399,15 +440,14 @@ sub ed2k_hash {
  198.     my $fh = file_open('<:mmap:raw', $file);
  199.     my $ed2k_sl;
  200.     if($conf->{show_hashing_progress}) {
  201. -       $ed2k_sl = Term::StatusLine::XofX->new(parent => $sl, label => 'Hashing', total_item_count => $size);
  202. -       while(my $bytes_read = read $fh, my $buffer, Digest::ED2K::CHUNK_SIZE) {
  203. -           $ctx->add($buffer);
  204. -           $ed2k_sl->update("+=$bytes_read");
  205. -       }
  206. -   }
  207. -   else {
  208. +       $ed2k_sl = Term::StatusLine::XofX->new(parent => $sl, label => 'Hashing', total_item_count => $size) ;
  209. +   } else {
  210.         $ed2k_sl = Term::StatusLine::Freeform->new(parent => $sl, value => 'Hashing');
  211. -       $ctx->addfile($fh);
  212. +   }
  213. +   while(my $bytes_read = read $fh, my $buffer, Digest::ED2K::CHUNK_SIZE) {
  214. +       $ctx->add($buffer);
  215. +       $ed2k_sl->update("+=$bytes_read") if($conf->{show_hashing_progress});
  216. +       cede;
  217.     }
  218.     close $fh;
  219.     my $ed2k = $ctx->hexdigest;
  220. @@ -530,6 +570,8 @@ use constant MYLIST_ANIME_ENUM => qw/anime_title episodes eps_with_state_unknown
  221.  sub new {
  222.     my($package, $opts) = @_;
  223.     my $self = bless $opts, $package;
  224. +   $self->{offline} = $main::offline_mode;
  225. +   return $self if ($self->{offline});
  226.     $self->{username} or die 'AniDB error: Need a username';
  227.     $self->{password} or die 'AniDB error: Need a password';
  228.     $self->{starttime} = time - 1;
  229. @@ -547,6 +589,7 @@ sub file_query {
  230.     if(my $r = $self->{db}->fetch("adbcache_file", ["*"], $query, 1)) {
  231.         return $r;
  232.     }
  233. +   return if ($self->{offline});
  234.  
  235.     my $file_sl = Term::StatusLine::Freeform->new(parent => $sl, value => 'Updating file information');
  236.  
  237. @@ -605,6 +648,7 @@ sub mylistedit {
  238.  
  239.  sub mylist_add_query {
  240.     my ($self, $params) = @_;
  241. +   return 0 if $self->{offline};
  242.     my $res;
  243.  
  244.     if ((!defined $params->{edit}) or $params->{edit} == 0) {
  245. @@ -660,6 +704,7 @@ sub mylist_file_by_ed2k_size {
  246.  
  247.  sub _mylist_file_query {
  248.     my($self, $query) = @_;
  249. +   return undef if $self->{offline};
  250.     my $mylist_sl = Term::StatusLine::Freeform->new(parent => $sl, value => 'Updating mylist information');
  251.     (my $msg = $self->_sendrecv("MYLIST", $query)) =~ s/.*\n//im;
  252.     $mylist_sl->finalize;
  253. @@ -683,6 +728,7 @@ sub mylist_anime_by_aid {
  254.  
  255.  sub _mylist_anime_query {
  256.     my($self, $query) = @_;
  257. +   return undef if $self->{offline};
  258.     my $mylist_anime_sl = Term::StatusLine::Freeform->new(parent => $sl, value => 'Updating mylist anime information');
  259.     my $msg = $self->_sendrecv("MYLIST", $query);
  260.     $mylist_anime_sl->finalize;
  261. @@ -726,6 +772,7 @@ sub _mylist_anime_query {
  262.  
  263.  sub login {
  264.     my($self) = @_;
  265. +   return undef if $self->{offline};
  266.     if(!defined $self->{skey} || (time - $self->{last_command}) > (35 * 60)) {
  267.         my $login_sl = Term::StatusLine::Freeform->new(parent => $sl, value => 'Logging in to AniDB');
  268.         my $msg = $self->_sendrecv("AUTH", {user => lc($self->{username}), pass => $self->{password}, protover => 3, client => CLIENT_NAME, clientver => CLIENT_VER, nat => 1, enc => "UTF8", comp => 1});
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!