Advertisement
Guest User

Fetch

a guest
Aug 14th, 2013
199
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/usr/bin/perl
  2. use strict;
  3. use Data::Dumper;
  4. require LWP::UserAgent;
  5. use File::Slurp;
  6.  
  7. # Constants.
  8. my $URL_BASE = 'http://www.bis.doc.gov';
  9. my $DB_FILE = 'lastfetch.db';
  10.  
  11. # Read existing data of last updated files.
  12. my %db;
  13. if (-f $DB_FILE) {
  14.   my $file = read_file($DB_FILE);
  15.   my $t;
  16.   {
  17.     no strict;
  18.     $t = eval($file);
  19.   }
  20.   if ($t) {
  21.     %db = %{$t};
  22.   }
  23. }
  24.  
  25. #exit;
  26.  
  27. # Find new list of all files.
  28. my $ua = LWP::UserAgent->new;
  29. my $index = $ua->get("$URL_BASE/index.php/regulations/export-administration-regulations-ear")->decoded_content;
  30. my %current;
  31. while ($index =~ m~<td\s+valign="top">\s+
  32. <p><a\s+class="doclink"\s+href="([^"]+)">.*?</a></p>\s+
  33. </td>\s+
  34. <td\s+valign="top">\s+
  35. <p>(.*?)</p>\s+
  36. </td>~xg){
  37.  $current{$URL_BASE . $1} = $2;
  38. }
  39.  
  40. # Figure out which ones to update.
  41. my @to_update;
  42. foreach my $key(keys %current) {
  43.  if ($db{$key} ne $current{$key}) {
  44.    push @to_update, $key;
  45.  }
  46. }
  47.  
  48. # Update them.
  49. for my $file(@to_update) {
  50.  my $response = $ua->get($file);
  51.  $response->header("content-disposition") =~ /filename="(.*?)"/;
  52.  my $filename = $1;
  53.  if ($filename) {
  54.    open F, ">", $filename;
  55.    binmode F;
  56.    print F $response->content;
  57.    close F;
  58.    $db{$file} = $current{$file};
  59.    print "Saved $filename\n";
  60.  }
  61. }
  62.  
  63. # Save new database of last updated files.
  64. open DB, ">", $DB_FILE;
  65. print DB Dumper(\%db);
  66. close DB;
Advertisement
Advertisement
Advertisement
RAW Paste Data Copied
Advertisement