Fetch

#!/usr/bin/perl
use strict;
use Data::Dumper;
require LWP::UserAgent;
use File::Slurp;

# Constants.
my $URL_BASE = 'http://www.bis.doc.gov';
my $DB_FILE = 'lastfetch.db';

# Read existing data of last updated files.
my %db;
if (-f $DB_FILE) {
  my $file = read_file($DB_FILE);
  my $t;
  {
    no strict;
    $t = eval($file);
  }
  if ($t) {
    %db = %{$t};
  }
}

#exit;

# Find new list of all files.
my $ua = LWP::UserAgent->new;
my $index = $ua->get("$URL_BASE/index.php/regulations/export-administration-regulations-ear")->decoded_content;
my %current;
while ($index =~ m~<td\s+valign="top">\s+
<p><a\s+class="doclink"\s+href="([^"]+)">.*?</a></p>\s+
</td>\s+
<td\s+valign="top">\s+
<p>(.*?)</p>\s+
</td>~xg){
  $current{$URL_BASE . $1} = $2;
}

# Figure out which ones to update.
my @to_update;
foreach my $key(keys %current) {
  if ($db{$key} ne $current{$key}) {
    push @to_update, $key;
  }
}

# Update them.
for my $file(@to_update) {
  my $response = $ua->get($file);
  $response->header("content-disposition") =~ /filename="(.*?)"/;
  my $filename = $1;
  if ($filename) {
    open F, ">", $filename;
    binmode F;
    print F $response->content;
    close F;
    $db{$file} = $current{$file};
    print "Saved $filename\n";
  }
}

# Save new database of last updated files.
open DB, ">", $DB_FILE;
print DB Dumper(\%db);
close DB;