urban_collect.pl

use strict;
use warnings;

use URI::Escape;
use JSON;
use File::Path qw(make_path);
use File::Copy qw(copy);

use IO::Handle qw();
STDERR->autoflush(1);

sub json_unescape($) {
    my $escaped = shift;
    my $unescaped;
    unless (eval { $unescaped = from_json("{\"term\":\"${escaped}\"}"); }) {
        die "Can't unescape JSON [$escaped].";
    } else { return $unescaped->{term}; }
}

sub prepare_search_term($) {
    my $term = shift;
    $term =~ s/^\s+//g;
    $term =~ s/\s+$//g;
    return lc($term);
}

my $time_start = time;

open my $fall, ">urban_all.txt" or die "Can't open urban_all.txt for writing.";
binmode $fall;

my $overall_dups = 0;
my $overall_unique = 0;

for my $char ('_', 'A'..'Z') {
    print STDERR "[$char]  ";
    my %entries;

    my $total = 0;

    open my $fdb, "<data/words/$char.txt" or die "Can't open [data/words/$char.txt] for reading.";
    binmode $fdb;
    while (<$fdb>) {
        next if /^\s*$/;
        my $line = $_;
        chomp $line;
        die "Can't parse line [$line]." unless ($line =~ /^{"query_term":"(.+?)","query_time":/);

        my $qt = prepare_search_term(json_unescape($1));

        $entries{$qt} = $line;
        ++$total;
    }
    close $fdb;

    my $unique = scalar keys %entries;
    my $dups = $total - $unique;

    $overall_dups += $dups;
    $overall_unique += $unique;

    printf STDERR ("Unique: %6d  Dups: %6d\n", $unique, $dups);

    for my $qt (sort keys %entries) {
        print $fall "$entries{$qt}\n";
    }
}

close $fall;

print STDERR "\n\nOverall Unique: $overall_unique, Overall Dups: $overall_dups\n";

my $time_end = time;

print STDERR "Total script running time: ".($time_end - $time_start)." sec.\n";