group_dump_by_symbols.pl

# A helper tool for hacking code pages of MT
# Outputs pretty-printed HTML grouping phrases examples by unknown symbols and highlighting those symbols
# Red symbols are unknown symbols of the current group, blue - of some other group

# All non-ru/en examples of use:
# perl group_dump_by_symbols.pl 24 dump/dutch.dump    >html/dutch.html
# perl group_dump_by_symbols.pl 26 dump/estonian.dump >html/estonian.html
# perl group_dump_by_symbols.pl  4 dump/french.dump   >html/french.html
# perl group_dump_by_symbols.pl  3 dump/german.dump   >html/german.html
# perl group_dump_by_symbols.pl 23 dump/italian.dump  >html/italian.html
# perl group_dump_by_symbols.pl 27 dump/latvian.dump  >html/latvian.html
# perl group_dump_by_symbols.pl  5 dump/spanish.dump  >html/spanish.html


use strict;
use warnings;
use URI::Escape;

our %desired_range = map { $_ => 1 } (0x80..0xFF); # Which range of bytes considered to be unknown set of symbols
our $max_num_examples = 20; # How many examples per group
our %examples;
our $lang_id = ($ARGV[0] + 0) or die "First argument should be language id.";

binmode $_ for *STDIN, *STDOUT;

$examples{$_} = [] for keys %desired_range;

open my $f, "<$ARGV[1]" or die "Can't open for reading [$ARGV[1]].";
binmode $f;

while (<$f>) {
    chomp;
    my $s = $_;
    my @bytes = unpack("C*", $s);

    for my $byte (@bytes) {
        push @{$examples{$byte}}, $s if $desired_range{$byte} and @{$examples{$byte}} < $max_num_examples;
    }
}

close $f;

# ISO-8859-1
print "<html><meta charset=\"windows-1251\"><style type=\"text/css\">a:link{text-decoration:none;}</style><body><font family=\"Courier\"><pre>\n";

for my $byte (sort keys %examples) {
    my $zbyte = sprintf("%02X", $byte);
    my $bytesym = chr($byte);
    print("<a name=\"sym_$zbyte\">$zbyte</a> [$bytesym]\n");

    for my $s (sort @{$examples{$byte}}) {
        my $orig_word = $s;

        for my $b (keys %desired_range) {
            my $bsym = chr($b);
            my $zb = sprintf("%02X", $b);
            my $color = ($b == $byte) ? 'red' : 'blue';
            $s =~ s/$bsym/<a href="#sym_$zb" title="$zb"><font color="$color">$bsym<\/font><\/a>/g;
        }

        print "\t<a href=\"http://multitran.ru/c/m.exe?CL=1&s=".uri_escape($orig_word)."&l1=$lang_id\" target=\"_blank\"><font color=forestgreen>MT</font></a> $s\n";
    }

    print "\n";
}

print "</pre></font></body></html>\n";