SHOW:
|
|
- or go back to the newest paste.
| 1 | #!/usr/bin/perl -w | |
| 2 | ||
| 3 | use strict; | |
| 4 | ||
| 5 | use Parse::MediaWikiDump; | |
| 6 | use Text::MediawikiFormat as => 'wiki2html'; | |
| 7 | use Time::Piece; | |
| 8 | use Unicode::Normalize; | |
| 9 | ||
| 10 | no warnings 'utf8'; | |
| 11 | ||
| 12 | # Get a list of musicals | |
| 13 | - | my @musicals_tmp = category_members('Broadway musicals',500);
|
| 13 | + | my @musicals_tmp = category_members('Broadway musicals');
|
| 14 | ||
| 15 | # Remove non-alphas from musicals | |
| 16 | my %musicals; | |
| 17 | my %hash_musicals; | |
| 18 | foreach my $musical (@musicals_tmp) | |
| 19 | {
| |
| 20 | #print "$musical\n"; | |
| 21 | $musical =~ s/[^A-Za-z ]//g; | |
| 22 | $musicals{lc $musical} = 1;
| |
| 23 | $hash_musicals{word2num($musical)} = lc $musical;
| |
| 24 | } | |
| 25 | #die; | |
| 26 | ||
| 27 | # Read in famous names | |
| 28 | my @names; | |
| 29 | my %last_to_first; | |
| 30 | open FILE, 'FamousNames.txt' or die $!; | |
| 31 | while (<FILE>) | |
| 32 | {
| |
| 33 | chomp; | |
| 34 | my $name = lc $_; | |
| 35 | my $score = 100; | |
| 36 | if ($name =~ /^(.*)\t(.*)$/) {$name = $1;$score = $2;}
| |
| 37 | - | #next unless $score >= 90; |
| 37 | + | next unless $score >= 90; |
| 38 | my $firstname; my $lastname; | |
| 39 | if ($name =~ /^([^ ]+) ([^ ]+)$/) {$firstname = $1; $lastname = $2;}
| |
| 40 | next unless $firstname; | |
| 41 | push(@names,"$firstname $lastname"); | |
| 42 | push(@{$last_to_first{$lastname}},$firstname);
| |
| 43 | } | |
| 44 | close FILE; | |
| 45 | ||
| 46 | # Go through the names and pull out anyone with the same first name as a musical | |
| 47 | my @musical_names; | |
| 48 | foreach my $n (@names) | |
| 49 | {
| |
| 50 | my ($fn, $ln) = split(/ /,$n); | |
| 51 | push(@musical_names,$n) if $musicals{$fn};
| |
| 52 | } | |
| 53 | ||
| 54 | foreach my $fullname (@musical_names) | |
| 55 | {
| |
| 56 | my ($fn, $ln) = split(/ /,$fullname); | |
| 57 | #print " $fn\n"; | |
| 58 | # Find all others with the same last name | |
| 59 | my $last_people_ref = $last_to_first{$ln};
| |
| 60 | # Check if any of these people have a first name that anagrams to a musical | |
| 61 | foreach my $new_fn (@$last_people_ref) | |
| 62 | {
| |
| 63 | next if $fn eq $new_fn; | |
| 64 | #print " $new_fn\n"; | |
| 65 | if ($hash_musicals{word2num($new_fn)} && $hash_musicals{word2num($new_fn)} ne $new_fn)
| |
| 66 | {
| |
| 67 | print "$fn and $new_fn $ln => " . $hash_musicals{word2num($new_fn)} . "\n";
| |
| 68 | } | |
| 69 | } | |
| 70 | } | |
| 71 | ||
| 72 | ||
| 73 | ###### | |
| 74 | # SUBS | |
| 75 | ###### | |
| 76 | ||
| 77 | sub category_members | |
| 78 | {
| |
| 79 | - | my ($category,$NUM) = @_; |
| 79 | + | use JSON; |
| 80 | - | $NUM = 500 unless $NUM; |
| 80 | + | use LWP::Simple; |
| 81 | - | my %wiki; |
| 81 | + | my ($category) = @_; |
| 82 | my @category_members; | |
| 83 | - | # Rather than parsing this every time, it is MUCH better to store and retrieve. |
| 83 | + | $category =~ s/ /_/g; |
| 84 | - | my $text_file = $category; |
| 84 | + | |
| 85 | - | $text_file =~ s/[^A-Za-z _]//g; |
| 85 | + | my $cmcontinue = ""; |
| 86 | - | $text_file =~ s/ /_/g; |
| 86 | + | do |
| 87 | - | $text_file .= ".txt"; |
| 87 | + | |
| 88 | my $url = "http://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:$category&cmlimit=500&format=json&cmnamespace=0&cmtype=page&cmcontinue=$cmcontinue"; | |
| 89 | - | if (-e $text_file) |
| 89 | + | my $json = get($url); |
| 90 | my $href = decode_json($json); | |
| 91 | - | my @ret; |
| 91 | + | my @arr = @{$href->{'query'}->{'categorymembers'}};
|
| 92 | - | open MYFILE, $text_file or die $!; |
| 92 | + | foreach my $a (@arr) |
| 93 | - | while (<MYFILE>) |
| 93 | + | |
| 94 | my $title = $a->{'title'};
| |
| 95 | - | chomp; |
| 95 | + | |
| 96 | - | push(@ret,$_); |
| 96 | + | |
| 97 | push(@category_members,$title); | |
| 98 | - | return @ret; |
| 98 | + | |
| 99 | ||
| 100 | - | else |
| 100 | + | # Decide whether to continue |
| 101 | if ($href->{'query-continue'}->{'categorymembers'}->{'cmcontinue'})
| |
| 102 | - | # This takes a LONG TIME (about an hour or more) |
| 102 | + | |
| 103 | - | my $xmlfile = "enwiki-latest-pages-articles.xml"; |
| 103 | + | $cmcontinue = $href->{'query-continue'}->{'categorymembers'}->{'cmcontinue'};
|
| 104 | - | my $pages = Parse::MediaWikiDump::Pages->new($xmlfile); |
| 104 | + | sleep(1); # Be nice to Wikipedia |
| 105 | } | |
| 106 | - | my @category_members; |
| 106 | + | else {$cmcontinue = "";}
|
| 107 | ||
| 108 | - | # Go through the XML and pull out interesting entries. |
| 108 | + | } while $cmcontinue; |
| 109 | - | my $ctr = 0; |
| 109 | + | |
| 110 | - | while(defined(my $page = $pages->next) && $ctr < $NUM) |
| 110 | + | return @category_members; |
| 111 | } | |
| 112 | - | #main namespace only |
| 112 | + | |
| 113 | - | next unless $page->namespace eq ''; |
| 113 | + | |
| 114 | - | |
| 114 | + | |
| 115 | - | # Exclude redirects |
| 115 | + | |
| 116 | - | next if defined($page->redirect); |
| 116 | + | |
| 117 | - | |
| 117 | + | |
| 118 | - | # Categories |
| 118 | + | |
| 119 | - | my $c = $page->categories; |
| 119 | + | |
| 120 | - | |
| 120 | + | |
| 121 | - | # Only take musicals |
| 121 | + | |
| 122 | - | next unless (grep {/$category/} @$c);
|
| 122 | + | |
| 123 | - | |
| 123 | + | |
| 124 | - | # If we've gotten this far we can proceed |
| 124 | + | |
| 125 | - | my $title = $page->title; |
| 125 | + | |
| 126 | # Remove any non-alphas | |
| 127 | $w =~ s/[^A-Z]//g; | |
| 128 | - | |
| 128 | + | |
| 129 | my $val = 1; | |
| 130 | - | $ctr++; |
| 130 | + | |
| 131 | {
| |
| 132 | next unless $convert{$l};
| |
| 133 | - | # Add this to a text file to pull up later |
| 133 | + | |
| 134 | - | open OUTFILE, ">$text_file" or die $!; |
| 134 | + | |
| 135 | - | foreach my $c (@category_members) {print OUTFILE "$c\n";}
|
| 135 | + | |
| 136 | - | close OUTFILE; |
| 136 | + |
