8th wowhead item loot scanner - 2015-08-11

#!/usr/bin/perl
# adjust path to fit your host

# Seventh generation item loot scanner
# Designed for use with Trinity DB 4.3.4
# Other DB formats should be easily implemented

# Output sorted in ascending order (lowest value first)
# Output is NOT sent to a file. Redirect output as desired.

# THE OUTPUT MUST BE MANUALLY EDITED BEFORE IMPORTING TO THE DATABASE!
# THE OUTPUT MUST BE MANUALLY EDITED BEFORE IMPORTING TO THE DATABASE!
# THE OUTPUT MUST BE MANUALLY EDITED BEFORE IMPORTING TO THE DATABASE!
# THE OUTPUT MUST BE MANUALLY EDITED BEFORE IMPORTING TO THE DATABASE!

# Use this only as a starting point for construction of loot templates
# `Reference`, `Chance`, `QuestRequired` and `GroupId` CANNOT be correctly set from wowhead data

# usage:
#        item.pl                   (takes input from STDIN, aka piped stream)
#        item.pl item#             (reads the specified item from www.wowhead.com)
#        item.pl startitem enditem (reads the given range of items from www.wowhead.com)

# Generation history:
#    First: based on my NPC script, but for the current (Nov 21, 2013) wowhead stream
#   Second: back-ported the line merging code update that was added while adjusting code for gameobjects
#    Third: changed to an updated "sourcemore" (sub-group in the wowhead item entry) filter
#    Forth: changed filter from "sourcemore" to "sourcemore"|"modelviewer" to strip both sub-groups
#           replaced split function with a regex method for splitting the items up
#    Fifth: altered comment header format
#    Sixth: fixed line merging code to properly filter Windows line termination (\r\n)
#  Seventh: updated for latest (last?) Trinity DB 4.3.4 database (Jan 14, 2015)
#   Eighth: updated for latest (for August 11 of 2015) wowhead page structure

use warnings;
use strict;
use LWP::Simple;

my $currentItem;
my $page;
my $line;
my @lines;
my $ItemName;
my $ItemID;

# item level then id sort order
sub itemcmp {
  $a =~ /^.*\,\"level\":(\d+)\,/;
  my $x = $1;
  $a =~ /^.*\"id\":(.+?)\,/;
  my $xx = $1;
  $b =~ /^.*\,\"level\":(\d+)\,/;
  my $y = $1;
  $b =~ /^.*\"id\":(.+?)\,/;
  my $yy = $1;
  if ($x eq $y) {
    return ($xx <=> $yy);
  } else {
    return ($x <=> $y);
  }
}

# build MySQL query
sub lootlist {
  my ($entry, $opened, $items) = @_;
  my $slot = 1;
  $items =~ s/\,\"sourcemore\":\[.+?\]\,|\,\"modelviewer\":\{.+?\}\,/\,/g;
  $items =~ s/\,pctstack:\'\{.+?\}\'//g;
  $items =~ s/\,context:.+?\,/\,/g;
  my @itemlist = ($items =~ m/\"classs\".+?\}/g);
  @itemlist = sort itemcmp @itemlist;
  print "-- wowhead submissions: $opened\n";
  print "DELETE FROM `item_loot_template` WHERE `Entry`=$entry;\n";
  print "INSERT INTO `item_loot_template` (`Entry`, `Item`, `Reference`, `Chance`, `QuestRequired`, `LootMode`, `GroupId`, `MinCount`, `MaxCount`, `Comment`) VALUES\n";
  foreach my $itemdata (@itemlist) {
    $itemdata =~ /^.*\"id\":(.+?)\,\"level\":(.+?)\,\"name\":\"(.+?)\".*\,count:(.+?)\,stack:\[(.+?)\,(.+?)\]/;
    my $id = $1;
    my $ilvl = $2;
    my $name = $3;
    my $count = $4;
    my $minval = $5;
    my $maxval = $6;
    my $pct = sprintf("%.2f", (($count / $opened) * 100));
    if ($name =~ /^[+-]?\d/) {
      $name =~ s/^.//;
    }
    print "\t($entry, $id, 0, $pct, 0, 1, 0, $minval, $maxval, '$name (ilvl $ilvl)')", ($slot < scalar(@itemlist)) ? "," : ";";
    print " -- $count time", ($count > 1) ? "s" : " ", "\n";
    $slot++;
  }
  print "\n";
}

# print name & ID as a comment header
sub header {
  my $dashes;
  foreach $line (@lines) {
    if (defined $line && length $line > 0) {
      chop($line);
      if ($line =~ /\$\.extend\(g_items/) {
        $line =~ /\$\.extend\(g_items\[(.+?)\].*\,\"name\":\"(.+?)\"/;
        $ItemID = $1;
        $ItemName = $2;
        if ($ItemName =~ /^[+-]?\d/) {
          $ItemName =~ s/^.//;
        }
        $dashes = length($ItemID) + length($ItemName) + 3;
        print "-- ---" . "-" x $dashes . "---\n";
        print "-- -- $ItemName \($ItemID\) --\n";
        print "-- ---" . "-" x $dashes . "---\n\n";
        print "-- EDIT ME - EDIT ME - EDIT ME - EDIT ME - EDIT ME - EDIT ME - EDIT ME - EDIT ME - EDIT ME\n";
        print "-- `Chance` and `GroupId` columns MUST be adjusted before query is executed\n";
        print "-- `Reference` and `QuestRequired` columns MAY need to be altered, too\n\n";
      }
    }
  }
}

# scan the page, obviously
sub parsepage {
  my $data;
  my $totalcount;
  my $linemerge = 0;
  foreach $line (@lines) {
    if (defined $line && length $line > 0) {
      $line =~ s/\s+$//;
    }
    if (defined $line && length $line > 0) {
      if ($linemerge == 0 && $line =~ /^\s+/) {
        $line =~ s/^\s+//;
        if ($line =~ /^new.Listview.*id:.\'contains\'/) {
          $linemerge = 4;
          $data = "";
        }
      }
      if ($linemerge > 0) {
        $line =~ s/^\s+//;
        $data .= "$line ";
        $linemerge--;
        if ($linemerge == 0) {
          $line = $data;
        }
      }
      if ($linemerge == 0 && $line =~ /^new.Listview.*id:.\'contains\'/) {
         $line =~ /^.*_totalCount:.(.+?)\,/;
         $totalcount = $1;
         $line =~ /^new.Listview.*id:.\'contains\'.*data: \[(.+?)\]\}\)/;
         $data = $1;
         header();
         lootlist($ItemID, $totalcount, $data);
      }
    }
  }
}

# generate a URL and get the HTML source
sub getpage {
  my ($Item) = @_;
  my $URL = "http://www.wowhead.com/item=$Item";
  $page = get($URL);
  $page =~ s/(\r\n)|(\r)/\n/g;
}

if ($#ARGV == -1) {
  while ($_ = <>) {
    push (@lines, $_);
  }
  parsepage();
  exit;
} elsif ($#ARGV == 0) {
  $currentItem = $ARGV[0];
  getpage($currentItem);
  @lines = split(/(\r)|(\n)|(\r\n)/, $page);
  parsepage();
  exit;
} elsif ($#ARGV == 1) {
  for ($currentItem = $ARGV[0]; $currentItem <= $ARGV[1]; $currentItem++) {
    getpage($currentItem);
    @lines = split(/(\r)|(\n)|(\r\n)/, $page);
    parsepage();
    if ($currentItem < $ARGV[1]) { sleep(5); }
  }
}