TibiaWiki Monster Scraper

<?php
    require("library.php");

    function xmlpp($xml, $html_output=false) {
        $xml_obj = new SimpleXMLElement($xml);
        $level = 4;
        $indent = 0; // current indentation level
        $pretty = array();

        // get an array containing each XML element
        $xml = explode("\n", preg_replace('/>\s*</', ">\n<", $xml_obj->asXML()));

        // shift off opening XML tag if present
        if (count($xml) && preg_match('/^<\?\s*xml/', $xml[0])) {
          $pretty[] = array_shift($xml);
        }

        foreach ($xml as $el) {
          if (preg_match('/^<([\w])+[^>\/]*>$/U', $el)) {
              // opening tag, increase indent
              $pretty[] = str_repeat(' ', $indent) . $el;
              $indent += $level;
          } else {
            if (preg_match('/^<\/.+>$/', $el)) {
              $indent -= $level;  // closing tag, decrease indent
            }
            if ($indent < 0) {
              $indent += $level;
            }
            $pretty[] = str_repeat(' ', $indent) . $el;
          }
        }
        $xml = implode("\n", $pretty);
        return ($html_output) ? htmlentities($xml) : $xml;
    }

    function getResult($results) {
        if($results) {
            foreach ($results as $item) {
                $result = trim($item->nodeValue);
                return $result;
            }
        }
        return false;
    }

    function writeLog($type, $msg) {
        file_put_contents("monsters/logs/".$type.".txt", $msg."\n", FILE_APPEND | LOCK_EX);
    }

    function loadMonster($name) {
        $path = "monsters/sample.xml";
        $new = true;
        if(file_exists("monsters/source/".$name.".xml")){
            $path = "monsters/source/".$name.".xml";
            $new = false;
        }
        $xml = simplexml_load_file($path);
        if ($xml) {
            return array($xml, $new);
        }
        return false;
    }

    function getMonsterLoot($link) {
        $dom = new DOMDocument();
        @$dom->loadHTMLFile($link);
        $lootitems = array();
        $i = 1;
        while ($i <= 60) {
            $xpath = new DOMXPath($dom);
            $base = "//*[@id='mw-content-text']/table[2]//tr[" . $i . "]//td//text()";
            $items = $xpath->query($base);
            $loot = array();
            foreach ($items as $item)
                $loot[] = $item->nodeValue;
            if(count($loot) > 0) {
                //handling count ranges
                $count_range = explode('-', $loot[0]);
                $count = ($count_range[1]) ? $count_range[1] : $loot[0];
                $itemname = strtolower(trim($loot[1]));
                $chance = (str_replace('%', '', $loot[4]) * 1000);
                if((strlen($itemname) > 1) && (strcasecmp($itemname, "empty") != 0) && is_numeric($count) && is_numeric($chance))
                    $lootitems[] = array('name'=>$itemname,'count'=>$count,'chance'=>$chance);
            }
            $i++;
        }
        return $lootitems;
    }

    function getMonsterDetails($name, $new) {
        $dom = new DOMDocument();
        @$dom->loadHTMLFile("http://tibia.wikia.com/wiki/" . str_ireplace(" ", "_", $name) . ((in_array($name, $duplicateCreatures)) ? '_(Creature)' : ''));
        $xpath = new DOMXPath($dom);
        $w_name = trim(str_ireplace("(creature)", "", getResult($xpath->query("//*[@id='WikiaPageHeader']//h1"))));
        if(strcasecmp($w_name, $name) == 0) { //correct page (hopefully)
            $w_pushable = getResult($xpath->query("//*[@id='mw-content-text']//tr[td/b[contains(text(),'Pushable')]]//td[2]//a//img/@alt"));
            $w_pushObjects = getResult($xpath->query("//*[@id='mw-content-text']//tr[td/b[contains(text(),'Push Objects')]]//td[2]//a//img/@alt"));
            $pushable = (strcasecmp($w_pushable, 'Tick.jpg') == 0) ? 1 : 0;
            $pushObjects = (strcasecmp($w_pushObjects, 'Cross.jpg') == 0) ? 0 : 1;

            //general stats
            $w_hp = trim(getResult($xpath->query("//*[@id='mw-content-text']/table//tr[2]/td[2]/text()[1]")));
            $w_exp = trim(getResult($xpath->query("//*[@id='mw-content-text']/table//tr[2]/td[2]/text()[2]")));
            $w_sumConString = trim(getResult($xpath->query("//*[@id='mw-content-text']//tr[td//a[@href='/wiki/Summon']]/td[2]/text()[1]")), " )");
            $summCon = explode('/', $w_sumConString);

            $hp = is_numeric($w_hp) ? $w_hp : 0;
            $exp = is_numeric($w_exp) ? $w_exp : 0;
            $summon = is_numeric($summCon[0]) ? $summCon[0] : 0;
            $convince = is_numeric($summCon[1]) ? $summCon[1] : 0;

            //immunities
            $immunities = array();
            $w_immunities = $xpath->query("//*[@id='mw-content-text']//tr[td/b[contains(text(),'Immun')]]//td[2]//a//text()");
            foreach ($w_immunities as $immunity)
                $immunities[] = $immunity->nodeValue;

            //elements
            $elements = array();
            $lastElement = NULL;
            $w_strengths = $xpath->query("//*[@id='mw-content-text']//tr[td/b[contains(text(),'Strong')]]//td[2]//span[contains(@class,'dmg')]");
            foreach ($w_strengths as $strength) {
                if(is_null($lastElement)) {
                    $lastElement = $strength->nodeValue;
                } else {
                    $elements[$lastElement] = intval(trim($strength->nodeValue, ' -+%'));
                    $lastElement = NULL;
                }
            }

            //log abilities
            writeLog((($new) ? 'added' : 'updated') . '-abilities', '[' . $name . '] - ' . getResult($xpath->query("//*[@id='mw-content-text']//tr[td/b[contains(text(),'Abilities')]]//td[2]")));

            $lastElement = NULL; //reset jic
            $w_weaknesses = $xpath->query("//*[@id='mw-content-text']//table[1]//tr//td//div//table//tr[td/b[contains(text(),'Weak')]]//td[2]//span[contains(@class,'dmg')]");
            foreach ($w_weaknesses as $weakness) {
                if(is_null($lastElement))
                    $lastElement = $weakness->nodeValue;
                else {
                    $elements[$lastElement] = intval('-' . trim($weakness->nodeValue, ' -+%'));
                    $lastElement = NULL;
                }
            }

            //sounds
            $w_sounds = explode(";", rtrim(getResult($xpath->query("//*[@id='mw-content-text']//span[@class='creaturesounds']")), '.'));
            $sounds = array();
            foreach ($w_sounds as $sound) {
                $trimmed = trim(str_ireplace(array('"', ';'), "", $sound));
                if(strlen($trimmed) > 1)
                    $sounds[] = $trimmed;
            }

            //loot
            $lootitems = @getMonsterLoot("http://tibia.wikia.com/wiki/Loot_Statistics:" . str_ireplace(" ", "_", $name) . ((in_array($name, $duplicateCreatures)) ? '_(Creature)' : ''));

            return array('exp'=>$exp, 'hp'=>$hp, 'convince'=>$convince, 'summon'=>$summon, 'pushable'=>$pushable, 'canpushitems'=>$pushObjects, 'sounds'=>$sounds, 'elements'=>$elements, 'immunities'=>$immunities, 'lootitems'=>$lootitems);
        } else {
            writeLog('404', "[Invalid Page] - http://tibia.wikia.com/wiki/" . $name);
        }
        return false;
    }

    /* Main */
    //TODO: getItemXMLDuplicates(); //loads xml and adds any reoccuring item name in an array

    $limit = 0; //set to 0 for all
    $n = 0;
    $itemlist = simplexml_load_file("items.xml");
    foreach ($monsterList as $name) {
        $n++;
        if($n >= $limit && $limit > 0) // for testing
            die("end of the line.");

        $load = loadMonster(strtolower($name));
        $new = $load[1];
        $monster = $load[0];
        $boss = in_array($name, $bossList);
        $illusionable = in_array($name, $illusionables);
        $details = getMonsterDetails($name, ($new) ? true : false);
        if($monster) {

            //basic notification triggers:
            if($details['exp'] <= 0)
                writeLog('missing-stats', "[Missing Experience] - " . $name);
            if($details['hp'] <= 0)
                writeLog('missing-stats', "[Missing Health] - " . $name);

            //main list stats
            $monster['name'] = $name;
            $monster['nameDescription'] = (($boss) ? '' : (strspn(strtolower($name), 'aeiou') ? 'an ' : 'a ')) . strtolower($name);
            $monster['experience'] = $details['exp'];
            $monster['manacost'] = $details['summon'];
            $monster->health['now'] = $details['hp'];
            $monster->health['max'] = $details['hp'];

            //defined attributes (looktype [, TODO: corpse, race])
            if($new) {
                $outfit = array_search(strtolower($name), $knownLookTypes);
                if($outfit !== false)
                    $monster->look['type'] = $outfit;
                else
                    writeLog('missing-looktypes', "[Missing Looktype] - " . $name);
            }

            //flags (bosses get the lureable 1 attribute)
            $old_flags = array();
            if(!$new) {
                //loop through all current flags of monster before clearing
                foreach ($monster->flags->children() as $old_flag)
                    foreach($old_flag->attributes() as $k => $v)
                        $old_flags[$k] = (string) $v;
                //it's safe to loop through and delete
                foreach ($monster->flags->children() as $old_flag)
                    unset($monster->flags->flag);
            }

            $monster->flags->flag[0]['attackable'] = (!$new && isset($old_flags['attackable'])) ? $old_flags['attackable'] : 1;
            $monster->flags->flag[1]['hostile'] = (!$new && isset($old_flags['hostile'])) ? $old_flags['hostile'] : 1;
            $monster->flags->flag[2]['summonable'] = ($details['summon'] > 0) ? 1 : 0;
            $monster->flags->flag[3]['convinceable'] = ($details['convince'] > 0) ? 1 : 0;
            $monster->flags->flag[4]['illusionable'] = ($illusionable) ? 1 : 0;
            $monster->flags->flag[5]['pushable'] = ($details) ? $details['pushable']: 0;
            $monster->flags->flag[6]['canpushitems'] = ($details) ? $details['canpushitems'] : 1;
            $monster->flags->flag[7]['canpushcreatures'] = (!$new && isset($old_flags['canpushcreatures'])) ? $old_flags['canpushcreatures'] : 1;
            $monster->flags->flag[8]['targetdistance'] = (!$new && isset($old_flags['targetdistance'])) ? $old_flags['targetdistance'] : 1;
            $monster->flags->flag[9]['staticattack'] = (!$new && isset($old_flags['staticattack'])) ? $old_flags['staticattack'] : 90;
            $monster->flags->flag[10]['runonhealth'] = (!$new && isset($old_flags['runonhealth'])) ? $old_flags['runonhealth'] : 0;
            $monster->flags->flag[11]['lureable'] = ($boss) ? 1 : 0;

            //elements
            unset($monster->elements);
            if(count($details['elements']) > 0) {
                $elements = $monster->addChild('elements', '');
                foreach ($details['elements'] as $element => $value) {
                    $knownElement = $knownElements[$element];
                    if(!is_null($knownElement)) {
                        $elementPercent = $elements->addChild('element', '');
                        $elementPercent->addAttribute($knownElement, $value);
                    }
                }
            }
            //immunities
            unset($monster->immunities);
            if(count($details['immunities']) > 0) {
                $immunities = $monster->addChild('immunities', '');
                foreach ($details['immunities'] as $immunity) {
                    $knownImmunity = $knownImmunities[$immunity];
                    if(!is_null($knownImmunity)) {
                        $monsterImmunity = $immunities->addChild('immunity', '');
                        $monsterImmunity->addAttribute($knownImmunity, 1);
                    }
                }
            }

            //voices (slightly buggy)
            unset($monster->voices);
            if($details){
                if(count($details['sounds']) > 0) {
                    $voices = $monster->addChild('voices', '');
                    $voices->addAttribute('interval', 5000);
                    $voices->addAttribute('chance', 10);
                    foreach ($details['sounds'] as $str) {
                        $voice = $monster->voices->addChild('voice', '');
                        $voice->addAttribute('sentence', trim($str));
                        if(strtoupper($str) == $str) //if all caps, must be yelling :)
                            $voice->addAttribute('yell', 1);
                    }
                }
            }

            //loot
            unset($monster->loot);
            $lootitems = $details['lootitems'];
            if($lootitems) {
                $monster->addChild('loot', '');
                foreach ($lootitems as $loot) {
                    $itemcount = count($itemlist->xpath('/items/item[@name="' . $loot['name'] . '"]'));

                    if($itemcount <= 0)
                        writeLog('missing-items', "[" . $loot['name'] . "] - " . $name);
                    elseif($itemcount > 1)
                        writeLog('duplicate-items', "[" . $loot['name'] . "][".$itemcount."] - " . $name);

                    $item = $monster->loot->addChild('item', '');
                    $knownDupe = $duplicateItems[strtolower($loot['name'])];
                    if(!is_null($knownDupe)) {
                        $item->addAttribute('id', $knownDupe);
                    } else {
                        $item->addAttribute('name', $loot['name']);
                    }
                    if($loot['count'] > 1)
                        $item->addAttribute('count', $loot['count']);
                    $item->addAttribute('chance', $loot['chance']);
                }
            }

            $color = ($new) ? 'green' : 'orange';
            $folder = ($new) ? 'added' : 'updated';

            //save monster
            $path = "monsters/" . $folder . "/" . (strtolower($name)) . ".xml";
            $string = xmlpp($monster->asXML());
            file_put_contents($path, $string);

            //verbose
            echo("<span style='color:" . $color . "'>[" . $folder . "]</span> monster: <b>" . (strtolower($name)) . ".xml</b><br/>");
        }
    }
?>