<?php
// 569512
include('simple_html_dom.php');

ini_set('memory_limit', '-1');
ini_set('max_execution_time', 3600000);

$page = 'http://www.getrandomthings.com/';
$html = new simple_html_dom();
$html->load_file($page);

$items = $html->find('[class=app_cat_ul] a');

//db

$servername = "localhost";
$username = "root";
$password = "";
$dbname = "getrandomthings";

// Create connection
$conn = new mysqli($servername, $username, $password, $dbname);
// Check connection
if ($conn->connect_error) {
    die("Connection failed: " . $conn->connect_error);
}

/*
$sql = "SELECT id, firstname, lastname FROM MyGuests";
$result = $conn->query($sql);

if ($result->num_rows > 0) {
    echo "<table><tr><th>ID</th><th>Name</th></tr>";
    // output data of each row
    while($row = $result->fetch_assoc()) {
        echo "<tr><td>".$row["id"]."</td><td>".$row["firstname"]." ".$row["lastname"]."</td></tr>";
    }
    echo "</table>";
} else {
    echo "0 results";
}*/


// curl
$ch = curl_init();

foreach ($items as $k => $item){
    echo $k.'<br>';
    $category = str_replace(' ','_',$item->innertext);//$item->innertext
    $sql_count = 'SELECT count(*) total FROM all_things ats
                      WHERE ats.category = "' . $category . '"';
    $_result = $conn->query($sql_count);
    $count = 0;
    while($row = $_result->fetch_assoc()) {
        $count = intval($row['total']);
    }

    // 545
    if($count < 10000 && $k > 260) {
    curl_setopt($ch, CURLOPT_URL,$page.'data/'.$item->href);//.$item->href
    curl_setopt($ch, CURLOPT_POST, 1);
    curl_setopt($ch, CURLOPT_POSTFIELDS,
        "num=100&add=address&unique=true");


    // in real life you should use something like:
    // curl_setopt($ch, CURLOPT_POSTFIELDS,
    //          http_build_query(array('postvar1' => 'value1')));

    // receive server response ...
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

    $max = $count;

    $server_output = curl_exec($ch);

        while (1) {

            $_check = true;

            $server_output = curl_exec($ch);

            $html_child = str_get_html($server_output);
            $f_child = $html_child->firstChild();
            if (!empty($f_child)) {
                $data = $f_child->children();
                foreach ($data as $_item) {
                    //  $value = $_item->plaintext;
                    //   echo 'value: '.$value.'<br>';
                    if ($_item->find('[class=sdes]', 0)) {
                        $sub_value = $_item->find('[class=sdes]', 0)->innertext;
                    } else {
                        $sub_value = '';
                    }
                    //      echo 'sub_value: '.$sub_value.'<br>';
                    $value = trim(str_replace($sub_value, '', $_item->plaintext));
                    //      echo 'value: '.$value.'<br>';
                    if ($_item->find('img', 0)) {
                        $image = $_item->find('img', 0)->src;
                    } else {
                        $image = '';
                    }
                    //       echo 'image: '.$image.'<br>';

                    $sql_exist = 'SELECT 1 FROM all_things ats
                          WHERE ats.value = "' . $value . '"
                                AND ats.category = "' . $category . '"
                          LIMIT 1';
                    $result = $conn->query($sql_exist);

                    if (empty($result) || $result->num_rows == 0) {
                        $_check = false;
                    }

                    $value = mysqli_real_escape_string($conn, $value);
                    $sub_value = mysqli_real_escape_string($conn, $sub_value);

                    if (empty($result) || $result->num_rows == 0) {
                        $sql = 'INSERT INTO all_things (value, category , sub_value, image, origin_image)
                                    VALUES ("' . $value . '", "' . $category . '","' . $sub_value . '","' . $image . '","' . $image . '")';
                        $h = $conn->query($sql);
                    }
                }
                if ($_check || $max >= 10000) {
                    break;
                }

                $max += count($data);
            }
        }
    }
  //  break;
}

echo 'done';

curl_close ($ch);

$conn->close();