ChickenSmoothie.py

## Last edited August 17, 2018
## Use for CS Rarity List

import io
import os
import sys
import urllib.request
import json
import requests
import re

from PIL import Image, ImageChops
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter

## pip install Pillow
## pip install requests
## pip install -U pyopenssl

REFERENCE_DIR = "reference_images"
PET_IMAGE_URL = "https://www.chickensmoothie.com/pet/%s&trans=1&noitems=1.jpg"
UNKNOWN_key = "UNKNOWN"
ERROR_key = "ERROR"
directory = {}
petCount = {}
ONLINE = "your account to get HTML";
VERYRARE = '/img/rarity/veryrare.png'
OMGSR = '/img/rarity/omgsorare.png'
onlineIDs = ''


def url(start,instancenum):     #grab URL of pet
    URL = "https://www.chickensmoothie.com/viewpet.php?id="+str(start)+".xml:443"
    # response = requests.get(URL,allow_redirects=False, stream=False,timeout=None)
    with requests.Session() as s:
        retry = Retry(connect=5, read=5, redirect=1, backoff_factor=0.5)
        adapter = HTTPAdapter(max_retries=retry)
        s.mount('http://',adapter)
        s.mount('https://',adapter)
        r = s.get(URL,allow_redirects=False, stream=False, timeout=3.05)
        with open('site_'+instancenum+'.xml','wb')as file:
            file.write(r.content)   #response.content
            file.close()
        s.close()


def defineOnline(start_id, end_id,onlineIDs,instancenum):
    isWanted = False
    isOnline = False
    start_id = int(start_id)
    end_id = int(end_id)+1
    while(start_id<end_id):
        url(start_id,instancenum)
        with open('site_'+instancenum+'.xml',encoding="utf8") as petfile:
            print ("Checking:",str(start_id))
            for line in petfile:
                if VERYRARE in line or OMGSR in line:
                    isWanted = True
                    print ('\tVR or OMGSR')
                if ONLINE in line:
                    isOnline = True
                    print ("\t"+str(start_id)+" is ONLINE")
            if isWanted is True and isOnline is True:
                onlineIDs += str(start_id)+"\n"
            petfile.close()
        start_id += 1
        isWanted = False
        isOnline = False
    with open('onlineIDs_'+str(instancenum)+'.txt','w')as idfile:
        for id in onlineIDs:
            idfile.write("%s"%id)
        idfile.close()


def processLine(line):
        id_key_list = line.split(',')
        #print (id_key_list)
        ids = id_key_list[0].split(',')
        id = ids[0]
        keys = id_key_list[1].split('\n')
        key = keys[0]
        print (key, id)
        id = directory.get(key)
        # print (id)
        count(key,id)


def createDirectory():      #creates ID directory
    with open('CS Code Directory.txt')as file: #or .csv
        for line in file:
            array = []
            updated = line.split(",")
            array.append(updated)
            directory[array[0][2]] = array[0][1]    #pet key : pet name
        file.close()

def count(key,id):
    if key in directory:
        if petCount.get(id) is None:
            petCount[id] = 0
        petCount[id] += 1

def countfile(start_id,end_id,instancenum):

    with open('onlineIDs_'+instancenum+'_out_['+str(start_id)+','+str(end_id)+'].txt','r') as f:
        for line in f:
            processLine(line)
        f.close()
    with open('['+start_id+','+end_id+'].txt', 'w') as file:
        file.write(json.dumps(petCount,sort_keys=True,separators=('',': '),indent=3))
        file.write('\n\nRANGE: ['+start_id+', '+end_id+']')
        file.close()
###############

def crop(img):
  """Remove part of the pet that says "___'s pet and transparent edges."""
  w, h = img.size

  h = h - 9
  img = img.crop((0, 0, w, h))

  # Get cropping border
  left = w
  top = h
  right = 0
  bottom = 0

  img_data = img.getdata()
  # Loop from left->right and top->bottom to get the left/top cropping borders.
  for i_h in range(h):
    for i_w in range(w):
      if img_data[i_w + i_h * w] == (0, 0, 0, 0):
        continue
      top = min(top, i_h)
      left = min(left, i_w)
      break

  # Loop from right->left and bottom->top to get the right/bottom cropping
  # borders.
  for i_h in range(h - 1, 0, -1):
    for i_w in range(w - 1, 0, -1):
      if img_data[i_w + i_h * w] == (0, 0, 0, 0):
        continue
      bottom = max(bottom, i_h)
      right = max(right, i_w)
      break

  img = img.crop((left, top, right, bottom))
  return img

def download_image(img_url):
    """Downloads an image from a URL and returns an `Image` object."""
    with urllib.request.urlopen(img_url) as url:
        f = io.BytesIO(url.read())
    return Image.open(f)


def equal(image1, image2):
    """Checks that 2 images are exactly the same."""
    return ImageChops.difference(image1, image2).getbbox() is None


class PetReference(object):
    def __init__(self, reference_dir):
        self.reference_dir = reference_dir

        # Map string pet key/code to an Image
        self.key_to_image = {}

        # Maps int tuple of (width, height) to a list of pet keys
        self.size_to_key = {}

        for img_file in os.listdir(reference_dir):
            pet_key = img_file.split('.')[0]
            img = crop(Image.open(os.path.join(reference_dir, img_file)))
            size = img.size

            self.key_to_image[pet_key] = img
            if size not in self.size_to_key:
                self.size_to_key[size] = []
            self.size_to_key[size].append(pet_key)

    def get_key(self, pet_id):
        # Assert pet_id is an integer
        assert isinstance(pet_id, int), "Pet ID must be an integer"
        image_url = PET_IMAGE_URL % pet_id
        downloaded_img = crop(download_image(image_url))

        size = downloaded_img.size

        if size in self.size_to_key:
            for pet_key in self.size_to_key[size]:
                img = self.key_to_image[pet_key]
                if equal(img, downloaded_img):
                    return pet_key

        return UNKNOWN_key


def main(args):
    # if len(args) < 2:
        # print("Please enter an input file. Quitting.")
        # sys.exit(0)
    ref = PetReference(REFERENCE_DIR)
    createDirectory()


    start_id = args[1]
    end_id = args[2]
    instancenum = args[3]
    input_file = 'onlineIDs_'+instancenum+'.txt'
    id = int(start_id)
    print (start_id, end_id)
    defineOnline(start_id,end_id,onlineIDs,instancenum)

    output_file = os.path.basename(input_file).split('.')[0] + "_out_["+start_id+","+end_id+"].txt"

    errors = []
    with open('inputfile.txt','w') as filename:
        while(id<int(end_id)):
            filename.write("%s\n"%str(id))
            id+=1
        filename.write("%s\n"%str(end_id))
    filename.close()

    with open(input_file, 'r') as f:
        with open(output_file, 'w') as w:
            for line in f:
                pet_id = line.strip()
                if not pet_id:
                    continue

                print("Identifying pet", pet_id)
                try:
                    pet_id = int(pet_id)
                    pet_key = ref.get_key(pet_id)
                except Exception as e:
                    print("\tSkipped %s because there was an error.")
                    errors.append((pet_id, e))
                    pet_key = ERROR_key
                if pet_key in directory:
                    w.write("%s,%s,%s\n" % (pet_id, pet_key, directory.get(pet_key)))
                else:
                    w.write("%s,%s\n" % (pet_id, pet_key))

    if errors:
        print("List of all errors")
        for pet_id, e in errors:
            print("\nPet ID:", pet_id)
            print(e)


# createDirectory()
    countfile(start_id,end_id,instancenum)
# print(directory)
# print(petCount)


    print ("FINISHED")
    print ('---> Check \"onlineIDs_'+instancenum+'_out_['+str(start_id)+','+str(end_id)+'].txt\" for any pet IDs with UNKNOWN')
    print ("---> Results: \"["+start_id+','+end_id+'].txt\"')





if __name__ == "__main__":
    main(sys.argv)