ebaypredict.py remade for Ch8 Programming Collective Intel..

"""  I couldn't get ebaypredict.py to return any data so I wrote a version that uses JSON.
     I haven't implemented getCategory(), but it's easy enough to add later :)
"""
import time
import re
import urllib as urllib
import json                        # we are after all importing json
from time import sleep             # sleep to slow the requests to the server

appKey = 'Your__Ebay__AppKey__Here'

def doSearch(query,pages=1):
    global appkey
    query1=query.replace(' ',"%20")
    results=[]
    for page in range(pages):
        try:
            url = "http://svcs.ebay.com/services/search/FindingService/v1?OPERATION-NAME=findItemsByKeywords"
            url += "&SERVICE-VERSION=1.0.0&SECURITY-APPNAME="+appKey
            url += "&RESPONSE-DATA-FORMAT=JSON&REST-PAYLOAD&keywords="+query1
            url += "&paginationInput.entriesPerPage=100&paginationOutput.pageNumber="+str(page+1)

            dic = {}
            conn = urllib.urlopen(url)
            for line in conn:
                dic = json.loads(line)     # load json into dic

            now = time.time()    # (current time in seconds)
            for a in range(len(dic['findItemsByKeywordsResponse'][0]['searchResult'][0]['item'])):
                itemId = dic['findItemsByKeywordsResponse'][0]['searchResult'][0]['item'][a]['itemId'][0]
                itemTitle = dic['findItemsByKeywordsResponse'][0]['searchResult'][0]['item'][a]['title'][0]
                itemPrice =  dic['findItemsByKeywordsResponse'][0]['searchResult'][0]['item'][a]['sellingStatus'][0]['convertedCurrentPrice'][0]['__value__']
                timeleft = dic['findItemsByKeywordsResponse'][0]['searchResult'][0]['item'][a]['sellingStatus'][0]['timeLeft'][0]
                d = re.split('(\d+)',timeleft)
                timeleft1 = (int(d[1])*24*60*60)+(int(d[3])*60*60)+(int(d[5])*60)+(int(d[7]))     # Convert to seconds
                endtime = now + timeleft1
                itemEnds = time.ctime(endtime)
                results.append((itemId,itemTitle,itemPrice,itemEnds))
        except: print "Unable to retrieve page",page+1
    return results

def getItem(itemId):
    global appkey
    try:
        url = "http://open.api.ebay.com/shopping?callname=GetSingleItem&responseencoding=JSON&appid=" + appKey
        url += "&version=515&ItemID=" + itemId
        url += "&IncludeSelector=ItemSpecifics,Details"

        dic = {}
        conn = urllib.urlopen(url)
        for line in conn:              # comes as a unicode string
            dic = json.loads(line)     # load json into dic

        result={}
        result['title']=dic['Item']['Title']
        result['price']=dic['Item']['CurrentPrice']['Value']
        result['bids']=dic['Item']['BidCount']
        result['feedback']=dic['Item']['Seller']['FeedbackScore']
        dic2={}
        for a in dic['Item']['ItemSpecifics']['NameValueList']:
            dic2[a['Name']]=a['Value'][0]
        result['attributes']=dic2
        return result
    except: print "Unable to retrieve",itemId

def makeLaptopDataset():
    searchResults=doSearch('laptop',7)
    result=[]
    for a in searchResults:
        sleep(0.7)
        print '.',
        try:
            item=getItem(a[0])
            hdd = item['attributes']['Hard Drive Capacity']
            d = re.split('(\d+)',hdd)
            mem = item['attributes']['Memory']
            e = re.split('(\d+)',mem)
            scr = item['attributes']['Screen Size']
            scr = scr.replace('"','')
            pro = item['attributes']['Processor Speed']
            pro = pro.replace('GHz','')
            data=(float(e[1]),float(pro),float(scr),float(d[1]),float(item['feedback']))
            entry={'input':data,'result':float(item['price'])}
            result.append(entry)
            if len(result)==300: return result
        except:
            try:
                print item['title']+' failed'   # We want a completed dataset
            except: print 'item failed'
    return result


"""
Example Usage
*************
import ebayjson1 as ebayjson
ebayjson.doSearch('laptop notebook')
ebayjson.getItem('321181609967')
data = ebayjson.makeLaptopDataset()
import numpredict2 as numpredict
numpredict.knnestimate(data,(2.0,2.5,14.1,500,1000))
sdata=numpredict.rescale(data,[1,1,1,1.5,0])
numpredict.weightedknn(sdata,(4.0, 1.3999999999999999, 11.6, 500.0, 158692.0),weightf=numpredict.inverseweight)

def knninverse(d,v):
    return numpredict.weightedknn(d,v,weightf=numpredict.inverseweight)
costf=numpredict.createcostfunction(knninverse,sdata)
optimization.geneticoptimize(knninverse,costf,popsize=5,step=1,elite=0.2,maxiter=20)


-------------------------------------------------------------------------------
makeLaptopDataset()     example output len=300

Notes: the 'feedback' score is a bit OTT examples 158693.0, 21.0, 1266.0
       the price is the current price converted into Dollar$

[{'input': (4.0, 2.3999999999999999, 15.6, 500.0, 5664.0), 'result': 288.0}, {'input': (4.0, 1.3999999999999999, 11.6, 500.0, 158692.0), 'result': 369.99000000000001}, {'input': (4.0, 1.0, 15.6, 320.0, 48646.0), 'result': 168.5}, {'input': (4.0, 2.7000000000000002, 13.300000000000001, 500.0, 52.0), 'result': 495.0}, {'input': (4.0, 2.3999999999999999, 14.1, 250.0, 69396.0), 'result': 350.0}, {'input': (2.0, 2.2000000000000002, 14.1, 80.0, 13347.0), 'result': 189.94999999999999}, {'input': (6.0, 2.7000000000000002, 17.300000000000001, 640.0, 5664.0), 'result': 379.99000000000001}, {'input': (2.0, 2.2599999999999998, 13.300000000000001, 160.0, 407.0), 'result': 400.0}, {'input': (4.0, 2.2999999999999998, 13.300000000000001, 320.0, 4621.0), 'result': 641.0}, {'input': (4.0, 2.5299999999999998, 17.0, 500.0, 4621.0), 'result': 909.02999999999997}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 19690.0), 'result': 394.99000000000001}, {'input': (2.0, 2.2599999999999998, 14.1, 160.0, 21.0), 'result': 124.98999999999999}, {'input': (4.0, 1.3, 15.6, 320.0, 19690.0), 'result': 334.99000000000001}, {'input': (1.0, 1.8600000000000001, 14.0, 40.0, 619.0), 'result': 99.989999999999995}, {'input': (4.0, 1.5, 15.6, 320.0, 19690.0), 'result': 324.99000000000001}, {'input': (2.0, 1.7, 15.6, 320.0, 5828.0), 'result': 309.94999999999999}, {'input': (4.0, 2.5, 17.300000000000001, 500.0, 196.0), 'result': 300.0}, {'input': (6.0, 3.1000000000000001, 15.6, 750.0, 5664.0), 'result': 398.88999999999999}, {'input': (3.0, 2.1000000000000001, 15.4, 60.0, 2637.0), 'result': 159.0}, {'input': (3.0, 2.5299999999999998, 14.1, 250.0, 69396.0), 'result': 147.5}, {'input': (2.0, 2.7999999999999998, 14.1, 160.0, 2428.0), 'result': 182.0}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 158693.0), 'result': 304.94999999999999}, {'input': (4.0, 1.7, 11.6, 500.0, 4494.0), 'result': 299.99000000000001}, {'input': (512.0, 1.8300000000000001, 14.1, 60.0, 1266.0), 'result': 129.99000000000001}, {'input': (4.0, 2.3999999999999999, 14.1, 160.0, 29729.0), 'result': 289.0}, {'input': (2.0, 1.8999999999999999, 15.4, 120.0, 2514.0), 'result': 159.99000000000001}, {'input': (4.0, 2.7999999999999998, 15.4, 250.0, 69397.0), 'result': 229.0}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 1368.0), 'result': 329.0}, {'input': (4.0, 2.7999999999999998, 17.0, 160.0, 93593.0), 'result': 329.99000000000001}, {'input': (1.0, 2.2000000000000002, 13.300000000000001, 160.0, 16.0), 'result': 100.0}, {'input': (4.0, 1.3999999999999999, 11.6, 500.0, 5664.0), 'result': 339.99000000000001}, {'input': (4.0, 2.5299999999999998, 15.6, 500.0, 13018.0), 'result': 284.99000000000001}, {'input': (1.0, 1.6000000000000001, 10.1, 160.0, 2.0), 'result': 112.5}, {'input': (4.0, 2.7999999999999998, 14.1, 250.0, 69397.0), 'result': 165.09999999999999}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 9801.0), 'result': 349.0}, {'input': (8.0, 2.2999999999999998, 14.0, 750.0, 1507.0), 'result': 1000.0}, {'input': (4.0, 2.6600000000000001, 13.300000000000001, 250.0, 69397.0), 'result': 299.99000000000001}, {'input': (2.0, 2.2000000000000002, 14.1, 120.0, 13347.0), 'result': 189.0}, {'input': (4.0, 2.5299999999999998, 15.6, 320.0, 1921.0), 'result': 249.99000000000001}, {'input': (3.0, 2.3999999999999999, 14.0, 250.0, 13347.0), 'result': 319.0}, {'input': (4.0, 2.6600000000000001, 14.1, 250.0, 434.0), 'result': 251.5}, {'input': (4.0, 2.7999999999999998, 14.1, 320.0, 15621.0), 'result': 259.25}, {'input': (4.0, 2.7999999999999998, 14.1, 250.0, 434.0), 'result': 145.0}, {'input': (4.0, 1.5, 13.300000000000001, 500.0, 35046.0), 'result': 359.99000000000001}, {'input': (4.0, 2.7999999999999998, 14.1, 320.0, 15621.0), 'result': 259.0}, {'input': (4.0, 1.3999999999999999, 13.300000000000001, 500.0, 773.0), 'result': 445.94999999999999}, {'input': (8.0, 1.73, 17.300000000000001, 500.0, 39.0), 'result': 550.0}, {'input': (4.0, 2.5299999999999998, 14.1, 160.0, 15621.0), 'result': 189.25}, {'input': (4.0, 2.2599999999999998, 15.4, 160.0, 1139.0), 'result': 73.0}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 158693.0), 'result': 294.94999999999999}, {'input': (4.0, 2.5299999999999998, 13.300000000000001, 250.0, 15621.0), 'result': 298.0}, {'input': (1.0, 1.6000000000000001, 14.1, 40.0, 6773.0), 'result': 117.0}, {'input': (2.0, 1.73, 15.4, 100.0, 199.0), 'result': 58.0}, {'input': (4.0, 2.5299999999999998, 14.0, 320.0, 15621.0), 'result': 269.25}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 5664.0), 'result': 288.0}, {'input': (4.0, 1.3999999999999999, 11.6, 500.0, 158693.0), 'result': 369.99000000000001}, {'input': (4.0, 1.0, 15.6, 320.0, 48646.0), 'result': 177.50999999999999}, {'input': (4.0, 2.7000000000000002, 13.300000000000001, 500.0, 52.0), 'result': 561.0}, {'input': (4.0, 2.3999999999999999, 14.1, 250.0, 69397.0), 'result': 350.0}, {'input': (2.0, 2.2000000000000002, 14.1, 80.0, 13347.0), 'result': 189.94999999999999}, {'input': (6.0, 2.7000000000000002, 17.300000000000001, 640.0, 5664.0), 'result': 379.99000000000001}, {'input': (2.0, 2.2599999999999998, 13.300000000000001, 160.0, 407.0), 'result': 461.0}, {'input': (4.0, 2.2999999999999998, 13.300000000000001, 320.0, 4622.0), 'result': 671.0}, {'input': (4.0, 2.5299999999999998, 17.0, 500.0, 4622.0), 'result': 909.02999999999997}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 19690.0), 'result': 394.99000000000001}, {'input': (2.0, 2.2599999999999998, 14.1, 160.0, 21.0), 'result': 124.98999999999999}, {'input': (4.0, 1.3, 15.6, 320.0, 19690.0), 'result': 334.99000000000001}, {'input': (1.0, 1.8600000000000001, 14.0, 40.0, 619.0), 'result': 99.989999999999995}, {'input': (4.0, 1.5, 15.6, 320.0, 19690.0), 'result': 324.99000000000001}, {'input': (2.0, 1.7, 15.6, 320.0, 5828.0), 'result': 309.94999999999999}, {'input': (4.0, 2.5, 17.300000000000001, 500.0, 196.0), 'result': 300.0}, {'input': (6.0, 3.1000000000000001, 15.6, 750.0, 5664.0), 'result': 398.88999999999999}, {'input': (3.0, 2.1000000000000001, 15.4, 60.0, 2637.0), 'result': 159.0}, {'input': (3.0, 2.5299999999999998, 14.1, 250.0, 69397.0), 'result': 175.0}, {'input': (2.0, 2.7999999999999998, 14.1, 160.0, 2428.0), 'result': 182.0}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 158693.0), 'result': 304.94999999999999}, {'input': (4.0, 1.7, 11.6, 500.0, 4494.0), 'result': 299.99000000000001}, {'input': (512.0, 1.8300000000000001, 14.1, 60.0, 1266.0), 'result': 129.99000000000001}, {'input': (4.0, 2.3999999999999999, 14.1, 160.0, 29730.0), 'result': 289.0}, {'input': (2.0, 1.8999999999999999, 15.4, 120.0, 2514.0), 'result': 159.99000000000001}, {'input': (4.0, 2.7999999999999998, 15.4, 250.0, 69397.0), 'result': 229.0}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 1368.0), 'result': 329.0}, {'input': (4.0, 2.7999999999999998, 17.0, 160.0, 93593.0), 'result': 329.99000000000001}, {'input': (1.0, 2.2000000000000002, 13.300000000000001, 160.0, 16.0), 'result': 145.49000000000001}, {'input': (4.0, 1.3999999999999999, 11.6, 500.0, 5664.0), 'result': 339.99000000000001}, {'input': (4.0, 2.5299999999999998, 15.6, 500.0, 13018.0), 'result': 284.99000000000001}, {'input': (1.0, 1.6000000000000001, 10.1, 160.0, 2.0), 'result': 112.5}, {'input': (4.0, 2.7999999999999998, 14.1, 250.0, 69397.0), 'result': 165.09999999999999}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 9802.0), 'result': 349.0}, {'input': (8.0, 2.2999999999999998, 14.0, 750.0, 1507.0), 'result': 1000.0}, {'input': (4.0, 2.6600000000000001, 13.300000000000001, 250.0, 69397.0), 'result': 304.99000000000001}, {'input': (2.0, 2.2000000000000002, 14.1, 120.0, 13347.0), 'result': 189.0}, {'input': (4.0, 2.5299999999999998, 15.6, 320.0, 1921.0), 'result': 249.99000000000001}, {'input': (3.0, 2.3999999999999999, 14.0, 250.0, 13347.0), 'result': 319.0}, {'input': (4.0, 2.6600000000000001, 14.1, 250.0, 434.0), 'result': 251.5}, {'input': (4.0, 2.7999999999999998, 14.1, 320.0, 15621.0), 'result': 259.25}, {'input': (4.0, 2.7999999999999998, 14.1, 250.0, 434.0), 'result': 145.0}, {'input': (4.0, 1.5, 13.300000000000001, 500.0, 35046.0), 'result': 359.99000000000001}, {'input': (4.0, 2.7999999999999998, 14.1, 320.0, 15621.0), 'result': 259.0}, {'input': (4.0, 1.3999999999999999, 13.300000000000001, 500.0, 773.0), 'result': 445.94999999999999}, {'input': (8.0, 1.73, 17.300000000000001, 500.0, 39.0), 'result': 550.0}, {'input': (4.0, 2.5299999999999998, 14.1, 160.0, 15621.0), 'result': 189.25}, {'input': (4.0, 2.2599999999999998, 15.4, 160.0, 1139.0), 'result': 73.0}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 158693.0), 'result': 294.94999999999999}, {'input': (4.0, 2.5299999999999998, 13.300000000000001, 250.0, 15621.0), 'result': 298.0}, {'input': (1.0, 1.6000000000000001, 14.1, 40.0, 6773.0), 'result': 117.0}, {'input': (2.0, 1.73, 15.4, 100.0, 199.0), 'result': 58.0}, {'input': (4.0, 2.5299999999999998, 14.0, 320.0, 15621.0), 'result': 269.25}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 5664.0), 'result': 288.0}, {'input': (4.0, 1.3999999999999999, 11.6, 500.0, 158693.0), 'result': 369.99000000000001}, {'input': (4.0, 1.0, 15.6, 320.0, 48646.0), 'result': 177.50999999999999}, {'input': (4.0, 2.7000000000000002, 13.300000000000001, 500.0, 52.0), 'result': 561.0}, {'input': (4.0, 2.3999999999999999, 14.1, 250.0, 69397.0), 'result': 350.0}, {'input': (2.0, 2.2000000000000002, 14.1, 80.0, 13347.0), 'result': 189.94999999999999}, {'input': (2.0, 2.2599999999999998, 13.300000000000001, 160.0, 407.0), 'result': 461.0}, {'input': (4.0, 2.2999999999999998, 13.300000000000001, 320.0, 4622.0), 'result': 671.0}, {'input': (6.0, 2.7000000000000002, 17.300000000000001, 640.0, 5664.0), 'result': 379.99000000000001}, {'input': (4.0, 2.5299999999999998, 17.0, 500.0, 4622.0), 'result': 909.02999999999997}, {'input': (4.0, 1.3, 15.6, 320.0, 19690.0), 'result': 334.99000000000001}, {'input': (4.0, 1.5, 15.6, 320.0, 19690.0), 'result': 324.99000000000001}, {'input': (2.0, 2.2599999999999998, 14.1, 160.0, 21.0), 'result': 124.98999999999999}, {'input': (1.0, 1.8600000000000001, 14.0, 40.0, 619.0), 'result': 99.989999999999995}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 19690.0), 'result': 394.99000000000001}, {'input': (6.0, 3.1000000000000001, 15.6, 750.0, 5664.0), 'result': 398.88999999999999}, {'input': (4.0, 2.5, 17.300000000000001, 500.0, 196.0), 'result': 300.0}, {'input': (2.0, 1.7, 15.6, 320.0, 5828.0), 'result': 309.94999999999999}, {'input': (3.0, 2.1000000000000001, 15.4, 60.0, 2637.0), 'result': 159.0}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 1368.0), 'result': 329.0}, {'input': (3.0, 2.5299999999999998, 14.1, 250.0, 69397.0), 'result': 175.0}, {'input': (4.0, 1.7, 11.6, 500.0, 4494.0), 'result': 299.99000000000001}, {'input': (2.0, 2.7999999999999998, 14.1, 160.0, 2428.0), 'result': 182.0}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 158693.0), 'result': 304.94999999999999}, {'input': (512.0, 1.8300000000000001, 14.1, 60.0, 1266.0), 'result': 129.99000000000001}, {'input': (4.0, 2.3999999999999999, 14.1, 160.0, 29730.0), 'result': 289.0}, {'input': (4.0, 2.7999999999999998, 15.4, 250.0, 69397.0), 'result': 229.0}, {'input': (2.0, 1.8999999999999999, 15.4, 120.0, 2515.0), 'result': 159.99000000000001}, {'input': (4.0, 2.7999999999999998, 17.0, 160.0, 93593.0), 'result': 329.99000000000001}, {'input': (1.0, 2.2000000000000002, 13.300000000000001, 160.0, 16.0), 'result': 145.49000000000001}, {'input': (4.0, 1.3999999999999999, 11.6, 500.0, 5664.0), 'result': 339.99000000000001}, {'input': (4.0, 2.5299999999999998, 15.6, 500.0, 13018.0), 'result': 284.99000000000001}, {'input': (1.0, 1.6000000000000001, 10.1, 160.0, 2.0), 'result': 112.5}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 9802.0), 'result': 349.0}, {'input': (4.0, 2.7999999999999998, 14.1, 250.0, 69398.0), 'result': 165.09999999999999}, {'input': (8.0, 2.2999999999999998, 14.0, 750.0, 1507.0), 'result': 1000.0}, {'input': (2.0, 2.2000000000000002, 14.1, 120.0, 13347.0), 'result': 189.0}, {'input': (4.0, 2.6600000000000001, 13.300000000000001, 250.0, 69398.0), 'result': 304.99000000000001}, {'input': (4.0, 2.5299999999999998, 15.6, 320.0, 1921.0), 'result': 249.99000000000001}, {'input': (3.0, 2.3999999999999999, 14.0, 250.0, 13347.0), 'result': 319.0}, {'input': (4.0, 2.3999999999999999, 15.6, 500.0, 158694.0), 'result': 294.94999999999999}, {'input': (4.0, 2.6600000000000001, 14.1, 250.0, 434.0), 'result': 251.5}, {'input': (4.0, 2.7999999999999998, 14.1, 320.0, 15621.0), 'result': 259.25}, {'input': (4.0, 2.7999999999999998, 14.1, 250.0, 434.0), 'result': 145.0}, {'input': (4.0, 1.5, 13.300000000000001, 500.0, 35046.0), 'result': 359.99000000000001}, {'input': (4.0, 2.7999999999999998, 14.1, 320.0, 15621.0), 'result': 259.0}, {'input': (8.0, 1.73, 17.300000000000001, 500.0, 39.0), 'result': 550.0}, {'input': (4.0, 1.3999999999999999, 13.300000000000001, 500.0, 773.0), 'result': 445.94999999999999}, {'input': (4.0, 2.2599999999999998, 15.4, 160.0, 1139.0), 'result': 73.0}, {'input': (4.0, 2.5299999999999998, 14.1, 160.0, 15621.0), 'result': 189.25}, {'input': (4.0, 2.5299999999999998, 14.0, 320.0, 15621.0), 'result': 269.25}, {'input': (1.0, 1.6000000000000001, 14.1, 40.0, 6773.0), 'result': 117.0}, {'input': (2.0, 1.73, 15.4, 100.0, 199.0), 'result': 58.0}, {'input': (2.0, 2.5299999999999998, 14.1, 80.0, 13347.0), 'result': 194.0}]


"""