get_catalog_master.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#

import sys,os,re
import time,datetime
import json,msgpack,yaml,csv
from pprint import pprint
import urllib
from urllib2 import Request, urlopen, URLError, HTTPError
from pprint import pprint

def get_result(url):
    fp = urlopen(url); contents = fp.read(); fp.close();
    return contents

# params
api_key = "XXXXXX" # https://webservice.recruit.co.jp/register/index.html よりAPI KEY を取得して下さい。
countries = ["JPN", "GER","USA","CAN","GBR","SWE","FRA","ITA","AUT","ESP","BEL","HOL","RUS","AUS","KOR","THA","MYS","IND","RSA","ZZZ"]
count = 100

# csv
filename = "catalog_master.csv"
writecsv = csv.writer(file(filename, 'w'), lineterminator='\n')

def get_items(jsn):
        dic = {}
        dic["width"]  = jsn["width"]
        dic["period"] = jsn["period"]
        dic["grade"]  = jsn["grade"]
        dic["model"]  = jsn["model"]

        brand_jsn   = jsn['brand']
        dic['name'] = brand_jsn['name']
        dic['code'] = brand_jsn['code']

        dic["height"] = jsn["height"]
        dic["series"] = jsn["series"]

        body_jsn         = jsn['body']
        dic["body_name"] = body_jsn["name"]
        dic["body_code"] = body_jsn["code"]

        dic["length"] = jsn["length"]
        dic["desc"]   = jsn["desc"]
        dic["person"] = jsn["person"]
        dic["price"]  = int(jsn["price"])
        return dic

##### main #####
for country in countries:
    print country
    contents = [0]
    start = 1
    header_flag = True
    # url info
    while 0 < len(contents):
        base_url = "http://webservice.recruit.co.jp/carsensor/catalog/v1/?key=%s&country=%s&format=json&count=%s&start=%s"%(api_key, country, count, start)
        print base_url
        res =  json.loads(get_result(base_url))
        contents = res["results"]["catalog"]
        if len(contents)==0: continue
        for jsn in contents:
            dic = get_items(jsn)
            keys = sorted(dic.keys())
            if header_flag:
              writecsv.writerow(keys)
              header_flag = False
            values = []
            for k in sorted(dic.keys()):
                # print "%s: %s"%(k,dic[k])
                k = k.encode('utf-8')
                if  isinstance(dic[k],unicode):
                    v = dic[k].encode('utf-8')
                else:
                    v = dic[k]
                values.append(v)
            writecsv.writerow(values)
        start += 100