Carmudi

# -*- coding: utf-8 -*-
from urlparse import urljoin

from scrapy.utils.response import get_base_url
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors import LinkExtractor

from entertainment.items import CarmudiItem


class CarmudiSpider(CrawlSpider):
    name = "carmudi"
    allowed_domains = ["carmudi.com.gh"]
    start_urls = ['http://www.carmudi.com.gh/cars/?page=1', 'http://www.carmudi.com.gh/cars']

    rules = [
        Rule(LinkExtractor(allow=["\?page=\d+"]),
             callback="parse_item",
             follow=True),
    ]

    def parse_item(self, response):
        catalog = response.xpath('//div[@class="catalog-item"]')
        items = []
        for index, catalog_item in enumerate(catalog):
            item = CarmudiItem()
            item['title'] = catalog_item.xpath('div/h5/a/text()').extract()
            relative_url = catalog_item.xpath('div/h5/a/@href').extract()
            item['description'] = catalog_item.xpath('div/div/p/text()').extract()
            item['url'] = [urljoin(get_base_url(response), url) for url in relative_url]
            _price = catalog_item.xpath('div[3]/div[2]/a/text()').extract()
            price, currency = ''.join(_price).split(" ")
            item['currency'] = currency
            item['price'] = price
            item['condition'] = catalog_item.xpath('div[2]/div[2]/ul/li[3]/text()').extract()
            item['transmission'] = catalog_item.xpath('div[2]/div[2]/ul/li[2]/text()').extract()
            item['gauge'] = catalog_item.xpath('div[2]/div[2]/ul/li[1]/text()').extract()
            item['location'] = catalog_item.xpath('div[3]/div[3]/ul/li[2]/span/@title').extract()
            item['fuel'] = catalog_item.xpath('div[2]/div[2]/ul/li[4]/text()').extract()
            item['images'] = ''

            # yield item
            items.append(item)
        print "{} items processed".format(len(items))
        return items