import scrapy

class TestSpider(scrapy.Spider):
    name = "testdoc1"
    allowed_domains = ['amazon.in']
    start_urls = ["https://www.amazon.in/s/ref=amb_link_46?ie=UTF8&bbn=1389432031&rh=i%3Aelectronics%2Cn%3A976419031%2Cn%3A%21976420031%2Cn%3A1389401031%2Cn%3A1389432031%2Cp_89%3AApple&pf_rd_m=A1VBAL9TL5WCBF&pf_rd_s=merchandised-search-leftnav&pf_rd_r=CYS25V3W021MSYPQ32FB&pf_rd_r=CYS25V3W021MSYPQ32FB&pf_rd_t=101&pf_rd_p=1ce3e975-c6e8-479a-8485-2e490b9f58a9&pf_rd_p=1ce3e975-c6e8-479a-8485-2e490b9f58a9&pf_rd_i=1389401031"]

    def parse(self, response):
        for post_link in response.xpath('//a/@href').extract():
            link = response.urljoin(post_link)
            yield scrapy.Request(link, callback=self.parse_post)

        # Checks if the main page has a link to next page if True keep parsing.
        next_page = response.xpath('(//a[@class="pagnNext"])[1]/@href').extract_first()
        if next_page:
            yield scrapy.Request(next_page, callback=self.parse)

    def parse_post(self, response):
        # Scrape title, content from post.
        for post in response.xpath('//li[contains(@class,"s-result-item  celwidget")]'):
            item = dict()
            item['Name'] = post.xpath('.//h2[contains(@class,"a-size-base s-inline  s-access-title  a-text-normal")]/text()').extract()
            item['Price'] = post.xpath('.//span[contains(@class,"a-size-base a-color-price s-price a-text-bold")]/text()').extract()
            item['Image'] = post.xpath('.//img[contains(@class,"s-access-image cfMarker")]/@src').extract()
            item['Link'] = post.xpath('.//a[contains(@class,"a-link-normal s-access-detail-page  s-color-twister-title-link a-text-normal")]/@href').extract()
            yield item

        # If the post page has a link to next page keep parsing.
        next_page = response.xpath('(//a[@class="pagnNext"])[1]/@href').extract_first()
        if next_page:
            yield scrapy.Request(next_page, callback=self.parse_post)