import scrapy
from scrapy import Selector
from scrapy_selenium import SeleniumRequest
from selenium.common.exceptions import NoSuchElementException

class WinesSpider(scrapy.Spider):
    name = 'wine'

    responses = []

    def start_requests(self):
        yield SeleniumRequest(
            url='https://www.getwines.com/category_Wine',
            callback=self.parse
        )

    def parse(self, response):
        driver = response.meta['driver']
        intial_page = driver.page_source
        self.responses.append(intial_page)
        found = True
        while found:
            try:
                next_page = driver.find_element_by_xpath("//a//b[text()= '>>' ")
                href = next_page.get_attribute('href')
                driver.click(href)
                driver.implicitly_wait(10)
                self.responses.append(driver.page_source)

            except NoSuchElementException:
                break

        for resp in self.responses:
            r = Selector(text=resp)
            products = r.xpath("(//div[@class='layMain']//tbody)[5]/tr")
            for product in products:
                yield {
                    'product_name':
                    product.xpath(".//a[@class='Srch-producttitle']/text()").get()
}