import scrapy
from selenium import webdriver
from time import sleep


class ProductSpider(scrapy.Spider):
    name = "card"
    allowed_domains = ['moneyfacts.co.uk']
    start_urls = ['https://moneyfacts.co.uk/credit-cards/balance-transfer-credit-cards/?fbclid=IwAR05-Sa1hIcYTRx8DXYYQd0UfDRjWF-jD2-u51jiLP-WKlkxSddKjzUcnWA']

    def __init__(self):
        self.driver = webdriver.Chrome()

    def parse(self, response):
        self.driver.get(response.url)

        while True:
            next = self.driver.find_element_by_xpath('//*[text()="Show More"]')

            try:
                next.click()

                lists = self.driver.find_elements_by_xpath(
                    '//ul[@id="finder-table"]/li')
                for list in lists:
                    yield{
                        'Name': list.xpath('./*[@class="table-item-heading-product-name"]/span/strong/text()').get(),
                        'Title': list.xpath('./*[@class="table-item-heading-product-name"]/span/text()').get()
                    }

            except:
                break

        self.driver.close()