import scrapy

from scrapy.selector import Selector

from selenium import webdriver

from selenium.webdriver.common.action_chains import ActionChains

from time import sleep


class ProductSpider(scrapy.Spider):

    name = "card"
    allowed_domains = ['moneyfacts.co.uk']
    start_urls = ['https://moneyfacts.co.uk/credit-cards/balance-transfer-credit-cards/?fbclid=IwAR05-Sa1hIcYTRx8DXYYQd0UfDRjWF-jD2-u51jiLP-WKlkxSddKjzUcnWA']


    def __init__(self):
        self.driver = webdriver.Chrome()


    def parse(self, response):
        self.driver.get(response.url)
        actions = ActionChains(self.driver)


        while True:
            next = self.driver.find_elements_by_css_selector("button#show-more")

            if next:
                last_height = self.driver.execute_script(
                    "return document.body.scrollHeight")
                self.driver.execute_script(
                    "window.scrollTo(0, document.body.scrollHeight);")
                actions.move_to_element(next[0]).click().perform()
                sleep(5)
                print(dir(self.driver))
                self.driver.maximize_window()

                lists = Selector(text=self.driver.page_source)

                for list in lists.xpath('//ul[@id="finder-table"]/li'):
                    yield{
                        'Name': list.xpath('.//*[@class="table-item-heading-product-name"]/span/strong/text()').get(),
                        'Title': list.xpath('.//*[@class="table-item-heading-product-name"]/span/text()').get()
                        }
            else:
                break

        self.driver.close()