Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- category_element = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, category.get_attribute('class'))))
- from selenium import webdriver
- from time import sleep
- from selenium.webdriver.common.keys import Keys
- from selenium.webdriver.support.ui import WebDriverWait
- from selenium.webdriver.common.by import By
- from selenium.webdriver.support import expected_conditions as EC
- def scrape_item_links(self, is_first = True):
- marketplace_button = self.driver.find_element_by_xpath('//div[contains(text(), "Marketplace")]')
- marketplace_button.click()
- #create a list of each section and loop through list
- sections = [self.driver.find_element_by_xpath('//div[contains(text(), "Home & Garden")]'),
- self.driver.find_element_by_xpath('//div[contains(text(), "Entertainment")]'),
- self.driver.find_element_by_xpath('//div[contains(text(), "Clothing & Accessories")]'),
- self.driver.find_element_by_xpath('//div[contains(text(), "Family")]'),
- self.driver.find_element_by_xpath('//div[contains(text(), "Electronics")]'),
- self.driver.find_element_by_xpath('//div[contains(text(), "Hobbies")]'),
- self.driver.find_element_by_xpath('//div[contains(text(), "Vehicles & Bicycles")]')]
- for category in sections:
- if is_first:
- category.click()
- is_first = False
- else:
- pass
- #try:
- for _ in range(10):
- self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
- sleep(1.5)
- full_items_list = self.driver.find_elements_by_xpath("//a[@class='_1oem']")
- if self.used_item_links == 0: #False or None?
- self.used_item_links = [item.get_attribute('href') for item in full_items_list]
- else:
- #append or extend or add?
- self.used_item_links.extend([item.get_attribute('href') for item in full_items_list])
- print(self.used_item_links)
- wait = WebDriverWait(self.driver, 10)
- category_element = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, category.get_attribute('class'))))
- category_element.click()
Add Comment
Please, Sign In to add comment