Advertisement
furas

Python - Selenium - click "Load more"

Jun 7th, 2017
162
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 0.83 KB | None | 0 0
  1. import selenium.webdriver
  2. from bs4 import BeautifulSoup
  3. import time
  4.  
  5. news_category = [
  6.     "http://e.vnexpress.net/news/news"
  7. ]    
  8. #    "http://e.vnexpress.net/news/business",
  9. #    "http://e.vnexpress.net/news/travel-life",
  10. #    "http://e.vnexpress.net/news/world"
  11. #]
  12.  
  13. driver = selenium.webdriver.Chrome()
  14.  
  15. for url in news_category:
  16.     driver.get(url)
  17.     for x in range(5):
  18.         driver.find_element_by_id('vnexpress_folder_load_more').click()
  19.         #time.sleep(0.2)
  20.     time.sleep(2)
  21.    
  22.     soup = BeautifulSoup(driver.page_source, "html.parser")
  23.    
  24.     links = soup.find_all('a')
  25.     url_list = []
  26.    
  27.     for link in links:
  28.         all_link = link.get('href')
  29.         if all_link.startswith(url):
  30.             url_list.append(all_link)
  31.            
  32.     print(url_list)
  33.     print('count:', len(url_list))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement