Advertisement
Guest User

Untitled

a guest
Sep 21st, 2017
56
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.96 KB | None | 0 0
  1. from selenium import webdriver
  2. driver= webdriver.Firefox()
  3. driver.get("https://www.website.com")
  4. f=open('output.txt','w')
  5. f.write(driver.page_source.encode('utf-8'))
  6. f.close()
  7. driver.quit()
  8.  
  9. import mechanize
  10. browser = mechanize.Browser()
  11. browser.set_handle_robots(False)
  12. cookies = mechanize.CookieJar()
  13. browser.set_cookiejar(cookies)
  14. browser.addheaders = [('User-agent', 'Mozilla/5.0')]
  15. browser.set_handle_refresh(False)
  16. browser.open("https://www.website.com")
  17.  
  18. from bs4 import BeautifulSoup as BS
  19. soup= BS(browser.response().read(),'lxml')
  20. print(soup.find(id="div_id"))
  21.  
  22. <div id="div_id" data referrer="div_id">
  23.  
  24. from lxml import etree
  25. parser= etree.HTMLParser()
  26. tree= etree.parse(open('source.txt'),parser)
  27. results= tree.xpath('//div[@id="div_id"]')
  28. print(etree.tostring(results[0]))
  29.  
  30. import requests
  31. from fake_useragent import UserAgent
  32. ua=UserAgent()
  33. url= 'https://www.website.com'
  34. headers= {'User-agent': str(ua.chrome)}
  35. page = requests.get(url, headers=headers)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement