Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import scrapy
- from scrapy_splash import SplashRequest
- class MySpider(scrapy.Spider):
- start_urls = ["http://example.com", "http://example.com/foo"]
- def start_requests(self):
- for url in self.start_urls:
- yield SplashRequest(url, self.parse,
- endpoint='render.html',
- args={'wait': 0.5},
- )
- def parse(self, response):
- # response.body is a result of render.html call; it
- # contains HTML processed by a browser.
- # …
- import scrapy
- from scrapy_splash import SplashRequest
- class MySpider(scrapy.Spider):
- first_page_url = "http://example.com"
- def get_page_contents(self):
- # fetch first page contents
- page_contents = extract_content(response)
- # Select the select control from page
- for element in selected_control:
- click_element(element, extract_content)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement