Advertisement
Guest User

Untitled

a guest
Apr 15th, 2016
93
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 0.80 KB | None | 0 0
  1. ## Spider
  2. import json
  3. import base64
  4. import scrapy
  5. import requests
  6. import logging
  7. from tutorial.items import ProxyTestItem
  8. from scrapy_splash import SplashRequest
  9.  
  10. class SplashSpider(scrapy.Spider):
  11.  
  12.     http_user = '5b0056be4659470ab4aaabfaf71239b1'
  13.     http_pass = ''
  14.  
  15.     name = "proxy_test"
  16.     download_delay = 2
  17.     start_urls = [ "http://scrapinghub.com" ]
  18.  
  19.     def start_requests(self):
  20.         for url in self.start_urls:
  21.             splash_args = {
  22.                 'html': 1,
  23.                 'png': 1,
  24.                 'width': 600,
  25.                 'render_all': 1,
  26.             }
  27.             yield SplashRequest(url, self.parse, endpoint='render.html', args=splash_args)
  28.  
  29.  
  30.  
  31.     def parse(self, response):
  32.         print(logging.info(response.text))
  33.         print('HELLOOOOOOOOOOOO')
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement