inspect_reponse

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from scrapy.http import FormRequest
from scrapy.http.cookies import CookieJar
from scrapy.shell import inspect_response       # Facilitates breaking out into a shell (debug)
from scrapy.utils.response import open_in_browser
from scrapy import log


class LKQSweepsEntrySpider(BaseSpider):
    '''
    Enters the LKQ daily contest automagically
    scrapy crawl --loglevel=WARNING lkqsweepsentry
    '''
    name = 'lkqsweepsentry'
    allowed_domains = ["lkqpickyourpartsweeps.com"]
    start_urls = ['http://www.lkqpickyourpartsweeps.com/lkq13c/pickyourpart/enter_picks']
    usernames = ['test1@gmail.com', 'test2@gmail.com']
    password = 'password'

    def parse(self, response):
        for jar,username in enumerate(self.usernames):
            self.log('Attempting entry with %s'%username, level=log.INFO)
            yield FormRequest.from_response(response,
                    formdata={'username': username, 'password': self.password},
                    callback=self.after_login,
                    dont_filter=True, # Disable filtering of duplicate URLs
                    meta = {'username': username, 'cookiejar': jar})

    def after_login(self, response):
        # verify login success and hit the submit button
        username = response.meta['username']
        self.log('Cookie jar for %s is #%s'%(username, response.meta['cookiejar']), level=log.INFO)
        hxs = HtmlXPathSelector(response)
        submitButtonXPath = '//*[@id="enter_picks-form-submit"]'

        if hxs.select(submitButtonXPath).extract():
            # We got the submit button, let's click it to enter
            self.log("Successfully logged in with %s"%username, level=log.INFO)
            yield FormRequest.from_response(response,
                    formname='enter_picks_form',
                    formdata={},
                    callback=self.after_submit,
                    dont_filter=True, # Disable filtering of duplicate URLs
                    meta={'username': username, 'cookiejar': response.meta['cookiejar']})
        else:
            # There was some sort of error
            loginStatus = hxs.select('//*[@id="user-access-note"]/text()').extract()[1]
            if "Invalid login" in loginStatus:
                self.log("Login failed for %s"%username, level=log.ERROR)
            elif "already entered" in loginStatus:
                self.log("%s already entered the contest, aborting!"%username, level=log.WARNING)

    def after_submit(self, response):
        # verify login success and hit the submit button
        print response.meta
        inspect_response(response)
        username = response.meta['username']
        hxs = HtmlXPathSelector(response)
        submissionStatus = hxs.select('//*[@id="pe-content"]/div[1]/p[1]').extract()
        if 'submission has been received' in submissionStatus:
            self.log("Successful Entry for %s"%username, level=log.INFO)
        else:
            self.log("Unsuccessful Entry for %s"%username, level=log.ERROR)
            print response.body
        return