quotes_spider_pt4.py

import scrapy
from lesson_03_scraping.items import Lesson03ScrapingItem
from scrapy.loader import ItemLoader

class QuotesSpider(scrapy.Spider):
    name = "quotes_pt4"

    start_urls = ['http://quotes.toscrape.com/page/1/']

    def parse(self, response):
        for quote in response.css('div.quote'):
            loader = ItemLoader(item=Lesson03ScrapingItem(), selector=quote)
            loader.add_css('quote_content', 'span.text::text')
            loader.add_css('author', 'small.author::text')
            loader.add_css('tags', 'div.tags a.tag::text')
            quote_item = loader.load_item()
            yield quote_item

        next_page = response.css('li.next a::attr(href)').get()
        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)