import scrapy
from scrapy import FormRequest
from scrapy.exceptions import CloseSpider
import json

class BaierlSpider(scrapy.Spider):
    name = 'rental'

    def start_requests(self):
        yield scrapy.Request(
            url='https://rent.591.com.hk/?hl=en-us#list',
            method = 'GET',
            headers={'X-Requested-With':'XMLHttpRequest',
                    'Referer':'https://rent.591.com.hk/?hl=en-us'},
            meta={
                'current_page': 1,
                'total_page': 275},
            callback=self.parse
             )

    def parse(self, response):
        resp = json.loads(response.body)
        hits = resp.get('items')
        for h in hits:
            yield {
                'Title': h.get('title'),
                'pricce':h.get('price')}


        total_page_num = response.meta['total_page']
        next_page = response.meta['current_page']+1

        payload = {
            'm':'home',
            'c':'search',
            'a':'rslist',
            'v':'new',
            'type':'1',
            'region':'6',
            'searchtype':'1',
            'p':str(next_page),
            'role':''
                }


        if next_page <= total_page_num:

            yield scrapy.Request(
                url='https://rent.591.com.hk/?hl=en-us#list',
                meta={
                    'current_page': next_page
                },
                callback=self.parse,
                method = 'GET',
                headers={'X-Requested-With':'XMLHttpRequest',
                        'Referer':'https://rent.591.com.hk/?hl=en-us'},
                body = payload
            )