Fazlul

https://pastebin.com/?fbclid=IwAR2vmDumim7mv-bDvj9JYk2vJARHG1zHXJcRuFe0l1UF90HgJp0ERnefQA0

May 24th, 2021
83
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.30 KB | None | 0 0
  1. import scrapy
  2. import json
  3.  
  4. class ProductsSpider(scrapy.Spider):
  5. name = 'products'
  6. page_num={'page':0}
  7.  
  8. body = '''
  9. {"requests":[{"indexName":"shopify_products","params":"highlightPreTag=%3Cais-highlight-0000000000%3E&highlightPostTag=%3C%2Fais-highlight-0000000000%3E&clickAnalytics=true&query=laptop&filters=(price%20%3E%200%20AND%20product_published%20%3D%201%20AND%20availability.displayProduct%20%3D%201)&hitsPerPage=36&distinct=true&maxValuesPerFacet=100&page=1&facets=%5B%22facets.Brand%22%2C%22facets.Price%22%2C%22facets.Category%22%2C%22facets.Availability%22%5D&tagFilters="}]}
  10. '''
  11.  
  12. def start_requests(self):
  13. yield scrapy.Request(
  14. url='https://vtvkm5urpx-1.algolianet.com/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20JavaScript%20(4.8.6)%3B%20Browser',
  15. method='POST',
  16. headers={
  17. 'content-type': 'application/x-www-form-urlencoded',
  18. 'x-algolia-api-key': 'a0c0108d737ad5ab54a0e2da900bf040',
  19. 'x-algolia-application-id': 'VTVKM5URPX'
  20. },
  21. body=self.body,
  22. callback=self.parse
  23. )
  24.  
  25. def parse(self, response):
  26. resp = json.loads(response.body)
  27. hits = resp.get('results')[0].get('hits')
  28. for h in hits:
  29. yield {
  30. 'title': h.get('primary_title')
  31. }
  32.  
  33. per_page= resp.get('results')[0].get('page')
  34. total_page = resp.get('results')[0].get('nbPages')
  35. per_hit=resp.get('results')[0].get('hitsPerPage')
  36. total_hits = resp.get('results')[0].get('nbHits')
  37.  
  38. if self.page_num['page'] <=total_page & per_hit<= total_hits:
  39. self.page_num['page'] +=1
  40.  
  41. yield scrapy.Request(
  42. url='https://vtvkm5urpx-1.algolianet.com/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20JavaScript%20(4.8.6)%3B%20Browser',
  43. method='POST',
  44. headers={
  45. 'content-type': 'application/x-www-form-urlencoded',
  46. 'x-algolia-api-key': 'a0c0108d737ad5ab54a0e2da900bf040',
  47. 'x-algolia-application-id': 'VTVKM5URPX'
  48. },
  49. body=self.body,
  50. callback=self.parse,
  51. )
  52.  
Advertisement
Add Comment
Please, Sign In to add comment