Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import scrapy
- from urllib.parse import urlencode
- def get_zenrows_api_url(url, api_key):
- # Define the necessary parameters
- payload = {
- 'url': url,
- 'js_render': 'true',
- 'js_instructions': '[{"wait": 500}]',
- 'premium_proxy': 'true',
- 'antibot': 'true',
- }
- # Construct the API URL
- api_url = f'https://api.zenrows.com/v1/?apikey={api_key}&{urlencode(payload)}'
- return api_url
- class TestSpider(scrapy.Spider):
- name = 'test'
- def start_requests(self):
- urls = [
- 'https://angular.io/docs',
- ]
- api_key = 'Your_API_Key'
- for url in urls:
- # make a GET request using the ZenRows API URL
- api_url = get_zenrows_api_url(url, api_key)
- yield scrapy.Request(api_url, callback=self.parse)
- def parse(self, response):
- # Extract and print the title tag
- title = response.css('title::text').get()
- yield {'title': title}
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement