Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import requests
- from bs4 import BeautifulSoup
- base_url = 'https://scrapingclub.com/exercise/list_infinite_scroll/'
- page_number = 1 # Start with the base URL
- total_pages = 6
- while True:
- # Construct ajax request URL
- url = f'{base_url}?page={page_number}'
- # Make GET request
- response = requests.get(url)
- # Retrieve the response content
- html_content = response.text
- # Move to the next page
- page_number += 1
- if page_number > total_pages:
- break # Stop the loop
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement