Advertisement
dereksir

Untitled

Aug 25th, 2023 (edited)
128
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 0.53 KB | None | 0 0
  1. import requests
  2. from bs4 import BeautifulSoup
  3.  
  4. base_url = 'https://scrapingclub.com/exercise/list_infinite_scroll/'
  5. page_number = 1  # Start with the base URL
  6. total_pages = 6
  7.  
  8. while True:
  9.     # Construct ajax request URL
  10.     url = f'{base_url}?page={page_number}'
  11.    
  12.     # Make GET request
  13.     response = requests.get(url)
  14.    
  15.     # Retrieve the response content
  16.     html_content = response.text
  17.        
  18.     # Move to the next page
  19.     page_number += 1
  20.  
  21.     if page_number > total_pages:
  22.         break  # Stop the loop
  23.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement