import requests
from bs4 import BeautifulSoup
import csv

base_url = 'https://scrapingclub.com/exercise/list_infinite_scroll/'
page_number = 1  # Start with the base URL
total_pages = 6
product_data = []

while page_number <= total_pages:
    # Construct ajax request URL
    url = f'{base_url}?page={page_number}'

    # Make GET request
    response = requests.get(url)

    # Retrieve the response content
    html_content = response.text

    # Parse the HTML content using Beautiful Soup
    soup = BeautifulSoup(html_content, 'html.parser')

    # Extract product names and prices
    products = soup.select('div.p-4 h4 > a')
    prices = soup.select('div.p-4 h5')

    # Collect product information
    for product, price in zip(products, prices):
        product_name = product.get_text(strip=True)
        product_price = price.get_text(strip=True)

        #add data to product_data []
        product_data.append({'name': product_name, 'price': product_price})

    # Move to the next page
    page_number += 1