Protergo Exercise #1 by Rama

const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs');

const websiteUrls = [
  'https://example.com/website1',
  'https://example.com/website2',
  // Add more URLs as needed
];

// Function to scrape product information from a website
async function scrapeWebsite(url) {
  try {
    const response = await axios.get(url);
    const $ = cheerio.load(response.data);

    // Extract product information using CSS selectors
    const products = [];
    $('.product').each((index, element) => {
      const name = $(element).find('.product-name').text().trim();
      const price = $(element).find('.product-price').text().trim();
      const details = $(element).find('.product-details').text().trim();

      products.push({ name, price, details });
    });

    return products;
  } catch (error) {
    console.error(`Error scraping ${url}: ${error.message}`);
    return [];
  }
}

// Function to scrape all websites and save the data in a structured way
async function scrapeWebsites() {
  try {
    const allProducts = [];

    for (const url of websiteUrls) {
      const products = await scrapeWebsite(url);
      allProducts.push(...products);
    }

    // Save the data as JSON
    const data = JSON.stringify(allProducts, null, 2);
    fs.writeFileSync('product_data.json', data);

    console.log('Scraping completed successfully!');
  } catch (error) {
    console.error('Error scraping websites:', error);
  }
}

// Start the scraping process
scrapeWebsites();