Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env python3
- """
- Targeted HTML Gift Card Scraper for GCX
- Extracts card value, discount, and your price using specific CSS classes
- """
- import time
- import re
- import logging
- import pandas as pd
- from typing import List, Dict
- from selenium import webdriver
- from selenium.webdriver.chrome.options import Options
- from selenium.webdriver.common.by import By
- from selenium.webdriver.support.ui import WebDriverWait
- from selenium.webdriver.support import expected_conditions as EC
- from selenium.common.exceptions import TimeoutException, NoSuchElementException
- from bs4 import BeautifulSoup
- # Set up logging
- logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
- logger = logging.getLogger(__name__)
- class TargetedHTMLGiftCardScraper:
- def __init__(self, headless: bool = False):
- self.headless = headless
- self.driver = None
- self.all_gift_cards = []
- self.base_url = "https://gcx.raise.com/buy-gift-cards"
- def setup_driver(self):
- """Set up Chrome driver with appropriate options"""
- chrome_options = Options()
- if self.headless:
- chrome_options.add_argument('--headless')
- # Options to avoid detection
- chrome_options.add_argument('--no-sandbox')
- chrome_options.add_argument('--disable-dev-shm-usage')
- chrome_options.add_argument('--disable-blink-features=AutomationControlled')
- chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
- chrome_options.add_experimental_option('useAutomationExtension', False)
- chrome_options.add_argument('--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
- chrome_options.add_argument('--window-size=1920,1080')
- try:
- self.driver = webdriver.Chrome(options=chrome_options)
- self.driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
- return True
- except Exception as e:
- logger.error(f"Failed to setup Chrome driver: {e}")
- return False
- def navigate_to_first_page(self):
- """Navigate to the first page of GCX gift cards"""
- try:
- logger.info(f"Navigating to first page: {self.base_url}")
- self.driver.get(self.base_url)
- # Wait for page to load completely
- WebDriverWait(self.driver, 20).until(
- EC.presence_of_element_located((By.TAG_NAME, "body"))
- )
- # Wait for gift card content to load
- try:
- WebDriverWait(self.driver, 10).until(
- EC.presence_of_element_located((By.XPATH, "//*[contains(text(), 'Gift Cards')]"))
- )
- except TimeoutException:
- logger.warning("Gift card content not found, continuing anyway")
- # Additional wait for dynamic content
- time.sleep(3)
- return True
- except Exception as e:
- logger.error(f"Error navigating to first page: {e}")
- return False
- def scroll_and_load_content(self):
- """Scroll the page to load all dynamic content"""
- try:
- # Get initial page height
- last_height = self.driver.execute_script("return document.body.scrollHeight")
- # Scroll down gradually
- scroll_pause_time = 2
- scroll_attempts = 0
- max_scroll_attempts = 5
- while scroll_attempts < max_scroll_attempts:
- # Scroll down to bottom
- self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
- # Wait to load page
- time.sleep(scroll_pause_time)
- # Calculate new scroll height and compare with last scroll height
- new_height = self.driver.execute_script("return document.body.scrollHeight")
- if new_height == last_height:
- break
- last_height = new_height
- scroll_attempts += 1
- # Scroll back to top
- self.driver.execute_script("window.scrollTo(0, 0);")
- time.sleep(1)
- except Exception as e:
- logger.warning(f"Error during scrolling: {e}")
- def extract_gift_card_links_from_first_page(self) -> List[str]:
- """Extract gift card links from the first page only"""
- links = []
- try:
- # Get page source and parse with BeautifulSoup
- soup = BeautifulSoup(self.driver.page_source, 'html.parser')
- # Look for gift card links - more specific selectors
- gift_card_links = soup.find_all('a', href=True)
- for link in gift_card_links:
- href = link.get('href', '')
- text = link.get_text(strip=True)
- # Check if this is a gift card link
- if 'buy-' in href and 'gift-cards' in href and 'Gift Cards' in text:
- # Make sure it's a full URL
- if href.startswith('/'):
- full_url = f"https://gcx.raise.com{href}"
- elif href.startswith('http'):
- full_url = href
- else:
- continue
- # Extract brand name from text
- brand = self.clean_brand_name(text)
- if brand and brand not in [card.get('Brand', '') for card in self.all_gift_cards]:
- links.append(full_url)
- logger.info(f"Found {len(links)} gift card links on first page")
- return links
- except Exception as e:
- logger.error(f"Error extracting links from first page: {e}")
- return []
- def clean_brand_name(self, text: str) -> str:
- """Clean brand name from link text"""
- # Remove "Buy Gift Card" prefix
- brand = re.sub(r'^Buy Gift Card\s+', '', text, flags=re.IGNORECASE)
- # Remove "Gift Cards" suffix
- brand = re.sub(r'\s+Gift Cards?.*$', '', brand, flags=re.IGNORECASE)
- # Clean up extra spaces and special characters
- brand = re.sub(r'\s+', ' ', brand).strip()
- return brand
- def scrape_individual_gift_card_page(self, url: str) -> List[Dict]:
- """Scrape individual gift card page using specific CSS classes"""
- try:
- logger.info(f"Scraping individual gift card page: {url}")
- self.driver.get(url)
- # Wait for page to load
- WebDriverWait(self.driver, 15).until(
- EC.presence_of_element_located((By.TAG_NAME, "body"))
- )
- # Additional wait for dynamic content
- time.sleep(3)
- # Get page source and parse
- soup = BeautifulSoup(self.driver.page_source, 'html.parser')
- # Extract brand name from URL
- brand = self.extract_brand_from_url(url)
- # Extract gift card details using specific CSS classes
- gift_cards = self.extract_gift_cards_with_css_classes(soup, brand)
- return gift_cards
- except Exception as e:
- logger.error(f"Error scraping individual gift card page {url}: {e}")
- return []
- def extract_brand_from_url(self, url: str) -> str:
- """Extract brand name from URL"""
- try:
- # Extract brand from URL like "buy-netflix-gift-cards"
- if 'buy-' in url and '-gift-cards' in url:
- brand_part = url.split('buy-')[1].split('-gift-cards')[0]
- # Convert to proper case
- brand = brand_part.replace('-', ' ').title()
- return brand
- return "Unknown Brand"
- except:
- return "Unknown Brand"
- def extract_gift_cards_with_css_classes(self, soup: BeautifulSoup, brand: str) -> List[Dict]:
- """Extract gift cards using the specific CSS classes provided"""
- gift_cards = []
- try:
- # Look for the specific CSS classes from the HTML structure provided
- # grouped-listing_listingRow__T3MMf - the main row container
- # grouped-listing_value__38ZMD - contains the card value and your price
- # grouped-listing_discount__2aokP - contains the discount percentage
- # Find all rows with the specific class
- listing_rows = soup.find_all('div', class_=re.compile(r'grouped-listing_listingRow__'))
- for row in listing_rows:
- try:
- # Extract card value (first grouped-listing_value__38ZMD)
- value_elements = row.find_all('div', class_=re.compile(r'grouped-listing_value__'))
- if len(value_elements) >= 2:
- card_value = value_elements[0].get_text(strip=True)
- your_price = value_elements[1].get_text(strip=True)
- else:
- continue
- # Extract discount percentage
- discount_element = row.find('div', class_=re.compile(r'grouped-listing_discount__'))
- if discount_element:
- discount_text = discount_element.get_text(strip=True)
- # Extract just the percentage number
- discount_match = re.search(r'(\d+(?:\.\d+)?)', discount_text)
- if discount_match:
- discount = discount_match.group(1)
- else:
- continue
- else:
- continue
- # Clean up the values
- card_value = card_value.replace('$', '').strip()
- your_price = your_price.replace('$', '').strip()
- gift_cards.append({
- 'Brand': brand,
- 'Card_Value': f"${card_value}",
- 'Discount': f"{discount}%",
- 'Your_Price': f"${your_price}"
- })
- except Exception as e:
- logger.warning(f"Error processing row: {e}")
- continue
- # Alternative method: Look for the specific class patterns in the entire page
- if not gift_cards:
- # Find all elements with the value class
- value_elements = soup.find_all('div', class_=re.compile(r'grouped-listing_value__'))
- discount_elements = soup.find_all('div', class_=re.compile(r'grouped-listing_discount__'))
- # Try to pair them up
- for i in range(0, len(value_elements), 2):
- if i + 1 < len(value_elements):
- card_value = value_elements[i].get_text(strip=True)
- your_price = value_elements[i + 1].get_text(strip=True)
- # Find the closest discount element
- if discount_elements:
- discount_text = discount_elements[i // 2].get_text(strip=True) if i // 2 < len(discount_elements) else "0%"
- discount_match = re.search(r'(\d+(?:\.\d+)?)', discount_text)
- if discount_match:
- discount = discount_match.group(1)
- else:
- continue
- else:
- continue
- # Clean up the values
- card_value = card_value.replace('$', '').strip()
- your_price = your_price.replace('$', '').strip()
- gift_cards.append({
- 'Brand': brand,
- 'Card_Value': f"${card_value}",
- 'Discount': f"{discount}%",
- 'Your_Price': f"${your_price}"
- })
- # Method 3: Look for the specific HTML structure pattern
- if not gift_cards:
- # Look for the exact pattern from the provided HTML
- page_text = soup.get_text()
- # Pattern: $200.00, 2.00% OFF, $196.00
- pattern = r'\$(\d+(?:\.\d+)?)\s*,\s*(\d+(?:\.\d+)?)%\s*OFF\s*,\s*\$(\d+(?:\.\d+)?)'
- matches = re.findall(pattern, page_text)
- for card_value, discount, your_price in matches:
- gift_cards.append({
- 'Brand': brand,
- 'Card_Value': f"${card_value}",
- 'Discount': f"{discount}%",
- 'Your_Price': f"${your_price}"
- })
- # Method 4: Look for any elements containing the specific class names
- if not gift_cards:
- # Find all elements with the specific class patterns
- all_elements = soup.find_all('div', class_=re.compile(r'grouped-listing_'))
- values = []
- discounts = []
- for element in all_elements:
- class_name = element.get('class', [])
- if any('value__' in cls for cls in class_name):
- values.append(element.get_text(strip=True))
- elif any('discount__' in cls for cls in class_name):
- discounts.append(element.get_text(strip=True))
- # Pair them up
- for i in range(0, len(values), 2):
- if i + 1 < len(values) and i // 2 < len(discounts):
- card_value = values[i].replace('$', '').strip()
- your_price = values[i + 1].replace('$', '').strip()
- discount_text = discounts[i // 2]
- discount_match = re.search(r'(\d+(?:\.\d+)?)', discount_text)
- if discount_match:
- discount = discount_match.group(1)
- gift_cards.append({
- 'Brand': brand,
- 'Card_Value': f"${card_value}",
- 'Discount': f"{discount}%",
- 'Your_Price': f"${your_price}"
- })
- # Remove duplicates
- seen = set()
- unique_gift_cards = []
- for card in gift_cards:
- key = (card['Brand'], card['Card_Value'], card['Your_Price'])
- if key not in seen:
- seen.add(key)
- unique_gift_cards.append(card)
- logger.info(f"Extracted {len(unique_gift_cards)} gift card variations for {brand}")
- return unique_gift_cards
- except Exception as e:
- logger.error(f"Error extracting gift card details: {e}")
- return []
- def scrape_first_page_only(self) -> List[Dict]:
- """Scrape only the first page and follow each gift card link for accurate information"""
- if not self.setup_driver():
- return []
- try:
- # Navigate to first page
- if not self.navigate_to_first_page():
- logger.error("Failed to navigate to first page")
- return []
- # Scroll to load content
- self.scroll_and_load_content()
- # Extract gift card links from first page
- gift_card_links = self.extract_gift_card_links_from_first_page()
- if gift_card_links:
- logger.info(f"Found {len(gift_card_links)} gift card links on first page")
- # Follow each gift card link
- for link in gift_card_links:
- try:
- gift_cards = self.scrape_individual_gift_card_page(link)
- if gift_cards:
- self.all_gift_cards.extend(gift_cards)
- logger.info(f"Added {len(gift_cards)} gift cards from {link}")
- # Be respectful with delays
- time.sleep(2)
- except Exception as e:
- logger.error(f"Error processing link {link}: {e}")
- continue
- else:
- logger.warning("No gift card links found on first page")
- return self.all_gift_cards
- except Exception as e:
- logger.error(f"Error during scraping: {e}")
- return self.all_gift_cards
- finally:
- if self.driver:
- self.driver.quit()
- def save_data(self, gift_cards: List[Dict], csv_filename: str = "gcx_targeted_html_gift_cards.csv", excel_filename: str = "gcx_targeted_html_gift_cards.xlsx"):
- """Save scraped data to CSV and Excel files"""
- if not gift_cards:
- logger.warning("No data to save")
- return None
- # Create DataFrame
- df = pd.DataFrame(gift_cards)
- # Remove duplicates based on brand name, card value, and your price
- df = df.drop_duplicates(subset=['Brand', 'Card_Value', 'Your_Price'], keep='first')
- # Sort by Brand name, then by Card Value
- df = df.sort_values(['Brand', 'Card_Value'], ascending=[True, True])
- # Save to CSV
- df.to_csv(csv_filename, index=False)
- logger.info(f"Saved {len(df)} gift cards to {csv_filename}")
- # Save to Excel
- df.to_excel(excel_filename, index=False, engine='openpyxl')
- logger.info(f"Saved {len(df)} gift cards to {excel_filename}")
- return df
- def analyze_results(self, gift_cards: List[Dict]):
- """Analyze the scraped results"""
- if not gift_cards:
- return
- print(f"\n" + "="*80)
- print("TARGETED HTML GIFT CARD SCRAPING RESULTS")
- print("="*80)
- print(f"Total gift cards scraped: {len(gift_cards)}")
- if gift_cards:
- # Show first 30
- print(f"\nFirst 30 gift cards:")
- print("-" * 100)
- print(f"{'Rank':<4} {'Brand':<25} {'Card Value':<12} {'Discount':<10} {'Your Price':<12}")
- print("-" * 100)
- for i, card in enumerate(gift_cards[:30], 1):
- print(f"{i:<4} {card['Brand']:<25} {card['Card_Value']:<12} {card['Discount']:<10} {card['Your_Price']:<12}")
- # Show summary by brand
- df = pd.DataFrame(gift_cards)
- brand_summary = df.groupby('Brand').agg({
- 'Card_Value': 'count',
- 'Discount': lambda x: f"{x.iloc[0]}" if len(x) > 0 else "N/A"
- }).rename(columns={'Card_Value': 'Card_Count'})
- print(f"\nSummary by Brand:")
- print("-" * 70)
- print(f"{'Brand':<25} {'Cards':<10} {'Sample Discount':<15}")
- print("-" * 70)
- for brand, row in brand_summary.iterrows():
- print(f"{brand:<25} {row['Card_Count']:<10} {row['Discount']:<15}")
- # Show examples of different prices for the same brand
- print(f"\nExamples of different prices for the same brand:")
- print("-" * 80)
- for brand in df['Brand'].unique()[:5]: # Show first 5 brands
- brand_cards = df[df['Brand'] == brand]
- if len(brand_cards) > 1:
- print(f"\n{brand}:")
- for _, card in brand_cards.head(5).iterrows():
- print(f" {card['Card_Value']} - {card['Discount']} - You pay {card['Your_Price']}")
- def main():
- """Main function"""
- logger.info("Starting targeted HTML gift card scraper for GCX...")
- # Try with visible browser first (better for debugging)
- scraper = TargetedHTMLGiftCardScraper(headless=False)
- gift_cards = scraper.scrape_first_page_only()
- if gift_cards:
- logger.info(f"Successfully scraped {len(gift_cards)} gift cards from first page")
- # Save data
- df = scraper.save_data(gift_cards)
- # Analyze results
- scraper.analyze_results(gift_cards)
- print(f"\nData saved to:")
- print(f"- gcx_targeted_html_gift_cards.csv")
- print(f"- gcx_targeted_html_gift_cards.xlsx")
- print(f"\nFormat: Brand | Card Value | Discount | Your Price")
- print(f"Where:")
- print(f"- Brand: Company name")
- print(f"- Card Value: Original gift card value (e.g., $5, $10, $25, $50, $100)")
- print(f"- Discount: Discount percentage for that specific amount")
- print(f"- Your Price: Actual price you pay for the gift card")
- else:
- logger.error("No gift cards found")
- print("\nNo gift cards found. This could be due to:")
- print("1. Website structure changes")
- print("2. Anti-bot protection")
- print("3. Network issues")
- print("4. Chrome driver not installed")
- if __name__ == "__main__":
- main()
Advertisement
Add Comment
Please, Sign In to add comment