jedtony

Scrape-Giftcards

Oct 9th, 2025
158
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 22.07 KB | None | 0 0
  1. #!/usr/bin/env python3
  2. """
  3. Targeted HTML Gift Card Scraper for GCX
  4. Extracts card value, discount, and your price using specific CSS classes
  5. """
  6.  
  7. import time
  8. import re
  9. import logging
  10. import pandas as pd
  11. from typing import List, Dict
  12. from selenium import webdriver
  13. from selenium.webdriver.chrome.options import Options
  14. from selenium.webdriver.common.by import By
  15. from selenium.webdriver.support.ui import WebDriverWait
  16. from selenium.webdriver.support import expected_conditions as EC
  17. from selenium.common.exceptions import TimeoutException, NoSuchElementException
  18. from bs4 import BeautifulSoup
  19.  
  20. # Set up logging
  21. logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
  22. logger = logging.getLogger(__name__)
  23.  
  24. class TargetedHTMLGiftCardScraper:
  25. def __init__(self, headless: bool = False):
  26. self.headless = headless
  27. self.driver = None
  28. self.all_gift_cards = []
  29. self.base_url = "https://gcx.raise.com/buy-gift-cards"
  30.  
  31. def setup_driver(self):
  32. """Set up Chrome driver with appropriate options"""
  33. chrome_options = Options()
  34.  
  35. if self.headless:
  36. chrome_options.add_argument('--headless')
  37.  
  38. # Options to avoid detection
  39. chrome_options.add_argument('--no-sandbox')
  40. chrome_options.add_argument('--disable-dev-shm-usage')
  41. chrome_options.add_argument('--disable-blink-features=AutomationControlled')
  42. chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
  43. chrome_options.add_experimental_option('useAutomationExtension', False)
  44. chrome_options.add_argument('--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
  45. chrome_options.add_argument('--window-size=1920,1080')
  46.  
  47. try:
  48. self.driver = webdriver.Chrome(options=chrome_options)
  49. self.driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
  50. return True
  51. except Exception as e:
  52. logger.error(f"Failed to setup Chrome driver: {e}")
  53. return False
  54.  
  55. def navigate_to_first_page(self):
  56. """Navigate to the first page of GCX gift cards"""
  57. try:
  58. logger.info(f"Navigating to first page: {self.base_url}")
  59. self.driver.get(self.base_url)
  60.  
  61. # Wait for page to load completely
  62. WebDriverWait(self.driver, 20).until(
  63. EC.presence_of_element_located((By.TAG_NAME, "body"))
  64. )
  65.  
  66. # Wait for gift card content to load
  67. try:
  68. WebDriverWait(self.driver, 10).until(
  69. EC.presence_of_element_located((By.XPATH, "//*[contains(text(), 'Gift Cards')]"))
  70. )
  71. except TimeoutException:
  72. logger.warning("Gift card content not found, continuing anyway")
  73.  
  74. # Additional wait for dynamic content
  75. time.sleep(3)
  76.  
  77. return True
  78. except Exception as e:
  79. logger.error(f"Error navigating to first page: {e}")
  80. return False
  81.  
  82. def scroll_and_load_content(self):
  83. """Scroll the page to load all dynamic content"""
  84. try:
  85. # Get initial page height
  86. last_height = self.driver.execute_script("return document.body.scrollHeight")
  87.  
  88. # Scroll down gradually
  89. scroll_pause_time = 2
  90. scroll_attempts = 0
  91. max_scroll_attempts = 5
  92.  
  93. while scroll_attempts < max_scroll_attempts:
  94. # Scroll down to bottom
  95. self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
  96.  
  97. # Wait to load page
  98. time.sleep(scroll_pause_time)
  99.  
  100. # Calculate new scroll height and compare with last scroll height
  101. new_height = self.driver.execute_script("return document.body.scrollHeight")
  102. if new_height == last_height:
  103. break
  104. last_height = new_height
  105. scroll_attempts += 1
  106.  
  107. # Scroll back to top
  108. self.driver.execute_script("window.scrollTo(0, 0);")
  109. time.sleep(1)
  110.  
  111. except Exception as e:
  112. logger.warning(f"Error during scrolling: {e}")
  113.  
  114. def extract_gift_card_links_from_first_page(self) -> List[str]:
  115. """Extract gift card links from the first page only"""
  116. links = []
  117.  
  118. try:
  119. # Get page source and parse with BeautifulSoup
  120. soup = BeautifulSoup(self.driver.page_source, 'html.parser')
  121.  
  122. # Look for gift card links - more specific selectors
  123. gift_card_links = soup.find_all('a', href=True)
  124.  
  125. for link in gift_card_links:
  126. href = link.get('href', '')
  127. text = link.get_text(strip=True)
  128.  
  129. # Check if this is a gift card link
  130. if 'buy-' in href and 'gift-cards' in href and 'Gift Cards' in text:
  131. # Make sure it's a full URL
  132. if href.startswith('/'):
  133. full_url = f"https://gcx.raise.com{href}"
  134. elif href.startswith('http'):
  135. full_url = href
  136. else:
  137. continue
  138.  
  139. # Extract brand name from text
  140. brand = self.clean_brand_name(text)
  141. if brand and brand not in [card.get('Brand', '') for card in self.all_gift_cards]:
  142. links.append(full_url)
  143.  
  144. logger.info(f"Found {len(links)} gift card links on first page")
  145. return links
  146.  
  147. except Exception as e:
  148. logger.error(f"Error extracting links from first page: {e}")
  149. return []
  150.  
  151. def clean_brand_name(self, text: str) -> str:
  152. """Clean brand name from link text"""
  153. # Remove "Buy Gift Card" prefix
  154. brand = re.sub(r'^Buy Gift Card\s+', '', text, flags=re.IGNORECASE)
  155.  
  156. # Remove "Gift Cards" suffix
  157. brand = re.sub(r'\s+Gift Cards?.*$', '', brand, flags=re.IGNORECASE)
  158.  
  159. # Clean up extra spaces and special characters
  160. brand = re.sub(r'\s+', ' ', brand).strip()
  161.  
  162. return brand
  163.  
  164. def scrape_individual_gift_card_page(self, url: str) -> List[Dict]:
  165. """Scrape individual gift card page using specific CSS classes"""
  166. try:
  167. logger.info(f"Scraping individual gift card page: {url}")
  168. self.driver.get(url)
  169.  
  170. # Wait for page to load
  171. WebDriverWait(self.driver, 15).until(
  172. EC.presence_of_element_located((By.TAG_NAME, "body"))
  173. )
  174.  
  175. # Additional wait for dynamic content
  176. time.sleep(3)
  177.  
  178. # Get page source and parse
  179. soup = BeautifulSoup(self.driver.page_source, 'html.parser')
  180.  
  181. # Extract brand name from URL
  182. brand = self.extract_brand_from_url(url)
  183.  
  184. # Extract gift card details using specific CSS classes
  185. gift_cards = self.extract_gift_cards_with_css_classes(soup, brand)
  186.  
  187. return gift_cards
  188.  
  189. except Exception as e:
  190. logger.error(f"Error scraping individual gift card page {url}: {e}")
  191. return []
  192.  
  193. def extract_brand_from_url(self, url: str) -> str:
  194. """Extract brand name from URL"""
  195. try:
  196. # Extract brand from URL like "buy-netflix-gift-cards"
  197. if 'buy-' in url and '-gift-cards' in url:
  198. brand_part = url.split('buy-')[1].split('-gift-cards')[0]
  199. # Convert to proper case
  200. brand = brand_part.replace('-', ' ').title()
  201. return brand
  202. return "Unknown Brand"
  203. except:
  204. return "Unknown Brand"
  205.  
  206. def extract_gift_cards_with_css_classes(self, soup: BeautifulSoup, brand: str) -> List[Dict]:
  207. """Extract gift cards using the specific CSS classes provided"""
  208. gift_cards = []
  209.  
  210. try:
  211. # Look for the specific CSS classes from the HTML structure provided
  212. # grouped-listing_listingRow__T3MMf - the main row container
  213. # grouped-listing_value__38ZMD - contains the card value and your price
  214. # grouped-listing_discount__2aokP - contains the discount percentage
  215.  
  216. # Find all rows with the specific class
  217. listing_rows = soup.find_all('div', class_=re.compile(r'grouped-listing_listingRow__'))
  218.  
  219. for row in listing_rows:
  220. try:
  221. # Extract card value (first grouped-listing_value__38ZMD)
  222. value_elements = row.find_all('div', class_=re.compile(r'grouped-listing_value__'))
  223. if len(value_elements) >= 2:
  224. card_value = value_elements[0].get_text(strip=True)
  225. your_price = value_elements[1].get_text(strip=True)
  226. else:
  227. continue
  228.  
  229. # Extract discount percentage
  230. discount_element = row.find('div', class_=re.compile(r'grouped-listing_discount__'))
  231. if discount_element:
  232. discount_text = discount_element.get_text(strip=True)
  233. # Extract just the percentage number
  234. discount_match = re.search(r'(\d+(?:\.\d+)?)', discount_text)
  235. if discount_match:
  236. discount = discount_match.group(1)
  237. else:
  238. continue
  239. else:
  240. continue
  241.  
  242. # Clean up the values
  243. card_value = card_value.replace('$', '').strip()
  244. your_price = your_price.replace('$', '').strip()
  245.  
  246. gift_cards.append({
  247. 'Brand': brand,
  248. 'Card_Value': f"${card_value}",
  249. 'Discount': f"{discount}%",
  250. 'Your_Price': f"${your_price}"
  251. })
  252.  
  253. except Exception as e:
  254. logger.warning(f"Error processing row: {e}")
  255. continue
  256.  
  257. # Alternative method: Look for the specific class patterns in the entire page
  258. if not gift_cards:
  259. # Find all elements with the value class
  260. value_elements = soup.find_all('div', class_=re.compile(r'grouped-listing_value__'))
  261. discount_elements = soup.find_all('div', class_=re.compile(r'grouped-listing_discount__'))
  262.  
  263. # Try to pair them up
  264. for i in range(0, len(value_elements), 2):
  265. if i + 1 < len(value_elements):
  266. card_value = value_elements[i].get_text(strip=True)
  267. your_price = value_elements[i + 1].get_text(strip=True)
  268.  
  269. # Find the closest discount element
  270. if discount_elements:
  271. discount_text = discount_elements[i // 2].get_text(strip=True) if i // 2 < len(discount_elements) else "0%"
  272. discount_match = re.search(r'(\d+(?:\.\d+)?)', discount_text)
  273. if discount_match:
  274. discount = discount_match.group(1)
  275. else:
  276. continue
  277. else:
  278. continue
  279.  
  280. # Clean up the values
  281. card_value = card_value.replace('$', '').strip()
  282. your_price = your_price.replace('$', '').strip()
  283.  
  284. gift_cards.append({
  285. 'Brand': brand,
  286. 'Card_Value': f"${card_value}",
  287. 'Discount': f"{discount}%",
  288. 'Your_Price': f"${your_price}"
  289. })
  290.  
  291. # Method 3: Look for the specific HTML structure pattern
  292. if not gift_cards:
  293. # Look for the exact pattern from the provided HTML
  294. page_text = soup.get_text()
  295.  
  296. # Pattern: $200.00, 2.00% OFF, $196.00
  297. pattern = r'\$(\d+(?:\.\d+)?)\s*,\s*(\d+(?:\.\d+)?)%\s*OFF\s*,\s*\$(\d+(?:\.\d+)?)'
  298. matches = re.findall(pattern, page_text)
  299.  
  300. for card_value, discount, your_price in matches:
  301. gift_cards.append({
  302. 'Brand': brand,
  303. 'Card_Value': f"${card_value}",
  304. 'Discount': f"{discount}%",
  305. 'Your_Price': f"${your_price}"
  306. })
  307.  
  308. # Method 4: Look for any elements containing the specific class names
  309. if not gift_cards:
  310. # Find all elements with the specific class patterns
  311. all_elements = soup.find_all('div', class_=re.compile(r'grouped-listing_'))
  312.  
  313. values = []
  314. discounts = []
  315.  
  316. for element in all_elements:
  317. class_name = element.get('class', [])
  318. if any('value__' in cls for cls in class_name):
  319. values.append(element.get_text(strip=True))
  320. elif any('discount__' in cls for cls in class_name):
  321. discounts.append(element.get_text(strip=True))
  322.  
  323. # Pair them up
  324. for i in range(0, len(values), 2):
  325. if i + 1 < len(values) and i // 2 < len(discounts):
  326. card_value = values[i].replace('$', '').strip()
  327. your_price = values[i + 1].replace('$', '').strip()
  328. discount_text = discounts[i // 2]
  329.  
  330. discount_match = re.search(r'(\d+(?:\.\d+)?)', discount_text)
  331. if discount_match:
  332. discount = discount_match.group(1)
  333.  
  334. gift_cards.append({
  335. 'Brand': brand,
  336. 'Card_Value': f"${card_value}",
  337. 'Discount': f"{discount}%",
  338. 'Your_Price': f"${your_price}"
  339. })
  340.  
  341. # Remove duplicates
  342. seen = set()
  343. unique_gift_cards = []
  344. for card in gift_cards:
  345. key = (card['Brand'], card['Card_Value'], card['Your_Price'])
  346. if key not in seen:
  347. seen.add(key)
  348. unique_gift_cards.append(card)
  349.  
  350. logger.info(f"Extracted {len(unique_gift_cards)} gift card variations for {brand}")
  351. return unique_gift_cards
  352.  
  353. except Exception as e:
  354. logger.error(f"Error extracting gift card details: {e}")
  355. return []
  356.  
  357. def scrape_first_page_only(self) -> List[Dict]:
  358. """Scrape only the first page and follow each gift card link for accurate information"""
  359. if not self.setup_driver():
  360. return []
  361.  
  362. try:
  363. # Navigate to first page
  364. if not self.navigate_to_first_page():
  365. logger.error("Failed to navigate to first page")
  366. return []
  367.  
  368. # Scroll to load content
  369. self.scroll_and_load_content()
  370.  
  371. # Extract gift card links from first page
  372. gift_card_links = self.extract_gift_card_links_from_first_page()
  373.  
  374. if gift_card_links:
  375. logger.info(f"Found {len(gift_card_links)} gift card links on first page")
  376.  
  377. # Follow each gift card link
  378. for link in gift_card_links:
  379. try:
  380. gift_cards = self.scrape_individual_gift_card_page(link)
  381. if gift_cards:
  382. self.all_gift_cards.extend(gift_cards)
  383. logger.info(f"Added {len(gift_cards)} gift cards from {link}")
  384.  
  385. # Be respectful with delays
  386. time.sleep(2)
  387.  
  388. except Exception as e:
  389. logger.error(f"Error processing link {link}: {e}")
  390. continue
  391. else:
  392. logger.warning("No gift card links found on first page")
  393.  
  394. return self.all_gift_cards
  395.  
  396. except Exception as e:
  397. logger.error(f"Error during scraping: {e}")
  398. return self.all_gift_cards
  399. finally:
  400. if self.driver:
  401. self.driver.quit()
  402.  
  403. def save_data(self, gift_cards: List[Dict], csv_filename: str = "gcx_targeted_html_gift_cards.csv", excel_filename: str = "gcx_targeted_html_gift_cards.xlsx"):
  404. """Save scraped data to CSV and Excel files"""
  405.  
  406. if not gift_cards:
  407. logger.warning("No data to save")
  408. return None
  409.  
  410. # Create DataFrame
  411. df = pd.DataFrame(gift_cards)
  412.  
  413. # Remove duplicates based on brand name, card value, and your price
  414. df = df.drop_duplicates(subset=['Brand', 'Card_Value', 'Your_Price'], keep='first')
  415.  
  416. # Sort by Brand name, then by Card Value
  417. df = df.sort_values(['Brand', 'Card_Value'], ascending=[True, True])
  418.  
  419. # Save to CSV
  420. df.to_csv(csv_filename, index=False)
  421. logger.info(f"Saved {len(df)} gift cards to {csv_filename}")
  422.  
  423. # Save to Excel
  424. df.to_excel(excel_filename, index=False, engine='openpyxl')
  425. logger.info(f"Saved {len(df)} gift cards to {excel_filename}")
  426.  
  427. return df
  428.  
  429. def analyze_results(self, gift_cards: List[Dict]):
  430. """Analyze the scraped results"""
  431. if not gift_cards:
  432. return
  433.  
  434. print(f"\n" + "="*80)
  435. print("TARGETED HTML GIFT CARD SCRAPING RESULTS")
  436. print("="*80)
  437. print(f"Total gift cards scraped: {len(gift_cards)}")
  438.  
  439. if gift_cards:
  440. # Show first 30
  441. print(f"\nFirst 30 gift cards:")
  442. print("-" * 100)
  443. print(f"{'Rank':<4} {'Brand':<25} {'Card Value':<12} {'Discount':<10} {'Your Price':<12}")
  444. print("-" * 100)
  445.  
  446. for i, card in enumerate(gift_cards[:30], 1):
  447. print(f"{i:<4} {card['Brand']:<25} {card['Card_Value']:<12} {card['Discount']:<10} {card['Your_Price']:<12}")
  448.  
  449. # Show summary by brand
  450. df = pd.DataFrame(gift_cards)
  451. brand_summary = df.groupby('Brand').agg({
  452. 'Card_Value': 'count',
  453. 'Discount': lambda x: f"{x.iloc[0]}" if len(x) > 0 else "N/A"
  454. }).rename(columns={'Card_Value': 'Card_Count'})
  455.  
  456. print(f"\nSummary by Brand:")
  457. print("-" * 70)
  458. print(f"{'Brand':<25} {'Cards':<10} {'Sample Discount':<15}")
  459. print("-" * 70)
  460.  
  461. for brand, row in brand_summary.iterrows():
  462. print(f"{brand:<25} {row['Card_Count']:<10} {row['Discount']:<15}")
  463.  
  464. # Show examples of different prices for the same brand
  465. print(f"\nExamples of different prices for the same brand:")
  466. print("-" * 80)
  467. for brand in df['Brand'].unique()[:5]: # Show first 5 brands
  468. brand_cards = df[df['Brand'] == brand]
  469. if len(brand_cards) > 1:
  470. print(f"\n{brand}:")
  471. for _, card in brand_cards.head(5).iterrows():
  472. print(f" {card['Card_Value']} - {card['Discount']} - You pay {card['Your_Price']}")
  473.  
  474. def main():
  475. """Main function"""
  476. logger.info("Starting targeted HTML gift card scraper for GCX...")
  477.  
  478. # Try with visible browser first (better for debugging)
  479. scraper = TargetedHTMLGiftCardScraper(headless=False)
  480. gift_cards = scraper.scrape_first_page_only()
  481.  
  482. if gift_cards:
  483. logger.info(f"Successfully scraped {len(gift_cards)} gift cards from first page")
  484.  
  485. # Save data
  486. df = scraper.save_data(gift_cards)
  487.  
  488. # Analyze results
  489. scraper.analyze_results(gift_cards)
  490.  
  491. print(f"\nData saved to:")
  492. print(f"- gcx_targeted_html_gift_cards.csv")
  493. print(f"- gcx_targeted_html_gift_cards.xlsx")
  494. print(f"\nFormat: Brand | Card Value | Discount | Your Price")
  495. print(f"Where:")
  496. print(f"- Brand: Company name")
  497. print(f"- Card Value: Original gift card value (e.g., $5, $10, $25, $50, $100)")
  498. print(f"- Discount: Discount percentage for that specific amount")
  499. print(f"- Your Price: Actual price you pay for the gift card")
  500.  
  501. else:
  502. logger.error("No gift cards found")
  503. print("\nNo gift cards found. This could be due to:")
  504. print("1. Website structure changes")
  505. print("2. Anti-bot protection")
  506. print("3. Network issues")
  507. print("4. Chrome driver not installed")
  508.  
  509. if __name__ == "__main__":
  510. main()
  511.  
Advertisement
Add Comment
Please, Sign In to add comment