24ur python scrape for keyword for 60 past days

#!/usr/bin/env python3
"""
Stream‐based script to count daily 24ur.com articles mentioning “Dončić” over the past 60 days,
writing two CSVs as soon as each day’s URLs are fully processed:

  • Stats CSV: date,count                  (`doncic_article_counts.csv`)
  • URLs  CSV:  date,url                    (`doncic_article_urls.csv`)

Algorithm:
 1. Remove old output files and write headers.
 2. Crawl sitemap pages from 1357 down to 1.
 3. Within each page, scan URLs in reverse order (so dates go newest→oldest).
 4. Track `current_date`; for each URL:
      - When the date changes, flush stats+URLs for the previous date.
      - Then start counting for the new date.
 5. For each URL whose **main article text** (only `<p>` paragraphs) contains “Dončić” at least once:
      - Increment that date’s counter.
      - Append the URL to that date’s URL CSV.
 6. After all pages, flush the last date’s data.

Requires:
    pip install requests beautifulsoup4 tqdm
"""
import os
import csv
import time
import unicodedata
import requests
import xml.etree.ElementTree as ET
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from requests.exceptions import HTTPError
from tqdm import tqdm

# ——— Configuration ———
KEYWORD = "Dončić"
KEYWORD_ASCII = (
    unicodedata.normalize("NFKD", KEYWORD)
    .encode("ascii", "ignore")
    .decode()
    .lower()
)
BASE_SITEMAP_PAGE = "https://www.24ur.com/sitemaps/sites/1/pages/{}"
FIRST_PAGE = 1357
LAST_PAGE = 1
WINDOW_DAYS = 60
STATS_CSV = "doncic_article_counts.csv"
URLS_CSV  = "doncic_article_urls.csv"
REQUEST_DELAY = 0.5  # seconds between HTTP requests
# ————————————————————

def parse_iso_date(s: str) -> datetime.date:
    return datetime.fromisoformat(s).date()

def fetch_xml(url: str) -> ET.Element:
    resp = session.get(url)
    resp.raise_for_status()
    return ET.fromstring(resp.text)

def extract_main_text(html: str) -> str:
    """
    Parse the HTML with BeautifulSoup, pull out only the <p> tags
    (which hold the article paragraphs), join them, then normalize
    to ASCII lowercase for matching.
    """
    soup = BeautifulSoup(html, "html.parser")
    paras = [p.get_text(" ") for p in soup.find_all("p")]
    text = " ".join(paras)
    return unicodedata.normalize("NFKD", text) \
                      .encode("ascii", "ignore") \
                      .decode() \
                      .lower()

if __name__ == "__main__":
    # Prepare window
    today = datetime.now().date()
    start_date = today - timedelta(days=WINDOW_DAYS)

    # Remove old outputs, write headers
    for path, header in [
        (STATS_CSV, ["date", "count"]),
        (URLS_CSV,  ["date", "url"])
    ]:
        if os.path.exists(path):
            os.remove(path)
        with open(path, "w", newline="", encoding="utf-8") as f:
            csv.writer(f).writerow(header)

    session = requests.Session()
    current_date = None
    current_count = 0

    # Crawl from page 1357 → 1
    for page_num in tqdm(range(FIRST_PAGE, LAST_PAGE - 1, -1),
                         desc="Sitemap pages", unit="page"):
        sitemap_url = BASE_SITEMAP_PAGE.format(page_num)
        try:
            root = fetch_xml(sitemap_url)
        except HTTPError as e:
            if e.response is not None and e.response.status_code == 404:
                continue
            break

        url_elems = root.findall(".//{*}url")
        if not url_elems:
            continue

        # Process URLs in reverse order (end→start)
        for url_elem in tqdm(reversed(url_elems),
                             desc=f" Articles (pg {page_num})",
                             unit="url", leave=False):
            loc = url_elem.find("{*}loc").text
            if not loc.startswith("https://www.24ur.com/"):
                continue

            pub_date = parse_iso_date(url_elem.find("{*}lastmod").text)
            if pub_date < start_date or pub_date > today:
                continue

            # Initialize on first URL
            if current_date is None:
                current_date = pub_date

            # When date rolls over, flush previous date
            if pub_date != current_date:
                with open(STATS_CSV, "a", newline="", encoding="utf-8") as sf:
                    csv.writer(sf).writerow([current_date.isoformat(), current_count])
                current_date = pub_date
                current_count = 0

            # Fetch article and search keyword in main text
            try:
                r = session.get(loc); r.raise_for_status()
            except:
                continue

            main_text = extract_main_text(r.text)
            if KEYWORD_ASCII in main_text:
                current_count += 1
                with open(URLS_CSV, "a", newline="", encoding="utf-8") as uf:
                    csv.writer(uf).writerow([pub_date.isoformat(), loc])

            time.sleep(REQUEST_DELAY)

    # Flush last date if any
    if current_date is not None:
        with open(STATS_CSV, "a", newline="", encoding="utf-8") as sf:
            csv.writer(sf).writerow([current_date.isoformat(), current_count])

    print("Done! Outputs written:")
    print("  ", STATS_CSV)
    print("  ", URLS_CSV)