News Scraper

#!/usr/bin/env python3
"""
Google News RSS search: enter a subject and a location, get a digest.

Usage:
    python news_search.py                      (opens the input window)
    python news_search.py "school board" "Phoenix Arizona" [days]
    python news_search.py "school board" "Phoenix, Arizona; Tucson, Arizona" [days]

Locations are separated by semicolons. A separate HTML digest is created
for each location in OUTPUT_DIR, and the pages open automatically in your browser.
"""

import sys
import io
import re
import html
import json
import webbrowser
import hashlib
import urllib.request
import urllib.parse
from datetime import datetime
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed

# With a windowed (no-console) build, stdout/stderr are None.
if sys.stdout is not None and sys.stdout.encoding and sys.stdout.encoding.lower() != 'utf-8':
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
if sys.stderr is not None and sys.stderr.encoding and sys.stderr.encoding.lower() != 'utf-8':
    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')

import feedparser

OUTPUT_DIR = Path(r"C:\Users\buzzl\OneDrive\Desktop\news output")
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

# Edition of Google News (language / country). Change for non-US searches.
HL, GL, CEID = 'en-US', 'US', 'US:en'


def build_url(subject, location, days):
    query = f"{subject} {location} when:{days}d"
    return (
        "https://news.google.com/rss/search?"
        + urllib.parse.urlencode({'q': query, 'hl': HL, 'gl': GL, 'ceid': CEID})
    )


def resolve_url(url, timeout=5):
    if 'news.google.com' not in url:
        return url
    try:
        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=timeout) as response:
            return response.url
    except Exception:
        return url


def resolve_all(urls, workers=10):
    resolved = {}
    with ThreadPoolExecutor(max_workers=workers) as ex:
        futures = {ex.submit(resolve_url, u): u for u in urls}
        for f in as_completed(futures):
            u = futures[f]
            try:
                resolved[u] = f.result()
            except Exception:
                resolved[u] = u
    return resolved


def fetch_articles(subject, location, days):
    url = build_url(subject, location, days)
    print(f"Fetching: {url}")
    feed = feedparser.parse(url)
    entries = feed.entries
    print(f"  {len(entries)} results; resolving links...")
    resolved = resolve_all([e.get('link', '') for e in entries if e.get('link')])

    articles, seen = [], set()
    for e in entries:
        title = e.get('title', 'No title')
        key = title.lower().strip()
        if key in seen:
            continue
        seen.add(key)
        link = resolved.get(e.get('link', ''), e.get('link', ''))
        src = e.get('source', {})
        source = src.get('title', '') if hasattr(src, 'get') else ''
        articles.append({
            'hash': hashlib.md5(f"{key}{link}".encode()).hexdigest()[:12],
            'title': title,
            'link': link,
            'source': source,
            'published': e.get('published', e.get('updated', '')),
        })
    return articles


def build_report(subject, location, days, articles):
    esc = html.escape
    items = []
    for a in articles:
        meta = " · ".join(esc(x) for x in (a['source'], a['published']) if x)
        items.append(
            f'<li><a href="{esc(a["link"], quote=True)}" target="_blank" rel="noopener noreferrer">'
            f'{esc(a["title"])}</a>'
            + (f'<div class="meta">{meta}</div>' if meta else '')
            + '</li>'
        )
    generated = datetime.now().strftime('%A %d %B %Y at %H:%M')
    return f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>News: {esc(subject)} — {esc(location)}</title>
<style>
  body {{ font-family: Segoe UI, Arial, sans-serif; max-width: 820px; margin: 2rem auto; padding: 0 1rem; color: #222; }}
  h1 {{ margin-bottom: .2rem; }}
  .sub {{ color: #666; margin-bottom: 1.5rem; }}
  ul {{ list-style: none; padding: 0; }}
  li {{ padding: .8rem 0; border-bottom: 1px solid #ddd; }}
  a {{ font-weight: 600; color: #0b57d0; text-decoration: none; }}
  a:hover {{ text-decoration: underline; }}
  .meta {{ color: #666; font-size: .85rem; margin-top: .2rem; }}
</style>
</head>
<body>
<h1>{esc(subject)} — {esc(location)}</h1>
<div class="sub">Generated {esc(generated)} &middot; last {esc(str(days))} days &middot; {len(articles)} articles</div>
<ul>
{chr(10).join(items)}
</ul>
</body>
</html>
"""


def slug(text):
    return re.sub(r'[^a-z0-9]+', '-', text.lower()).strip('-')


def parse_locations(text):
    locations = []
    seen = set()
    for location in text.split(';'):
        location = location.strip()
        key = location.casefold()
        if location and key not in seen:
            locations.append(location)
            seen.add(key)
    return locations


def run_search(subject, locations, days, status=print):
    date = datetime.now().strftime('%Y-%m-%d')
    used_bases = set()
    html_files = []
    for i, location in enumerate(locations, 1):
        status(f"[{i}/{len(locations)}] Searching {location}...")
        articles = fetch_articles(subject, location, days)
        base = f"news_{slug(subject)}_{slug(location)}_{date}"
        unique_base = base
        suffix = 2
        while unique_base in used_bases:
            unique_base = f"{base}_{suffix}"
            suffix += 1
        used_bases.add(unique_base)
        html_file = OUTPUT_DIR / f"{unique_base}.html"
        html_file.write_text(
            build_report(subject, location, days, articles), encoding='utf-8'
        )
        html_files.append((html_file, len(articles)))

    for html_file, article_count in html_files:
        status(f"{article_count} articles -> {html_file}")
        webbrowser.open(html_file.as_uri())
    return html_files


def run_gui():
    import queue
    import threading
    import tkinter as tk
    from tkinter import messagebox

    root = tk.Tk()
    root.title("News Search")
    root.columnconfigure(0, weight=1)

    pad = {'padx': 12, 'sticky': 'ew'}
    tk.Label(root, text="Subject").grid(row=0, column=0, pady=(12, 2), **pad)
    subject_box = tk.Entry(root, width=50)
    subject_box.grid(row=1, column=0, **pad)

    tk.Label(root, text="Locations (separate with semicolons)").grid(row=2, column=0, pady=(10, 2), **pad)
    locations_box = tk.Entry(root, width=50)
    locations_box.grid(row=3, column=0, **pad)

    tk.Label(root, text="Number of days").grid(row=4, column=0, pady=(10, 2), **pad)
    days_box = tk.Entry(root, width=50)
    days_box.insert(0, '7')
    days_box.grid(row=5, column=0, **pad)

    status_var = tk.StringVar()
    search_btn = tk.Button(root, text="Search")
    search_btn.grid(row=6, column=0, pady=12)
    tk.Label(root, textvariable=status_var, fg='#555', wraplength=420).grid(row=7, column=0, pady=(0, 12), **pad)

    messages = queue.Queue()

    def worker(subject, locations, days):
        try:
            run_search(subject, locations, days, status=lambda m: messages.put(('msg', m)))
            messages.put(('done', f"Done: {len(locations)} file(s) created."))
        except Exception as exc:
            messages.put(('done', f"Error: {exc}"))

    def poll():
        try:
            while True:
                kind, text = messages.get_nowait()
                status_var.set(text)
                if kind == 'done':
                    search_btn.config(state='normal')
        except queue.Empty:
            pass
        root.after(100, poll)

    def on_search(event=None):
        subject = subject_box.get().strip()
        locations = parse_locations(locations_box.get())
        days = days_box.get().strip() or '7'
        if not subject or not locations:
            messagebox.showwarning("Missing input", "Enter a subject and at least one location.")
            return
        if not days.isdigit() or int(days) < 1:
            messagebox.showwarning("Invalid days", "Number of days must be a positive whole number.")
            return
        search_btn.config(state='disabled')
        status_var.set("Starting...")
        threading.Thread(target=worker, args=(subject, locations, days), daemon=True).start()

    search_btn.config(command=on_search)
    root.bind('<Return>', on_search)
    subject_box.focus_set()
    poll()
    root.mainloop()


def main():
    args = sys.argv[1:]
    if len(args) < 2:
        run_gui()
        return
    subject = args[0]
    rest = args[1:]
    days = '7'
    if len(rest) > 1 and rest[-1].isdigit():
        days = rest.pop()
    locations = parse_locations(';'.join(rest))
    if not locations:
        print("Both a subject and at least one location are required.")
        return
    if not days.isdigit():
        print("Days must be a number; using 7.")
        days = '7'
    run_search(subject, locations, days)


if __name__ == '__main__':
    main()