parse dataset9

#!/usr/bin/env python3
"""
Scrape Dataset 9 file list from DOJ website and download PDFs one by one.
Uses Playwright to handle age verification and pagination.
"""

import asyncio
import os
import json
from pathlib import Path
from playwright.async_api import async_playwright

BASE_URL = "https://www.justice.gov/epstein/doj-disclosures/data-set-9-files"
OUTPUT_DIR = Path("~/epstein-doj-2026-01-30/DataSet_9_individual")
INDEX_FILE = Path("~/epstein-doj-2026-01-30/dataset9_index.json")

async def scrape_all_pages(start_page=0):
    """Scrape all pages to build index of PDF files."""
    all_files = []

    # Load existing index if resuming
    if start_page > 0 and INDEX_FILE.exists():
        with open(INDEX_FILE, 'r') as f:
            all_files = json.load(f)
        print(f"Resuming from page {start_page}, loaded {len(all_files)} existing files")

    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=False)  # Set to True for headless
        context = await browser.new_context()

        # Set the age verification cookie
        await context.add_cookies([{
            'name': 'justiceGovAgeVerified',
            'value': 'true',
            'domain': 'www.justice.gov',
            'path': '/'
        }])

        page = await context.new_page()

        # First, find the last page number
        print("Finding total pages...")
        await page.goto(f"{BASE_URL}?page=0")
        await page.wait_for_load_state('networkidle')

        # Find the "Last" link to get max page
        last_link = await page.query_selector('a[aria-label="Last page"]')
        if last_link:
            href = await last_link.get_attribute('href')
            max_page = int(href.split('page=')[1])
            print(f"Found {max_page + 1} pages (0 to {max_page})")
        else:
            print("Could not find last page link, using manual max...")
            max_page = 20500  # Dataset 9 has ~20,450 pages

        # Scrape each page
        current_page = start_page
        while current_page <= max_page:
            print(f"Scraping page {current_page}/{max_page}...")
            await page.goto(f"{BASE_URL}?page={current_page}")
            await page.wait_for_load_state('networkidle')

            # Extract PDF links
            links = await page.query_selector_all('a[href*="/epstein/files/DataSet%209/"]')

            if not links:
                print(f"No files found on page {current_page}, stopping.")
                break

            for link in links:
                href = await link.get_attribute('href')
                filename = href.split('/')[-1]
                if filename not in [f['filename'] for f in all_files]:
                    all_files.append({
                        'filename': filename,
                        'url': f"https://www.justice.gov{href}",
                        'downloaded': False
                    })

            print(f"  Found {len(links)} links, total unique files: {len(all_files)}")
            current_page += 1

            # Save progress every 100 pages
            if current_page % 100 == 0:
                with open(INDEX_FILE, 'w') as f:
                    json.dump(all_files, f, indent=2)
                print(f"  Progress saved at page {current_page}")

            # Small delay to be polite
            await asyncio.sleep(0.5)

        await browser.close()

    # Save index
    with open(INDEX_FILE, 'w') as f:
        json.dump(all_files, f, indent=2)

    print(f"\nSaved index with {len(all_files)} files to {INDEX_FILE}")
    return all_files


async def download_files(start_from=0):
    """Download PDFs from the index."""

    if not INDEX_FILE.exists():
        print("Index file not found. Run scrape first.")
        return

    with open(INDEX_FILE, 'r') as f:
        all_files = json.load(f)

    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=False)
        context = await browser.new_context(accept_downloads=True)

        # Set the age verification cookie
        await context.add_cookies([{
            'name': 'justiceGovAgeVerified',
            'value': 'true',
            'domain': 'www.justice.gov',
            'path': '/'
        }])

        page = await context.new_page()

        downloaded = 0
        skipped = 0
        failed = 0

        for i, file_info in enumerate(all_files[start_from:], start=start_from):
            filename = file_info['filename']
            url = file_info['url']
            output_path = OUTPUT_DIR / filename

            if output_path.exists():
                skipped += 1
                continue

            print(f"[{i+1}/{len(all_files)}] Downloading {filename}...")

            try:
                # Navigate to the file URL - this will trigger a download
                async with page.expect_download(timeout=60000) as download_info:
                    await page.goto(url)

                download = await download_info.value
                await download.save_as(output_path)

                file_info['downloaded'] = True
                downloaded += 1

                # Save progress periodically
                if downloaded % 10 == 0:
                    with open(INDEX_FILE, 'w') as f:
                        json.dump(all_files, f, indent=2)
                    print(f"  Progress saved. Downloaded: {downloaded}, Skipped: {skipped}")

                await asyncio.sleep(0.3)  # Small delay between downloads

            except Exception as e:
                print(f"  FAILED: {e}")
                failed += 1

                # If we hit too many failures in a row, pause
                if failed > 5:
                    print("Too many failures, pausing for 30 seconds...")
                    await asyncio.sleep(30)
                    failed = 0

        await browser.close()

    # Final save
    with open(INDEX_FILE, 'w') as f:
        json.dump(all_files, f, indent=2)

    print(f"\nDone! Downloaded: {downloaded}, Skipped: {skipped}, Failed: {failed}")


async def main():
    import sys

    if len(sys.argv) < 2:
        print("Usage:")
        print("  python scrape_dataset9.py scrape         - Build index of all files")
        print("  python scrape_dataset9.py scrape 500     - Resume scraping from page 500")
        print("  python scrape_dataset9.py download       - Download all files from index")
        print("  python scrape_dataset9.py download 100   - Resume download from file #100")
        return

    cmd = sys.argv[1]

    if cmd == 'scrape':
        start_page = int(sys.argv[2]) if len(sys.argv) > 2 else 0
        await scrape_all_pages(start_page)
    elif cmd == 'download':
        start = int(sys.argv[2]) if len(sys.argv) > 2 else 0
        await download_files(start)
    else:
        print(f"Unknown command: {cmd}")


if __name__ == '__main__':
    asyncio.run(main())