pastebin archiver (update 1)

(function () {
    const RAW_PREFIX = "https://pastebin.com/raw/";
    const PROXIES = [
        "https://api.allorigins.win/raw?url=",
        "https://cors.isomorphic-git.org/",
        "https://api.codetabs.com/v1/proxy?quest="
    ];
    const THREADS = 4;
    const PREVIEW_LIMIT = 2000;
    const LOG_LIMIT = 200;
    const MAX_RETRIES = 3;

    // ---------------- IndexedDB Storage Engine ----------------
    const DB_NAME = "PastebinArchiveDB";
    const DB_VERSION = 1;
    const STORE_NAME = "archive_state";

    function openDB() {
        return new Promise((resolve, reject) => {
            const req = indexedDB.open(DB_NAME, DB_VERSION);
            req.onupgradeneeded = e => {
                const db = e.target.result;
                if (!db.objectStoreNames.contains(STORE_NAME)) {
                    db.createObjectStore(STORE_NAME);
                }
            };
            req.onsuccess = e => resolve(e.target.result);
            req.onerror = e => reject(e.target.error);
        });
    }

    async function saveIDB(key, val) {
        try {
            const db = await openDB();
            const tx = db.transaction(STORE_NAME, "readwrite");
            tx.objectStore(STORE_NAME).put(val, key);
            return new Promise(r => tx.oncomplete = r);
        } catch (e) {
            console.warn("IndexedDB Save Error:", e);
        }
    }

    async function getIDB(key) {
        try {
            const db = await openDB();
            const tx = db.transaction(STORE_NAME, "readonly");
            const req = tx.objectStore(STORE_NAME).get(key);
            return new Promise(r => req.onsuccess = () => r(req.result));
        } catch (e) {
            console.warn("IndexedDB Load Error:", e);
            return null;
        }
    }

    async function clearIDB() {
        try {
            const db = await openDB();
            const tx = db.transaction(STORE_NAME, "readwrite");
            tx.objectStore(STORE_NAME).clear();
            return new Promise(r => tx.oncomplete = r);
        } catch (e) {
            console.warn("IndexedDB Clear Error:", e);
        }
    }

    // ---------------- State Variables ----------------
    function sleep(ms) {
        return new Promise(resolve => setTimeout(resolve, ms));
    }

    let users = [],
        currentUserIndex = 0,
        user = "";
    let proxyIndex = 0;
    let running = false;
    let active = 0;
    let found = new Set();
    let queue = [];
    let results = [];
    let metadataMap = new Map(); // Stores scraped metadata per paste ID
    let searchDebounceTimer = null;

    // ---------------- Helper Functions ----------------
    function escapeHTML(t) {
        return (t || "").replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
    }

    async function proxyFetch(url) {
        for (let i = 0; i < PROXIES.length; i++) {
            const proxy = PROXIES[(proxyIndex + i) % PROXIES.length];
            try {
                const targetUrl = proxy.includes("isomorphic-git")
                    ? proxy + url
                    : proxy + encodeURIComponent(url);
                const r = await fetch(targetUrl);
                if (r.ok) return r;
            } catch {}
        }
        proxyIndex++;
        throw new Error("All proxies failed");
    }

    async function saveProgress() {
        await saveIDB("found", [...found]);
        await saveIDB("queue", queue);
        await saveIDB("results", results);
        await saveIDB("users", users);
        await saveIDB("currentUserIndex", currentUserIndex);
        await saveIDB("metadataMap", [...metadataMap.entries()]);
    }

    async function loadProgress() {
        const f = await getIDB("found");
        const q = await getIDB("queue");
        const res = await getIDB("results");
        const u = await getIDB("users");
        const ci = await getIDB("currentUserIndex");
        const mm = await getIDB("metadataMap");

        if (f) found = new Set(f);
        if (q) queue = q;
        if (res) results = res;
        if (mm) metadataMap = new Map(mm);
        if (u) {
            users = u;
            const usersInput = document.getElementById("ah-users");
            if (usersInput) usersInput.value = users.join("\n");
        }
        if (ci !== null && ci !== undefined) currentUserIndex = parseInt(ci, 10) || 0;
    }

    // ---------------- HUD Injection ----------------
    if (document.getElementById("archiveHUD")) return;
    const hud = document.createElement("div");
    hud.id = "archiveHUD";
    hud.style = `
        position:fixed;top:20px;left:20px;width:440px;height:580px;
        background:#111;color:#0f0;font-family:monospace;
        border:1px solid #0f0;padding:10px;z-index:999999;
        resize:both;overflow:hidden;box-sizing:border-box;
    `;
    hud.innerHTML = `
        <div id="ah-header" style="cursor:move;font-weight:bold;display:flex;justify-content:space-between;user-select:none;">
            <span>Pastebin Archive HUD (IndexedDB)</span>
            <button id="ah-close" style="background:#222;color:#0f0;border:1px solid #0f0;cursor:pointer;">X</button>
        </div>
        <textarea id="ah-users" placeholder="Enter usernames, one per line" style="width:100%;height:60px;background:#000;color:#0f0;border:1px solid #0f0;margin-top:5px;box-sizing:border-box;"></textarea>
        <div style="font-size:12px;margin-top:5px;">
            User: <span id="ah-currentUser">-</span> | Indexed: <span id="ah-found">0</span> | Queue: <span id="ah-queue">0</span><br>
            Threads: <span id="ah-active">0</span> | Running: <span id="ah-state">false</span> | Bytes: <span id="ah-bytes">0</span>
        </div>
        <hr style="border-color:#0f0;margin:8px 0;">
        <button id="ah-start" style="background:#222;color:#0f0;border:1px solid #0f0;cursor:pointer;">START</button>
        <button id="ah-stop" style="background:#222;color:#0f0;border:1px solid #0f0;cursor:pointer;">STOP</button>
        <button id="ah-export" style="background:#222;color:#0f0;border:1px solid #0f0;cursor:pointer;">EXPORT JSON</button>
        <button id="ah-clear" style="background:#222;color:#0f0;border:1px solid #0f0;cursor:pointer;">CLEAR DATA</button>
        <br><label style="font-size:12px;"><input type="checkbox" id="ah-autodownload"> Auto-download each paste (.txt)</label>
        <br><br>
        <input id="ah-search" placeholder="Search title, content, or syntax..." style="width:100%;background:#000;color:#0f0;border:1px solid #0f0;box-sizing:border-box;">
        <hr style="border-color:#0f0;margin:8px 0;">
        <div id="ah-log" style="height:270px;overflow:auto;font-size:12px;background:#050505;padding:4px;border:1px solid #004400;"></div>
    `;
    document.body.appendChild(hud);

    const logBox = document.getElementById("ah-log");
    const usersInput = document.getElementById("ah-users");

    function updateStats() {
        const getEl = id => document.getElementById(id);
        if (!getEl("archiveHUD")) return;

        if (getEl("ah-found")) getEl("ah-found").textContent = found.size;
        if (getEl("ah-queue")) getEl("ah-queue").textContent = queue.length;
        if (getEl("ah-active")) getEl("ah-active").textContent = active;
        if (getEl("ah-state")) getEl("ah-state").textContent = running;
        if (getEl("ah-currentUser")) {
            getEl("ah-currentUser").textContent =
                (user || "-") + " (" + (users.length ? currentUserIndex + 1 : 0) + "/" + users.length + ")";
        }
        const totalBytes = results.reduce((a, b) => a + (b.length || 0), 0);
        if (getEl("ah-bytes")) getEl("ah-bytes").textContent = totalBytes;
    }

    function log(msg, data) {
        if (!logBox) return;
        const line = document.createElement("div");
        line.textContent = msg;
        if (data) {
            line.style.cursor = "pointer";
            line.style.textDecoration = "underline";
            line.onclick = () => preview(data);
        }
        logBox.appendChild(line);
        if (logBox.children.length > LOG_LIMIT) logBox.removeChild(logBox.firstChild);
        logBox.scrollTop = logBox.scrollHeight;
    }

    // ---------------- Drag Handler ----------------
    let drag = false, dx = 0, dy = 0;
    const header = document.getElementById("ah-header");
    header.onmousedown = e => {
        drag = true;
        dx = e.clientX - hud.offsetLeft;
        dy = e.clientY - hud.offsetTop;
    };

    function onMouseMove(e) {
        if (drag) {
            hud.style.left = (e.clientX - dx) + "px";
            hud.style.top = (e.clientY - dy) + "px";
        }
    }
    function onMouseUp() { drag = false; }

    document.addEventListener("mousemove", onMouseMove);
    document.addEventListener("mouseup", onMouseUp);

    document.getElementById("ah-close").onclick = () => {
        running = false;
        document.removeEventListener("mousemove", onMouseMove);
        document.removeEventListener("mouseup", onMouseUp);
        hud.remove();
    };

    // ---------------- Preview Modal ----------------
    function preview(data) {
        const win = document.createElement("div");
        win.style = `
            position:fixed;top:80px;left:80px;width:620px;height:440px;
            background:#111;color:#0f0;border:1px solid #0f0;padding:12px;
            overflow:auto;z-index:999999;font-family:monospace;box-sizing:border-box;
        `;
        win.innerHTML = `
            <div style="display:flex;justify-content:space-between;margin-bottom:8px;border-bottom:1px dashed #0f0;padding-bottom:5px;">
                <b>[${escapeHTML(data.id)}] ${escapeHTML(data.title || "Untitled")}</b>
                <button onclick="this.parentNode.parentNode.remove()" style="background:#222;color:#0f0;border:1px solid #0f0;cursor:pointer;">Close</button>
            </div>
            <div style="font-size:11px;margin-bottom:8px;color:#8f8;">
                Syntax: ${escapeHTML(data.syntax || "Plain Text")} | Date: ${escapeHTML(data.date || "Unknown")} | Views: ${escapeHTML(data.views || "0")}
            </div>
            <pre style="margin:0;white-space:pre-wrap;word-wrap:break-word;background:#050505;padding:8px;border:1px solid #004400;">${escapeHTML(data.content || data.preview || "")}</pre>
        `;
        document.body.appendChild(win);
    }

    // ---------------- Download File ----------------
    function downloadPaste(id, text) {
        const blob = new Blob([text], { type: "text/plain" });
        const a = document.createElement("a");
        a.href = URL.createObjectURL(blob);
        a.download = id + ".txt";
        a.click();
        URL.revokeObjectURL(a.href);
    }

    // ---------------- Page Scraper (DOMParser + Metadata) ----------------
    async function getPage(page) {
        const BASE = `https://pastebin.com/u/${user}`;
        const url = page === 1 ? BASE : `${BASE}/${page}`;
        const excludeList = new Set(["settings", "contacts", "messages", "pro_signup", "doc_api", "tools_v2"]);

        for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
            try {
                const r = await proxyFetch(url);
                const html = await r.text();

                if (!html.includes("pastebin.com") && !html.includes("Pastebin") && !html.includes("href=")) {
                    throw new Error("Invalid response markup");
                }

                const parser = new DOMParser();
                const doc = parser.parseFromString(html, "text/html");
                const extracted = [];

                const rows = doc.querySelectorAll("table tr");
                rows.forEach(tr => {
                    const link = tr.querySelector('a[href^="/"]');
                    if (!link) return;

                    const href = link.getAttribute("href") || "";
                    const match = href.match(/^\/([A-Za-z0-9]{8})$/);
                    if (!match) return;

                    const id = match[1];
                    if (excludeList.has(id.toLowerCase())) return;

                    const title = link.textContent.trim() || "Untitled";
                    let syntax = "Plain Text";
                    let date = "Unknown";
                    let views = "0";

                    const cells = tr.querySelectorAll("td");
                    cells.forEach(td => {
                        const txt = td.textContent.trim();
                        if (td.querySelector('a[href*="/archive/"]') || td.querySelector(".i_p0")) {
                            syntax = txt;
                        } else if (txt.includes("ago") || txt.includes("202")) {
                            date = txt;
                        } else if (/^\d+([\.,]\d+)?\s*(KB|MB|B|views)?$/i.test(txt)) {
                            views = txt;
                        }
                    });

                    const meta = { id, title, syntax, date, views };
                    metadataMap.set(id, meta);
                    extracted.push(id);
                });

                return [...new Set(extracted)];
            } catch (err) {
                if (attempt === MAX_RETRIES) {
                    log(`Error fetching page ${page} after ${MAX_RETRIES} attempts`);
                    return null;
                }
                await sleep(1000);
            }
        }
        return null;
    }

    // ---------------- Thread Worker ----------------
    async function worker() {
        while (running) {
            const id = queue.shift();
            if (!id) break;

            active++;
            updateStats();

            await sleep(1200);

            let success = false;
            for (let attempt = 1; attempt <= MAX_RETRIES && running; attempt++) {
                try {
                    const r = await proxyFetch(RAW_PREFIX + id);
                    const text = await r.text();
                    const meta = metadataMap.get(id) || {};

                    const data = {
                        id,
                        title: meta.title || "Untitled",
                        syntax: meta.syntax || "Plain Text",
                        date: meta.date || "Unknown",
                        views: meta.views || "0",
                        preview: text.slice(0, PREVIEW_LIMIT),
                        content: text,
                        length: text.length
                    };

                    results.push(data);
                    log(`Indexed [${id}] ${data.title} (${text.length} B)`, data);

                    const autoDlCheck = document.getElementById("ah-autodownload");
                    if (autoDlCheck && autoDlCheck.checked) {
                        downloadPaste(id, text);
                    }
                    success = true;
                    break;
                } catch {
                    if (attempt < MAX_RETRIES) await sleep(1000);
                }
            }

            if (!success) {
                log("Failed to fetch paste " + id);
            }

            active--;
            updateStats();
            await saveProgress();
        }
    }

    // ---------------- Crawl Single User ----------------
    async function crawlUser(u) {
        user = u;
        let page = 1;

        while (running) {
            const ids = await getPage(page);

            if (ids === null) {
                log(`Skipping page ${page} due to connection error.`);
                page++;
                continue;
            }
            if (ids.length === 0) break;

            for (const id of ids) {
                if (!found.has(id)) {
                    found.add(id);
                    queue.push(id);
                }
            }

            updateStats();

            const workers = [];
            for (let i = 0; i < THREADS; i++) workers.push(worker());
            await Promise.all(workers);

            page++;
        }
        await saveProgress();
    }

    // ---------------- Crawl Multiple Users ----------------
    async function crawlAllUsers() {
        running = true;
        updateStats();

        const inputUsers = usersInput.value.split("\n").map(u => u.trim()).filter(u => u);
        if (inputUsers.length > 0) {
            if (JSON.stringify(inputUsers) !== JSON.stringify(users)) {
                users = inputUsers;
                currentUserIndex = 0;
            }
        }

        while (currentUserIndex < users.length && running) {
            user = users[currentUserIndex];
            log("Starting crawl for user: " + user);
            await crawlUser(user);

            if (running) {
                currentUserIndex++;
                await saveProgress();
            }
        }

        running = false;
        updateStats();
        if (currentUserIndex >= users.length && users.length > 0) {
            log("All users completed.");
        }
    }

    // ---------------- Debounced Metadata Search ----------------
    document.getElementById("ah-search").oninput = e => {
        clearTimeout(searchDebounceTimer);
        const q = e.target.value.toLowerCase().trim();

        searchDebounceTimer = setTimeout(() => {
            if (!q) return;
            const hits = results.filter(r =>
                r.id.toLowerCase().includes(q) ||
                (r.title && r.title.toLowerCase().includes(q)) ||
                (r.syntax && r.syntax.toLowerCase().includes(q)) ||
                (r.content && r.content.toLowerCase().includes(q))
            );
            log("Search '" + q + "' -> " + hits.length + " hits");
            hits.slice(0, 15).forEach(h => log("Hit: [" + h.id + "] " + h.title, h));
        }, 300);
    };

    // ---------------- Export ----------------
    document.getElementById("ah-export").onclick = () => {
        if (results.length === 0) {
            alert("No results to export.");
            return;
        }
        const blob = new Blob([JSON.stringify(results, null, 2)], { type: "application/json" });
        const a = document.createElement("a");
        a.href = URL.createObjectURL(blob);
        a.download = "pastebin_archive_full.json";
        a.click();
        URL.revokeObjectURL(a.href);
    };

    // ---------------- Clear ----------------
    document.getElementById("ah-clear").onclick = async () => {
        if (confirm("Clear all IndexedDB data and reset tasks?")) {
            running = false;
            found = new Set();
            queue = [];
            results = [];
            users = [];
            metadataMap = new Map();
            currentUserIndex = 0;

            await clearIDB();

            if (usersInput) usersInput.value = "";
            updateStats();
            log("IndexedDB data cleared.");
        }
    };

    // ---------------- Buttons ----------------
    document.getElementById("ah-start").onclick = () => {
        if (!running) {
            log("Starting archive...");
            crawlAllUsers();
        }
    };

    document.getElementById("ah-stop").onclick = () => {
        running = false;
        log("Stopping execution...");
    };

    // ---------------- Init ----------------
    (async () => {
        await loadProgress();
        updateStats();
    })();
})();