Grok: download all files

// ==UserScript==
// @name         Grok: download all files
// @namespace    local.grok-tools
// @version      1.0.0
// @description  Scans every Grok asset page and saves all versions to a folder selected once by the user.
// @match        https://grok.com/files*
// @run-at       document-idle
// @grant        none
// ==/UserScript==

(() => {
  'use strict';

  const PAGE_SIZE = 50;
  const PARALLEL_DOWNLOADS = 3;
  const RETRIES = 3;
  const LIST_REQUEST_INTERVAL_MS = 1250;
  const MAX_RATE_LIMIT_RETRIES = 8;
  const MANIFEST_FILE = 'grok-download-manifest.json';

  /** @type {FileSystemDirectoryHandle | null} */
  let destination = null;
  /** @type {Array<Record<string, unknown>>} */
  let assets = [];
  /** @type {Map<string, { status: string, error?: string }>} */
  const results = new Map();
  let shouldStop = false;
  let isBusy = false;
  let nextListRequestAt = 0;

  const root = document.createElement('section');
  root.id = 'grok-all-files-downloader';
  root.innerHTML = `
    <style>
      #grok-all-files-downloader { position: fixed; right: 18px; bottom: 18px; z-index: 2147483647; width: 350px; color: #f4f4f5; font: 13px/1.4 system-ui, sans-serif; }
      #grok-all-files-downloader .box { background: #18181b; border: 1px solid #3f3f46; border-radius: 12px; box-shadow: 0 12px 40px #0009; padding: 14px; }
      #grok-all-files-downloader h2 { margin: 0 0 8px; font-size: 15px; }
      #grok-all-files-downloader p { margin: 7px 0; color: #d4d4d8; }
      #grok-all-files-downloader .folder { overflow-wrap: anywhere; color: #a1a1aa; }
      #grok-all-files-downloader .buttons { display: grid; grid-template-columns: 1fr 1fr; gap: 7px; margin: 10px 0; }
      #grok-all-files-downloader button { border: 0; border-radius: 7px; padding: 8px; background: #3f3f46; color: white; cursor: pointer; font: inherit; }
      #grok-all-files-downloader button.primary { background: #2563eb; }
      #grok-all-files-downloader button.stop { background: #b91c1c; }
      #grok-all-files-downloader button:disabled { opacity: .48; cursor: not-allowed; }
      #grok-all-files-downloader progress { width: 100%; height: 9px; }
      #grok-all-files-downloader details { margin-top: 9px; }
      #grok-all-files-downloader pre { max-height: 110px; overflow: auto; margin: 6px 0 0; padding: 7px; border-radius: 6px; background: #09090b; white-space: pre-wrap; color: #d4d4d8; }
    </style>
    <div class="box">
      <h2>Grok — выгрузка файлов</h2>
      <p class="folder">Папка не выбрана</p>
      <div class="buttons">
        <button data-action="folder">Выбрать папку</button>
        <button data-action="scan">Сканировать всё</button>
        <button class="primary" data-action="download">Скачать всё</button>
        <button class="stop" data-action="stop" disabled>Остановить</button>
      </div>
      <progress value="0" max="1"></progress>
      <p class="status">Готово. Будут запрошены все версии файлов.</p>
      <details>
        <summary>Журнал</summary>
        <pre></pre>
      </details>
    </div>`;
  document.body.append(root);

  const ui = {
    folder: root.querySelector('.folder'),
    status: root.querySelector('.status'),
    progress: root.querySelector('progress'),
    log: root.querySelector('pre'),
    buttons: Object.fromEntries([...root.querySelectorAll('button')].map((button) => [button.dataset.action, button])),
  };

  function setBusy(value) {
    isBusy = value;
    ui.buttons.folder.disabled = value;
    ui.buttons.scan.disabled = value;
    ui.buttons.download.disabled = value;
    ui.buttons.stop.disabled = !value;
  }

  function setStatus(message) {
    ui.status.textContent = message;
  }

  function setProgress(value, maximum) {
    ui.progress.max = Math.max(maximum, 1);
    ui.progress.value = Math.min(value, ui.progress.max);
  }

  function log(message) {
    const time = new Date().toLocaleTimeString();
    ui.log.textContent = `[${time}] ${message}\n${ui.log.textContent}`.slice(0, 12000);
  }

  function assertNotStopped() {
    if (shouldStop) throw new DOMException('Операция остановлена пользователем.', 'AbortError');
  }

  async function waitWithStop(milliseconds) {
    const deadline = Date.now() + milliseconds;
    while (Date.now() < deadline) {
      assertNotStopped();
      await new Promise((resolve) => setTimeout(resolve, Math.min(250, deadline - Date.now())));
    }
  }

  function retryAfterMilliseconds(value, retry) {
    if (value) {
      const seconds = Number(value);
      if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000;
      const date = Date.parse(value);
      if (!Number.isNaN(date)) return Math.max(0, date - Date.now());
    }
    return Math.min(60_000, 5_000 * 2 ** retry);
  }

  async function chooseFolder() {
    if (!window.showDirectoryPicker) {
      throw new Error('Этот браузер не поддерживает выбор папки. Нужен Chrome или Edge последней версии.');
    }
    destination = await window.showDirectoryPicker({ mode: 'readwrite' });
    ui.folder.textContent = `Папка: ${destination.name}`;
    log(`Выбрана папка «${destination.name}».`);
  }

  function listUrl(isLatest, pageToken) {
    const query = new URLSearchParams({
      pageSize: String(PAGE_SIZE),
      orderBy: 'ORDER_BY_LAST_USE_TIME',
      source: 'SOURCE_ANY',
      isLatest: String(isLatest),
      includeImagineFiles: 'true',
    });
    if (pageToken) query.set('pageToken', pageToken);
    return `/rest/assets?${query}`;
  }

  async function fetchPage(isLatest, pageToken) {
    for (let retry = 0; retry <= MAX_RATE_LIMIT_RETRIES; retry += 1) {
      const intervalWait = Math.max(0, nextListRequestAt - Date.now());
      if (intervalWait) {
        setStatus(`Пауза ${Math.ceil(intervalWait / 1000)} с перед следующим запросом списка…`);
        await waitWithStop(intervalWait);
      }
      assertNotStopped();
      nextListRequestAt = Date.now() + LIST_REQUEST_INTERVAL_MS;
      const response = await fetch(listUrl(isLatest, pageToken), { credentials: 'same-origin' });
      if (response.status === 429) {
        if (retry === MAX_RATE_LIMIT_RETRIES) break;
        const wait = retryAfterMilliseconds(response.headers.get('Retry-After'), retry);
        nextListRequestAt = Math.max(nextListRequestAt, Date.now() + wait);
        setStatus(`Grok ограничил частоту запросов. Повтор через ${Math.ceil(wait / 1000)} с…`);
        log(`HTTP 429 для страницы списка; ожидание ${Math.ceil(wait / 1000)} с, попытка ${retry + 1}/${MAX_RATE_LIMIT_RETRIES}.`);
        await waitWithStop(wait);
        continue;
      }
      if (!response.ok) {
        throw new Error(`Не удалось получить список файлов: HTTP ${response.status}. Запрос: ${listUrl(isLatest, pageToken)}`);
      }
      const body = await response.json();
      if (!Array.isArray(body.assets)) throw new Error('Ответ /rest/assets не содержит массива assets.');
      return body;
    }
    throw new Error(`Grok продолжает возвращать HTTP 429 после ${MAX_RATE_LIMIT_RETRIES} повторов. Запустите выгрузку позже.`);
  }

  function uniqueAssets(items) {
    const byId = new Map();
    for (const asset of items) {
      if (asset?.assetId) byId.set(asset.assetId, asset);
    }
    return [...byId.values()];
  }

  async function scan() {
    shouldStop = false;
    setBusy(true);
    try {
      const collected = [];
      let page = 0;
      // isLatest is an equality filter, not a switch for disabling the filter.
      // Scan both values so the union contains current and historical assets.
      for (const isLatest of [true, false]) {
        const seenTokens = new Set();
        let token = undefined;
        do {
          assertNotStopped();
          page += 1;
          setStatus(`Получение ${isLatest ? 'актуальных' : 'старых'} версий, страница ${page}…`);
          const body = await fetchPage(isLatest, token);
          collected.push(...body.assets);
          setProgress(collected.length, collected.length + (body.nextPageToken ? PAGE_SIZE : 0));
          token = body.nextPageToken || undefined;
          if (token && seenTokens.has(token)) throw new Error(`API вернул повторяющийся nextPageToken (${token}); выгрузка остановлена, чтобы не зациклиться.`);
          if (token) seenTokens.add(token);
        } while (token);
      }

      assets = uniqueAssets(collected);
      results.clear();
      setProgress(assets.length, assets.length);
      setStatus(`Найдено ${assets.length} уникальных файлов. Можно запускать скачивание.`);
      log(`Сканирование завершено: страниц ${page}, файлов ${assets.length}.`);
      await writeManifest();
    } finally {
      setBusy(false);
    }
  }

  function sanitizeFilename(value) {
    const name = String(value || 'unnamed').replace(/[<>:"/\\|?*\u0000-\u001F]/g, '_').replace(/[. ]+$/g, '');
    return (name || 'unnamed').slice(0, 170);
  }

  function filenameFor(asset) {
    const prefix = asset.rRated === true ? 'r_' : '';
    return `${prefix}${asset.assetId}_${sanitizeFilename(asset.name)}`;
  }

  async function writeFile(name, blob) {
    const handle = await destination.getFileHandle(name, { create: true });
    const writable = await handle.createWritable();
    try {
      await writable.write(blob);
    } finally {
      await writable.close();
    }
  }

  async function writeManifest() {
    if (!destination || !assets.length) return;
    const manifest = {
      generatedAt: new Date().toISOString(),
      filter: { isLatest: [true, false], source: 'SOURCE_ANY', includeImagineFiles: true },
      total: assets.length,
      completed: [...results.entries()].filter(([, value]) => value.status === 'done').length,
      failed: [...results.entries()].filter(([, value]) => value.status === 'failed').length,
      assets,
      results: Object.fromEntries(results),
    };
    await writeFile(MANIFEST_FILE, new Blob([JSON.stringify(manifest, null, 2)], { type: 'application/json' }));
  }

  async function downloadOne(asset) {
    assertNotStopped();
    if (!asset.key) throw new Error(`У ${asset.assetId} отсутствует key для скачивания.`);
    const name = filenameFor(asset);
    const existing = await destination.getFileHandle(name, { create: true });
    const existingFile = await existing.getFile();
    // A non-empty file from an earlier run is considered complete. A file that
    // failed during this session is retried and overwritten instead.
    if (existingFile.size > 0 && results.get(asset.assetId)?.status !== 'failed') return 'skipped';

    let lastError;
    for (let attempt = 1; attempt <= RETRIES; attempt += 1) {
      try {
        assertNotStopped();
        const response = await fetch(`https://assets.grok.com/${asset.key}`, { credentials: 'include' });
        if (!response.ok) throw new Error(`HTTP ${response.status}`);
        const blob = await response.blob();
        if (!blob.size) throw new Error('сервер вернул пустой файл');
        await writeFile(name, blob);
        return 'done';
      } catch (error) {
        lastError = error;
        if (attempt < RETRIES) await new Promise((resolve) => setTimeout(resolve, attempt * 1000));
      }
    }
    throw lastError;
  }

  async function downloadAll() {
    if (!destination) throw new Error('Сначала выберите папку назначения.');
    if (!assets.length) await scan();
    if (!assets.length) return;

    shouldStop = false;
    setBusy(true);
    let cursor = 0;
    let done = 0;
    let skipped = 0;
    let failed = 0;
    setProgress(0, assets.length);

    async function worker() {
      while (true) {
        assertNotStopped();
        const asset = assets[cursor++];
        if (!asset) return;
        const previous = results.get(asset.assetId);
        if (previous?.status === 'done' || previous?.status === 'skipped') {
          done += 1;
          setProgress(done + skipped + failed, assets.length);
          continue;
        }
        try {
          const outcome = await downloadOne(asset);
          results.set(asset.assetId, { status: outcome });
          if (outcome === 'skipped') skipped += 1;
          else done += 1;
        } catch (error) {
          if (error?.name === 'AbortError') throw error;
          failed += 1;
          results.set(asset.assetId, { status: 'failed', error: String(error?.message || error) });
          log(`Ошибка ${asset.assetId}: ${error?.message || error}`);
        }
        setProgress(done + skipped + failed, assets.length);
        setStatus(`Скачивание: ${done} готовы, ${skipped} уже есть, ${failed} ошибок из ${assets.length}.`);
        if ((done + skipped + failed) % 10 === 0) await writeManifest();
      }
    }

    try {
      await Promise.all(Array.from({ length: PARALLEL_DOWNLOADS }, worker));
      await writeManifest();
      setStatus(`Готово: ${done} скачано, ${skipped} уже было, ошибок: ${failed}.`);
      log(`Скачивание завершено. Manifest: ${MANIFEST_FILE}`);
    } finally {
      setBusy(false);
    }
  }

  root.addEventListener('click', async (event) => {
    const action = event.target.closest('button')?.dataset.action;
    if (!action || isBusy && action !== 'stop') return;
    try {
      if (action === 'folder') await chooseFolder();
      if (action === 'scan') await scan();
      if (action === 'download') await downloadAll();
      if (action === 'stop') {
        shouldStop = true;
        setStatus('Остановка после завершения текущих операций…');
        log('Запрошена остановка.');
      }
    } catch (error) {
      const message = error?.name === 'AbortError' ? 'Операция остановлена.' : `Ошибка: ${error?.message || error}`;
      setStatus(message);
      log(message);
    }
  });
})();