FA Conglomerator 3.2.1 [2018-06-12]

// ==UserScript==
// @name           FA Conglomerator
// @namespace      http://lulz.net/
// @version        3.2.1
// @description    Asynchronously extracts all submissions from a FurAffinity gallery or scraps collection and displays them as a list of links in a new window. A button "Conglomerate" is inserted into the gallery/scraps collection page which activates the script. Additional input boxes allow the specification of the maximum number of concurrent requests used, a request delay time and the number of retries. Install guide, updates, changelog and other scripts @ http://pastebin.com/u/lulzScript
// @include        http*://*.furaffinity.net/gallery/*
// @include        http*://*.furaffinity.net/scraps/*
//
// @grant GM_setValue
// @grant GM_getValue
//
// ==/UserScript==


//Some constants
//Default is 48 submissions per page. Valid values can be taken from the drop-down box on the gallery page.
var congPerPage = 48;
//Default is up to 10 retries for a page/submission to load.
var congRetries = 10;

//Maximum number of concurrent requests. Default is 4.
//Note: Your browser may still generally limit the maximum concurrent requests per server according to its system settings.
var maxConcReq = 4;

//The polling times in miliseconds for submission requests and submission list requests.
//Each request will check for an available request slot using this interval.
//This is needed to limit concurrent request to maxConcReq.
//The value of submListPollTime should be lower than submPollTime, to give page loading priority over submission loading.
var submPollTime = 40;
var submListPollTime = 20;

//Nummber of miliseconds to wait before a request slot is freed for further request
//Can be used to reduce the stress on the server but links will take longer to conglomerate
var congFreeSlotDelay = 500;


//When each request either loaded or gave an error, the rCount should be back at 0.
//Pending downlaods lead to rCount > 0
function checkIfDone()
{
  if (rCount == 0)
    lStatus.innerHTML = '<b> Done!</b>';
}

//Frees a used request slot
function freeRequestSlot()
{
  concReq--;
}

//Function called when a submission page was successfully loaded.
//req = XMLHttpRequest object of the loaded page
//linkObj = link object within the new link window
function onSubm (req, linkObj)
{
  //Extract download link from submission page
  var submFile_ = req.responseText.match(/< *a *href=\"([^>\"]*)\" *> *Download *< *\/a *>/i);
  if ( submFile_ )
  {
    var wProto = submFile_[1].replace(/^\/\//, faProto[0]);
    //Insert extracted download link into link list
    //Link displayed as text
    linkObj.innerHTML = wProto;
    //Also set the href attribut, to make the link clickable
    linkObj.setAttribute('href', wProto);
  }

  //Request finished successfully. Decrease request counter and check if rCount = 0.
  rCount--;
  checkIfDone();

}

//Function called when a submission page failed to load.
//req = XMLHttpRequest object of the failed page
//linkObj = link object within the new link window
//retry = counter that indicates the current number of retries.
function onSubmErr (req, linkObj, retry)
{
  //Display "Error" in the linklist if the link couldn't be extracted.
  linkObj.innerHTML = '<b>Error (retry = ' + retry + '/' + congRetries + ')</b>';

  //Request finished with error. Decrease request counter and check if rCount = 0.
  rCount--;
  checkIfDone();
}

//Loads a submission page asynchronously.
//submURL = Full URL of submission.
//linkObj = link object within the new link window
//retry = counter that indicates the current number of retries.
//After [congRetries] failed retries, the function fails.
function getSubm (submURL, linkObj, retry)
{
  //Check for free request slot
  if (concReq < maxConcReq)
  {
    //Reserve request slot
    concReq++;

    //Display loading message and retries needed if any.
    if (retry == 0)
      linkObj.innerHTML = '<b>loading...</b>';
    else
      linkObj.innerHTML = '<b>loading...(retry = ' + retry + '/' + congRetries + ')</b>';

    var req = new XMLHttpRequest();
    //Prepare request with full URL, containing the desired page number
    req.open('GET', submURL, true);
    //Function to call when loading state of page changes
    req.onreadystatechange = function (evt)
    {

      //Check if page was loaded (or not)
      if (req.readyState == 4)
      {
        //Free reserved request slot
        if (congFreeSlotDelay > 0)
          setTimeout (freeRequestSlot, congFreeSlotDelay);
        else
          freeRequestSlot();

        if (req.status == 200)
        {
          //Page was sucessfully loaded
          //Call handler for successful requests.
          onSubm (req, linkObj);
        }
        else
        {
          //If page failed to load, retry up to [congRetries] times.
          if ( retry < congRetries )
          {
            linkObj.innerHTML = 'waiting...[queued for retry ' + (retry+1) + '/' + congRetries + ']'
            //The retry is done with setTimeout, instead of calling getSubm directly.
            //This will prevent multiple recursive calls for each retry
            setTimeout (getSubm, 0, submURL, linkObj, retry+1);
          }
          else
          {
            //Page could not be loaded even after retrying
            //Call error handler.
            onSubmErr (req, linkObj, retry);
          }
        }
      }
    };

    //Sends request and starts loading of the page.
    req.send(null);
  }
  else
    //If no request slot is available, poll for free request slot by calling this request again after a delay of [submPollTime] miliseconds.
    setTimeout (getSubm, submPollTime, submURL, linkObj, retry)
}

//Function called when a submission list page was successfully loaded.
//req = XMLHttpRequest object of the loaded page
//no = number of loaded page
//retry = counter that indicates the current number of retries.
function onSubmList (req, no, retry)
{
  //Extract all submission links
  var rawSubmLinks = req.responseText.match(/<a href=\"\/view\/\d+\/\">/gi);

  //Continue as long as there are submission links found
  //An empty page indicates the end of a gallery/scraps collection
  if ( rawSubmLinks )
  {
    //Display page number
    var div = congLinkWin.document.createElement('div');

    //Display page number an retries needed if any.
    if (retry == 0)
      div.innerHTML = '</br></br><b>Page ' + no + '</b></br></br>';
    else
      div.innerHTML = '</br></br><b>Page ' + no + ' (retry = ' + retry + '/' + congRetries + ')</b></br></br>';

    congLinkWin.document.body.insertBefore(div, null);

    //Iterate through all found links.
    for (var i=0; i < rawSubmLinks.length; i++)
    {
      //Extract actual link from list of matched links
      var submLink = rawSubmLinks[i].match(/<a href=\"\/view\/(\d+)\/\">/);
      if ( submLink )
      {
        //Make full URL of submission page from link.
        var submURL = faBaseURL + '/view/' + submLink[1] + '/';

        //Add new line for submission in link window and set status to "loading..."
        var div = congLinkWin.document.createElement('div');
        div.innerHTML = '<b>' + sCount + ': </b>';
        var linkObj = congLinkWin.document.createElement('a');
        linkObj.innerHTML = 'waiting...';
        linkObj.setAttribute('target', '_blank');
        div.insertBefore(linkObj, null);
        congLinkWin.document.body.insertBefore(div, null);

        //New request. Increase request counter.
        rCount++;
        getSubm (submURL, linkObj, 0);

        //Increase submission counter
        sCount++;

      }
    }

    //New request. Increase request counter.
    rCount++;
    //Load next submission list page
    getSubmList (submBaseURL, no+1, congPerPage, 0);
  }
  else
  {
    //There are no more pages. Display message.
    var div = congLinkWin.document.createElement('div');
    div.innerHTML = '</br></br><b>No more pages!</b>';
    congLinkWin.document.body.insertBefore(div, null);
  }

  //Request finished successfully. Decrease request counter and check if rCount = 0.
  rCount--;
  checkIfDone();

}

//Function called when a submission list page failed to load.
//req = XMLHttpRequest object of the failed page
//no = number of loaded page
//retry = counter that indicates the current number of retries.
function onSubmListErr (req, no, retry)
{
  //Page could not be loaded. Display Error message.
  var div = congLinkWin.document.createElement('div');
  div.innerHTML = '</br></br><b>Error - cannot load page ' + no + ' (retry = ' + retry + '/' + congRetries + ')!<br/>Further conglomeration stoppped.</b><br/>';
  congLinkWin.document.body.insertBefore(div, null);

  //Request finished with error. Decrease request counter and check if rCount = 0.
  rCount--;
  checkIfDone();
}

//Loads a submission list page from the gallery/scraps collection asynchronously.
//baseURL = Base URL of gallery/scraps collection page.
//no = Nummer of list page to load
//perpage = submissions per page {24, 36, 48, 60}
//retry = counter that indicates the current number of retries.
//After [congRetries] failed retries, the function fails.
function getSubmList (baseURL, no, perpage, retry)
{
  //Check for free request slot
  if (concReq < maxConcReq)
  {
    //Reserve request slot
    concReq++;

    var req = new XMLHttpRequest();
    //Prepare request with full URL, containing the desired page number
    req.open('POST', baseURL + no, true);
    //Set additional request header
    req.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');

    //Function to call when loading state of page changes
    req.onreadystatechange = function (evt)
    {
      //Check if page was loaded (or not)
      if (req.readyState == 4)
      {
        //Free reserved request slot
        if (congFreeSlotDelay > 0)
          setTimeout (freeRequestSlot, congFreeSlotDelay);
        else
          freeRequestSlot();

        if (req.status == 200)
        {
          //Page was sucessfully loaded
          //Call handler for successful requests.
          onSubmList (req, no, retry);
        }
        else
        {
          //If page failed to load, retry up to [congRetries] times.
          if ( retry < congRetries )
            //The retry is done with setTimeout, instead of calling getSubmList directly.
            //This will prevent multiple recursive calls for each retry and therefore save stack resources.
            setTimeout (getSubmList, 0, baseURL, no, perpage, retry+1);
          else
          {
            //Page could not be loaded even after retrying
            //Call error handler.
            onSubmListErr (req, no, retry);
          }
        }
      }
    };

    //Sends request and starts loading of the page.
    req.send('perpage=' + perpage);
  }
  else
    //If no request slot is available, poll for free request slot by calling this request again after a delay of [submListPollTime] miliseconds.
    setTimeout(getSubmList, submListPollTime, baseURL, no, perpage, retry);
}

//Global variables:

//Holds the object of the drop-down box containing the selected number of submissions per page
var submPerPage;
//Base URL of FA, including protocol and subdomain
var faBaseURL;
//Base URL of gallery/scraps
var submBaseURL;
//Reference to new link window
var congLinkWin;
//Counter for submission index
var sCount;
//Name of FA user whose galery/scraps are being conglomerated
var faUsername;
//Source of submission ("gallery" or "scraps")
var submSrc;
//Reference to status message in link window
var lStatus;
//Request counter. Used to determine when all submissions are loaded.
var rCount;
//Protocol http or https
var faProto;
//Counter for concurrent connections
var concReq;

//Main script function. Called when conglomerator button is clicked
function congStart(e)
{
  //Prevents the form's default action for a button
  e.preventDefault();

  //Get FA base URL (including protocol and subdomain); gallery/scraps base URL; submission source and username
  var baseURLs = document.location.href.match(/((.*furaffinity\.net)\/(gallery|scraps)\/([^\/]*))(?:\/|)/);
  if ( baseURLs )
  {
    submBaseURL = baseURLs[1] + '/'; //Submission base URL
    faBaseURL = baseURLs[2];         //FA base URL
    submSrc = baseURLs[3];           //submission source
    faUsername = baseURLs[4];        //FA username

    faProto = document.location.href.match(/(.*):\/\//);

    //Open new window for extracted submission links
    congLinkWin = window.open();
    if ( congLinkWin )
    {
      //Set title of new window
      congLinkWin.document.title = 'FA conglomerator';
      //Add title to link window
      var span = congLinkWin.document.createElement('span');
      span.innerHTML = '<b>List of ' + faUsername + '\'s ' + submSrc + ':</b>';
      congLinkWin.document.body.insertBefore(span, null);

      //Add status message to link Window
      lStatus = congLinkWin.document.createElement('span');
      lStatus.innerHTML = '<b> Loading...</b>';
      congLinkWin.document.body.insertBefore(lStatus, null);

      //Reset submission-, request and connection counter
      sCount = 1;
      rCount = 1;
      concReq = 0;

      //Set maximum concurrent requests according to the input field
      maxConcReq = parseInt(congMCRInput.value, 10);

      //Set free slot delay according to the input field
      congFreeSlotDelay = parseInt(congFSDInput.value, 10);

      //Set retries according to the input field
      congRetries = parseInt(congRetInput.value, 10);

      //Set number sumbissions per page using the selected value of the drop-down box if it is valid, otherwise uses default value
      if (submPerPage)
      {
        var congPerPage_ = parseInt(submPerPage.value, 10);
        if (!isNaN(congPerPage))
          congPerPage = congPerPage_;
      }

      //At this point faBaseURL and submBaseURL both matched correctly.
      //Load first submission list page
      getSubmList (submBaseURL, 1, congPerPage, 0, 0);
    }
  }
}

function insertButton()
{
  //Find location of gallery control buttons
  var browsebar = document.getElementsByClassName('page-options');

  //If found, insert conglomerator button
  if ( browsebar.length >= 1 )
  {
    var bParent = browsebar[0];
    bParent.insertBefore(congButton, bParent.firstChild);
    bParent.insertBefore(congInputSpan, congButton.nextSibling);
    bParent.insertBefore(document.createElement('br'), congInputSpan.nextSibling);
    bParent.insertBefore(document.createElement('br'), congInputSpan.nextSibling);

    //Find drop-down box containing the valid values for the number of submissions per page
    var submPerPage_ = document.getElementsByName('perpage');
    if (submPerPage_.length > 0)
    {
      submPerPage = submPerPage_[0];
      var submPerPageTitle = 'NOTE: This input box is part of the original webpage and lets you set the number of submissions you can view per page.\nThe conglomerator will also use this value for the number of submissions per page it will request from the server.';

      //Insert oder add tooltip text to original input field
      if (!submPerPage.title)
        submPerPage.title = submPerPageTitle;
      else
        submPerPage.title = submPerPage.title + '\n\n' + submPerPageTitle;
    }

    buttonInserted = true;
  }
}

//Flag that is later set to true if button was inserted.
var buttonInserted = false;

function onCongMCRInputChange(e)
{
  GM_setValue ('maxConcReq', congMCRInput.value);
}

//Create button for conglomerator script
var congButton = document.createElement('button');
//Set button styles/attributes
congButton.setAttribute('class', 'button');
congButton.innerHTML = 'Conglomerate';
congButton.style.marginRight = '10px';
congButton.title = 'Start conglomeration';

//Set function to call on a click
congButton.addEventListener ('click', congStart, false);

//Create span to group all conglomerator input fields
var congInputSpan = document.createElement('span');

//Create and insert text label
var congInputLabel = document.createElement('span');
congInputLabel.innerHTML = 'Concurrent requests:';
congInputSpan.insertBefore (congInputLabel, null);

//Create input field for maximum number of concurrent requests
var congMCRInput = document.createElement('input');
congInputSpan.insertBefore (congMCRInput, null);

//Create and insert text label
congInputLabel = document.createElement('span');
congInputLabel.innerHTML = 'Request delay [ms]:';
congInputSpan.insertBefore (congInputLabel, null);

//Create input field for request delay
var congFSDInput = document.createElement('input');
congInputSpan.insertBefore (congFSDInput, null);

congInputLabel = document.createElement('span');
congInputLabel.innerHTML = 'Retries:';
congInputSpan.insertBefore (congInputLabel, null);

//Create input field for number of retries
var congRetInput = document.createElement('input');
congInputSpan.insertBefore (congRetInput, null);



//Set input styles/attributes
congMCRInput.setAttribute('class', 'button');
congMCRInput.setAttribute ('type', 'number');
congMCRInput.setAttribute ('size', '3');
congMCRInput.setAttribute ('min', '1');
congMCRInput.style.width = '30px';
congMCRInput.style.marginLeft = '5px';
congMCRInput.style.marginRight = '10px';
congMCRInput.title = 'Maximum number of concurrent requests made to the server.\nIf this number is too high it might trigger request restrictions and timeouts on the server side, leading to high numbers of retries and possibly incomplete conglomeration.';

//Load last used value from storage
var maxConcReq_ = GM_getValue ('maxConcReq');
//If value is valid, set as initial value
if (maxConcReq_)
  congMCRInput.value = maxConcReq_;
else
  congMCRInput.value = maxConcReq;

congMCRInput.addEventListener ('change', onCongMCRInputChange, false);


function onCongFSDInputChange(e)
{
  GM_setValue ('congFreeSlotDelay', congFSDInput.value);
}

//Set input styles/attributes
congFSDInput.setAttribute('class', 'button');
congFSDInput.setAttribute ('type', 'number');
congFSDInput.setAttribute ('size', '3');
congFSDInput.setAttribute ('min', '0');
congFSDInput.style.width = '50px';
congFSDInput.style.marginLeft = '5px';
congFSDInput.style.marginRight = '10px';
congFSDInput.title = 'Number of milliseconds [ms] to wait before a request slot is freed for further requests.\nIncreasing this number can reduce the stress on the server but conglomeration will take longer';


//Load last used value from storage
var congFreeSlotDelay_ = GM_getValue ('congFreeSlotDelay');
//If value is valid, set as initial value
if (congFreeSlotDelay_)
  congFSDInput.value = congFreeSlotDelay_;
else
  congFSDInput.value = congFreeSlotDelay;

congFSDInput.addEventListener ('change', onCongFSDInputChange, false);


function onCongRetInputChange(e)
{
  GM_setValue ('congRetries', congRetInput.value);
}

//Set input styles/attributes
congRetInput.setAttribute('class', 'button');
congRetInput.setAttribute ('type', 'number');
congRetInput.setAttribute ('size', '3');
congRetInput.setAttribute ('min', '0');
congRetInput.style.width = '30px';
congRetInput.style.marginLeft = '5px';
congRetInput.title = 'The number of retries for a single request if an error occurs.\nIf there are many requests with a high number of retries in the conglomerator window, try to decreas the number of concurrent requests and/or increase the request delay.';

//Load last used value from storage
var congRetries_ = GM_getValue ('congRetries');
//If value is valid, set as initial value
if (congRetries_)
  congRetInput.value = congRetries_;
else
  congRetInput.value = congRetries;

congRetInput.addEventListener ('change', onCongRetInputChange, false);


//Try to insert button into page if page is already available at this time
insertButton();

//Function onDCL is called when DOM tree has completely been loaded.
//This is the safest way to start a script.
document.addEventListener ('DOMContentLoaded',
  function onDCL(e)
  {
    //If the button wasn't inserted before because the page wasn't
    //available at that time, it will be inserted now.
    if ( !buttonInserted )
      insertButton();

  }, false
);