FA NewSubs Conglomerator 1.1.0 [2018-06-26]

// ==UserScript==
// @name           FA NewSubs Conglomerator
// @namespace      http://lulz.net/
// @version        1.1.0
// @description    Asynchronously extracts submissions from a FurAffinity "New Submissions" list and displays them as a list of links in a new window. A button "Conglomerate" is inserted into the "New Submissions" page which activates the script. Additional input boxes allows the specification of the maximum number of concurrent requests, request delay time and number of retries. Install guide, updates, changelog and other scripts @ http://pastebin.com/u/lulzScript
// @include        http*://*.furaffinity.net/msg/submissions*
//
// @grant GM_setValue
// @grant GM_getValue
//
// @run-at document-start
// ==/UserScript==


//Retry constant
//Default is up to 10 retries for a page/submission to load.
var congRetries = 10;

//Maximum number of concurrent requests. Default is 4.
//Note: Your browser may still generally limit the maximum concurrent requests per server according to its system settings.
var maxConcReq = 4;

//The polling times in miliseconds for submission requests and submission list requests.
//Each request will check for an available request slot using this interval.
//This is needed to limit concurrent request to maxConcReq.
//The value of listPollTime should be lower than pollTime, to give page loading priority over submission loading.
var pollTime = 40;
var listPollTime = 20;

//Nummber of miliseconds to wait before a request slot is freed for further request
//Can be used to reduce the stress on the server but links will take longer to conglomerate
var congFreeSlotDelay = 500;


//When each request either loaded or gave an error, the rCount should be back at 0.
//Pending downlaods lead to rCount > 0
function checkIfDone()
{
  if (rCount == 0)
    lStatus.innerHTML = '<b> Done!</b>';
}

//Frees a used request slot
function freeRequestSlot()
{
  concReq--;
}

//Function called when a submission page was successfully loaded.
//req = XMLHttpRequest object of the loaded page
//linkObj = link object within the new link window
function onNS (req, linkObj)
{
  //Extract download link from submission page
  var submFile_ = req.responseText.match(/< *a *href=\"([^>\"]*)\" *> *Download *< *\/a *>/i);
  if ( submFile_ )
  {
    //Add protocol if missing
    var wProto = submFile_[1].replace(/^\/\//, faProto[0]);
    //Insert extracted download link into link list
    //Link displayed as text
    linkObj.innerHTML = wProto;
    //Also set the href attribut, to make the link clickable
    linkObj.setAttribute('href', wProto);
  }

  //Request finished successfully. Decrease request counter and check if rCount = 0.
  rCount--;
  checkIfDone();

}

//Function called when a submission page failed to load.
//req = XMLHttpRequest object of the failed page
//linkObj = link object within the new link window
//retry = counter that indicates the current number of retries.
function onNSErr (req, linkObj, retry)
{
  //Display error message in the linklist if the link couldn't be extracted.
  linkObj.innerHTML = 'Error (retry = ' + retry + ')';

  //Request finished with error. Decrease request counter and check if rCount = 0.
  rCount--;
  checkIfDone();
}

//Loads a submission page asynchronously.
//submURL = Full URL of submission.
//linkObj = link object within the new link window
//retry = counter that indicates the current number of retries.
//After [congRetries] failed retries, the function fails.
function getNS (submURL, linkObj, retry)
{
  //Check for free request slot
  if (concReq < maxConcReq)
  {
    //Reserve request slot
    concReq++;

    //Display loading message and retries needed if any.
    if (retry == 0)
      linkObj.innerHTML = '<b>loading...</b>';
    else
      linkObj.innerHTML = '<b>loading...(retry = ' + retry + '/' + congRetries +')</b>';

    var req = new XMLHttpRequest();
    //Prepare request with full URL, containing the desired page number
    req.open('GET', submURL, true);
    //Function to call when loading state of page changes
    req.onreadystatechange = function (evt)
    {
      //Check if page was loaded (or not)
      if (req.readyState == 4)
      {
        //Free reserved request slot
        if (congFreeSlotDelay > 0)
          setTimeout (freeRequestSlot, congFreeSlotDelay);
        else
          freeRequestSlot();

        if (req.status == 200)
        {
          //Page was sucessfully loaded
          //Free reserved request slot
          concReq--;
          //Call handler for successful requests.
          onNS (req, linkObj);
        }
        else
        {
          //If page failed to load, retry up to [congRetries] times.
          if ( retry < congRetries )
          {
            linkObj.innerHTML = 'waiting...[queued for retry ' + (retry+1) + '/' + congRetries + ']'
            //The retry is done with setTimeout, instead of calling getSubm directly.
            //This will prevent multiple recursive calls for each retry and therefore save stack resources.
            setTimeout (getNS, 0, submURL, linkObj, retry+1);
          }
          else
          {
            //Page could not be loaded even after retrying
            //Free reserved request slot
            concReq--;
            //Call error handler.
            onNSErr (req, linkObj, retry);
          }
        }
      }
    };

    //Sends request and starts loading of the page.
    req.send(null);
  }
  else
    //If no request slot is available, poll for free request slot by calling this request again after a delay of [submPollTime] miliseconds.
    setTimeout (getNS, pollTime, submURL, linkObj, retry)

}



//Function called when a submission list page was successfully loaded.
//req = XMLHttpRequest object of the loaded page
//no = number of loaded page
//retry = counter that indicates the current number of retries.
function onNSList (req, no, retry)
{
  //Extract all submission links
  var rawSubmLinks = req.responseText.match(/<a href=\"\/view\/\d+\/\">/gi);

  //If submission links are found
  if ( rawSubmLinks )
  {
    //Display page number
    var div = NScongLinkWin.document.createElement('div');

    //Display page number an retries needed if any.
    if (retry == 0)
      div.innerHTML = '</br></br><b>Page ' + no + '</b></br></br>';
    else
      div.innerHTML = '</br></br><b>Page ' + no + ' (retry = ' + retry + '/' + congRetries + ')</b></br></br>';

    NScongLinkWin.document.body.insertBefore(div, null);

    //Iterate through all found links.
    for (var i=0; i < rawSubmLinks.length; i++)
    {
      //Extract actual link from list of matched links
      var submLink = rawSubmLinks[i].match(/<a href=\"\/view\/(\d+)\/\">/);
      if ( submLink )
      {
        //Make full URL of submission page from link.
        var submURL = faBaseURL + '/view/' + submLink[1] + '/';

        //Add new line for submission in link window and set status to "loading..."
        var div = NScongLinkWin.document.createElement('div');
        div.innerHTML = '<b>' + sCount + ': </b>';
        var linkObj = NScongLinkWin.document.createElement('a');
        linkObj.innerHTML = 'waiting...';
        linkObj.setAttribute('target', '_blank');
        div.insertBefore(linkObj, null);
        NScongLinkWin.document.body.insertBefore(div, null);

        //New request. Increase request counter.
        rCount++;
        getNS (submURL, linkObj, 0);

        //Increase submission counter
        sCount++;

      }
    }
  }

  //<a class="more" href="/msg/submissions/new~27778975@48/">&gt;&gt;&gt; Next 48 &gt;&gt;&gt;</a>

  //Extract link to next page, if there is one.
  var nLink = req.responseText.match(/<a[^>]*class=\"more(?:-half)?\"[^>]*?href=\"\/msg\/submissions(\/new~\d+@\d+\/)\"[^>]*?/i);

  if (nLink)
  {
    //New request. Increase request counter.
    rCount++;
    //Load next submission list page
    getNSList (NSBaseURL + nLink[1], no+1, 0);
  }
  else
  {
    //There are no more pages. Display message.
    var div = NScongLinkWin.document.createElement('div');
    div.innerHTML = '</br></br><b>No more pages!<b>';
    NScongLinkWin.document.body.insertBefore(div, null);
  }

  //Request finished successfully. Decrement request counter and check if rCount = 0.
  rCount--;
  checkIfDone();

}

//Function called when a submission list page failed to load.
//req = XMLHttpRequest object of the failed page
//no = number of loaded page
//retry = counter that indicates the current number of retries.
function onNSListErr (req, no, retry)
{
  //Page could not be loaded. Display Error message.
  var div = NScongLinkWin.document.createElement('div');
  div.innerHTML = '</br></br><b>Error - cannot load page ' + no + ' (retry = ' + retry + '/' + congRetries + ')!<br/>Further conglomeration stoppped.</b><br/>';
  NScongLinkWin.document.body.insertBefore(div, null);

  //Request finished with error. Decrease request counter and check if rCount = 0.
  rCount--;
  checkIfDone();
}


//Loads a page of new submissions asynchronously.
//URL = URL of the submission list page.
//no = number of loaded page
//retry = counter that indicates the current number of retries.
//After [congRetries] failed retries, the function fails.
function getNSList (URL, no, retry)
{
  //Check for free request slot
  if (concReq < maxConcReq)
  {
    //Reserve request slot
    concReq++;

    var req = new XMLHttpRequest();
    //Prepare request with full URL, containing the desired page number
    req.open('GET', URL, true);

    //Function to call when loading state of page changes
    req.onreadystatechange = function (evt)
    {
      //Check if page was loaded (or not)
      if (req.readyState == 4)
      {
        //Free reserved request slot
        if (congFreeSlotDelay > 0)
          setTimeout (freeRequestSlot, congFreeSlotDelay);
        else
          freeRequestSlot();

        if (req.status == 200)
        {
          //Page was sucessfully loaded
          //Free reserved request slot
          concReq--;
          //Call handler for successful requests.
          onNSList (req, no, retry);
        }
        else
        {
          //If page failed to load, retry up to [congRetries] times.
          if ( retry < congRetries )
            //The retry is done with setTimeout, instead of calling getSubmList directly.
            //This will prevent multiple recursive calls for each retry and therefore save stack resources.
            setTimeout(getNSList, 0, URL, no, retry+1);
          else
          {
            //Page could not be loaded even after retrying
            //Free reserved request slot
            concReq--;
            //Call error handler.
            onNSListErr (req, no, retry);
          }
        }
      }
    };

    //Sends request and starts loading of the page.
    req.send(null);
  }
  else
    //If no request slot is available, poll for free request slot by calling this request again after a delay of [listPollTime] miliseconds.
    setTimeout(getNSList, listPollTime, URL, no, retry);
}

//Global variables:

//Base URL of FA, including protocol and subdomain
var faBaseURL;
//Base URL of the "New Submissions" page
var NSBaseURL;
//Reference to new link window
var NScongLinkWin;
//Counter for submission index
var sCount;
//Reference to status message in link window
var lStatus;
//Request counter. Used to determine when all submissions are loaded.
var rCount;
//Protocol http or https
var faProto;
//Counter for concurrent requests
var concReq;



//Main script function. Called when conglomerator button is clicked
//e = event object
function NScongStart(e)
{
  //Prevents the form's default action for a button
  e.preventDefault();

  //Get FA base URL (including protocol and subdomain)
  var faBaseURL_ = document.location.href.match(/(.*furaffinity\.net)/);
  if ( faBaseURL_ )
  {
    faBaseURL = faBaseURL_[1]
    NSBaseURL = faBaseURL + '/msg/submissions';

    faProto = document.location.href.match(/(.*):\/\//);


    //Open new window for extracted submission links
    NScongLinkWin = window.open();
    if ( NScongLinkWin )
    {
      //Set title of new window
      NScongLinkWin.document.title = 'FA NewSubs conglomerator';
      //Add title to link window
      var span = NScongLinkWin.document.createElement('span');
      span.innerHTML = '<b>List of new submissions:<b>';
      NScongLinkWin.document.body.insertBefore(span, null);

      //Add status message to link Window
      lStatus = NScongLinkWin.document.createElement('span');
      lStatus.innerHTML = '<b> Loading...</b>';
      NScongLinkWin.document.body.insertBefore(lStatus, null);

      //Reset submission, request and connection counters
      sCount = 1;
      rCount = 1;
      concReq = 0;

      //Set maximum concurrent requests according to the input field
      maxConcReq = parseInt(NScongMCRInput.value, 10);

      //Set free slot delay according to the input field
      congFreeSlotDelay = parseInt(NScongFSDInput.value, 10);

      //Set retries according to the input field
      congRetries = parseInt(NScongRetInput.value, 10);


      //Load first submission list page
      getNSList (NSBaseURL, 1, 0);
    }
  }
}


function insertButton()
{
  var msgform = document.getElementById('messages-form');
  if (msgform)
  {
    var inputInserted = false;

    var divs = msgform.getElementsByTagName('div');
    for (var i=0; i < divs.length; i++)
    {
      if ( divs[i].hasAttribute('class') && (divs[i].getAttribute('class') == 'actions') )
      {
        //Create button for conglomerator script
        var NScongButton = document.createElement('button');
        //Set button styles/attributes
        NScongButton.setAttribute('class', 'button');
        NScongButton.innerHTML = 'Conglomerate';
        NScongButton.style.backgroundColor = "lightgreen";
        NScongButton.style.marginRight = '10px';
        NScongButton.title = 'Start conglomeration';
        //Set function to call on a click
        NScongButton.addEventListener ('click', NScongStart, false);

        //Only insert one input box for limit of concurrent request
        if (inputInserted)
        {
          NScongInputDiv2.insertBefore (NScongButton, null);
          msgform.insertBefore(NScongInputDiv2, divs[i]);

          //Set flag to true
          buttonInserted = true;
          return;
        }
        else
        {
          NScongInputDiv.insertBefore (NScongButton, NScongInputDiv.firstChild);
          msgform.insertBefore(NScongInputDiv, divs[i].nextSibling);
          inputInserted = true;
        }

        //Set flag to true
        buttonInserted = true;
      }
    }
  }
}

//Flag that is later set to true if button was inserted.
var buttonInserted = false;


//Create div to group all conglomerator input fields in the top action bar
var NScongInputDiv = document.createElement('div');
//Use FA alignment styles
NScongInputDiv.style.textAlign = 'left';
NScongInputDiv.style.maxWidth = '700px';
NScongInputDiv.style.margin = '7px auto 7px auto';
NScongInputDiv.style.overflowX = 'hidden';
NScongInputDiv.style.overflowY = 'hidden';

//Create second div for bottom action bar
var NScongInputDiv2 = document.createElement('div');
//Use FA alignment styles
NScongInputDiv2.style.textAlign = 'left';
NScongInputDiv2.style.maxWidth = '700px';
NScongInputDiv2.style.margin = '7px auto 7px auto';
NScongInputDiv2.style.overflowX = 'hidden';
NScongInputDiv2.style.overflowY = 'hidden';

//Create and insert text label
var NScongInputLabel = document.createElement('span');
NScongInputLabel.innerHTML = 'Concurrent requests:';
NScongInputDiv.insertBefore (NScongInputLabel, null);

//Create input field for maximum number of concurrent requests
var NScongMCRInput = document.createElement('input');
NScongInputDiv.insertBefore (NScongMCRInput, null);

//Create and insert text label
NScongInputLabel = document.createElement('span');
NScongInputLabel.innerHTML = 'Request delay [ms]:';
NScongInputDiv.insertBefore (NScongInputLabel, null);

//Create input field for request delay
var NScongFSDInput = document.createElement('input');
NScongInputDiv.insertBefore (NScongFSDInput, null);

NScongInputLabel = document.createElement('span');
NScongInputLabel.innerHTML = 'Retries:';
NScongInputDiv.insertBefore (NScongInputLabel, null);

//Create input field for number of retries
var NScongRetInput = document.createElement('input');
NScongInputDiv.insertBefore (NScongRetInput, null);


//Set input styles/attributes
NScongMCRInput.setAttribute('class', 'button');
NScongMCRInput.style.backgroundColor = 'lightgreen';
NScongMCRInput.setAttribute ('type', 'number');
NScongMCRInput.setAttribute ('size', '3');
NScongMCRInput.setAttribute ('min', '1');
NScongMCRInput.style.width = '30px';
NScongMCRInput.style.marginLeft = '5px';
NScongMCRInput.style.marginRight = '10px';
NScongMCRInput.title = 'Maximum number of concurrent requests made to the server.\nIf this number is too high it might trigger request restrictions and timeouts on the server side, leading to high numbers of retries and possibly incomplete conglomeration.';

//Load last used value from storage
var maxConcReq_ = GM_getValue ('maxConcReq');
//If value is valid, set as initial value
if (maxConcReq_)
  NScongMCRInput.value = maxConcReq_;
else
  NScongMCRInput.value = maxConcReq;

NScongMCRInput.addEventListener ('change', onNSCongMCRInputChange, false);

function onNSCongMCRInputChange(e)
{
  GM_setValue ('maxConcReq', NScongMCRInput.value);
}


//Set input styles/attributes
NScongFSDInput.setAttribute('class', 'button');
NScongFSDInput.style.backgroundColor = 'lightgreen';
NScongFSDInput.setAttribute ('type', 'number');
NScongFSDInput.setAttribute ('size', '3');
NScongFSDInput.setAttribute ('min', '0');
NScongFSDInput.style.width = '50px';
NScongFSDInput.style.marginLeft = '5px';
NScongFSDInput.style.marginRight = '10px';
NScongFSDInput.title = 'Number of milliseconds [ms] to wait before a request slot is freed for further requests.\nIncreasing this number can reduce the stress on the server but conglomeration will take longer';


//Load last used value from storage
var congFreeSlotDelay_ = GM_getValue ('congFreeSlotDelay');
//If value is valid, set as initial value
if (congFreeSlotDelay_)
  NScongFSDInput.value = congFreeSlotDelay_;
else
  NScongFSDInput.value = congFreeSlotDelay;

NScongFSDInput.addEventListener ('change', onNSCongFSDInputChange, false);

function onNSCongFSDInputChange(e)
{
  GM_setValue ('congFreeSlotDelay', NScongFSDInput.value);
}


//Set input styles/attributes
NScongRetInput.setAttribute('class', 'button');
NScongRetInput.style.backgroundColor = 'lightgreen';
NScongRetInput.setAttribute ('type', 'number');
NScongRetInput.setAttribute ('size', '3');
NScongRetInput.setAttribute ('min', '0');
NScongRetInput.style.width = '30px';
NScongRetInput.style.marginLeft = '5px';
NScongRetInput.title = 'The number of retries for a single request if an error occurs.\nIf there are many requests with a high number of retries in the conglomerator window, try to decreas the number of concurrent requests and/or increase the request delay.';

//Load last used value from storage
var congRetries_ = GM_getValue ('congRetries');
//If value is valid, set as initial value
if (congRetries_)
  NScongRetInput.value = congRetries_;
else
  NScongRetInput.value = congRetries;

NScongRetInput.addEventListener ('change', onNSCongRetInputChange, false);

function onNSCongRetInputChange(e)
{
  GM_setValue ('congRetries', NScongRetInput.value);
}


//Try to insert button into page if page is already available at this time
insertButton();

//Function onDCL is called when DOM tree has completely been loaded.
//This is the safest way to start a script.
document.addEventListener ('DOMContentLoaded',
  function onDCL(e)
  {
    //If the button wasn't inserted before because the page wasn't
    //available at that time, it will be inserted now.
    if ( !buttonInserted )
      insertButton();

  }, false
);