[mTurk userscript] Pending Earnings

// ==UserScript==
// @name           Pending Earnings
// @namespace      http://bluesky.software.com/turkscripts
// @description    Adds a pending earnings item to mturk dashboard
// @include        https://www.mturk.com/mturk/dashboard
// ==/UserScript==

//
// We are on the dashboard page. We want to go to the status_detail
// pages for the last 30 days and total the dollar amount of the
// HITs still pending and add it to the dashboard page like the today's projected earnings script
// does. We will use the XMLHttpRequest Object to get the pages and then
// process them one by one until we have done them all. An enhancement I just
// thought of is that we don't have to do all 30 days if we read the number of
// HITs pending and just process dates that have pending HITs.
//
// This will only be run if someone clicks on it as it is going to be really slow if they
// have a lot of HITs pending from a long time ago.
//
//
// 08/22/2011 Coding started
//
// 08/23/2011 Beta version ready but I only have 1 HIT pending so no way to do
//            much testing, will have to do some turking and then test
//
// 08/25/2011 There is the potential for a race condition in that while I'm
//            added up pending HITs to get to the total that was present on the
//            status page will have been approved before I've added them all up.
//            So I need to add some new logic otherwise it would be possible to
//            end up in an infinite loop trying to add in some pending HITs that
//            are no longer pending. I'm going to implement this by keeping track
//            of the number of HITs submitted and when reading detailed status
//            pages abort trying to add up x pending HITs if we exceed the number
//            of HITs submitted for that day.
//
// 08/30/2011 Not working for some so changed the way getNumberOfPending() works
//            and added an error message alert if this function can't find the
//            number of pending HITs
//
// 08/31/2011 Made a test version, the problem is located in
//            process_status_page()
//
// 09/01/2011 Found the problem: If there was no link in a row then trying to
//            to work on links[0] that doesn't exist would cause the script to blow
//            up, I put in a check that links has a length before trying to work with
//            it.
//
// 10/03/2011 Modified to save a new cookie which is a history of the previous time we
//            summed up the pendingEarnings and we use that to skip processing dates where
//            the pendingHits hasn't changed since the last time we were invoked. This should
//            minimize the times the page request exceeded error is encountered.
//
// 10/29/2011 Changed the expiration date of the pendingEarnings cookie to 30 days.
//
// 11/03/2011 Moved the setting of the pendingEarnings cookie outside the if loop to fix the
//            change of 10/29.
//
// The normal functioning of the script is to just get the cookie
// of PendingEarnings and display the link and total.
//
// If we are clicked on then processPendingHits is called and we will
// add up the pending HITs. Note there could still be no pending HITs
// the user just clicked on it anyway so we can't ignore the case of
// zero HITs to process. The total is saved in a cookie so it is
// available for all the times we haven't been clicked on.
//

// Insert the Pending Earnings in the dashboard.
// Copied from current_earnings script - Copyright (c) 2008, Mr. Berserk
//
// Modified to suit
//

var allTds, thisTd;
allTds = document.getElementsByTagName('td');
for (var i = 0; i < allTds.length; i++)
{
   thisTd = allTds[i];
   if ( thisTd.innerHTML.match(/Total Earnings/) && thisTd.className.match(/metrics\-table\-first\-value/) )
   {
      var row = document.createElement('tr');
      row.className = "even";


      var pendingEarningsLink = document.createElement('a');
      pendingEarningsLink.href =  "https://www.mturk.com/mturk/dashboard";
      pendingEarningsLink.innerHTML = "Pending HITs";
      pendingEarningsLink.addEventListener('click',processPendingHits,false);
      var cellLeft = document.createElement('td');
      cellLeft.className = "metrics-table-first-value";
      cellLeft.appendChild(pendingEarningsLink);
      row.appendChild(cellLeft);

      var cellRight = document.createElement('td');
      if(getCookie("MturkPendingEarnings"))
      {
         var pendingEarnings = parseFloat(getCookie("MturkPendingEarnings"));
         cellRight.innerHTML = "$" + pendingEarnings.toFixed(2);
      }
      else
      {
         cellRight.innerHTML = "$?.??";
      }
      row.appendChild(cellRight);

      thisTd.parentNode.parentNode.insertBefore(row,thisTd.parentNode.nextSibling);
   }
}

//
// Functions
//

//
// User has clicked on us, so add up all the pending HITs. The first thing
// we do is get a copy of the status page, this contains the summary of the
// past 30 days. We scan through this looking at the Pending HITs column and
// saving the dates that have pending HITs which we will subsequently use to
// access the appropriate detailed status pages :)
//

function processPendingHits()
{
   var pendingEarnings = 0;            // Dollar amount of pendingHITs
   var pendingHits = getNumberOfPending();
   if(pendingHits>0)   // only process pages if there is at least one pending HIT
   {
      var oldDatesToDo = new Array();           // this array will hold the history of the last
                                                // time we were clicked
      var datesToDo = process_status_page();    // get dates that have pending HITs

      //
      // Ok we have a list of dates to process and the number of pending HITS
      // for that date is appended to the encoded date. To make the script faster and
      // more efficient we save the array datesToDo as a cookie and compare the new values
      // just retrieved from the status page to skip processing those dates where the number of
      // pending HITs hasn't changed since the last time we were called. The one exception is today
      // where the pending HIT count might not of changed but the hits pending might have since more
      // hits could of been added. To catch this we also check the submitted HITs to the old submitted HITs
      //

      if(getCookie("MturkPendingDates"))   // retrieve history if it exists
      {
         oldDatesToDo = getCookie("MturkPendingDates").split(",");
      }

      //
      // Now we want to compare the old array with the new array and only process those dates where we don't have
      // the subtotal in the old array. But if the date is today, the pending HIT count could be the same but
      // but they could be different HITs because pending HITs could be added and subtracted so we have to check the
      // submitted HITs to catch this condition.
      //

      var subtotal = 0;
      for(n = 0; n < datesToDo.length; n++)
      {
         var dateProcessed = 0;
         var encodedDate = datesToDo[n].substr(0,8);
         var index1 = datesToDo[n].indexOf('$');
         var pendingHits = datesToDo[n].substring(8,index1); // the next part of the string up to the $
                                                            // is the pending Hits total
         var submittedHits = datesToDo[n].substring(index1+1);

         for(var m = 0; m < oldDatesToDo.length; m++)            // check if we have this date in the history
         {
            var old_encodedDate = oldDatesToDo[m].substr(0,8);
            if(encodedDate == old_encodedDate)
            {
               index1 = oldDatesToDo[m].indexOf('$');
               var old_pendingHits = oldDatesToDo[m].substring(8,index1); // the next part of the string up to the $
                                                                          // is the pending Hits total
               var index2 = oldDatesToDo[m].indexOf('%');                 // the next part of the string up to the %
                                                                          //  is the submitted HITs total
               var old_submittedHits = oldDatesToDo[m].substring(index1+1,index2);

               var old_subtotal = oldDatesToDo[m].substring(index2+1);   // the rest of the string is the subtotal

               // So since we have this date in the history we check if the pending HITs and the submitted HITs are the
               // same and if they are we don't bother to process this date by NOT adding it to the processDates array
               // and we add the existing subtotal for that date into the pendingHITs dollar amount right now.

               if (submittedHits == old_submittedHits && pendingHits == old_pendingHits)
               {

                  pendingEarnings += parseFloat(old_subtotal);           // use the old value since it is still good
                  datesToDo[n] = datesToDo[n] + '%' + old_subtotal;      // add old subtotal into the new array
                  dateProcessed = 1;
                  break;                                                 // found the date so exit loop
               }
            }
         }
         if(dateProcessed < 1)                                           // if the date wasn't in the history or the
         {                                                               // pending HITs has changed process the date
            subtotal = process_detailed_status_pages(encodedDate, pendingHits, submittedHits);
            datesToDo[n] = datesToDo[n] + '%' + subtotal;
            pendingEarnings += subtotal;
         }
      }
      // now overwrite the oldDatesToDo with the new one if the cookie already existed else create the cookie
      setCookie("MturkPendingDates",datesToDo.join(","),1);   // Save the array datesToDo as a cookie by converting
                                                              //  it to a string first

   }
      setCookie("MturkPendingEarnings",pendingEarnings,30);    // save total in cookie - move out here so we set it
                                                               // to zero if no HITs pending
}


//
// Get total pending HITs
//
// As of now there are 8 'metrics-table's on the dashboard
// but the last two are subsets of table 6 which has 26 td's
// but again we have subsets within subsets so that was the confusion
// So when you get the td's for table 5 you are also getting the td's
// for table 7 which is what we want. The confusion is when I try to
// match on innerHTML for ... Pending, I get a match for the superset
// td's not on the individual td. So td 14 contains ... Pending but it
// also contains all the other td's that are part of the second column
//
// To handle the recursive tables I'm now checking that the td is
// numeric, if it isn't we continue to scan tds.
//
function getNumberOfPending()
{
   var tables = document.getElementsByClassName('metrics-table');
   for (var m = 0; m <tables.length; m++)
   {
      var table_data = tables[m].getElementsByTagName('td');  // 26 data
      for (var n = 0; n <table_data.length; n++)
      {
         if(table_data[n].innerHTML.match('... Pending'))
         {
            if(isNumber(table_data[n+1].innerHTML))
            {
               return table_data[n+1].innerHTML;
            }
         }
      }
   }
   alert("Script Bug: Can't find how many pending HITs you have");
   return -1;           // didn't find it - This is a bug exit!
}

//
// Process the status page by getting all those dates that have pending HITs
//
// The first status page contains 30 days worth of HITs so there is no need to
// check the second page as there should be no pending HITS on that page
//
// Note: If the person is a total newbie then maybe the status page doesn't even
// have 30 days worth of data so don't hard code the 30 bozo :)
//
//
// We need to process the DOM in a recursive manner so that I can associate the
// correct date with the correct pending HIT count. The days listed aren't necessarly
// in sequential order if he took some days off there will be missing days.
//
// This main function just grabs the status page and then calls the function
// pending_dates to do the actual work
//

function process_status_page()
{
   // use XMLHttpRequest to fetch the entire page, use async mode for now because I understand it
   var page = getHTTPObject();
   page.open("GET",'https://www.mturk.com/mturk/status',false);
   page.send(null);
   return pending_dates(page.responseText);
}

//
// First of all we have to turn the grabbed status page back into a DOM
// object so we can process it with javascript. We do this with the
// create div trick.
//
// Now get all the tables
// Some of these tables won't be what we are looking for but there is no
// way to distingush at the table level, we have to look at the table data to
// know if this table is of interest to us.
//
// There is a problem that the table we are interested in is embedded in another
// table so we are processing the table we want twice, how to avoid this?
//

function pending_dates(page_text)
{
   var page_html = document.createElement('div');
   page_html.innerHTML = page_text;

   var datesIndex = 0;
   var activeDates = new Array();
   var tableProcessed = 0;

   var tables = page_html.getElementsByTagName('table');
   for (var m = 0; m < tables.length; m++) // process each table
   {
      var table_rows = tables[m].getElementsByTagName('tr');
      for (var n = 0; n < table_rows.length; n++) // process each row
      {
         //
         // if we are in a row we are interested in, its first td will contain a link
         // to a detailed status page, we look for a match on that link
         //
         var links = table_rows[n].getElementsByTagName('a');
         if(links.length>0 && links[0].href.substr(0,40)=='https://www.mturk.com/mturk/statusdetail')
         {
            //
            // OK we found an interesting row, now does this date have any pending HITs?
            //
            tableProcessed = 1; // Indicate that we have processed the table
            var table_data = table_rows[n].getElementsByClassName('statusPendingColumnValue');
            var pendingHits = table_data[0].innerHTML;  // This is a number, if greater than zero we have pending HITs
            table_data = table_rows[n].getElementsByClassName('statusSubmittedColumnValue');
            var submittedHits = table_data[0].innerHTML; // Number of HITs submitted for this date
            if(pendingHits > 0) //then save the date in the activeDates array
            {
               var encodedDate = links[0].href.substr(links[0].href.search(/Date=/)+5,8);
               // as a hack I'm appending the number of pending HITs to the encoded date so we
               // can return both pieces of data through the one dimensional array
               // now I want to add in the number of HITs submitted also but I have to be able
               // to parse the string later to distingush the two numbers
               activeDates[datesIndex++] = encodedDate + pendingHits + '$' + submittedHits;
            }
         }
      }
      if(tableProcessed>0) return activeDates;  // bail if we already processed the table, don't do it again!
   }
   alert("Script Bug: Couldn't find any dates to process");
}

//
// Process the detailed status pages associated with this date until we have
// found all the pending HITs for this date
//

function process_detailed_status_pages(encodedDate, pendingHits, submittedHits)
{
   var subtotal = 0;
   var pagenum = 1;          // detailed status page number, we start with page 1
   while (pendingHits > 0)
   {
      // use XMLHttpRequest to fetch the entire page, use async mode for now because I understand it
      var page = getHTTPObject();
      link = "https://www.mturk.com/mturk/statusdetail?sortType=All&pageNumber=" + pagenum + "&encodedDate=" + encodedDate;
      page.open("GET",link,false);
      page.send(null);
      var page_html = document.createElement('div');
      page_html.innerHTML = page.responseText;
      var amounts = page_html.getElementsByClassName('statusdetailAmountColumnValue');
      var statuses = page_html.getElementsByClassName('statusdetailStatusColumnValue');
      for(var k = 0; k < amounts.length; k++)
      {
         if(statuses[k].innerHTML == 'Pending Approval')
         {
            pendingHits--;
            index = amounts[k].innerHTML.indexOf('$');
            subtotal += parseFloat(amounts[k].innerHTML.substring(index+1));
         }
      }
      submittedHits -= 25;                       // 25 HITs to a page
      if (submittedHits <= 0) return subtotal;   // We have done all the HITs for this date
                                                 // But the pendingHits count isn't zero
                                                 // So we must of encountered a race condition
                                                 // exit with the subtotal
      pagenum++; // go do another page if necessary
   }
   return subtotal; // This is the dollar amount of pending HITs for this date
}

//
// XMLHttpRequest wrapper from web
//

function getHTTPObject()
{
   if (typeof XMLHttpRequest != 'undefined')
   {
      return new XMLHttpRequest();
   }
   try
   {
      return new ActiveXObject("Msxml2.XMLHTTP");
   }
   catch (e)
   {
      try
      {
         return new ActiveXObject("Microsoft.XMLHTTP");
      }
      catch (e) {}
   }
   return false;
}

//
// Is the variable a number or a string that parses to a number?
//

function isNumber (o)
{
   return ! isNaN(o-0);
}


//
//  Cookie functions copied from http://www.w3schools.com/JS/js_cookies.asp
//

function setCookie(cookie_name,value,exdays)
{
   var exdate = new Date();
   exdate.setDate(exdate.getDate() + exdays);
   var cookie_value = escape(value) + ((exdays==null) ? "" : "; expires="+exdate.toUTCString());
   document.cookie = cookie_name + "=" + cookie_value;
}


function getCookie(cookie_name)
{
   var i,x,y
   var ARRcookies = document.cookie.split(";");
   for (i=0; i<ARRcookies.length; i++)
   {
      x = ARRcookies[i].substr(0,ARRcookies[i].indexOf("="));
      y = ARRcookies[i].substr(ARRcookies[i].indexOf("=")+1);
      x = x.replace(/^\s+|\s+$/g,"");
      if (x == cookie_name)
      {
         return unescape(y);
      }
   }
}