Outfit Class Breakdown Scraper

#!/usr/bin/python
# Script to get class-partitioned playtimes of all members of an outfit
# Author: Nerylix - [3GIS] Waterson
# Start by running 'python get_playtimes.py -h'
# Licensed under the MIT License:
# http://opensource.org/licenses/MIT

import urllib2, json, time, sys

# SOE SID (aka API key) from http://census.soe.com/#devSignup
soeSID = ''

# outfit alias, aka tag (case sensitive!)
outfitAlias = '3GIS'

# ensure the presence of an SID
if soeSID == '':
  print 'ERROR: No SOE SID provided'
  print 'Enter your SOE SID in the variable soeSID'
  sys.exit(1)

def showHelp():
  print (
'''Usage: get_playtimes.py [<tag>]
Script to get class-partitioned playtimes of all members of PlanetSide 2 outfit
with given outfit tag. No tag defaults to 3GIS.
Author: Nerylix - [3GIS] Waterson''')
  sys.exit(0)

# get override outfit alias if present
if len(sys.argv) == 2:
  if '-h' in sys.argv[1]:
    showHelp()
  else:
    outfitAlias = sys.argv[1]
else:
  print 'WARNING: using default outfit alias "3GIS"'
  print 'NOTE: Run with "-h" for usage.'

baseAddr = 'http://census.soe.com/s:' + soeSID + '/'

# helper to make the calls
def makeAPICall(collection, callArgs={}, count=False):
  if count: addr = baseAddr + 'count/'
  else:     addr = baseAddr + 'get/'
  addr += 'ps2/' + collection + '/'
  if len(callArgs) != 0:
    addr += '?'
    for k, v in callArgs.items():
      addr += str(k) + '=' + str(v) + '&'
    addr = addr[:-1]

    return json.loads(urllib2.urlopen(addr).read())

def overwrite(s):
  sys.stdout.write('\r{0}'.format(s).ljust(70))
  sys.stdout.flush()

# this is the mapping of profile_type_id to profile_id, indicating the classes
# for which this data is available; this is a poorly designed feature of the
# API.  See
# http://census.soe.com/get/ps2/profile/?c:limit=100&c:lang=en&faction_id=1&c:show=name,profile_type_id&c:sort=profile_type_id
# and
# https://forums.station.sony.com/soe/index.php?threads/class-profile-mis-match-in-character_stat.11500063121/#post-11500338702
profileIDList = [1, 3, 4, 5, 6, 7]

# store the data in a list with items of format:
# [<name of player>, IN, LA, CM, EN, HA, MX]
# where each two-letter identifier maps to the total playtime of that player
# on the class indicated by the identifier (e.g. IN is the total playtime of
# the player on the infiltrator class)
playtimeList = []

# start by grabbing 3GIS's outfit ID, just in case it changes (I don't think it
# can, but better safe than sorry)
callArgs = {'alias':outfitAlias, 'c:show':'outfit_id,member_count'}
print 'Grabbing {0} outfit ID...   '.format(outfitAlias)
responseDict = makeAPICall('outfit', callArgs)
try:
  outfitID = responseDict['outfit_list'][0]['outfit_id'] # yuck
  memberCount = responseDict['outfit_list'][0]['member_count']
except KeyError:
  print 'Error in API call!\nExiting...'
  sys.exit(1)
except IndexError:
  # if IndexError, that almost certainly means there were no items in the
  # returned list of results
  print 'No outfit with tag {0} exists!\nExiting...'.format(outfitAlias)
  sys.exit(1)
print 'Done.'
print 'Outfit ID:', outfitID
print 'Member count:', memberCount

# use outfit key to get member list, resolved against playtime list from
# characters_stat and against character name list from character
# this is a scary API call :)
callArgs = {'outfit_id':outfitID, 'c:limit':memberCount,
            'c:join':'characters_stat^on:character_id^to:character_id^'
            'terms:stat_name=play_time^list:1^inject_at:play_times^'
            'show:profile_id\'value_forever,character^on:character_id^'
            'to:character_id^inject_at:name^show:name.first',
            'c:hide':'member_since,member_since_date,outfit_id,rank,rank_ordinal'}
print 'Grabbing all {0} members\' playtime history...   '.format(outfitAlias)
responseDict = makeAPICall('outfit_member', callArgs)
memberList = responseDict['outfit_member_list']
malformed = 0
for member in memberList:
  try:
    name = member['name']['name']['first']
    memberDataList = [name] + [0] * 6
    memberPlaytimes = member['play_times']
    for time in memberPlaytimes:
      profileID = time['profile_id']
      timePlayed = time['value_forever']
      memberDataList[profileIDList.index(int(profileID)) + 1] = float(timePlayed)/3600.0
  except KeyError:
    malformed += 1
    overwrite('WARNING: {0} malformed results encountered...'.format(malformed))
    continue
  playtimeList.append(memberDataList)

if malformed != 0:
  print
print 'Done.'

playtimeListSorted = sorted(playtimeList, key=lambda x: sum(x[1:]),
                            reverse=True)

print 'Writing CSV...   '
with open('{0}_playtimes.csv'.format(outfitAlias), 'w') as outFile:
  outFile.write('Name,Infiltrator,Light Assault,Combat Medic,Engineer,'
                'Heavy Assault,MAX,Total\n')
  for playtime in playtimeListSorted:
    playtime.append(sum(playtime[1:]))
    playtime = map(lambda x: str(x), playtime)
    outFile.write(','.join(playtime) + '\n')

print 'Done.'
print 'Check "{0}_playtimes.csv"'.format(outfitAlias)