jtvStreamsUpdater.py

# ================================
# Ensure env variables are correct
# ================================

from os.path import abspath, dirname, join
from os import environ
from sys import path

# Get absolute paths from relative paths
honsappPath = abspath(dirname(__file__))
honstreamsPath = abspath(join(honsappPath, '../'))

# Make sure they are in sys.path
if honsappPath not in path:
    path.append(honsappPath)
if honstreamsPath not in path:
    path.append(honstreamsPath)

# Make sure DJANGO_SETTINGS_MODULE is set
if 'DJANGO_SETTINGS_MODULE' not in environ.keys():
    environ['DJANGO_SETTINGS_MODULE'] = 'settings'

# =======
# Imports
# =======

from jtvSettings import *
from honsapp.models import Stream
import sys
import time
import math
import datetime
import json
import logging
import socket

# =======
# Globals
# =======

DEBUG = False

# Interval length is 5 minutes in seconds
INTERVAL_LENGTH = 60 * 5

# Update return codes
UPDATED = 1
SKIPPED = 2
UPDATED_TO_ONLINE = 3
UPDATED_TO_OFFLINE = 4
NOTHING_TO_DO = 5

# =============
# Configuration
# =============

# Set up logging
logging.basicConfig(filename=join(honsappPath, 'jtvStreamsUpdater.log'),
                    level=logging.ERROR,
                    format='%(asctime)s:%(levelname)s: %(message)s')

# ====================
# Function definitions
# ====================

def getJtvClient():
    "Fetch an authenticated jtv client"

    from jtvClient import JtvClient
    from jtvSettings import CONSUMER_KEY, CONSUMER_SECRET

    return JtvClient(CONSUMER_KEY, CONSUMER_SECRET)

def updateStreamStatus(stream):

    returncode = SKIPPED

    # Create a jtv client
    client = getJtvClient()

    # Poll the jtv API
    response = None
    attempts = 0

    while response == None:
        try:
            # Try to fetch status
            response = client.get('/stream/list.json?channel=%s' % stream.name)
        except socket.timeout:
            # Debug message
            if DEBUG:
                print 'Failed fetching status for %s. Retrying...' \
                       % stream.name

            if attempts < 3:
                # Try 3 times
                attempts += 1
                continue
            else:
                # Then give up
                error = 'Failed fetching status %d times in a row. Exiting'
                print error
                logging.critical(error)
                exit()

    result = json.loads(response.read())

    # Check for errors
    if 'error' in result:
        print result['error']
        logging.error(result['error'])
        exit()

    if len(result) > 0:
        # The stream is live
        if stream.online != True:
            stream.online = True
            returncode = UPDATED_TO_ONLINE
    else:
        # The stream is not live
        if stream.online != False:
            stream.online = False
            returncode = UPDATED_TO_OFFLINE

    # Set the stream to updated now
    stream.statusUpdated = datetime.datetime.now()

    # Save the stream
    stream.save()

    return returncode

def updateStreamInfo(stream):

    # Import StreamInfo
    from honsapp.models import StreamInfo

    returncode = SKIPPED

    # Get a jtv client
    client = getJtvClient()

    # Poll jtv API for channel info
    response = client.get('/channel/show/%s.json' % stream.name)

    result = json.loads(response.read())

    # Check for errors
    if 'error' in result:
        print result['error']
        logging.error(result['error'])
        exit()

    # Create StreamInfo object
    sinfo = StreamInfo()
    sinfo.title = result['title']
    sinfo.description = result['description']
    sinfo.about = result['about']

    sinfo.stream_url = result['channel_url']

    sinfo.backgroundImage = result['channel_background_image_url']
    sinfo.headerImage = result['channel_header_image_url']
    sinfo.mediumChannelImage = result['image_url_medium']
    sinfo.largeChannelImage = result['image_url_large']

    sinfo.mediumScreenCap = result['screen_cap_url_medium']
    sinfo.largeScreenCap = result['screen_cap_url_large']

    sinfo.backgroundColor = result['channel_background_color']
    sinfo.columnColor = result['channel_column_color']
    sinfo.textColor = result['channel_text_color']
    sinfo.linkColor = result['channel_link_color']

    # Compare the new stream info to the old
    if stream.info == sinfo:
        # If identical, set returncode to skipped
        returncode = SKIPPED
    else:
        # Save the new steam info
        sinfo.save()

        # If there's an old one, delete it
        if stream.info != None:
            stream.info.delete()

        # Link the new stream info to the stream
        stream.info = sinfo

        # Set the returncode to updated
        returncode = UPDATED

    # Mark stream as updated
    stream.infoUpdated = datetime.datetime.now()
    stream.save()

    return returncode

def mainUpdateLoop():
    """
    Updates the status of all streams to either online or offline, as well as
    updating the stream info, respecting the global restrictions.
    """

    def streamText(stream):
        """
        Returns a string representation of the stream displaying last updated
        information
        """
        # return '<Stream name="%s", statusUpdated="%s", infoUpdated="%s">' \
        #             % (stream.name, stream.statusUpdated, stream.infoUpdated)

        return '<Stream name="%s">' % stream.name

    # Run for ever!
    while True:
        # Start time
        intervalStart = time.time()

        # The total amount of ticks this interval
        totalTicks = MAINLOOP_STATUSUPDATES + MAINLOOP_INFOUPDATES

        # Get all jtv streams sorted by last updated status
        statusStreams = Stream.objects.filter(network__name='justin.tv')\
                                      .order_by('statusUpdated', 'id')

        # Get all jtv streams sorted by last updated info
        infoStreams = Stream.objects.filter(network__name='justin.tv')\
                                    .order_by('infoUpdated', 'id')

        # Update the status of the streams with the least recently updated
        # status
        for i, stream in enumerate(statusStreams):

            # Remember the limits
            if i > (MAINLOOP_STATUSUPDATES - 1):
                break

            print 'Updating status on %s...' % streamText(stream),

            # Update stream here
            result = updateStreamStatus(stream)

            # Print out what happened
            if result == SKIPPED:
                print 'Unchanged'
            elif result == UPDATED_TO_ONLINE:
                print 'Updated to online'
            elif result == UPDATED_TO_OFFLINE:
                print 'Updated to offline'

        # Update the info of the streams with the least recently updated info
        for i, stream in enumerate(infoStreams):

            # Remember the limits
            if i > (MAINLOOP_INFOUPDATES - 1):
                break

            print 'Updating info on %s...' % streamText(stream),

            # Update stream here
            result = updateStreamInfo(stream)

            # Print out what happened
            if result == SKIPPED:
                print 'Unchanged'
            else:
                print 'Saved new info'

        # Wait 5 minutes
        while True:
            timeWaited = math.ceil(time.time() - intervalStart)

            # If we've waited over 5 minutes
            # if timeWaited > 10:
            if timeWaited > INTERVAL_LENGTH:
                break

            if DEBUG:
                # Print a message every 5 seconds
                if timeWaited % 30 == 0:
                    print '### %d seconds to next interval...' \
                          % (INTERVAL_LENGTH - timeWaited)

            time.sleep(1)

def fetchNewInfo():
    # Get all streams that has no info
    streams = Stream.objects.filter(info__isnull=True)

    if(len(streams) < 1):
        # If there are no streams to update
        return NOTHING_TO_DO

    for i, stream in enumerate(streams):
        print 'Updating info on %s...' % stream,

        # Update the info of this stream
        result = updateStreamInfo(stream)

        # Output result
        if result == SKIPPED:
            print 'Unchanged'
        else:
            print 'Saved new information'

        # Respect global limits
        if (i + 1) >= MAX_REQUESTS_PER_5MIN:
            # Pause every time the max request limit is met
            if i % MAX_REQUESTS_PER_5MIN == 0:
                print 'Limit met! Sleeping for 5 minutes before continuing...'

                # Count down
                for i in xrange(INTERVAL_LENGTH):
                    sys.stdout.write('\r' + ' ' * 80)
                    sys.stdout.write('\r' + 'Countdown: %d' % (INTERVAL_LENGTH - i))
                    sys.stdout.flush()
                    time.sleep(1)

# ===============
# Run this script
# ===============
if(__name__ == '__main__'):
    # Dependencies
    import argparse
    import textwrap

    # Some constants
    commandchoices = ['scrapeloop', 'newinfo']
    commandhelp = '''This module handles refreshing Justin TV streams

Commands:
  scrapeloop    Starts a loop that updates all the streams statuses, respecting
                the global limits. Also updated stream information, but at a
                much slower rate

  newinfo       Fetches stream information for streams that still has no
                information row'''

    # Create a parser
    parser = argparse.ArgumentParser(description=commandhelp,
                formatter_class=argparse.RawDescriptionHelpFormatter)

    # Take exactly one argument out of the command choices
    parser.add_argument('command', metavar='command', type=str,
                        choices=commandchoices,
                        help='The command to give the script')

    # Flag for debug
    parser.add_argument('-d', '--debug', action='store_true',
                        help='Print debug messages')

    # Get arguments
    args = parser.parse_args()
    command = args.command

    # Debug?
    if args.debug:
        DEBUG = True

    # Start the if loop
    if(command == 'scrapeloop'):
        mainUpdateLoop()

    elif(command == 'newinfo'):
        print 'Now making sure all jtv streams has info'

        # Fetch new info
        result = fetchNewInfo()

        # Print results
        if result == NOTHING_TO_DO:
            print 'All streams already has info'
        else:
            print 'All streams now has info'