PW_intake.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
import cookielib
import urllib
import urllib2
import os
import re

# ソコニハ、意志ハ無イ。在ルノハプログラムダケ。
# (pwOpener, pwCookies, pwHH) = PW_Connect()
# songIDList = PW_ParseSongIDs_IIDX(pwOpener)
# chartsRead = PW_BruteForce_IIDX(pwOpener, songIDList)
# chartsRead = PW_BruteForce_DDR(pwOpener)

def PW_Connect(pwOutputDir=r"C:\Temp\PW_IIDX"):
    # With appreciation to @blokeley at http://stackoverflow.com/questions/2910221/how-can-i-login-to-a-website-with-python
    loginURL = 'https://programmedworld.net/login'
    loginTel1 = {'username' : 'Telperion_1',
             'password' : 'end_of_world'}

    loginData = urllib.urlencode(loginTel1)
    pwCookies = cookielib.CookieJar()

    pwOpener = urllib2.build_opener(
        urllib2.HTTPRedirectHandler(),
        urllib2.HTTPHandler(debuglevel=0),
        urllib2.HTTPSHandler(debuglevel=0),
        urllib2.HTTPCookieProcessor(pwCookies))

    pwResponse = pwOpener.open(loginURL, loginData)
    pwLandingPage = pwResponse.read()
    pwHTTPHeaders = pwResponse.info()

    try:
        localFIDLP = open(pwOutputDir + "\\landingPage.html", 'w')
    except:
        os.makedirs(pwOutputDir)
        localFIDLP = open(pwOutputDir + "\\landingPage.html", 'w')
    localFIDLP.write(pwLandingPage)
    localFIDLP.close()

##  try:
##      localFIDHH = open(pwOutputDir + "\\HTTPHeaders.html", 'w')
##  except:
##      os.makedirs(pwOutputDir)
##      localFIDHH = open(pwOutputDir + "\\HTTPHeaders.html", 'w')
##  localFIDHH.write(pwHTTPHeaders)
##  localFIDHH.close()

    return (pwOpener, pwCookies, pwHTTPHeaders)

def PW_ReadChartScorePage_IIDX(pwOpener, songID, chartDifficulty, mixIIDX=22, pwOutputDir=r"C:\Temp\PW_IIDX"):
    try:
        pwFilename = 'https://programmedworld.net/iidx/{2:n}/music/{0:n}/{1:s}'.format(songID, chartDifficulty, mixIIDX)
        pwFID = pwOpener.open(pwFilename)
        songPage = pwFID.read()
    except:
        print "Unable to locate: song ID {0:05n}, chart {1:s}".format(songID, chartDifficulty)
        return 0
    else:
        localFilename = (pwOutputDir + '\\IIDX{2:n}\\{0:05n}_{1:s}.html').format(songID, chartDifficulty, mixIIDX)
        try:
            os.makedirs( (pwOutputDir + '\\IIDX{2:n}').format(songID, chartDifficulty, mixIIDX) )
        except:
            pass
        finally:
            localFID = open(localFilename, 'w')
            localFID.write(songPage)
            localFID.close()

            print("Successfully read song ID {0:05n}, chart {1:s}!".format(songID, chartDifficulty))
            return 1

def PW_ParseSongIDs_IIDX(pwOpener, mixIIDX=22, pwOutputDir=r"C:\Temp\PW_IIDX"):
    try:
        pwFilename = 'https://programmedworld.net/iidx/{0:n}/music'.format(mixIIDX)
        pwFID = pwOpener.open(pwFilename)
        songPage = pwFID.read()
    except:
        print "Unable to locate music listing page"
        return []
    else:
        songIDList = re.findall(r"/iidx/\d+/music/(\d+)/", songPage, re.UNICODE)

        localFilename = (pwOutputDir + '\\IIDX{0:n}\\songIDList.txt').format(mixIIDX)
        try:
            os.makedirs( (pwOutputDir + '\\IIDX{0:n}').format(mixIIDX) )
        except:
            pass
        finally:
            localFID = open(localFilename, 'w')
            for songID in songIDList:
                localFID.write(str(songID) + '\n')
            localFID.close()

            print("Successfully downloaded and parsed music listing page!")
        return songIDList


def PW_BruteForce_IIDX(pwOpener, songIDList=None, mixIIDX=22, pwOutputDir=r"C:\Temp\PW_IIDX"):
    songsRead = 0;

    try:
        digestFID = open(pwOutputDir + "\\readResults.csv", 'w')
    except:
        os.makedirs(pwOutputDir)
        digestFID = open(pwOutputDir + "\\readResults.csv", 'w')

    if songIDList == None:
        for mixID in range(1, 22):
            songIDsSkipped = 0
            for songIndex in range(150):
                songID = mixID*1000 + songIndex;
                for chartID in ['n7', 'h7', 'a7', 'n14', 'h14', 'a14']:
                    try:
                        pwResult = PW_ReadChartScorePage_IIDX(pwOpener, songID, chartID, mixIIDX, pwOutputDir)
                    except:
                        pwResult = 0
                        songIDsSkipped = songIDsSkipped + 1
                    finally:
                        digestFID.write('{0:05n},{1:s},{2:n}\n'.format(songID, chartID, pwResult))
                        songsRead = songsRead + pwResult
            digestFID.flush()
    else:
        for songID in songIDList:
            for chartID in ['n7', 'h7', 'a7', 'n14', 'h14', 'a14']:
                try:
                    pwResult = PW_ReadChartScorePage_IIDX(pwOpener, int(songID), chartID, mixIIDX, pwOutputDir)
                except:
                    pwResult = 0
                    songIDsSkipped = songIDsSkipped + 1
                finally:
                    digestFID.write('{0:05n},{1:s},{2:n}\n'.format(int(songID), chartID, pwResult))
                    songsRead = songsRead + pwResult
        digestFID.flush()

    digestFID.close()

    return songsRead

def PW_ReadChartScorePage_DDR(pwOpener, songID, chartDifficulty, mixDDR=15, pwOutputDir=r"C:\Temp\PW_DDR"):
    try:
        pwFilename = 'https://programmedworld.net/ddr/{2:n}/music/{0:n}/{1:s}'.format(songID, chartDifficulty, mixDDR)
        pwFID = pwOpener.open(pwFilename)
        songPage = pwFID.read()
    except:
        print "Unable to locate: song ID {0:05n}, chart {1:s}".format(songID, chartDifficulty)
        return 0
    else:
        localFilename = (pwOutputDir + '\\DDR{2:n}\\{0:05n}_{1:s}.html').format(songID, chartDifficulty, mixDDR)
        try:
            os.makedirs( (pwOutputDir + '\\DDR{2:n}').format(songID, chartDifficulty, mixDDR) )
        except:
            pass
        finally:
            localFID = open(localFilename, 'w')
            localFID.write(songPage)
            localFID.close()

            print("Successfully read song ID {0:05n}, chart {1:s}!".format(songID, chartDifficulty))
            return 1

def PW_BruteForce_DDR(pwOpener, mixDDR=15, pwOutputDir=r"C:\Temp\PW_DDR"):
    songsRead = 0;

    try:
        digestFID = open(pwOutputDir + "\\readResults.csv", 'w')
    except:
        os.makedirs(pwOutputDir)
        digestFID = open(pwOutputDir + "\\readResults.csv", 'w')

    for songID in range(1000):
        for chartID in ['sp-bas', 'sp-dif', 'sp-exp', 'sp-cha',
                'dp-bas', 'dp-dif', 'dp-exp', 'dp-cha']:
            try:
                pwResult = PW_ReadChartScorePage_DDR(pwOpener, songID, chartID, mixDDR, pwOutputDir)
            except:
                pwResult = 0
            finally:
                digestFID.write('{0:05n},{1:s},{2:n}\n'.format(songID, chartID, pwResult))
                songsRead = songsRead + pwResult
        digestFID.flush()

    digestFID.close()

    return songsRead