Guest User

Master data.lst Updater v2

a guest
May 29th, 2015
545
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #MASTER data.lst update tool. Works for all Falcom PSP games!
  2. #Tagline: All Falcom games. All files. One tool.
  3. #v2 - Add UMDgen file list update
  4. #v1.1 Fix regarding root directory data.lst file
  5. #v1 - Initial version
  6.  
  7. #How to use
  8. #1) Create your "production folder" with a dump from UMDgen (or other tool) with all
  9. #   game files
  10. #2) Modify files as needed
  11. #3) Drop exported filelist from UMDgen with filename 'filelist.txt'
  12. #4) Drop this tool into the USRDIR folder and run
  13. #5) Insert modified data.lst with UMDgen and other files, import new file list, and test!
  14.  
  15. #Non-basic uses
  16. #You can run the script into interactive mode from the command prompt:
  17. #Python -i script.py
  18. #Or run from IDLE
  19. #If you do this, after the program is finished running:
  20. #Type updatelist to see a list of the filenames whose sizes were updated
  21. #Type missingfiles to see a list of missing files
  22. #Type LBAfiles to see a list of files that required LBA expansion
  23.  
  24. #Limitations
  25. #1) Can't modify LBAs for EBOOT.BIN
  26.  
  27. import os
  28. import struct
  29.  
  30. dirname = 'Updated data.lst'        #Directory into which the updated data.lst goes
  31.  
  32. def get_data(filename):     #Gets data
  33.     totalbytes = os.path.getsize(filename)
  34.     infile = open(filename, 'rb')
  35.     totalfiledata = infile.read(totalbytes)
  36.     infile.close()
  37.     return totalfiledata
  38.  
  39. def ensure_dir(directory):  #Check if directory exists and create if necessary
  40.     if not os.path.exists(directory):
  41.         os.mkdir(directory)
  42.  
  43. def replacestr(origstr,replacestr,startpos,replacelen):
  44.     '''Returns a string with a replaced sub-string'''
  45. #origstr - the original string, replacestr = the string to replace
  46. #startpos - where the replacement string should go
  47. #replacelen - how many characters of the original string to replace
  48.     return origstr[:startpos] + replacestr + origstr[startpos+replacelen:]
  49.  
  50.  
  51. #Initialize extension dictionary
  52. #Extension table is found in data.lst at the beginning
  53. #Any number of extensions up to 3 characters long separated by nulls
  54. ExtDict = {}                
  55. i = 0
  56. with open('data.lst','rb') as f:
  57.     while True:
  58.         Ext = f.read(4)
  59.         if Ext == '\x00'*4:
  60.             break
  61.         while Ext[-1] == '\x00':
  62.             Ext = Ext[:-1]
  63.         ExtDict[i] = Ext
  64.         i += 1
  65.  
  66. #My try at object-oriented programming
  67. class DataListLine:
  68.     def __init__(self, f, pos):         #Reads a 'data line' from data.lst
  69.         self.Addr = pos
  70.         origpos = f.tell()
  71.         f.seek(pos)                     #Move to start of line
  72.         self.Name = f.read(8)           #Get the name
  73.         while self.Name[-1] == '\x00':  
  74.             self.Name = self.Name[:-1]
  75.         self.Size = struct.unpack('<I',f.read(4))[0]    #Get the file size
  76.         self.LBA = struct.unpack('<I',f.read(3) + '\x00')[0]    #Get the LBA
  77.         self.ExtByte = ord(f.read(1))                   #Get the extension byte
  78.         self.IsFolder = False           #Folders have 00 as their ext. byte
  79.         if self.ExtByte == 0:
  80.             self.IsFolder = True
  81.         self.Ext = '.' + ExtDict[self.ExtByte]  #Get the ext. name from the dictionary
  82.         self.FileName = self.Name + self.Ext    #The filename is name + ext
  83.         f.seek(origpos)
  84.     def binary(self):                   #Converts object data back into a line for data.lst
  85.         return self.Name + '\x00'*(8-len(self.Name)) + \
  86.                struct.pack('<I',self.Size) + \
  87.                struct.pack('<I',self.LBA)[:3] + chr(self.ExtByte)
  88.  
  89. class FileListLine:                     #For files in the UMD but not present in data.lst
  90.     def __init__(self, LBA, PathName):
  91.         self.LBA = LBA
  92.         self.PathName = PathName
  93.         try:
  94.             self.Size = os.path.getsize(PathName)
  95.         except WindowsError:
  96.             self.Size = 0
  97.  
  98. #Wow, so many variables
  99. pos = 0x410         #A magic number. The first line in data.lst.
  100. updates = 0         #Keep track of # of updates to data.lst sizes
  101. updatelist = []     #Keep track of filenames whose size was updated
  102. #Names and counts is a little more complicated
  103. #When the program gets to a folder entry, it pushes the name of the folder onto
  104. #the name stack (to capture path name information) and pushes the number of
  105. #files + folders in that folder (incl. subfolders) onto the counts stack.
  106. #Each time the program processes a line, it subtracts 1 from each number on the
  107. #counts stack. After this, new values are pushed onto the stack if the current
  108. #line is a folder.
  109. #If the last entry on the counts stack is 0, then that counts entry and the
  110. #corresponding name are popped off of each stack.
  111. #I hope that made some sense
  112. names = []
  113. counts = []
  114. lines = []          #For storage of line data for writing a new file later
  115. missingfiles = []   #Keep track of files that couldn't be updated because they
  116.                     #are missing
  117. with open('data.lst','rb') as f:
  118.     datalstbase = f.read(0x410)                 #For making a new file later
  119.     while pos < os.path.getsize('data.lst'):    #Main loop
  120.         line = DataListLine(f, pos)             #Initialize line data
  121.         pos += 0x10                             #Increment position counter
  122.         #Has to be done in this order-
  123.         #Update counts, process line, pop counts (if applicable)
  124.         #Folder counts include subfolders, and exclude the line itself is on
  125.         for i, count in enumerate(counts):      #Subtract one from each count
  126.             counts[i] -= 1                      #on the counts stack
  127.         if line.IsFolder:                       #If current entry is a folder
  128.             counts.append(line.Size)            #push new values to the count
  129.             names.append(line.Name)             #and name stacks
  130.         else:                                   #Current entry is a file
  131.             filename = ''
  132.             if len(names) > 0:                  #Form the file path
  133.                 line.PathName = '\\'.join(names) + '\\'
  134.             else:
  135.                 line.PathName = ''
  136.             line.PathName += line.FileName
  137. ##            print line.PathName             #Uncomment for printing
  138. #os.path.getsize can fail if the file doesn't exist (WindowsError in that case)
  139.             try:
  140.                 if os.path.getsize(line.PathName) != line.Size:  #Update required
  141.                     updates += 1                            #Increment counter
  142.                     updatelist.append(line.PathName)        #Add to list of updated entries
  143.                     line.Size = os.path.getsize(line.PathName)   #Update object with new size
  144.             except WindowsError:                            #File doesn't exist
  145.                 missingfiles.append(line.PathName)          #Update list of missing files
  146.         #Has to be done in this order
  147.         while len(counts) != 0:     #As long as the counts stack isn't empty
  148.             if counts[-1] != 0:     #and the last entry isn't zero, pop off the
  149.                 break               #last count and name entries
  150.             if counts[-1] == 0:
  151.                 del names[-1]
  152.                 del counts[-1]
  153.         lines.append(line)
  154.  
  155. #Updating LBAs portion of the program
  156. if os.path.isfile('filelist.txt'):  #Original file list
  157.     #Sort "file lines" (not folder lines) by LBA
  158.     filelines = sorted([l for l in lines if l.IsFolder == False], key= lambda x: x.LBA)
  159.     #Read original filelist file
  160.     filelistheader = ''                 #Read header from file list
  161.     flag = True                         #For marking the header
  162.     pathlist = [x.PathName for x in filelines]
  163.     filelistlines = []                  #Container for file list line objects
  164.     with open('filelist.txt') as f:
  165.         for line in f:
  166.             if 'USRDIR' in line:        #Break on first USRDIR line
  167.                 flag = False
  168.             if flag:
  169.                 filelistheader += line  #Add line to header
  170.             else:                       #Determine if line is in data.lst or not
  171.                 line = line.translate(None,'\r\n')
  172.                 line = line.split(' , ')
  173.                 #If the line is not in data.lst, create FileListLine object for it
  174.                 if line[1][17:] not in pathlist:    #Removes '\PSP_GAME\USRDIR\' text
  175.                     filelistlines.append(FileListLine(int(line[0]),line[1][17:]))
  176.     #Integrate and sort lines by LBA
  177.     filelines = sorted(filelines + filelistlines, key= lambda x: x.LBA)
  178.     firstpass = True
  179.     LBAfiles = []   #List of files that needed updated LBAs
  180.     LBAcount = 0    #Count of files that needed updated LBAs
  181.     for i in range(len(filelines)):     #Loop and...
  182.         if firstpass:                   #From the 2nd pass onward...
  183.             firstpass = False
  184.         else:
  185.             avail_blocks = (filelines[i].LBA - filelines[i-1].LBA) / 0x10   #Blocks available based on LBAs
  186.             if filelines[i-1].Size % (2048*0x10) != 0:
  187.                 required_blocks = filelines[i-1].Size / (2048*0x10) + 1    
  188.             else:
  189.                 required_blocks = filelines[i-1].Size / (2048*0x10)     #Blocks needed based on file size
  190.             if required_blocks > avail_blocks:                          #Insufficient blocks
  191.                 LBAcount += 1                                           #Increment counter
  192.                 LBAfiles.append(filelines[i-1].PathName)                #Add name of file needing LBAs
  193.                 #User messages
  194.                 print 'Insufficient LBAs at position {}, {}.'.format(hex(filelines[i-1].Addr),filelines[i-1].FileName)
  195.                 print 'LBA: {}, Required blocks {}, available blocks {}.'.format(filelines[i-1].LBA, required_blocks, avail_blocks)
  196.                 for j, l in enumerate(filelines):   #Update LBAs for current file and everything after it
  197.                     if j >= i:
  198.                         l.LBA += (required_blocks - avail_blocks) * 0x10
  199.     ##        In case you want to know about files that have too many LBAs
  200.     ##        elif required_blocks < avail_blocks:
  201.     ##            print 'Extra LBAs at position {}, {}.'.format(hex(filelines[i-1].Addr),filelines[i-1].FileName)
  202.     ##            print 'Required blocks {}, available blocks {}.'.format(required_blocks, avail_blocks)
  203.            
  204.     ensure_dir(dirname)                 #Create directory if it doesn't exist
  205.     with open(dirname + '\\filelist.txt','w') as f: #Open new file list file for writing
  206.         f.write(filelistheader)                     #Write the header
  207.         #File lines (not folder lines) sorted by LBA
  208.         for line in sorted([l for l in lines if l.IsFolder == False] + filelistlines, key= lambda x: x.LBA):
  209.             f.write('{:0>7d}'.format(line.LBA))     #The LBA, zero-padded to 7 digits
  210.             f.write(' , \\PSP_GAME\\USRDIR\\')      #Separator
  211.             f.write(line.PathName + '\n') #Path and file name
  212.     print 'Updated {} entries in data.lst for additional LBAs.'.format(LBAcount)
  213. else:
  214.     print 'filelist.txt not found in folder. LBAs not updated.'
  215.    
  216. ensure_dir(dirname)                 #Create directory if it doesn't exist
  217. with open(dirname + '\\data.lst','wb') as f:
  218.     f.write(datalstbase)            #Write first 0x410 of file
  219.     for line in lines:              #Write data for each line
  220.         f.write(line.binary())
  221. #User messages
  222. print 'Updated {} entries in data.lst for file size.'.format(updates)
  223. print "There were {} missing files that couldn't be updated.".format(len(missingfiles))
RAW Paste Data