Guest User

id_helper.py

a guest
Jul 4th, 2013
74
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. # walk through all config files and get a list of ids with collisions
  2. # fix them
  3.  
  4. # author michael clark
  5. # date June 21 2013
  6. # licence, GNUv3
  7. # version 3
  8. import os, re
  9. import numpy as np
  10. from shutil import move
  11. from os import remove, close
  12. from tempfile import mkstemp
  13. import itertools
  14. #import raw_input
  15. import random
  16.  
  17. i=re.compile(r'[IB]:(.+)=(\d+)') # use this to find numbers with regular expressions
  18.  
  19. MAX_ID=31743+1 # add one for pythons range function
  20. ids=dict() # the id's we find
  21. collisions=dict() # list of collisions we find
  22. configdir=os.path.join(os.path.abspath(os.curdir),'config')
  23. IGNORE_IF_SAME_CONFIG_FILE=True
  24. mark_item_section=['id','item']
  25. def used_ids(ids):
  26.     return list(sorted(ids.keys()))
  27.  
  28. def free_ids(ids):
  29.     return list(set(range(159, MAX_ID)) - set(ids.keys()))
  30.  
  31. def collision_ids(collisions):
  32.     return list(sorted(collisions.keys()))
  33.  
  34. def main():
  35.     # run one time to get the list of ids and collisions
  36.     # walk the files in the config directory
  37.    
  38.     for root, dirs, files in os.walk(configdir):
  39.         files = [os.path.join(root, f) for f in files]
  40.         for f in files: # look in each file
  41.             ff=open(f,'r')
  42.             first_line=ff.readline()
  43.             if not "# Configuration file" in first_line:
  44.                 continue # skip this file
  45.             l=0
  46.             cur_ids=[]
  47.             in_items=False
  48.             for line in ff: # look at each line of the file
  49.                 ls=line.strip()
  50.                 l+=1
  51.                 if '#' == line.strip()[:1]: # skip as its a comment
  52.                     continue
  53.                 if (('item' in ls) or ('id' in ls)) and ('{' in ls):
  54.                     in_items=True
  55.                     continue
  56.                 if '}' in ls :
  57.                     in_items=False
  58.                     continue
  59.                 if in_items:
  60.                     if ('I:' in line) or ('B:' in line):
  61.                         id=get_id(f,l,line.strip()) # also records the id into ids and collisions
  62.                         if id: cur_ids.append(id)
  63.             if len(cur_ids)>0: print f, " contained ids:", str_ranges(list(set(cur_ids))), "\n"
  64.             ff.close()
  65.    
  66.     # now write to command line, and a csv file
  67.     write_to_stdout()
  68.     write_to_csv()
  69.    
  70.     # now interactivly fix each one
  71.     print ("\nFound {} conflicts, interactivly fixing now:".format(len(collisions.keys())))
  72.  
  73.     for key in sorted(collisions.keys()):
  74.         for collision in collisions[key][1:]: # ignore the first which is the first in first served
  75.             fix(key,collision)
  76.  
  77. def fix(key,collision):
  78.     # To autofix collisions: uncomment this
  79.     # watch out this might change other numbers in your config!
  80.     fixed=False
  81.     r=0
  82.     fs=collision[0]
  83.     while not fixed:
  84.         new_id=n_free_block(key,r) # get a free block nearby
  85.         q=''
  86.         q+="\nFix (y,n,r(Reroll))?\n {}->{}\t in file {} for line: \n{}\nAll collisions:".format(key,new_id,fs,collision[2])
  87.         for c in collisions[key]:
  88.             q+="\n    file:{},line:{}".format(c[0],c[2])
  89.         q+="\n:"
  90.         a = raw_input(q)
  91.         if a=='y':
  92.             print os.path.join(os.curdir,collision[0])
  93.             replace(os.path.join(os.curdir,collision[0]), str(key), str(new_id) )
  94.             ids[new_id]=collision
  95.             fixed=True
  96.             print "fixed collision ({}->{}) at {} line {}".format(key,new_id,fs,collision[1])
  97.         elif a=='n':
  98.             return 0
  99.         elif a=='r':
  100.             r+=1
  101.             pass
  102.         else:
  103.             print "User input was not a y,n, or r. You need to enter one of these."
  104.  
  105. def n_free_block(old_id,n=0):
  106.     "Return a nearest free block"
  107.     freeids=free_ids(ids)
  108.     s_fids=[(i,np.abs(i-old_id)) for i in freeids]
  109.     dtype=[('id', int), ('distance', int)]
  110.     a = np.array(s_fids, dtype=dtype)
  111.     a = np.sort(a,order=['distance','id']) # now we have an array of free ids sorted by how close they are to our old id
  112.     free_id=a[n][0]
  113.     while free_id in ids.keys(): # check it worked, if not make a new one
  114.         n+=1
  115.         free_id=a[n][0]
  116.     return free_id
  117.  
  118. def get_id(f,l,line):
  119.     "this will grab an id from a line and check it against the ids we have found"
  120.     #fs=f.replace(configdir,'')
  121.     fs=os.path.relpath(f)
  122.     if i.search(line):
  123.         try:
  124.             item,id=i.search(line).groups()
  125.         except:
  126.             print "could not parse ", line
  127.             return ''
  128.         #print id, item
  129.         id=int(id) # convert to int
  130.         if id in range(159, MAX_ID):
  131.             if id in ids.keys():
  132.                 if ids[id][3]==item: # if they just repeat a previous statement
  133.                     ids[id]=[fs,l,line,item]
  134.                 elif IGNORE_IF_SAME_CONFIG_FILE and (ids[id][0]==fs):
  135.                     ids[id]=[fs,l,line,item] # if they come from the same config file
  136.                 else:
  137.                     if not id in collisions.keys(): # if it doesn't exist add the previous config that registered this id
  138.                         collisions[id]=[ids[id]]
  139.                     collisions[id].append([fs,l,line,item]) # add the latest conflict
  140.             else:
  141.                 ids[id]=[fs,l,line,item] # remember the id registration
  142.         return id
  143.          
  144. def write_to_stdout():
  145.     print collisions
  146.     # print results to standard out      
  147.     for key in collision_ids(collisions):
  148.         print "Collisions for id {}:".format(key)
  149.         for f,l,line,item in collisions[key]:
  150.             print "    {}\t at line#: {} in file: \t{}".format(line,l,f)
  151.         print '\n'
  152.     print "Found these keys:  \t", str_ranges(used_ids(ids)), '\n'
  153.     print "Free keys are:  \t", str_ranges(free_ids(ids)), '\n'
  154.     print "Please wait a moment..."
  155.  
  156. def write_to_csv():
  157.     # write output to here
  158.     output=os.path.join(os.curdir,'id_help_py_report.csv')
  159.     o=open(output,'w')
  160.    
  161.     # cache this
  162.     usedids=used_ids(ids)
  163.     collisions_keys=sorted(collisions.keys())
  164.     ids_keys=ids.keys()
  165.    
  166.     # write to output
  167.     o.write('ID table\nID, Used?, In Collision?, file, line#\n')
  168.     outstr=''
  169.     for key in range(159, MAX_ID):
  170.         if key in ids_keys:
  171.             f,l,line,item = ids[key]
  172.         else:
  173.             f,l,line,item='',-999,'',''
  174.         outstr+='{},{},{},{},{},{}\n'.format(key, 1*(key in usedids),1*(key in collisions_keys),f,l,line)
  175.     o.write(outstr)
  176.    
  177.     outstr=''
  178.     for key in collision_ids(collisions):
  179.         outstr+="\nCollisions for id {}: (id,line,line$,file)\n".format(key)
  180.         for f,l,line,item in collisions[key]:
  181.             outstr+=" {},{},{},{}\n".format(key,line,l,f )
  182.         outstr+='\n'
  183.     o.write(outstr)
  184.     o.close()
  185.    
  186.     print "\nOutput writen as csv's to", output
  187.    
  188. def replace(file_path, pattern, subst):
  189.     "Replace a string in a file from http://stackoverflow.com/questions/39086/search-and-replace-a-line-in-a-file-in-python"
  190.     #Create temp file
  191.     fh, abs_path = mkstemp()
  192.     new_file = open(abs_path,'w')
  193.     old_file = open(file_path)
  194.     done=False
  195.     for line in old_file:
  196.         if (pattern in line) and not done:
  197.             new_file.write(line.replace(pattern, subst))
  198.             done=True
  199.         else:
  200.             new_file.write(line.replace(pattern, subst))
  201.     #close temp file
  202.     new_file.close()
  203.     close(fh)
  204.     old_file.close()
  205.     #Remove original file
  206.     remove(file_path)
  207.     #Move new file
  208.     move(abs_path, file_path)
  209.  
  210. def ranges(i):
  211.     "Return a list of tuples summarising range of numbers. E.g [1,2,3,6,7,9] would be [(1,3),(6,7),(9,9)]. From stackoverflow"
  212.     for a, b in itertools.groupby(enumerate(i), lambda (x, y): y - x):
  213.         b = list(b)
  214.         yield b[0][1], b[-1][1]
  215.  
  216. def str_ranges(i):
  217.     "Return a string summarising range of numbers. E.g [1,2,3,6,7,9,13,14,14] would be '1-3,6-7,9,1-14,14'."
  218.     out=''
  219.     rng=list(ranges(i))
  220.     for a,b in rng:
  221.         if a==b:
  222.             out+="{}, ".format(a)
  223.         else:
  224.             out+="{}-{}, ".format(a,b)
  225.     return out
  226.  
  227. if __name__ == "__main__":
  228.    
  229.     main()
RAW Paste Data