rekvijem

Language Detection using gzip

Jan 2nd, 2014
67
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 2.02 KB | None | 0 0
  1. import gzip
  2. import glob
  3. import os
  4.  
  5. #Programmed by Glob/Rekvijem/Catalyst
  6.  
  7. base = os.getcwd()
  8.  
  9. def findLanguage(text):
  10.     #Get language files
  11.     os.chdir(base+'/LanguageFiles')
  12.     languagefiles = glob.glob('*.txt')
  13.  
  14.     #Get the sizes of the gzipped language files
  15.     base_sizes = {}
  16.     for lang in languagefiles:
  17.         #Make a gzipped version of the file if it doesn't exist
  18.         if not os.path.exists(lang+'.gz'):
  19.             i = open(lang,'rb')
  20.             o = gzip.open(lang+'.gz','wb')
  21.             o.writelines(i)
  22.             i.close()
  23.             o.close()
  24.        
  25.         #Get the size of the file and append to dictionary
  26.         name = lang.split('.')[0]
  27.         base_sizes[name] = os.path.getsize(lang+'.gz')
  28.  
  29.     #Get the sizes of the gzipped language files with the text to be identified
  30.     new_sizes = {}
  31.     for lang in languagefiles:
  32.         #Make a temporary file holding the gzipped data
  33.         i = open(lang,'rb')
  34.         o = gzip.open('temp.gz','wb')
  35.         o.write(i.read()+text)
  36.         o.close()
  37.         i.close()
  38.        
  39.         #Get the size of the file
  40.         name = lang.split('.')[0]
  41.         new_sizes[name] = os.path.getsize('temp.gz')
  42.    
  43.         #Close the temporary file
  44.         os.remove('temp.gz')
  45.  
  46.     #Calculate the difference in bytes of each file
  47.     difference = {key : new_sizes[key] - base_sizes[key] for key in base_sizes.keys()}
  48.    
  49.     #Get the language(languages in a rare rare rare case) with the largest difference
  50.     dif = min(difference.values())
  51.     language = [k for k,v in difference.iteritems() if v == dif]
  52.    
  53.     #Return the language(s)
  54.     return language
  55.  
  56.     #Alternatively return a string if only one language found:
  57.     #return language[0] if len(language) == 1 else language
  58. print findLanguage("In friendship diminution instrument so. Son sure paid door with say them. Two among sir sorry men court.")
  59. print findLanguage("Gut gassen durren nun wirtin weg art hinaus neckte. Davon ri zu gutes haute in zahne wesen waren. Ja drehte arbeit wo la ja kugeln.")
  60. print findLanguage("Terrasse et ai etrangle te bataille dissiper musiques bravoure. Continue preparer pressait que etroites empecher habitent oui. Vit ere infiniment descendons simplement eux.")
Advertisement
Add Comment
Please, Sign In to add comment