Don't like ads? PRO users don't see any ads ;-)
Public Pastes
Guest

Untitled

By: a guest on May 5th, 2012  |  syntax: None  |  size: 7.66 KB  |  hits: 11  |  expires: Never
download  |  raw  |  embed  |  report abuse  |  print
Text below is selected. Please press Ctrl+C to copy to your clipboard. (⌘+C on Mac)
  1. from tvchecker.models import *
  2. from xml.etree import ElementTree as et
  3. import urllib
  4. import zipfile
  5. import os
  6.  
  7. MIRROR = "http://www.thetvdb.com/"
  8. API_KEY = "4E5D5C8EFC4175A1"
  9. REMOVE_OLD = False
  10.  
  11. def getIdFor(name):
  12.     print "Retrieving info about %s" % name
  13.     filehandle = urllib.urlopen('%sapi/GetSeries.php?seriesname=%s' % (MIRROR, name.replace('&', '')))
  14.     tvxml = et.fromstring(filehandle.read())
  15.     series = tvxml.find("Series")
  16.     serName = series.findtext("SeriesName")
  17.     print "Found %s" % serName
  18.     if serName == name:
  19.         sId = int(series.findtext("seriesid"))
  20.         print 'Id = %d' % sId
  21.     else:
  22.         print 'Id not found'
  23.         return None
  24.     return sId
  25.    
  26. def getInfoFile(ser_id):
  27.     d = '%s/seriesZip' % os.getcwd()
  28.     if not os.path.exists(d):
  29.         print "Creating %s" % d
  30.         os.makedirs(d)
  31.     if ser_id:
  32.         print "Retrieving zipfile for %s" % ser_id
  33.         tvdbZip = "seriesZip/%s.zip" % ser_id
  34.         folder = "seriesZip/%s" % ser_id
  35.         info = "%s/en.xml" % folder
  36.         #print "Checking %s/en.xml" % folder
  37.         if os.path.isfile(info) and REMOVE_OLD:
  38.             print "Removing old files"
  39.             if os.path.isfile(tvdbZip):
  40.                 print "Removing old zipfile from %s" % tvdbZip
  41.                 os.remove(tvdbZip)
  42.             if os.path.exists(folder):
  43.                 print "Removing contents from %s" % folder
  44.                 for the_file in os.listdir(folder):
  45.                     file_path = os.path.join(folder, the_file)
  46.                     print "Removing %s" % file_path
  47.                     try:
  48.                         if os.path.isfile(file_path):
  49.                             os.unlink(file_path)
  50.                     except Exception, e:
  51.                         print e
  52.             else:
  53.                 os.makedirs(folder)
  54.         if not os.path.isfile(info):
  55.             urllib.urlretrieve('http://www.thetvdb.com/api/%s/series/%d/all/en.zip' % (API_KEY, ser_id), tvdbZip)
  56.             if os.path.isfile(tvdbZip):
  57.                 print "Extracting files from %s" % tvdbZip
  58.                 z = zipfile.ZipFile(tvdbZip, 'r')
  59.                 z.extractall(folder)
  60.         else:
  61.             print "found en.xml"
  62.         return info
  63.        
  64. def getLanguage_id(lang):
  65.     old = Language.query.filter_by(caption=lang).first()
  66.     if old:
  67.         return old
  68.     newLang = Language(lang)
  69.     db.session.add(newLang)
  70.     return newLang
  71.    
  72. def getGenre_id(genre):
  73.     old = Genre.query.filter_by(caption=genre).first()
  74.     if old:
  75.         return old
  76.     newGen = Genre(genre)
  77.     db.session.add(newGen)
  78.     #db.session.commit()
  79.     return newGen
  80.    
  81. def getGenres(genre):
  82.     genres = []
  83.     for gen in genre.split('|'):
  84.         if gen:
  85.             genres.append(getGenre_id(gen))
  86.     return genres
  87.    
  88. def getAllInfo(series):
  89.     if series:
  90.         print "Retrieving all info about %d" % series.tvdb_id
  91.         info = getInfoFile(series.tvdb_id)
  92.         if os.path.isfile(info):
  93.             print "Parsing en.xml"
  94.             infoFile = open(info, "r")
  95.             # -----------------------
  96.             tvxml = et.fromstring(infoFile.read())
  97.             header = tvxml.find("Series")
  98.             # = = =
  99.             overview = header.findtext("Overview")
  100.             if overview:
  101.                 print "Saving overview.."
  102.                 series.overview = overview
  103.             # = = =
  104.             lang = header.findtext("Language")
  105.             if lang:
  106.                 print "Saving language.."
  107.                 series.lang_id = getLanguage_id(lang)
  108.             # = = =
  109.             first_air = header.findtext("FirstAired")
  110.             if first_air:
  111.                 print "Saving first aired.."
  112.                 series.first_aired = first_air.replace('-', '')
  113.             # = = =
  114.             air_dow = header.findtext("Airs_DayOfWeek")
  115.             if air_dow:
  116.                 print "Saving airs day of week.."
  117.                 series.airs_dow = air_dow[:2]
  118.             # = = =
  119.             air_time = header.findtext("Airs_Time")
  120.             if air_time:
  121.                 print "Saving air time.."
  122.                 series.airs_time = air_time.replace(' ', '')
  123.             # = = =
  124.             genre = header.findtext("Genre")
  125.             if genre:
  126.                 print "Saving genres.."
  127.                 series.genre_id = getGenres(genre)
  128.             # -----------------------
  129.             infoFile.close()
  130.  
  131. print "Checking series for missing info"
  132. series = Series.query.all()
  133. for s in series:
  134.     print "Checking %s" % s.name
  135.     if not s.tvdb_id:
  136.         print "---------------------------------------"
  137.         print "Missing id for - %s" % s.name
  138.         newId = getIdFor(s.name)
  139.         if newId:
  140.             s.tvdb_id = newId
  141.             db.session.commit()
  142.     if (s.genre_id is None) and s.tvdb_id:
  143.         print "---------------------------------------"
  144.         print "Missing overview - %s" % s.name
  145.         getAllInfo(s)
  146.         print "Committing..."
  147.         db.session.commit()
  148.  
  149. -----------------------------------------------------------------------
  150. ERRORS:
  151.  
  152. Missing overview - Dexter
  153. Retrieving all info about 79349
  154. Retrieving zipfile for 79349
  155. found en.xml
  156. Parsing en.xml
  157. Saving overview..
  158. Saving language..
  159. Saving first aired..
  160. Saving airs day of week..
  161. Saving air time..
  162. Saving genres..
  163. Committing...
  164. Traceback (most recent call last):
  165.   File "checker.py", line 147, in <module>
  166.     db.session.commit()
  167.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/scoping.py", line 114, in do
  168.     return getattr(self.registry(), name)(*args, **kwargs)
  169.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 655, in commit
  170.     self.transaction.commit()
  171.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 313, in commit
  172.     self._prepare_impl()
  173.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 297, in _prepare_impl
  174.     self.session.flush()
  175.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1587, in flush
  176.     self._flush(objects)
  177.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1658, in _flush
  178.     flush_context.execute()
  179.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 331, in execute
  180.     rec.execute(self)
  181.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 475, in execute
  182.     uow
  183.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 59, in save_obj
  184.     mapper, table, update)
  185.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 485, in _emit_update_statements
  186.     execute(statement, params)
  187.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1450, in execute
  188.     params)
  189.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1583, in _execute_clauseelement
  190.     compiled_sql, distilled_params
  191.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1697, in _execute_context
  192.     context)
  193.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1690, in _execute_context
  194.     context)
  195.   File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 335, in do_execute
  196.     cursor.execute(statement, parameters)
  197.   File "/usr/lib/pymodules/python2.7/MySQLdb/cursors.py", line 166, in execute
  198.     self.errorhandler(self, exc, value)
  199.   File "/usr/lib/pymodules/python2.7/MySQLdb/connections.py", line 35, in defaulterrorhandler
  200.     raise errorclass, errorvalue
  201. sqlalchemy.exc.OperationalError
  202. furby@furby-VirtualBox:~/Flask/tvchecker-project$
create a new version of this paste RAW Paste Data